diff --git a/.coderabbit.yaml b/.coderabbit.yaml deleted file mode 100644 index 160bda5f0f6..00000000000 --- a/.coderabbit.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# yaml-language-server: $schema=https://coderabbit.ai/integrations/schema.v2.json -language: "en-US" - -# Only comment on Critical/Major bugs. No Minor, Trivial, or style comments. -tone_instructions: "Only comment on Critical or Major bugs. Never comment on Minor issues, style, refactoring, or suggestions. When in doubt, stay silent." - -reviews: - # Use chill profile - filters out nitpicks automatically - profile: "chill" - - # Disable all summary features - high_level_summary: false - high_level_summary_in_walkthrough: false - - # Disable walkthrough comment entirely - collapse_walkthrough: true - changed_files_summary: false - sequence_diagrams: false - - # Disable status/effort estimates - review_status: false - commit_status: false - estimate_code_review_effort: false - - # Disable auto-suggestions for labels/reviewers - suggested_labels: false - suggested_reviewers: false - - # Disable related issues/PRs lookup - assess_linked_issues: false - related_issues: false - related_prs: false - - # Auto-review disabled - only review when explicitly requested via @coderabbitai review - auto_review: - enabled: false - -chat: - auto_reply: true diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 1e35e0c496b..00000000000 --- a/.flake8 +++ /dev/null @@ -1,4 +0,0 @@ -[flake8] -max-line-length = 100 -extend-ignore = E203,E501,F401,E402,E714 -per-file-ignores = __init__.py:F401 \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 9662160da10..a4fce9a17a7 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -9,7 +9,7 @@ assignees: '' **Describe the bug** -A clear and concise description of what the bug is. Tag the [@mcore-oncall](https://github.com/orgs/NVIDIA/teams/mcore-oncall) +A clear and concise description of what the bug is. Tag @NVIDIA/mcore-oncall to get oncall's attention to this issue. **Steps/Code to reproduce bug** @@ -26,4 +26,4 @@ A clear and concise description of what you expected to happen. **Additional context** -Add any other context about the problem here. +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index b0da6789a8e..329b7292949 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -10,7 +10,7 @@ assignees: '' **Is your feature request related to a problem? Please describe.** A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] -Tag the [@mcore-oncall](https://github.com/orgs/NVIDIA/teams/mcore-oncall) +Tag @NVIDIA/mcore-oncall to get oncall's attention to this issue. **Describe the solution you'd like** diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md index 899ff44d6a6..f4a95b16c77 100644 --- a/.github/ISSUE_TEMPLATE/question.md +++ b/.github/ISSUE_TEMPLATE/question.md @@ -9,5 +9,5 @@ assignees: '' --- **Your question** -Ask a clear and concise question about Megatron-LM. Tag the [@mcore-oncall](https://github.com/orgs/NVIDIA/teams/mcore-oncall) -to get oncall's attention to this issue. \ No newline at end of file +Ask a clear and concise question about Megatron-LM. Tag @NVIDIA/mcore-oncall +to get oncall's attention to this issue. diff --git a/.github/ISSUE_TEMPLATE/regression.md b/.github/ISSUE_TEMPLATE/regression.md index 180db633cb8..0e0a34fddc6 100644 --- a/.github/ISSUE_TEMPLATE/regression.md +++ b/.github/ISSUE_TEMPLATE/regression.md @@ -8,7 +8,7 @@ assignees: '' --- **Describe the regression** -A clear and concise description of what the regression is. Tag the [@mcore-oncall](https://github.com/orgs/NVIDIA/teams/mcore-oncall) +A clear and concise description of what the regression is. Tag @NVIDIA/mcore-oncall to get oncall's attention to this issue. **To Reproduce** diff --git a/.github/actions/action.yml b/.github/actions/action.yml index 18ffa50ac9a..b4261cf1ce9 100644 --- a/.github/actions/action.yml +++ b/.github/actions/action.yml @@ -65,6 +65,10 @@ inputs: description: "Trigger cadence for cadence filter (pr|nightly|mergegroup). Empty disables filter." required: false default: "" + sha: + description: "Git ref to check out. Must match the SHA used by the upstream parse step so recipes don't diverge between scheduling and execution." + required: false + default: "" runs: using: "composite" steps: @@ -74,10 +78,20 @@ runs: - name: Checkout repository uses: actions/checkout@v6 + with: + ref: ${{ inputs.sha }} - name: Change ownership of /home/runner/ shell: bash - run: sudo chown -R $(whoami) /home/runner/ + # Tolerate vanishing `.git/objects/pack/.tmp-*` files: the prior + # `actions/checkout` may leave a background `git gc --auto` running, + # whose `git pack-objects` renames/deletes temp files while `chown` + # is walking the tree. On failure, wait 5 s for gc to settle, retry, + # then succeed unconditionally. + run: | + sudo chown -R $(whoami) /home/runner/ 2>/dev/null && exit 0 + sleep 5 + sudo chown -R $(whoami) /home/runner/ 2>/dev/null || true - name: Setup python uses: actions/setup-python@v5 @@ -113,6 +127,9 @@ runs: export PYTHONPATH=$(pwd) export NEMORUN_HOME=$(pwd) export NCCL_DEBUG=INFO + export PIP_DEFAULT_TIMEOUT=120 + export PIP_RETRIES=5 + export UV_HTTP_TIMEOUT=120 uv venv .venv uv cache clean uv sync --no-cache --only-group test @@ -155,6 +172,9 @@ runs: export PYTHONPATH=$(pwd) export NEMORUN_HOME=$(pwd) + export PIP_DEFAULT_TIMEOUT=120 + export PIP_RETRIES=5 + export UV_HTTP_TIMEOUT=120 uv venv .venv uv cache clean uv sync --no-cache --only-group test @@ -209,6 +229,8 @@ runs: if: always() env: IS_UNIT_TEST: ${{ inputs.is_unit_test == 'true' }} + MAIN_CONCLUSION: ${{ steps.run-main-script.conclusion }} + MAIN_EXIT_CODE: ${{ steps.run-main-script.outputs.exit_code }} run: | logs_report=logs-${{ inputs.test_case }}-${{ github.run_id }}-$(cat /proc/sys/kernel/random/uuid) echo "logs_report=$logs_report" | sed 's/\//-/g' | sed 's/\*/-/g' | tee -a "$GITHUB_OUTPUT" @@ -219,8 +241,12 @@ runs: fi echo "coverage_report=$coverage_report" | tee -a "$GITHUB_OUTPUT" - EXIT_CODE=${{ steps.run-main-script.outputs.exit_code }} - IS_SUCCESS=$([[ "$EXIT_CODE" -eq 0 ]] && echo "true" || echo "false") + EXIT_CODE="${MAIN_EXIT_CODE:-${MAIN_CONCLUSION}}" + if [[ "$MAIN_CONCLUSION" == "success" ]]; then + IS_SUCCESS=true + else + IS_SUCCESS=false + fi if [[ "$IS_SUCCESS" == "false" && "${{ inputs.is-optional }}" == "true" ]]; then echo "::warning::Test failed but is marked optional — treating as success." diff --git a/.github/copy-pr-bot.yaml b/.github/copy-pr-bot.yaml index c4f81fbcb7d..afff481a552 100644 --- a/.github/copy-pr-bot.yaml +++ b/.github/copy-pr-bot.yaml @@ -1,4 +1,4 @@ enabled: true auto_sync_draft: false auto_sync_ready: true -trustees_override: ["AAnoosheh", "ArEsKay3", "Autumn1998", "BestJuly", "BoxiangW", "CarlosGomes98", "ChenhanYu", "Connor-XY", "FDecaYed", "HaochenYuan", "ISEEKYAN", "JRD971000", "Mellonta", "Phlip79", "QiZhangNV", "RPrenger", "ShriyaRishab", "Victarry", "WanZzzzzz", "Wohox", "YangFei1990", "ZhiyuLi-Nvidia", "ahmadki", "aklife97", "ananthsub", "aroshanghias-nvd", "asolergi-nv", "balasaajay", "buptzyb", "chtruong814", "cjld", "cspades", "cuichenx", "deepakn94", "dimapihtar", "dingqingy-nv", "duncanriach", "erhoo82", "ericharper", "fanshiqing", "faradawn", "fitsumreda", "frsun-nvda", "gautham-kollu", "gdengk", "guihong-nv", "guyueh1", "hexinw-nvidia", "huvunvidia", "hxbai", "ilml", "jalbericiola", "janEbert", "jaredcasper", "jenchen13", "jiemingz", "jingqiny-99", "jkamalu", "jon-barker", "jstjohn", "kajalj22", "kanz-nv", "kevalmorabia97", "ko3n1g", "ksivaman", "kunlunl", "kvareddy", "kwyss-nvidia", "layalir", "lhb8125", "lmcafee-nvidia", "maanug-nv", "mathemakitten", "matthieule", "mchrzanowski", "mehraakash", "minitu", "mkhona-nvidia", "nanz-nv", "ntajbakhsh", "parthmannan", "prajwal1210", "pthombre", "rhewett-nv", "rogerwaleffe", "sajadn", "sanandaraj5597", "sancha", "santhnm2", "sbak5", "shanmugamr1992", "sharathts", "sheliang-nv", "shengf-nv", "shifangx", "shjwudp", "sidsingh-nvidia", "skyw", "sraman-rgb", "sudhakarsingh27", "tdene", "theothermike", "thomasdhc", "tomlifu", "trintamaki", "tylerpoon", "wdykas", "wplf", "wujingyue", "xiaoyao0115", "xuantengh", "xuwchen", "yanring", "yaox12", "yaoyu-33", "yashaswikarnati", "yeyu-nvidia", "yobibyte", "youngeunkwon0405", "yueshen2016", "yuzhongw-nvidia", "zhongbozhu"] +trustees_override: ["AAnoosheh", "ArEsKay3", "Autumn1998", "BestJuly", "BoxiangW", "CarlosGomes98", "ChenhanYu", "Connor-XY", "FDecaYed", "HaochenYuan", "ISEEKYAN", "JRD971000", "Mellonta", "Phlip79", "QiZhangNV", "RPrenger", "ShriyaRishab", "Victarry", "WanZzzzzz", "Wohox", "YangFei1990", "ZhiyuLi-Nvidia", "adistomar", "ahmadki", "aklife97", "alokpathy", "ananthsub", "anlthms", "aroshanghias-nvd", "ashehper", "asolergi-nv", "athitten", "balasaajay", "buptzyb", "chtruong814", "cjld", "cspades", "cuichenx", "deepakn94", "dimapihtar", "dingqingy-nv", "duncanriach", "erhoo82", "ericharper", "fanshiqing", "faradawn", "fitsumreda", "frsun-nvda", "gautham-kollu", "gdengk", "guihong-nv", "guyueh1", "hexinw-nvidia", "huvunvidia", "hxbai", "ilml", "jalbericiola", "janEbert", "jaredcasper", "jenchen13", "jiemingz", "jingqiny-99", "jkamalu", "jon-barker", "jstjohn", "kajalj22", "kamran-nvidia", "kevalmorabia97", "ko3n1g", "ksivaman", "kunlunl", "kvareddy", "kwyss-nvidia", "layalir", "lhb8125", "liding-nv", "lmcafee-nvidia", "maanug-nv", "macandro96", "mathemakitten", "matthieule", "mchrzanowski", "mehraakash", "minitu", "mkhona-nvidia", "nanz-nv", "ntajbakhsh", "parthmannan", "philipcmonk", "prajwal1210", "pthombre", "rapatel", "rhewett-nv", "rogerwaleffe", "sajadn", "sanandaraj5597", "sancha", "santhnm2", "sbak5", "shanmugamr1992", "sharathts", "sheliang-nv", "shengf-nv", "shifangx", "shjwudp", "sidsingh-nvidia", "skyw", "sraman-rgb", "sudhakarsingh27", "tdene", "theothermike", "thomasdhc", "tomlifu", "trintamaki", "tylerpoon", "wdykas", "wplf", "wujingyue", "xiaoyao0115", "xuantengh", "xuwchen", "yaox12", "yaoyu-33", "yashaswikarnati", "yeyu-nvidia", "yobibyte", "youngeunkwon0405", "yueshen2016", "yuzhongw-nvidia", "zhongbozhu"] diff --git a/.github/oncall_schedule.json b/.github/oncall_schedule.json index 2f6e01c786c..eea6acdef57 100644 --- a/.github/oncall_schedule.json +++ b/.github/oncall_schedule.json @@ -1,50 +1,50 @@ [ { - "user": "janEbert", - "date": "2026-05-06" + "user": "Phlip79", + "date": "2026-06-17" }, { - "user": "dimapihtar", - "date": "2026-05-13" + "user": "asolergi-nv", + "date": "2026-06-24" }, { - "user": "ilml", - "date": "2026-05-20" + "user": "maanug-nv", + "date": "2026-07-01" }, { "user": "wujingyue", - "date": "2026-05-27" + "date": "2026-07-08" }, { "user": "Connor-XY", - "date": "2026-06-03" + "date": "2026-07-15" }, { - "user": "guihong-nv", - "date": "2026-06-10" + "user": "Phlip79", + "date": "2026-07-22" }, { - "user": "Phlip79", - "date": "2026-06-17" + "user": "YangFei1990", + "date": "2026-07-29" }, { "user": "asolergi-nv", - "date": "2026-06-24" + "date": "2026-08-05" }, { - "user": "maanug-nv", - "date": "2026-07-01" + "user": "dimapihtar", + "date": "2026-08-12" }, { - "user": "wujingyue", - "date": "2026-07-08" + "user": "guihong-nv", + "date": "2026-08-19" }, { - "user": "Connor-XY", - "date": "2026-07-15" + "user": "ilml", + "date": "2026-08-26" }, { - "user": "Phlip79", - "date": "2026-07-22" + "user": "janEbert", + "date": "2026-09-02" } ] diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 8f319e66f87..9cde56ccc49 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,7 +1,9 @@ +- [ ] I, the PR author, have personally reviewed every line of this PR. + # What does this PR do ? -:warning: For major changes (either in lines of code or in its impact), please make sure to first share a design doc with the team. If you're unsure what's the best way to do so, contact the @mcore-oncall. +:warning: For major changes (either in lines of code or in its impact), please make sure to first share a design doc with the team. If you're unsure what's the best way to do so, contact @NVIDIA/mcore-oncall. ## Issue tracking @@ -24,7 +26,7 @@ Linked issue: ### Code review -Feel free to message or comment the [@mcore-oncall](https://github.com/orgs/NVIDIA/teams/mcore-oncall) to help accelerate your merge into main. The less complex your PR is, the faster it will be approved and merged! +Feel free to message or comment @NVIDIA/mcore-oncall to help accelerate your merge into main. The less complex your PR is, the faster it will be approved and merged! All PRs start as **draft**. If you open a non-draft PR, it will be automatically converted to draft. @@ -50,10 +52,3 @@ Once all required reviewers have approved, the `Approved` label is applied **aut ### Merge Any member of [mcore-engineers](https://github.com/orgs/NVIDIA/teams/mcore-engineers) will be able to merge your PR. - -
-For MRs into `dev` branch -The proposed review process for `dev` branch is under active discussion. - -MRs are mergable after one approval by either `eharper@nvidia.com` or `zijiey@nvidia.com`. -
diff --git a/.github/scripts/oncall_manager.py b/.github/scripts/oncall_manager.py index 332fcb1c8cc..d8d0ed74187 100644 --- a/.github/scripts/oncall_manager.py +++ b/.github/scripts/oncall_manager.py @@ -12,13 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import argparse +import json import os import sys -import json -import requests -import argparse from datetime import datetime, timedelta, timezone +import requests from slack_sdk import WebClient from slack_sdk.errors import SlackApiError @@ -28,26 +28,26 @@ ROTATION_TEAM_SLUG = "mcore-oncall-rotation" ACTIVE_ONCALL_TEAM_SLUG = "mcore-oncall" SLACK_USERGROUP_HANDLE = "mcore-oncall" +COMMUNITY_REQUEST_LABEL = "community-request" TARGET_WEEKS = 12 # Caches for email and Slack lookups _email_cache = {} _slack_id_cache = {} + def get_headers(): token = os.environ.get("GH_TOKEN") if not token: # Fallback to GITHUB_TOKEN if GH_TOKEN not set token = os.environ.get("GITHUB_TOKEN") - + if not token: print("Error: GH_TOKEN or GITHUB_TOKEN not set") sys.exit(1) - - return { - "Authorization": f"token {token}", - "Accept": "application/vnd.github.v3+json" - } + + return {"Authorization": f"token {token}", "Accept": "application/vnd.github.v3+json"} + def get_repo_info(): """Returns (owner, repo) from GITHUB_REPOSITORY env var.""" @@ -58,11 +58,12 @@ def get_repo_info(): parts = repo_env.split("/") return parts[0], parts[1] + def get_team_members(org, team_slug): """Fetches members of the GitHub team.""" url = f"{GITHUB_API_URL}/orgs/{org}/teams/{team_slug}/members" headers = get_headers() - + members = set() page = 1 while True: @@ -70,31 +71,32 @@ def get_team_members(org, team_slug): if resp.status_code != 200: print(f"Error fetching team members: {resp.status_code} {resp.text}") sys.exit(1) - + data = resp.json() if not data: break - + members.update([m['login'] for m in data]) if len(data) < 100: break page += 1 - + return members + def get_user_email(username): """Get user's email from GitHub, prioritizing @nvidia.com emails. - + Checks in order: 1. Public profile email 2. Recent commits in the repository """ if username in _email_cache: return _email_cache[username] - + headers = get_headers() public_email = None - + try: # 1. Try to get user's public profile email first resp = requests.get(f"{GITHUB_API_URL}/users/{username}", headers=headers) @@ -107,12 +109,12 @@ def get_user_email(username): return email # Store non-nvidia email as fallback public_email = email - + # 2. Check recent commits in the repository for @nvidia.com email repo_env = os.environ.get("GITHUB_REPOSITORY", "NVIDIA/Megatron-LM") commits_url = f"{GITHUB_API_URL}/repos/{repo_env}/commits?author={username}&per_page=10" resp = requests.get(commits_url, headers=headers) - + if resp.status_code == 200: commits = resp.json() for commit in commits: @@ -120,7 +122,7 @@ def get_user_email(username): commit_data = commit.get('commit', {}) author_data = commit_data.get('author', {}) email = author_data.get('email') - + if email and not email.endswith("@users.noreply.github.com"): if email.endswith("@nvidia.com"): _email_cache[username] = email @@ -128,41 +130,43 @@ def get_user_email(username): return email elif public_email is None: public_email = email - + # 3. Use public email if found, otherwise fallback if public_email: _email_cache[username] = public_email print(f"Using public email for {username}: {public_email}") return public_email - + # Fallback to noreply email fallback = f"{username}@users.noreply.github.com" _email_cache[username] = fallback print(f"Warning: No email found for {username}, using fallback: {fallback}") return fallback - + except Exception as e: print(f"Warning: Could not get email for {username}: {e}") fallback = f"{username}@users.noreply.github.com" _email_cache[username] = fallback return fallback + def get_slack_client(): """Get Slack WebClient if token is available.""" slack_token = os.environ.get("SLACK_TOKEN") if not slack_token: return None - + return WebClient(token=slack_token) + def get_slack_user_id(slack_client, email): """Get Slack user ID from email.""" if not slack_client: return None - + if email in _slack_id_cache: return _slack_id_cache[email] - + try: response = slack_client.users_lookupByEmail(email=email) user_id = response["user"]["id"] @@ -173,11 +177,12 @@ def get_slack_user_id(slack_client, email): _slack_id_cache[email] = None return None + def get_slack_usergroup_id(slack_client, handle): """Get Slack usergroup ID from handle.""" if not slack_client: return None - + try: response = slack_client.usergroups_list(include_users=True) for usergroup in response.get("usergroups", []): @@ -189,6 +194,7 @@ def get_slack_usergroup_id(slack_client, handle): print(f"Warning: Could not list Slack usergroups: {e.response['error']}") return None, [] + def update_slack_usergroup(new_oncall_username, old_members_usernames): """ Updates the Slack usergroup to contain only the new oncall user. @@ -198,43 +204,44 @@ def update_slack_usergroup(new_oncall_username, old_members_usernames): if not slack_client: print("Slack token not configured, skipping Slack usergroup update") return - + # Get the new oncall's email and Slack user ID new_email = get_user_email(new_oncall_username) new_slack_id = get_slack_user_id(slack_client, new_email) - + if not new_slack_id: - print(f"Could not find Slack user ID for {new_oncall_username} ({new_email}), skipping Slack update") + print( + f"Could not find Slack user ID for {new_oncall_username} ({new_email}), skipping Slack update" + ) return - + # Get the usergroup ID and current members - usergroup_id, current_slack_members = get_slack_usergroup_id(slack_client, SLACK_USERGROUP_HANDLE) - + usergroup_id, current_slack_members = get_slack_usergroup_id( + slack_client, SLACK_USERGROUP_HANDLE + ) + if not usergroup_id: print(f"Could not find Slack usergroup '{SLACK_USERGROUP_HANDLE}', skipping Slack update") return - + try: # Step 1: Add new oncall first (include current members to avoid removing anyone yet) # This ensures usergroup always has at least one member if new_slack_id not in current_slack_members: updated_members = list(set(current_slack_members + [new_slack_id])) - slack_client.usergroups_users_update( - usergroup=usergroup_id, - users=updated_members - ) + slack_client.usergroups_users_update(usergroup=usergroup_id, users=updated_members) print(f"Added {new_oncall_username} to Slack usergroup '{SLACK_USERGROUP_HANDLE}'") - + # Step 2: Now set the usergroup to contain only the new oncall - slack_client.usergroups_users_update( - usergroup=usergroup_id, - users=[new_slack_id] + slack_client.usergroups_users_update(usergroup=usergroup_id, users=[new_slack_id]) + print( + f"Updated Slack usergroup '{SLACK_USERGROUP_HANDLE}' to contain only {new_oncall_username}" ) - print(f"Updated Slack usergroup '{SLACK_USERGROUP_HANDLE}' to contain only {new_oncall_username}") - + except SlackApiError as e: print(f"Failed to update Slack usergroup: {e.response['error']}") + def load_schedule(): if not os.path.exists(SCHEDULE_FILE): return [] @@ -252,44 +259,55 @@ def load_schedule(): except (json.JSONDecodeError, FileNotFoundError): return [] + def save_schedule(schedule): with open(SCHEDULE_FILE, 'w') as f: json.dump(schedule, f, indent=4) - f.write('\n') # trailing newline + f.write('\n') # trailing newline + def update_active_oncall_team(org, new_oncall): """Updates the active oncall team to contain only the new oncall user.""" # 1. Get current members of the active team current_members = get_team_members(org, ACTIVE_ONCALL_TEAM_SLUG) - + # 2. Add the new oncall if not present if new_oncall not in current_members: - url = f"{GITHUB_API_URL}/orgs/{org}/teams/{ACTIVE_ONCALL_TEAM_SLUG}/memberships/{new_oncall}" + url = ( + f"{GITHUB_API_URL}/orgs/{org}/teams/{ACTIVE_ONCALL_TEAM_SLUG}/memberships/{new_oncall}" + ) resp = requests.put(url, headers=get_headers()) if resp.status_code == 200: print(f"Added {new_oncall} to {ACTIVE_ONCALL_TEAM_SLUG}") else: - print(f"Failed to add {new_oncall} to {ACTIVE_ONCALL_TEAM_SLUG}: {resp.status_code} {resp.text}") + print( + f"Failed to add {new_oncall} to {ACTIVE_ONCALL_TEAM_SLUG}: {resp.status_code} {resp.text}" + ) # 3. Remove everyone else old_members = [] for member in current_members: if member not in [new_oncall, 'svcnvidia-nemo-ci']: old_members.append(member) - url = f"{GITHUB_API_URL}/orgs/{org}/teams/{ACTIVE_ONCALL_TEAM_SLUG}/memberships/{member}" + url = ( + f"{GITHUB_API_URL}/orgs/{org}/teams/{ACTIVE_ONCALL_TEAM_SLUG}/memberships/{member}" + ) resp = requests.delete(url, headers=get_headers()) if resp.status_code == 204: print(f"Removed {member} from {ACTIVE_ONCALL_TEAM_SLUG}") else: - print(f"Failed to remove {member} from {ACTIVE_ONCALL_TEAM_SLUG}: {resp.status_code} {resp.text}") - + print( + f"Failed to remove {member} from {ACTIVE_ONCALL_TEAM_SLUG}: {resp.status_code} {resp.text}" + ) + # 4. Update Slack usergroup (add new oncall first, then remove old members) update_slack_usergroup(new_oncall, old_members) + def rotate_schedule(repo_owner, dry_run=False): schedule = load_schedule() print(f"Current schedule length: {len(schedule)}") - + # 1. Rotate (Remove past week) # Only if schedule is not empty. if schedule: @@ -300,26 +318,28 @@ def rotate_schedule(repo_owner, dry_run=False): # The shift ends 7 days later. start_date = datetime.strptime(first_entry['date'], "%Y-%m-%d").date() end_date = start_date + timedelta(days=7) - + today = datetime.now(timezone.utc).date() - + # If today is >= end_date, the shift is over. # (e.g. Started last Wed, ends today Wed. If today is Wed, we rotate) if today >= end_date: removed = schedule.pop(0) print(f"Rotated out: {removed} (Ended {end_date})") else: - print(f"First entry {first_entry} has not ended yet (Ends {end_date}). Not removing.") + print( + f"First entry {first_entry} has not ended yet (Ends {end_date}). Not removing." + ) except ValueError: - # Fallback if date is invalid, rotate anyway - removed = schedule.pop(0) - print(f"Rotated out (invalid date): {removed}") + # Fallback if date is invalid, rotate anyway + removed = schedule.pop(0) + print(f"Rotated out (invalid date): {removed}") else: print("Schedule empty, nothing to rotate.") # 2. Replenish ensure_schedule_filled(schedule, repo_owner) - + # 3. Update active oncall team if schedule: current_oncall = schedule[0]['user'] @@ -327,8 +347,10 @@ def rotate_schedule(repo_owner, dry_run=False): if not dry_run: update_active_oncall_team(repo_owner, current_oncall) else: - print(f"Dry run: Would update {ACTIVE_ONCALL_TEAM_SLUG} to contain only {current_oncall}") - + print( + f"Dry run: Would update {ACTIVE_ONCALL_TEAM_SLUG} to contain only {current_oncall}" + ) + if not dry_run: save_schedule(schedule) print("Schedule updated and saved.") @@ -336,12 +358,14 @@ def rotate_schedule(repo_owner, dry_run=False): print("Dry run: Schedule not saved.") print(json.dumps(schedule, indent=4)) + def get_last_wednesday(): today = datetime.now(timezone.utc).date() # Monday=0, Wednesday=2 offset = (today.weekday() - 2) % 7 return today - timedelta(days=offset) + def ensure_schedule_filled(schedule, repo_owner): """Appends users to schedule until it reaches TARGET_WEEKS.""" members = get_team_members(repo_owner, ROTATION_TEAM_SLUG) @@ -352,20 +376,20 @@ def ensure_schedule_filled(schedule, repo_owner): members.remove('svcnvidia-nemo-ci') members = list(members) - members.sort() # Deterministic order - + members.sort() # Deterministic order + while len(schedule) < TARGET_WEEKS: # Determine start date for the new entry if not schedule: # Start with the most recent Wednesday if list is empty next_date = get_last_wednesday() - + # Start with the first member alphabetically if list is empty next_user = members[0] else: last_entry = schedule[-1] last_user = last_entry['user'] - + # Parse last date and add 7 days try: last_date = datetime.strptime(last_entry['date'], "%Y-%m-%d").date() @@ -385,45 +409,81 @@ def ensure_schedule_filled(schedule, repo_owner): next_user = members[0] except ValueError: next_user = members[0] - + new_entry = {"user": next_user, "date": next_date.strftime("%Y-%m-%d")} schedule.append(new_entry) print(f"Appended: {new_entry}") + def assign_reviewer(pr_number): - """Assigns the mcore-oncall team as the reviewer for the PR.""" + """Assigns mcore-oncall if no reviewers are set or community-request is applied.""" owner, repo = get_repo_info() + + pr_url = f"{GITHUB_API_URL}/repos/{owner}/{repo}/pulls/{pr_number}" + pr_resp = requests.get(pr_url, headers=get_headers()) + + if pr_resp.status_code != 200: + print(f"Failed to fetch PR: {pr_resp.status_code} {pr_resp.text}") + sys.exit(1) + + pr_data = pr_resp.json() + requested_reviewers = pr_data.get("requested_reviewers", []) + requested_teams = pr_data.get("requested_teams", []) + labels = {label.get("name") for label in pr_data.get("labels", [])} + requested_team_slugs = {team.get("slug") for team in requested_teams} + has_community_request_label = COMMUNITY_REQUEST_LABEL in labels + + if ACTIVE_ONCALL_TEAM_SLUG in requested_team_slugs: + print( + f"Skipping reviewer request: team NVIDIA/{ACTIVE_ONCALL_TEAM_SLUG} " + "is already requested" + ) + return + + if not has_community_request_label and (requested_reviewers or requested_teams): + print( + "Skipping reviewer request: PR already has " + f"{len(requested_reviewers)} user reviewer(s) and " + f"{len(requested_teams)} team reviewer(s)" + ) + return + url = f"{GITHUB_API_URL}/repos/{owner}/{repo}/pulls/{pr_number}/requested_reviewers" - + # Assign the oncall team as reviewer data = {"team_reviewers": [ACTIVE_ONCALL_TEAM_SLUG]} resp = requests.post(url, headers=get_headers(), json=data) - + if resp.status_code in [201, 200]: print(f"Successfully requested review from team NVIDIA/{ACTIVE_ONCALL_TEAM_SLUG}") else: print(f"Failed to request review: {resp.status_code} {resp.text}") sys.exit(1) + def main(): parser = argparse.ArgumentParser(description="Manage Oncall Schedule") subparsers = parser.add_subparsers(dest="command", required=True) - + # Rotate command - parser_rotate = subparsers.add_parser("rotate", help="Rotate the schedule (remove first, append new)") + parser_rotate = subparsers.add_parser( + "rotate", help="Rotate the schedule (remove first, append new)" + ) parser_rotate.add_argument("--dry-run", action="store_true", help="Do not save changes") # Fill command (just fill up to 12 without rotating - useful for init) - parser_fill = subparsers.add_parser("fill", help="Fill the schedule to 12 weeks without rotating") - + parser_fill = subparsers.add_parser( + "fill", help="Fill the schedule to 12 weeks without rotating" + ) + # Assign command parser_assign = subparsers.add_parser("assign", help="Assign current oncall to PR") parser_assign.add_argument("--pr", type=int, required=True, help="PR number") args = parser.parse_args() - + owner, _ = get_repo_info() - + if args.command == "rotate": rotate_schedule(owner, dry_run=args.dry_run) elif args.command == "fill": @@ -434,6 +494,6 @@ def main(): elif args.command == "assign": assign_reviewer(args.pr) + if __name__ == "__main__": main() - diff --git a/.github/workflows/_build_test_publish_wheel.yml b/.github/workflows/_build_test_publish_wheel.yml index 21e8d59f881..9e37e068b6d 100644 --- a/.github/workflows/_build_test_publish_wheel.yml +++ b/.github/workflows/_build_test_publish_wheel.yml @@ -18,7 +18,7 @@ on: default: true secrets: TWINE_PASSWORD: - required: true + required: false jobs: build-and-test-wheels: @@ -72,9 +72,14 @@ jobs: pushd $BUILD_DIR rm LICENSE || true + for i in 1 2 3; do + docker pull "$IMAGE" && break + echo "docker pull attempt $i failed, retrying..." + sleep 10 + done docker run --rm -e NO_VCS_VERSION=1 -v $(pwd):/workspace -w /workspace $IMAGE bash -c '\ for python_version in cp311 cp312 cp313; do \ - /opt/python/${python_version}-${python_version}/bin/pip install --upgrade "setuptools<80.0.0,>=77.0.0" build; \ + /opt/python/${python_version}-${python_version}/bin/pip install --upgrade "setuptools>=80" build; \ done && \ for python_version in cp311 cp312 cp313; do \ /opt/python/${python_version}-${python_version}/bin/python -m build; \ @@ -144,7 +149,8 @@ jobs: publish-wheels: needs: [build-and-test-wheels] runs-on: ubuntu-latest - if: inputs.no-publish == false + environment: + name: ${{ inputs.no-publish && 'public' || 'main' }} strategy: fail-fast: false matrix: @@ -171,6 +177,7 @@ jobs: TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} TWINE_REPOSITORY: ${{ (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/r')) && 'pypi' || 'testpypi' }} PLATFORM: ${{ matrix.PLATFORM }} + DRY_RUN: ${{ inputs.no-publish }} run: | # Delete sdist for arm64 since we already upload it with amd64. @@ -180,9 +187,15 @@ jobs: ls -al dist/ pip install twine - twine upload \ - --verbose \ - -r $TWINE_REPOSITORY \ - -u $TWINE_USERNAME \ - -p $TWINE_PASSWORD \ - dist/* + + if [[ "$DRY_RUN" == "false" ]]; then + [[ -z "$TWINE_PASSWORD" ]] && { echo "::error::TWINE_PASSWORD unset"; exit 1; } + twine upload \ + --verbose \ + -r $TWINE_REPOSITORY \ + -u $TWINE_USERNAME \ + -p $TWINE_PASSWORD \ + dist/* + else + echo "[dry-run] would execute: twine upload --verbose -r $TWINE_REPOSITORY -u -p dist/*" + fi diff --git a/.github/workflows/_release_library.yml b/.github/workflows/_release_library.yml deleted file mode 100644 index 954d81c8259..00000000000 --- a/.github/workflows/_release_library.yml +++ /dev/null @@ -1,510 +0,0 @@ -# Copyright (c) 2020-2021, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: "Release" - -defaults: - run: - shell: bash -x -e -u -o pipefail {0} - -on: - workflow_call: - inputs: - release-ref: - required: true - description: Ref (SHA or branch) to release - type: string - dry-run: - type: boolean - required: true - description: Do not publish a wheel and GitHub release. - version-bump-branch: - type: string - required: true - description: Branch to target for version bump - create-gh-release: - required: false - description: Create a GitHub release - type: boolean - default: true - gh-release-use-changelog-builder: - required: false - description: Use release-changelog-builder-action to dynamically build changelog - type: boolean - default: true - gh-release-changelog-config: - required: false - description: Path to changelog builder configuration file - type: string - default: ".github/workflows/config/changelog-config.json" - gh-release-from-tag: - required: false - description: Starting tag for changelog builder (leave empty for auto-detect) - type: string - default: "" - publish-docs: - required: false - description: Publish documentation to S3 after release - type: boolean - default: true - secrets: - TWINE_PASSWORD: - required: true - SLACK_WEBHOOK: - required: true - PAT: - required: true - AWS_ASSUME_ROLE_ARN: - required: true - AWS_ACCESS_KEY_ID: - required: true - AWS_SECRET_ACCESS_KEY: - required: true - AKAMAI_HOST: - required: true - AKAMAI_CLIENT_TOKEN: - required: true - AKAMAI_CLIENT_SECRET: - required: true - AKAMAI_ACCESS_TOKEN: - required: true - S3_BUCKET_NAME: - required: true - -permissions: - contents: write # To read repository content - pull-requests: write # To create PR(s) - -jobs: - build-test-publish-wheels-dry-run: - uses: ./.github/workflows/_build_test_publish_wheel.yml - with: - dry-run: true - ref: ${{ inputs.release-ref }} - no-publish: true - secrets: - TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} - - bump-next-version: - runs-on: ubuntu-latest - needs: build-test-publish-wheels-dry-run - if: | - ( - success() || !failure() - ) - && !cancelled() - outputs: - release-version: ${{ steps.bump-version-mcore.outputs.release-version }} - env: - IS_DRY_RUN: ${{ inputs.dry-run }} - steps: - - name: Checkout repository - uses: actions/checkout@v6 - with: - path: ${{ github.run_id }} - token: ${{ secrets.PAT }} - fetch-depth: 0 - fetch-tags: true - ref: ${{ inputs.release-ref }} - - name: Bump version MCore - id: bump-version-mcore - env: - SRC_DIR: "" - PYPROJECT_NAME: "megatron.core" - run: | - set +u - cd ${{ github.run_id }} - - PACKAGE_INFO_FILE="$SRC_DIR${PYPROJECT_NAME//.//}/package_info.py" - - MAJOR=$(cat $PACKAGE_INFO_FILE | awk '/^MAJOR = /' | awk -F"= " '{print $2}') - MINOR=$(cat $PACKAGE_INFO_FILE | awk '/^MINOR = /' | awk -F"= " '{print $2}') - PATCH=$(cat $PACKAGE_INFO_FILE | awk '/^PATCH = /' | awk -F"= " '{print $2}') - PRERELEASE=$(cat $PACKAGE_INFO_FILE | awk '/^PRE_RELEASE = /' | awk -F"= " '{print $2}' | tr -d '"' | tr -d "'") - - echo "release-version=$MAJOR.$MINOR.$PATCH$PRERELEASE" | tee -a "$GITHUB_OUTPUT" - - if [[ "$PRERELEASE" != "" ]]; then - if [[ "$PRERELEASE" == *rc* ]]; then - NEXT_PATCH=$PATCH - NEXT_PRERELEASE=rc$((${PRERELEASE#rc} + 1)) - elif [[ "$PRERELEASE" == *a* ]]; then - NEXT_PATCH=$PATCH - NEXT_PRERELEASE=a$((${PRERELEASE#a} + 1)) - else - echo "Unknown pre-release: $PRERELEASE" - exit 1 - fi - else - NEXT_PATCH=$((${PATCH} + 1)) - NEXT_PRERELEASE=$PRERELEASE - fi - - sed -i "/^PATCH/c\PATCH = $NEXT_PATCH" $PACKAGE_INFO_FILE - sed -i "/^PRE_RELEASE/c\PRE_RELEASE = \"$NEXT_PRERELEASE\"" $PACKAGE_INFO_FILE - - echo "version=$MAJOR.$MINOR.$NEXT_PATCH$NEXT_PRERELEASE" | tee -a "$GITHUB_OUTPUT" - - - name: Bump version MFSDP - id: bump-version-mfsdp - env: - SRC_DIR: "megatron/core/distributed/fsdp/src/" - PYPROJECT_NAME: "megatron_fsdp" - run: | - set +u - - cd ${{ github.run_id }} - - PACKAGE_INFO_FILE="$SRC_DIR${PYPROJECT_NAME//.//}/package_info.py" - - MAJOR=$(cat $PACKAGE_INFO_FILE | awk '/^MAJOR = /' | awk -F"= " '{print $2}') - MINOR=$(cat $PACKAGE_INFO_FILE | awk '/^MINOR = /' | awk -F"= " '{print $2}') - PATCH=$(cat $PACKAGE_INFO_FILE | awk '/^PATCH = /' | awk -F"= " '{print $2}') - PRERELEASE=$(cat $PACKAGE_INFO_FILE | awk '/^PRE_RELEASE = /' | awk -F"= " '{print $2}' | tr -d '"' | tr -d "'") - - if [[ "$PRERELEASE" != "" ]]; then - if [[ "$PRERELEASE" == *rc* ]]; then - NEXT_PATCH=$PATCH - NEXT_PRERELEASE=rc$((${PRERELEASE#rc} + 1)) - elif [[ "$PRERELEASE" == *a* ]]; then - NEXT_PATCH=$PATCH - NEXT_PRERELEASE=a$((${PRERELEASE#a} + 1)) - else - echo "Unknown pre-release: $PRERELEASE" - exit 1 - fi - else - NEXT_PATCH=$((${PATCH} + 1)) - NEXT_PRERELEASE=$PRERELEASE - fi - - sed -i "/^PATCH/c\PATCH = $NEXT_PATCH" $PACKAGE_INFO_FILE - sed -i "/^PRE_RELEASE/c\PRE_RELEASE = \"$NEXT_PRERELEASE\"" $PACKAGE_INFO_FILE - - echo "version=$MAJOR.$MINOR.$NEXT_PATCH$NEXT_PRERELEASE" | tee -a "$GITHUB_OUTPUT" - - - name: Create and push deployment branch - env: - GH_TOKEN: ${{ secrets.PAT }} - run: | - cd ${{ github.run_id }} - - TMP_BRANCH="deploy-release/$(uuidgen)" - git config --global user.name "github-actions[bot]" - git config --global user.email "github-actions[bot]@users.noreply.github.com" - git checkout -b "$TMP_BRANCH" - git add -A . - git commit -m "beep boop 🤖: Bumping versions" || echo "No changes to commit" - git push -u origin "$TMP_BRANCH" - echo "TMP_BRANCH=$TMP_BRANCH" | tee -a $GITHUB_ENV - - # Create PR to collect app based status checks that run on PRs only - # (like DCO check) - PR_URL=$(gh pr create \ - --base ${{ inputs.version-bump-branch }} \ - --head $TMP_BRANCH \ - --title "beep boop 🤖: Bumping versions" \ - --body "This is an automated PR to bump versions.") - - # Extract PR number from URL - PR_NUMBER=$(echo $PR_URL | grep -o '[0-9]*$') - - - name: Wait for status checks on tmp branch - uses: actions/github-script@v8 - id: wait-status - with: - github-token: ${{ secrets.PAT }} - script: | - const branch = process.env.TMP_BRANCH; - const owner = context.repo.owner; - const repo = context.repo.repo; - - // Get latest commit SHA of branch - const { data: refData } = await github.rest.git.getRef({ - owner, - repo, - ref: `heads/${branch}`, // note: no 'refs/' prefix here - }); - - const sha = refData.object.sha; - - console.log(`Polling status for commit SHA: ${sha}`); - - let checksPassed = false; - let maxAttempts = 30; - let attempt = 0; - const delay = ms => new Promise(res => setTimeout(res, ms)); - - while (!checksPassed && attempt < maxAttempts) { - attempt++; - - // Use commit SHA instead of branch ref - const { data: status } = await github.rest.repos.getCombinedStatusForRef({ - owner, - repo, - ref: sha, - }); - - const { data: checks } = await github.rest.checks.listForRef({ - owner, - repo, - ref: sha, - }); - - const allStatuses = status.statuses; - const allChecks = checks.check_runs; - - if (allStatuses.length === 0 && allChecks.length === 0) { - console.log(`Attempt ${attempt}: No checks or statuses yet. Waiting...`); - await delay(10000); - continue; - } - - const statusesOk = allStatuses.every(s => s.state === 'success'); - const checksOk = allChecks.every(c => c.status === 'completed'); - - if (statusesOk && checksOk) { - console.log('✅ All checks passed.'); - checksPassed = true; - break - } - - console.log(`Attempt ${attempt}: Checks not complete yet. Waiting...`); - await delay(10000); - } - - if (!checksPassed) { - core.setFailed('❌ Status checks did not pass in time'); - } - - - name: Merge into ${{ inputs.version-bump-branch }} - run: | - cd ${{ github.run_id }} - - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - - CMD=$(echo -E 'git push origin ${{ inputs.version-bump-branch }}') - - if [[ "$IS_DRY_RUN" == "true" ]]; then - echo "dry-run enabled, would have run: $CMD" - else - # Here we account for potential race conditions from multiple concurrent releases. - # Those can be legit (operating on different packages within the monorepo, for example) - # but the pushes would be still rejected purely because of git's inability to - # push non-fast-forward updates to the branch. In this case we would need to let - # a retry. - git fetch origin ${{ inputs.version-bump-branch }} - git checkout ${{ inputs.version-bump-branch }} - git merge ${{ env.TMP_BRANCH }} - - for attempt in {1..3}; do - if eval "$CMD"; then - echo "Git push succeeded on attempt $attempt" - break - else - echo "Git push failed on attempt $attempt" - if [[ $attempt -lt 3 ]]; then - sleep $((RANDOM % 3 + 1)) - # We refetch, reset and re-merge. Note resetting because the local - # branch is "contaminated" with previous merge attempt. - git fetch origin ${{ inputs.version-bump-branch }} - git reset --hard origin/${{ inputs.version-bump-branch }} - git merge ${{ env.TMP_BRANCH }} - else - echo "Git push failed after 3 attempts" - exit 1 - fi - fi - done - fi - - - name: Delete ${{ env.TMP_BRANCH }} branch - if: always() - run: | - cd ${{ github.run_id }} - git push -d origin ${{ env.TMP_BRANCH }} - - build-test-publish-wheels: - needs: [bump-next-version] - uses: ./.github/workflows/_build_test_publish_wheel.yml - with: - dry-run: false - ref: ${{ inputs.release-ref }} - no-publish: false - secrets: - TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} - - create-gh-release: - needs: [build-test-publish-wheels, bump-next-version] - runs-on: ubuntu-latest - if: | - ( - success() || !failure() - ) - && inputs.create-gh-release == true - && !cancelled() - outputs: - is-release-candidate: ${{ steps.version-number.outputs.is-release-candidate }} - env: - REPOSITORY: ${{ github.repository }} - PROJECT_NAME: Megatron Core - VERSION: ${{ needs.bump-next-version.outputs.release-version }} - TAG_PREFIX: core_ - steps: - - name: Checkout repository - uses: actions/checkout@v6 - with: - path: ${{ github.run_id }} - ref: ${{ inputs.release-ref }} - token: ${{ secrets.PAT || secrets.GITHUB_TOKEN }} - - - name: Determine fromTag for changelog - id: determine-from-tag - if: inputs.gh-release-use-changelog-builder == true - run: | - cd ${{ github.run_id }} - - # If gh-release-from-tag is provided, use it - if [[ -n "${{ inputs.gh-release-from-tag }}" ]]; then - FROM_TAG="${{ inputs.gh-release-from-tag }}" - echo "Using provided fromTag: $FROM_TAG" - else - # Get the most recent tag - FROM_TAG=$(git describe --tags --abbrev=0 2>/dev/null || echo "") - if [[ -z "$FROM_TAG" ]]; then - echo "No previous tags found, leaving fromTag empty" - else - echo "Auto-detected most recent tag: $FROM_TAG" - fi - fi - - echo "from-tag=$FROM_TAG" >> $GITHUB_OUTPUT - - - name: Build Changelog - id: build-changelog - if: inputs.gh-release-use-changelog-builder == true - uses: mikepenz/release-changelog-builder-action@v6.1.0 - env: - GITHUB_TOKEN: ${{ secrets.PAT || secrets.GITHUB_TOKEN }} - with: - configuration: ${{ github.run_id }}/${{ inputs.gh-release-changelog-config }} - owner: ${{ github.repository_owner }} - repo: ${{ github.event.repository.name }} - ignorePreReleases: "false" - failOnError: "false" - fromTag: ${{ steps.determine-from-tag.outputs.from-tag }} - toTag: ${{ inputs.release-ref }} - mode: ${{ inputs.gh-release-changelog-mode }} - - - name: Create release - id: version-number - env: - SHA: ${{ inputs.release-ref }} - GH_TOKEN: ${{ secrets.PAT }} - IS_DRY_RUN: ${{ inputs.dry-run }} - BUILT_CHANGELOG: ${{ steps.build-changelog.outputs.changelog }} - run: | - cd ${{ github.run_id }} - - IS_RELEASE_CANDIDATE=$([[ "$VERSION" == *rc* ]] && echo "true" || echo "false") - IS_ALPHA=$([[ "$VERSION" == *a* ]] && echo "true" || echo "false") - IS_PRERELEASE=$([[ "$IS_RELEASE_CANDIDATE" == "true" || "$IS_ALPHA" == "true" ]] && echo "true" || echo "false") - NAME="NVIDIA $PROJECT_NAME ${VERSION}" - - # Use built changelog if available, otherwise fall back to CHANGELOG.md - if [[ -n "$BUILT_CHANGELOG" ]]; then - CHANGELOG="$BUILT_CHANGELOG" - elif [[ "$IS_RELEASE_CANDIDATE" == "true" ]]; then - DATE=$(date +"%Y-%m-%d") - CHANGELOG="Prerelease: $NAME ($DATE)" - else - CHANGELOG=$(awk '/^## '"$NAME"'/{flag=1; next} /^## /{flag=0} flag' CHANGELOG.md) - CHANGELOG=$(echo "$CHANGELOG" | sed '/./,$!d' | sed ':a;N;$!ba;s/\n$//') - fi - - echo "is-release-candidate=$IS_RELEASE_CANDIDATE" | tee -a "$GITHUB_OUTPUT" - - PAYLOAD=$(jq -nc \ - --arg TAG_NAME "${TAG_PREFIX}v${VERSION}" \ - --arg CI_COMMIT_BRANCH "$SHA" \ - --arg NAME "$NAME" \ - --arg BODY "$CHANGELOG" \ - --argjson PRERELEASE "$IS_PRERELEASE" \ - '{ - "tag_name": $TAG_NAME, - "target_commitish": $CI_COMMIT_BRANCH, - "name": $NAME, - "body": $BODY, - "draft": false, - "prerelease": $PRERELEASE, - "generate_release_notes": false - }' - ) - echo -E "$PAYLOAD" > payload.txt - - CMD=$(echo -E 'curl -L \ - -X POST \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer '"$GH_TOKEN"'" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - https://api.github.com/repos/'"$REPOSITORY"'/releases \ - -d @payload.txt - ') - - if [[ "$IS_DRY_RUN" == "true" ]]; then - echo -E "$CMD" - else - eval "$CMD" - fi - - publish-docs: - needs: [bump-next-version, create-gh-release] - uses: ./.github/workflows/release-docs.yml - if: | - ( - success() || !failure() - ) - && inputs.publish-docs == true - && !cancelled() - with: - dry-run: ${{ inputs.dry-run }} - publish-as-latest: true - docs-version-override: ${{ needs.bump-next-version.outputs.release-version }} - build-docs-ref: ${{ inputs.release-ref }} - secrets: inherit - - notify: - needs: [build-test-publish-wheels, create-gh-release, bump-next-version] - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - repository: NVIDIA-NeMo/FW-CI-templates - ref: v0.17.0 - path: send-slack-alert - - - name: Send Slack alert - uses: ./send-slack-alert/.github/actions/send-slack-alert - env: - MESSAGE: | - ${{ inputs.dry-run == true && 'This is a dry-run, nothing actually happened: ' || '' }}We have released `${{ needs.bump-next-version.outputs.release-version }}` of `NVIDIA Megatron Core` 🚀✨🎉 - - • - • - - with: - message: ${{ env.MESSAGE }} - webhook: ${{ secrets.SLACK_WEBHOOK }} diff --git a/.github/workflows/_update_dependencies.yml b/.github/workflows/_update_dependencies.yml index 903d773edbd..b8410f8fc00 100644 --- a/.github/workflows/_update_dependencies.yml +++ b/.github/workflows/_update_dependencies.yml @@ -117,6 +117,7 @@ jobs: base: ${{ env.TARGET_BRANCH }} title: ${{ env.title }} token: ${{ secrets.PAT }} + labels: Run functional tests body: | 🚀 PR to bump `uv.lock` in `${{ inputs.target-branch }}`. diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml deleted file mode 100644 index f77e665d22f..00000000000 --- a/.github/workflows/build-docs.yml +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Build docs - -on: - push: - branches: - - main - - "pull-request/[0-9]+" - - "deploy-release/*" - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.event.label.name || 'main' }}-${{ github.event_name }} - cancel-in-progress: true - -jobs: - pre-flight: - uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_cicd_preflight.yml@v1.0.0 - - build-docs: - needs: [pre-flight] - if: needs.pre-flight.outputs.is_deployment_workflow != 'true' - uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_build_docs.yml@v1.0.0 - - build-docs-summary: - needs: [pre-flight, build-docs] - if: | - ( - needs.pre-flight.outputs.is_deployment_workflow == 'true' - || always() - ) - && !cancelled() - runs-on: ubuntu-latest - steps: - - name: Get workflow result - id: result - shell: bash -x -e -u -o pipefail {0} - env: - GH_TOKEN: ${{ github.token }} - RUN_ID: ${{ github.run_id }} - SKIPPING_IS_ALLOWED: ${{ needs.pre-flight.outputs.docs_only == 'true' || needs.pre-flight.outputs.is_deployment_workflow == 'true' }} - run: | - FAILED_JOBS=$(gh run view $GITHUB_RUN_ID --json jobs --jq '[.jobs[] | select(.status == "completed" and .conclusion != "success")] | length') || echo 0 - - if [ "${FAILED_JOBS:-0}" -eq 0 ] || [ "$SKIPPING_IS_ALLOWED" == "true" ]; then - echo "✅ All previous jobs completed successfully" - exit 0 - else - echo "❌ Found $FAILED_JOBS failed job(s)" - # Show which jobs failed - gh run view $GITHUB_RUN_ID --json jobs --jq '.jobs[] | select(.status == "completed" and .conclusion != "success") | .name' - exit 1 - fi diff --git a/.github/workflows/build-test-publish-wheel.yml b/.github/workflows/build-test-publish-wheel.yml deleted file mode 100644 index 6817f611485..00000000000 --- a/.github/workflows/build-test-publish-wheel.yml +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright (c) 2019-2025, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Build, test, and publish a PyPi wheel (to testpypi). - -on: - push: - branches: - - main - - "pull-request/[0-9]+" - - "deploy-release/*" - merge_group: - types: [checks_requested] - -defaults: - run: - shell: bash -x -e -u -o pipefail {0} - -permissions: - id-token: write - contents: read - -jobs: - pre-flight: - uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_cicd_preflight.yml@v1.0.0 - if: github.repository == 'NVIDIA/Megatron-LM' - - build-test-publish-wheels: - needs: [pre-flight] - uses: ./.github/workflows/_build_test_publish_wheel.yml - with: - no-publish: true - secrets: - TWINE_PASSWORD: ${{ (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/r')) && secrets.SVC_PYPI_TOKEN || secrets.SVC_PYPI_TEST_TOKEN }} - - build-test-publish-wheel-summary: - needs: [pre-flight, build-test-publish-wheels] - if: | - ( - needs.pre-flight.outputs.docs_only == 'true' - || needs.pre-flight.outputs.is_merge_group == 'true' - || needs.pre-flight.outputs.is_deployment_workflow == 'true' - || always() - ) - && github.repository == 'NVIDIA/Megatron-LM' - && !cancelled() - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v6 - - - name: Result - env: - GH_TOKEN: ${{ github.token }} - GITHUB_RUN_ID: ${{ github.run_id }} - SKIPPING_IS_ALLOWED: false - run: | - FAILED_JOBS=$(gh run view $GITHUB_RUN_ID --json jobs --jq '[.jobs[] | select(.status == "completed" and .conclusion != "success" and (.name | test("build-and-test-wheels")))] | length') || echo 0 - - if [ "${FAILED_JOBS:-0}" -eq 0 ] || [ "$SKIPPING_IS_ALLOWED" == "true" ]; then - echo "✅ All build-and-test-wheels jobs completed successfully" - exit 0 - else - echo "❌ Found $FAILED_JOBS failed build-and-test-wheels job(s)" - # Show which jobs failed - gh run view $GITHUB_RUN_ID --json jobs --jq '.jobs[] | select(.status == "completed" and .conclusion != "success" and (.name | test("build-and-test-wheels"))) | .name' - exit 1 - fi diff --git a/.github/workflows/cicd-main.yml b/.github/workflows/cicd-main.yml index c814bf3106f..1efd3f9e34f 100644 --- a/.github/workflows/cicd-main.yml +++ b/.github/workflows/cicd-main.yml @@ -145,15 +145,38 @@ jobs: lightweight: ${{ steps.configure.outputs.lightweight }} lts: ${{ steps.configure.outputs.lts }} mbridge_suite: ${{ steps.configure.outputs.mbridge_suite }} + run_mbridge: ${{ steps.configure.outputs.run_mbridge }} dev: ${{ steps.configure.outputs.dev }} cadence: ${{ steps.configure.outputs.cadence }} cadence_bypass: ${{ steps.configure.outputs.cadence_bypass }} + sha: ${{ steps.resolve-sha.outputs.sha }} steps: - name: Get PR info id: get-pr-info if: startsWith(github.ref, 'refs/heads/pull-request/') && github.event_name == 'push' uses: nv-gha-runners/get-pr-info@main + # Resolve a single SHA used by the build, every test job, and every + # downstream checkout so that the container image, golden values, and + # test recipes always come from the same commit. For PR pushes this is + # the synthetic PR `merge_commit_sha`; for merge_group it is the merge + # queue head_sha; otherwise it falls back to github.sha. + - name: Resolve SHA + id: resolve-sha + shell: bash -x -e -u -o pipefail {0} + env: + IS_PR: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && github.event_name == 'push' }} + IS_MERGE_GROUP: ${{ github.event_name == 'merge_group' }} + run: | + if [[ "$IS_PR" == "true" ]]; then + SHA='${{ fromJSON(steps.get-pr-info.outputs.pr-info || '{}').merge_commit_sha }}' + elif [[ "$IS_MERGE_GROUP" == "true" ]]; then + SHA='${{ github.event.merge_group.head_sha }}' + else + SHA='${{ github.sha }}' + fi + echo "sha=${SHA}" | tee -a "$GITHUB_OUTPUT" + - name: Configure id: configure shell: bash -x -e -u -o pipefail {0} @@ -197,6 +220,19 @@ jobs: MBRIDGE_SUITE="unit-only" fi + # MBridge job gating: PR pushes skip the downstream MBridge trigger + # by default. The historical triggers (merge_group, schedule, + # workflow_dispatch) continue to run it, and PR authors can opt in + # by adding the `Run MBridge tests` label. + if [ "$HAS_MBRIDGE" == "true" ] \ + || [ "$IS_MERGE_GROUP" == "true" ] \ + || [ "$EVENT_NAME" == "schedule" ] \ + || [ "$EVENT_NAME" == "workflow_dispatch" ]; then + RUN_MBRIDGE=true + else + RUN_MBRIDGE=false + fi + # Cadence: trigger-driven test selection axis (see filter_by_cadence # in tests/test_utils/python_scripts/recipe_parser.py). PR labels # `Run tests` and `Run functional tests` bypass the cadence filter so @@ -224,6 +260,7 @@ jobs: echo "lightweight=$LIGHTWEIGHT" | tee -a $GITHUB_OUTPUT echo "lts=$HAS_LTS" | tee -a $GITHUB_OUTPUT echo "mbridge_suite=$MBRIDGE_SUITE" | tee -a $GITHUB_OUTPUT + echo "run_mbridge=$RUN_MBRIDGE" | tee -a $GITHUB_OUTPUT echo "dev=$DEV" | tee -a $GITHUB_OUTPUT echo "cadence=$CADENCE_OUTPUT" | tee -a $GITHUB_OUTPUT echo "cadence_bypass=$CADENCE_BYPASS" | tee -a $GITHUB_OUTPUT @@ -250,6 +287,7 @@ jobs: | \`lightweight\` | \`$LIGHTWEIGHT\` | | \`lts\` | \`$HAS_LTS\` | | \`dev\` | \`$DEV\` | + | \`run_mbridge\` | \`$RUN_MBRIDGE\` | | \`mbridge_suite\` | \`$MBRIDGE_SUITE\` | | \`cadence\` | \`$CADENCE\` | | \`cadence_bypass\` | \`$CADENCE_BYPASS\` | @@ -286,6 +324,7 @@ jobs: - **\`lts\`**: uses the Long Term Support container base image instead of the latest dev image - **\`dev\`**: uses the latest development container base image (default) - **\`cadence\`**: per-test trigger filter (recipe \`cadence:\` field). Recipes default to \`[pr, nightly, mergegroup]\`. + - **\`run_mbridge\`**: whether to trigger the Megatron-Bridge downstream CI. Off for PR pushes by default; flip on by adding the _Run MBridge tests_ label. SUMMARY linting: @@ -376,12 +415,21 @@ jobs: runs-on: ubuntu-latest needs: - pre-flight + - configure - cicd-wait-in-queue - cicd-parse-downstream-testing + # skip downstream mbridge testing on PR pushes by + # default. They still run for merge_group and nightly (schedule / + # workflow_dispatch) triggers, and PR authors can opt in by adding the + # "Run MBridge tests" label — all three cases set + # configure.outputs.run_mbridge == 'true'. if: | needs.pre-flight.result != 'cancelled' + && needs.configure.result != 'cancelled' && needs.cicd-wait-in-queue.result != 'cancelled' && needs.cicd-parse-downstream-testing.result != 'cancelled' + && vars.ENABLE_CICD_MBRIDGE_TESTING == 'true' + && needs.configure.outputs.run_mbridge == 'true' && ( success() || needs.pre-flight.outputs.is_ci_workload == 'true' @@ -412,22 +460,6 @@ jobs: git checkout -b ${{ env.MBRIDGE_BRANCH_NAME }} origin/main git push origin ${{ env.MBRIDGE_BRANCH_NAME }} --force - - name: Get merge commit sha - shell: bash -x -e -u -o pipefail {0} - id: sha - env: - IS_PR: ${{ startsWith(github.ref, 'refs/heads/pull-request/') }} - IS_MERGE_GROUP: ${{ github.event_name == 'merge_group' }} - run: | - if [[ "$IS_PR" == "true" ]]; then - SHA=${{ fromJSON(steps.get-pr-info.outputs.pr-info || '{}').merge_commit_sha }} - elif [[ "$IS_MERGE_GROUP" == "true" ]]; then - SHA=${{ github.event.merge_group.head_sha }} - else - SHA=${GITHUB_SHA} - fi - echo "main=${SHA}" | tee -a "$GITHUB_OUTPUT" - - name: Trigger MBridge tests uses: convictional/trigger-workflow-and-wait@v1.6.5 env: @@ -442,7 +474,7 @@ jobs: propagate_failure: true client_payload: | { - "mcore_ref": "${{ steps.sha.outputs.main }}", + "mcore_ref": "${{ needs.configure.outputs.sha }}", "test_suite": "${{ needs.cicd-parse-downstream-testing.outputs.mbridge-test-suite }}", "triggered_by": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" } @@ -455,6 +487,25 @@ jobs: cd megatron-bridge git push origin --delete ${{ env.MBRIDGE_BRANCH_NAME }} + cicd-mbridge-testing-notify: + runs-on: ubuntu-latest + needs: [cicd-mbridge-testing] + # Notify on both success and failure of the MBridge downstream tests. + # Skipped/cancelled runs are intentionally not announced. + if: | + always() + && (needs.cicd-mbridge-testing.result == 'success' || needs.cicd-mbridge-testing.result == 'failure') + steps: + - name: Send Slack alert + uses: NVIDIA-NeMo/FW-CI-templates/.github/actions/send-slack-alert@main + with: + webhook: ${{ secrets.SLACK_WH_MLM_MB_ALERTS }} + message: | + ${{ needs.cicd-mbridge-testing.result == 'success' && ':white_check_mark: *MBridge downstream tests passed*' || ':rotating_light: *MBridge downstream tests failed*' }} + • Trigger: `${{ github.event_name }}` on `${{ github.ref_name }}` + • Run: + ${{ needs.cicd-mbridge-testing.result == 'failure' && format('cc ', secrets.SLACK_NEMO_MB_CODEOWNERS_GROUP_ID) || '' }} + cicd-compute-build-matrix: runs-on: ubuntu-latest needs: [is-not-external-contributor] @@ -465,6 +516,7 @@ jobs: id: compute env: IS_MAINTAINER: ${{ needs.is-not-external-contributor.outputs.is_maintainer }} + ENABLE_GB200_TESTING: ${{ vars.ENABLE_GB200_TESTING }} SELECTED_RUNNER: ${{ needs.is-not-external-contributor.outputs.selected_runner }} SELECTED_RUNNER_GB200: ${{ needs.is-not-external-contributor.outputs.selected_runner_gb200 }} REGISTRY_AWS: ${{ env.container-registry }} @@ -472,7 +524,7 @@ jobs: run: | AWS_ENTRY=$(jq -nc --arg registry "$REGISTRY_AWS" --arg runner "$SELECTED_RUNNER" \ '{"cloud": "aws", "registry": $registry, "runner": $runner}') - if [ "$IS_MAINTAINER" == "true" ]; then + if [ "$IS_MAINTAINER" == "true" ] && [ "$ENABLE_GB200_TESTING" == "true" ]; then GCP_ENTRY=$(jq -nc --arg registry "$REGISTRY_GCP" --arg runner "$SELECTED_RUNNER_GB200" \ '{"cloud": "gcp", "registry": $registry, "runner": $runner}') MATRIX=$(jq -nc --argjson aws "$AWS_ENTRY" --argjson gcp "$GCP_ENTRY" \ @@ -506,26 +558,10 @@ jobs: if: startsWith(github.ref, 'refs/heads/pull-request/') && github.event_name == 'push' uses: nv-gha-runners/get-pr-info@main - - name: Get merge commit sha - shell: bash -x -e -u -o pipefail {0} - id: sha - env: - IS_PR: ${{ startsWith(github.ref, 'refs/heads/pull-request/') }} - IS_MERGE_GROUP: ${{ github.event_name == 'merge_group' }} - run: | - if [[ "$IS_PR" == "true" ]]; then - SHA=${{ fromJSON(steps.get-pr-info.outputs.pr-info || '{}').merge_commit_sha }} - elif [[ "$IS_MERGE_GROUP" == "true" ]]; then - SHA=${{ github.event.merge_group.head_sha }} - else - SHA=${GITHUB_SHA} - fi - echo "main=${SHA}" | tee -a "$GITHUB_OUTPUT" - - name: Checkout uses: actions/checkout@v6 with: - ref: ${{ steps.sha.outputs.main }} + ref: ${{ needs.configure.outputs.sha }} - name: Setup python uses: actions/setup-python@v6 @@ -581,10 +617,12 @@ jobs: NGC_VERSION=$(cat docker/.ngc_version.lts) echo "version=$NGC_VERSION" | tee -a $GITHUB_OUTPUT echo "image_type=lts" | tee -a $GITHUB_OUTPUT + echo "dockerfile=./docker/Dockerfile.ci.lts" | tee -a $GITHUB_OUTPUT else NGC_VERSION=$(cat docker/.ngc_version.dev) echo "version=$NGC_VERSION" | tee -a $GITHUB_OUTPUT echo "image_type=dev" | tee -a $GITHUB_OUTPUT + echo "dockerfile=./docker/Dockerfile.ci.dev" | tee -a $GITHUB_OUTPUT fi - name: Set up Docker Buildx @@ -593,7 +631,7 @@ jobs: - name: Build and push uses: docker/build-push-action@v7.1.0 with: - file: ./docker/Dockerfile.ci.dev + file: ${{ steps.base-image.outputs.dockerfile }} push: true context: . target: main @@ -609,7 +647,7 @@ jobs: no-cache: false tags: | ${{ matrix.registry }}/megatron-lm:${{ fromJSON(steps.get-pr-info.outputs.pr-info || '{}').number || 0 }} - ${{ matrix.registry }}/megatron-lm:${{ github.sha }} + ${{ matrix.registry }}/megatron-lm:${{ needs.configure.outputs.sha }} secrets: | GH_TOKEN=${{ secrets.PAT }} @@ -619,10 +657,12 @@ jobs: unit-tests: ${{ steps.parse-unit-tests.outputs.unit-tests }} needs: - pre-flight + - configure - cicd-wait-in-queue - cicd-container-build if: | needs.pre-flight.result != 'cancelled' + && needs.configure.result != 'cancelled' && needs.cicd-wait-in-queue.result != 'cancelled' && needs.cicd-container-build.result != 'cancelled' && ( @@ -635,6 +675,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v6 + with: + ref: ${{ needs.configure.outputs.sha }} - name: Parse unit tests id: parse-unit-tests run: | @@ -649,6 +691,7 @@ jobs: needs: - is-not-external-contributor - pre-flight + - configure - cicd-wait-in-queue - cicd-container-build - cicd-parse-unit-tests @@ -658,6 +701,7 @@ jobs: if: | needs.is-not-external-contributor.result != 'cancelled' && needs.pre-flight.result != 'cancelled' + && needs.configure.result != 'cancelled' && needs.cicd-wait-in-queue.result != 'cancelled' && needs.cicd-container-build.result != 'cancelled' && needs.cicd-parse-unit-tests.result != 'cancelled' @@ -672,9 +716,13 @@ jobs: PIP_DISABLE_PIP_VERSION_CHECK: 1 PIP_NO_PYTHON_VERSION_WARNING: 1 PIP_ROOT_USER_ACTION: ignore + PIP_DEFAULT_TIMEOUT: 120 + PIP_RETRIES: 5 steps: - name: Checkout uses: actions/checkout@v6 + with: + ref: ${{ needs.configure.outputs.sha }} - name: main uses: ./.github/actions with: @@ -683,9 +731,21 @@ jobs: timeout: ${{ matrix.timeout || 30 }} is_unit_test: "true" PAT: ${{ secrets.PAT }} - container-image: ${{ env.container-registry }}/megatron-lm:${{ github.sha }} - - cicd-parse-integration-tests-h100: + container-image: ${{ env.container-registry }}/megatron-lm:${{ needs.configure.outputs.sha }} + sha: ${{ needs.configure.outputs.sha }} + + # Single source of truth for "should integration tests run?". + # Encodes two independent gates: + # (A) Approval gate — `cicd-wait-in-queue` must have succeeded + # (PR-push env approval), OR we're in a regime where it skips by + # design: merge_group, ci_workload (schedule / workflow_dispatch), + # or an explicit force_run_all override. + # (B) Unit-test gate — unit tests must have succeeded on PR push and + # merge_group; scheduled / force-run workflows bypass this for + # full nightly coverage. + # Downstream integration jobs consume `outputs.should_run` instead of + # duplicating this logic four times. + cicd-integration-gate: runs-on: ubuntu-latest needs: - pre-flight @@ -699,18 +759,60 @@ jobs: && needs.cicd-wait-in-queue.result != 'cancelled' && needs.cicd-container-build.result != 'cancelled' && needs.cicd-unit-tests-latest.result != 'cancelled' - && ( - success() - || needs.pre-flight.outputs.is_ci_workload == 'true' - || needs.pre-flight.outputs.force_run_all == 'true' - || needs.pre-flight.outputs.is_merge_group == 'true' - ) && !cancelled() + outputs: + should_run: ${{ steps.gate.outputs.should_run }} + steps: + - id: gate + env: + WAIT_RESULT: ${{ needs.cicd-wait-in-queue.result }} + UNIT_RESULT: ${{ needs.cicd-unit-tests-latest.result }} + IS_MERGE_GROUP: ${{ needs.pre-flight.outputs.is_merge_group }} + IS_CI_WORKLOAD: ${{ needs.pre-flight.outputs.is_ci_workload }} + FORCE_RUN_ALL: ${{ needs.pre-flight.outputs.force_run_all }} + shell: bash + run: | + # (A) Approval gate + approval=false + if [ "$WAIT_RESULT" = "success" ] \ + || [ "$IS_MERGE_GROUP" = "true" ] \ + || [ "$IS_CI_WORKLOAD" = "true" ] \ + || [ "$FORCE_RUN_ALL" = "true" ]; then + approval=true + fi + # (B) Unit-test gate + unit=false + if [ "$UNIT_RESULT" = "success" ] \ + || [ "$IS_CI_WORKLOAD" = "true" ] \ + || [ "$FORCE_RUN_ALL" = "true" ]; then + unit=true + fi + if [ "$approval" = "true" ] && [ "$unit" = "true" ]; then + should_run=true + else + should_run=false + fi + echo "should_run=$should_run" >> "$GITHUB_OUTPUT" + echo "approval=$approval unit=$unit -> should_run=$should_run" + echo " (wait-in-queue=$WAIT_RESULT, unit-tests=$UNIT_RESULT," + echo " is_merge_group=$IS_MERGE_GROUP, is_ci_workload=$IS_CI_WORKLOAD," + echo " force_run_all=$FORCE_RUN_ALL)" + + cicd-parse-integration-tests-h100: + runs-on: ubuntu-latest + needs: + - configure + - cicd-integration-gate + if: | + !cancelled() + && needs.cicd-integration-gate.outputs.should_run == 'true' outputs: integration-tests-h100: ${{ steps.main.outputs.integration-tests-h100 }} steps: - name: Checkout uses: actions/checkout@v6 + with: + ref: ${{ needs.configure.outputs.sha }} - name: Parse functional tests id: main @@ -756,34 +858,26 @@ jobs: include: ${{ fromJson(needs.cicd-parse-integration-tests-h100.outputs.integration-tests-h100) }} needs: - is-not-external-contributor - - pre-flight - configure - - cicd-wait-in-queue + - cicd-integration-gate - cicd-parse-integration-tests-h100 - - cicd-unit-tests-latest runs-on: ${{ needs.is-not-external-contributor.outputs.selected_runner }} name: "${{ matrix.model }}/${{ matrix.test_case }} - latest" env: PIP_DISABLE_PIP_VERSION_CHECK: 1 PIP_NO_PYTHON_VERSION_WARNING: 1 PIP_ROOT_USER_ACTION: ignore + PIP_DEFAULT_TIMEOUT: 120 + PIP_RETRIES: 5 if: | - needs.is-not-external-contributor.result != 'cancelled' - && needs.pre-flight.result != 'cancelled' - && needs.configure.result != 'cancelled' - && needs.cicd-wait-in-queue.result != 'cancelled' - && needs.cicd-parse-integration-tests-h100.result != 'cancelled' - && needs.cicd-unit-tests-latest.result != 'cancelled' - && ( - success() - || needs.pre-flight.outputs.is_ci_workload == 'true' - || needs.pre-flight.outputs.force_run_all == 'true' - || needs.pre-flight.outputs.is_merge_group == 'true' - ) - && !cancelled() + !cancelled() + && needs.cicd-integration-gate.outputs.should_run == 'true' + && needs.cicd-parse-integration-tests-h100.result == 'success' steps: - name: Checkout uses: actions/checkout@v6 + with: + ref: ${{ needs.configure.outputs.sha }} - name: main uses: ./.github/actions with: @@ -793,40 +887,31 @@ jobs: timeout: ${{ matrix.timeout || 30 }} is_unit_test: "false" PAT: ${{ secrets.PAT }} - container-image: ${{ env.container-registry }}/megatron-lm:${{ github.sha }} + container-image: ${{ env.container-registry }}/megatron-lm:${{ needs.configure.outputs.sha }} scope: ${{ needs.configure.outputs.scope }} n_repeat: ${{ needs.configure.outputs.n_repeat }} lightweight: ${{ needs.configure.outputs.lightweight }} cadence: ${{ needs.configure.outputs.cadence }} + sha: ${{ needs.configure.outputs.sha }} cicd-parse-integration-tests-gb200: runs-on: ubuntu-latest needs: - is-not-external-contributor - - pre-flight - configure - - cicd-wait-in-queue - - cicd-container-build - - cicd-unit-tests-latest + - cicd-integration-gate if: | - needs.is-not-external-contributor.outputs.is_maintainer == 'true' - && needs.pre-flight.result != 'cancelled' - && needs.configure.result != 'cancelled' - && needs.cicd-wait-in-queue.result != 'cancelled' - && needs.cicd-container-build.result != 'cancelled' - && needs.cicd-unit-tests-latest.result != 'cancelled' - && ( - success() - || needs.pre-flight.outputs.is_ci_workload == 'true' - || needs.pre-flight.outputs.force_run_all == 'true' - || needs.pre-flight.outputs.is_merge_group == 'true' - ) - && !cancelled() + !cancelled() + && needs.cicd-integration-gate.outputs.should_run == 'true' + && needs.is-not-external-contributor.outputs.is_maintainer == 'true' + && vars.ENABLE_GB200_TESTING == 'true' outputs: integration-tests-gb200: ${{ steps.main.outputs.integration-tests-gb200 }} steps: - name: Checkout uses: actions/checkout@v6 + with: + ref: ${{ needs.configure.outputs.sha }} - name: Parse functional tests id: main @@ -872,35 +957,28 @@ jobs: include: ${{ fromJson(needs.cicd-parse-integration-tests-gb200.outputs.integration-tests-gb200) }} needs: - is-not-external-contributor - - pre-flight - configure - - cicd-wait-in-queue + - cicd-integration-gate - cicd-parse-integration-tests-gb200 - - cicd-unit-tests-latest runs-on: ${{ needs.is-not-external-contributor.outputs.selected_runner_gb200 }} name: "${{ matrix.model }}/${{ matrix.test_case }} - latest" env: PIP_DISABLE_PIP_VERSION_CHECK: 1 PIP_NO_PYTHON_VERSION_WARNING: 1 PIP_ROOT_USER_ACTION: ignore + PIP_DEFAULT_TIMEOUT: 120 + PIP_RETRIES: 5 if: | - needs.is-not-external-contributor.outputs.is_maintainer == 'true' - && needs.is-not-external-contributor.result != 'cancelled' - && needs.pre-flight.result != 'cancelled' - && needs.configure.result != 'cancelled' - && needs.cicd-wait-in-queue.result != 'cancelled' - && needs.cicd-parse-integration-tests-gb200.result != 'cancelled' - && needs.cicd-unit-tests-latest.result != 'cancelled' - && ( - success() - || needs.pre-flight.outputs.is_ci_workload == 'true' - || needs.pre-flight.outputs.force_run_all == 'true' - || needs.pre-flight.outputs.is_merge_group == 'true' - ) - && !cancelled() + !cancelled() + && needs.cicd-integration-gate.outputs.should_run == 'true' + && needs.cicd-parse-integration-tests-gb200.result == 'success' + && needs.is-not-external-contributor.outputs.is_maintainer == 'true' + && vars.ENABLE_GB200_TESTING == 'true' steps: - name: Checkout uses: actions/checkout@v6 + with: + ref: ${{ needs.configure.outputs.sha }} - name: main uses: ./.github/actions with: @@ -910,12 +988,13 @@ jobs: timeout: ${{ matrix.timeout || 30 }} is_unit_test: "false" PAT: ${{ secrets.PAT }} - container-image: ${{ env.container-registry-gb200 }}/megatron-lm:${{ github.sha }} + container-image: ${{ env.container-registry-gb200 }}/megatron-lm:${{ needs.configure.outputs.sha }} scope: ${{ needs.configure.outputs.scope }} n_repeat: ${{ needs.configure.outputs.n_repeat }} lightweight: ${{ needs.configure.outputs.lightweight }} platform: dgx_gb200 cadence: ${{ needs.configure.outputs.cadence }} + sha: ${{ needs.configure.outputs.sha }} Nemo_CICD_Test: needs: @@ -949,6 +1028,9 @@ jobs: DOCS_ONLY: ${{ needs.pre-flight.outputs.docs_only }} IS_DEPLOYMENT: ${{ needs.pre-flight.outputs.is_deployment_workflow }} IS_MAINTAINER: ${{ needs.is-not-external-contributor.outputs.is_maintainer }} + IS_CI_WORKLOAD: ${{ needs.pre-flight.outputs.is_ci_workload }} + FORCE_RUN_ALL: ${{ needs.pre-flight.outputs.force_run_all }} + ENABLE_GB200_TESTING: ${{ vars.ENABLE_GB200_TESTING }} UNIT_RESULT: ${{ needs.cicd-unit-tests-latest.result }} H100_RESULT: ${{ needs.cicd-integration-tests-latest-h100.result }} GB200_RESULT: ${{ needs.cicd-integration-tests-latest-gb200.result }} @@ -961,26 +1043,40 @@ jobs: FAILED=false - # Unit tests must always succeed (never skipped or cancelled) + # Unit tests are required on PR-push and merge_group, but scheduled + # / force-run workflows still want integration to run (and be + # judged) even when unit tests failed — for full nightly coverage. + FORCE_INTEGRATION=false + if [ "$IS_CI_WORKLOAD" == "true" ] || [ "$FORCE_RUN_ALL" == "true" ]; then + FORCE_INTEGRATION=true + fi + if [ "$UNIT_RESULT" != "success" ]; then echo "❌ cicd-unit-tests-latest: $UNIT_RESULT" FAILED=true + # On PR-push / merge_group, integration was skipped by design — + # don't double-fail on H100/GB200 below. + if [ "$FORCE_INTEGRATION" != "true" ]; then + H100_RESULT=skipped-by-unit-failure + GB200_RESULT=skipped-by-unit-failure + fi fi - # H100 integration tests must always succeed - if [ "$H100_RESULT" != "success" ]; then + if [ "$H100_RESULT" != "success" ] && [ "$H100_RESULT" != "skipped-by-unit-failure" ]; then echo "❌ cicd-integration-tests-latest-h100: $H100_RESULT" FAILED=true fi - # GB200 integration tests may be skipped only for non-maintainer PRs - # (no GB200 runners available); maintainer runs must always succeed - if [ "$GB200_RESULT" == "skipped" ] && [ "$IS_MAINTAINER" == "true" ]; then - echo "❌ cicd-integration-tests-latest-gb200: skipped unexpectedly for a maintainer run" - FAILED=true - elif [ "$GB200_RESULT" != "success" ] && [ "$GB200_RESULT" != "skipped" ]; then - echo "❌ cicd-integration-tests-latest-gb200: $GB200_RESULT" - FAILED=true + # GB200 integration tests are required only when explicitly enabled. + if [ "$ENABLE_GB200_TESTING" == "true" ]; then + # GB200 integration tests may be skipped only for non-maintainer PRs + # (no GB200 runners available); maintainer runs must always succeed. + if [ "$GB200_RESULT" == "skipped" ] && [ "$IS_MAINTAINER" == "true" ]; then + echo "❌ cicd-integration-tests-latest-gb200: skipped unexpectedly for a maintainer run" + FAILED=true + fi + else + echo "✅ GB200 integration tests disabled by ENABLE_GB200_TESTING" fi # Broad scan: catch any individual job failures or cancellations @@ -1146,3 +1242,10 @@ jobs: - name: Taint node for cleanup shell: bash run: taint-node.sh + + DCO_merge_group: + name: DCO + if: github.event_name == 'merge_group' + runs-on: ubuntu-latest + steps: + - run: echo "The real DCO check happens on PRs only. This is a placeholder for the merge queue to keep the DCO check as a required status check." diff --git a/.github/workflows/claude-copy-to-main.yml b/.github/workflows/claude-copy-to-main.yml index 7bde3941bb8..3905a276fd0 100644 --- a/.github/workflows/claude-copy-to-main.yml +++ b/.github/workflows/claude-copy-to-main.yml @@ -10,6 +10,7 @@ jobs: if: | github.event_name == 'issue_comment' && github.event.issue.pull_request && + github.event.comment.user.login != 'svcnvidia-nemo-ci' && contains(github.event.comment.body, '/claude copy') runs-on: ubuntu-latest permissions: @@ -28,7 +29,7 @@ jobs: run: | PERMISSION=$(gh api repos/$REPO/collaborators/$COMMENTER/permission --jq .permission) if [[ "$PERMISSION" != "admin" && "$PERMISSION" != "write" ]]; then - gh pr comment $PR_NUMBER --repo $REPO --body "❌ You do not have write access to use \`/claude copy\`." + gh pr comment $PR_NUMBER --repo $REPO --body "❌ You do not have write access to use the Claude copy command." exit 1 fi @@ -39,12 +40,12 @@ jobs: PR_MERGED=$(echo "$PR_JSON" | jq -r .mergedAt) if [ "$PR_BASE" = "main" ]; then - gh pr comment $PR_NUMBER --repo $REPO --body "❌ This PR already targets \`main\`. \`/claude copy\` only works on PRs targeting non-main branches." + gh pr comment $PR_NUMBER --repo $REPO --body "❌ This PR already targets \`main\`. The Claude copy command only works on PRs targeting non-main branches." exit 1 fi if [ "$PR_MERGED" = "null" ] || [ -z "$PR_MERGED" ]; then - gh pr comment $PR_NUMBER --repo $REPO --body "❌ This PR has not been merged yet. \`/claude copy\` only works on merged PRs." + gh pr comment $PR_NUMBER --repo $REPO --body "❌ This PR has not been merged yet. The Claude copy command only works on merged PRs." exit 1 fi @@ -110,7 +111,7 @@ jobs: --base main \ --head copy-pr-${PR_NUMBER}-to-main \ --title "[Copy to main] " \ - --body "🤖 **This PR was auto-generated by Claude** via the \`/claude copy\` command.\n\nCherry-picked from #${PR_NUMBER}.\n\n---\n\n" + --body "🤖 **This PR was auto-generated by Claude** via the Claude copy workflow.\n\nCherry-picked from #${PR_NUMBER}.\n\n---\n\n" 8. Comment on the original PR with a link to the newly created PR. diff --git a/.github/workflows/claude_review.yml b/.github/workflows/claude_review.yml index b7d5f1217c0..c4ca7423eff 100644 --- a/.github/workflows/claude_review.yml +++ b/.github/workflows/claude_review.yml @@ -73,6 +73,13 @@ jobs: - If the PR adds a new feature or significant functionality without corresponding tests, suggest adding tests - If the PR fixes a bug that was not caught by an existing unit test, suggest adding a regression test to prevent recurrence - Outdated or inaccurate documentation affected by the changes + - New direct global process group access in `megatron/core` production code + - Flag added calls to `parallel_state.get_*_group()` or directly imported + `get_*_group()` helpers unless they are in `parallel_state.py`, + `process_groups_config.py`, initialization/bootstrap code that materializes a + `ProcessGroupCollection`, tests, docs, or an explicitly documented migration fallback + - Prefer passing a `ProcessGroupCollection` or explicit + `torch.distributed.ProcessGroup` from the caller Do NOT comment on: - Style preferences or formatting @@ -212,6 +219,18 @@ jobs: - **API contract changes**: Changed function signatures, return types, or side effects in megatron/core/ without backward-compat shim - **Model architecture changes**: Altered layer ordering, initialization, or normalization placement — existing pretrained weights become incompatible + ### Megatron Core Process Group Usage + - In `megatron/core` production code, treat new direct reads of global process groups + from `parallel_state` as review findings unless they are clearly compatibility-only. + - Flag added calls to `parallel_state.get_*_group()` or directly imported + `get_*_group()` helpers when the surrounding code could instead receive a + `ProcessGroupCollection` or explicit `torch.distributed.ProcessGroup` from its caller. + - Do not flag `megatron/core/parallel_state.py`, `megatron/core/process_groups_config.py`, + tests, docs, initialization/bootstrap code that materializes a `ProcessGroupCollection` + from MPU globals, or explicitly documented migration fallbacks. + - This guidance is advisory and targets Megatron Core library code; do not apply it to + `megatron/training` or other training-loop code unless the PR opts into that migration. + ### Mandatory Check: Unused New Variables / Arguments - For each changed file, list newly added identifiers (function args, config fields, locals). - Verify each has a meaningful read/use path — not just declaration/docstring or discard assignment (_ = new_arg). diff --git a/.github/workflows/nightly-sync-main-to-dev.yml b/.github/workflows/nightly-sync-main-to-dev.yml index d7c9e46811d..4be18456f1a 100644 --- a/.github/workflows/nightly-sync-main-to-dev.yml +++ b/.github/workflows/nightly-sync-main-to-dev.yml @@ -17,9 +17,10 @@ name: Nightly Sync Main to Dev on: workflow_dispatch: schedule: - # 21:00 UTC = 2 PM PDT (1 PM PST during winter — GitHub Actions cron - # is UTC-only and does not follow DST). - - cron: '0 21 * * *' + # Twice-weekly cadence: Monday and Thursday at 15:00 UTC. + # 15:00 UTC = 8 AM PDT (7 AM PST during winter — GitHub Actions cron + # is UTC-only and does not follow DST). Days-of-week: 1=Mon, 4=Thu. + - cron: '0 15 * * 1,4' concurrency: group: nightly-sync-main-to-dev @@ -55,8 +56,10 @@ jobs: sync-main-to-dev: if: github.event_name == 'workflow_dispatch' && github.repository == 'NVIDIA/Megatron-LM' - runs-on: ubuntu-latest - timeout-minutes: 360 + # GitHub-hosted runners are capped at 6h; use an NVIDIA runner so the + # sync bot can wait through long CI queues and retries. + runs-on: linux-amd64-cpu16 + timeout-minutes: 720 env: GH_TOKEN: ${{ secrets.PAT }} steps: @@ -108,6 +111,76 @@ jobs: echo "skip=false" >> "$GITHUB_OUTPUT" fi + - name: Install pre-push merge guard + if: steps.check-sync.outputs.skip != 'true' + run: | + cat > .git/hooks/pre-push <<'HOOK' + #!/usr/bin/env bash + set -euo pipefail + + echo "=== nightly-sync pre-push guard ===" + + merge_commit=$(git rev-list --min-parents=2 --max-count=1 HEAD || true) + if [ -n "$merge_commit" ]; then + dev_ref="${merge_commit}^1" + main_ref="${merge_commit}^2" + else + dev_ref="origin/dev" + main_ref="origin/main" + fi + + if ! git diff --quiet "$dev_ref" HEAD -- .github/CODEOWNERS; then + echo "ABORT: .github/CODEOWNERS differs from dev. Restore it before pushing." + exit 1 + fi + + for f in pyproject.toml uv.lock docker/Dockerfile.ci.dev; do + if ! git diff --quiet "$dev_ref" HEAD -- "$f"; then + echo "WARNING: $f differs from dev" + fi + done + + if [ -z "$merge_commit" ]; then + echo "No merge commit found in HEAD history; skipping dev-feature audit." + exit 0 + fi + + intentional_override_regex='^(megatron/training/training\.py|megatron/training/initialize\.py|megatron/training/utils\.py|megatron/training/datasets/data_samplers\.py|megatron/core/optimizer/layer_wise_optimizer\.py)$' + skip_regex='^(pyproject\.toml|uv\.lock|docker/Dockerfile\.ci\.dev|\.github/CODEOWNERS)$' + + violations=0 + while IFS= read -r f; do + [[ "$f" =~ $skip_regex ]] && continue + [[ "$f" =~ $intentional_override_regex ]] && continue + git cat-file -e "HEAD:$f" 2>/dev/null || continue + + missing=$(comm -23 \ + <(git show "$dev_ref:$f" 2>/dev/null | sort -u) \ + <(git show "$main_ref:$f" 2>/dev/null | sort -u) \ + | comm -23 - <(git show "HEAD:$f" 2>/dev/null | sort -u) \ + | grep -E '[[:alnum:]_]' \ + || true) + + if [ -n "$missing" ]; then + echo "=== $f ===" + printf '%s\n' "$missing" + violations=$((violations + $(printf '%s\n' "$missing" | grep -c .))) + fi + done < <(git diff --name-only "$dev_ref"..HEAD \ + -- '*.py' '*.md' '*.yaml' '*.yml' '*.toml' \ + '*.sh' '*.cpp' '*.cu' '*.h' \ + | sort -u) + + if [ "$violations" -gt 0 ]; then + echo "ABORT: $violations dev-only line(s) were dropped by the merge." + echo "Restore the dev-only code, or document the exact main commit that intentionally removed it." + exit 1 + fi + + echo "nightly-sync pre-push guard passed" + HOOK + chmod +x .git/hooks/pre-push + - name: Run Claude Code to merge, fix, and iterate if: steps.check-sync.outputs.skip != 'true' uses: anthropics/claude-code-action@v1 @@ -159,6 +232,21 @@ jobs: conversation in between — that wastes `--max-turns` and creates windows where the agent could forget the loop. + **Pre-push guard:** The workflow installs a local git pre-push + hook that enforces CODEOWNERS, dependency-triple, and dev-feature + preservation checks. You MUST NOT bypass it with `--no-verify`. + If a push fails, read the hook output, restore the dropped dev + code unless main explicitly removed it, and push again only after + the hook passes. + + **Merge strategy:** Start from `origin/dev` and run + `git merge origin/main --no-edit`. Do NOT use global + `git merge -X theirs`. Main's version may be taken wholesale only + for files explicitly listed in the nightly-sync skill's + "Files to Override from Main" section. For other conflicts, + preserve recent dev-only additions and combine them with main's + incoming changes. + **Source of truth for CI status:** `gh pr view --repo $REPO --json statusCheckRollup` This lists every required check — GitHub Actions jobs AND diff --git a/.github/workflows/release-freeze.yml b/.github/workflows/release-freeze.yml index 97def9dc2f7..8037a8cb4bc 100644 --- a/.github/workflows/release-freeze.yml +++ b/.github/workflows/release-freeze.yml @@ -34,13 +34,15 @@ on: default: true jobs: code-freeze: - uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_code_freeze.yml@v0.86.0 + uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_code_freeze.yml@v1.4.2 with: library-name: Megatron-Core python-package: megatron.core release-type: ${{ inputs.release-type }} freeze-commit: ${{ inputs.freeze-commit }} dry-run: ${{ inputs.dry-run }} + next-pre-release: "" + next-dev: "" release-branch-prefix: core_ use-pat: true secrets: diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index a756d49eb20..cd193d819eb 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,4 +1,4 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2019-2026, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,9 +11,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -name: "Release Megatron-Core" +name: "Build, validate, and release Megatron-Core" on: + push: + branches: + - main + - "pull-request/[0-9]+" + - "deploy-release/*" + merge_group: + types: [checks_requested] workflow_dispatch: inputs: release-ref: @@ -21,7 +28,7 @@ on: required: true type: string dry-run: - description: Do not publish a wheel and GitHub release. + description: Compute the release but do not publish wheel, GH release, or docs. required: true default: true type: boolean @@ -51,29 +58,106 @@ on: default: "" permissions: - contents: write # To read repository content - pull-requests: write # To create PRs + id-token: write + contents: write + pull-requests: write + +defaults: + run: + shell: bash -x -e -u -o pipefail {0} + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.event_name }} + cancel-in-progress: ${{ github.event_name == 'push' }} jobs: - release: - uses: ./.github/workflows/_release_library.yml + pre-flight: + uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_cicd_preflight.yml@v0.94.1 + if: github.repository == 'NVIDIA/Megatron-LM' && github.event_name != 'workflow_dispatch' + + bump: + needs: [pre-flight] + if: | + !cancelled() && !failure() + && github.repository == 'NVIDIA/Megatron-LM' + && !(needs.pre-flight.outputs.docs_only == 'true' + || needs.pre-flight.outputs.is_merge_group == 'true' + || needs.pre-flight.outputs.is_deployment_workflow == 'true') + uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_release_bump.yml@v1.4.0 with: + release-branch-pattern: "core_[rv][0-9]*.[0-9]*.[0-9]*" release-ref: ${{ inputs.release-ref || github.sha }} + validate-only: ${{ github.event_name != 'workflow_dispatch' }} dry-run: ${{ inputs.dry-run || false }} version-bump-branch: ${{ inputs.version-bump-branch || github.ref_name }} + restrict-to-admins: true + app-id: ${{ vars.BOT_ID }} + library-name: Megatron Core + bump-targets: | + [ + {"python-package": "megatron.core", "src-dir": ""}, + {"python-package": "megatron_fsdp", "src-dir": "megatron/core/distributed/fsdp/src/"} + ] + secrets: inherit # pragma: allowlist secret + + build-test-publish-wheels: + needs: [pre-flight, bump] + if: | + !cancelled() && !failure() && needs.bump.result == 'success' + && github.repository == 'NVIDIA/Megatron-LM' + && ( + github.event_name == 'workflow_dispatch' + || !(needs.pre-flight.outputs.docs_only == 'true' + || needs.pre-flight.outputs.is_deployment_workflow == 'true') + ) + uses: ./.github/workflows/_build_test_publish_wheel.yml + with: + ref: ${{ inputs.release-ref || github.sha }} + dry-run: ${{ inputs.dry-run || false }} + no-publish: ${{ github.event_name != 'workflow_dispatch' || inputs.dry-run }} + secrets: inherit # pragma: allowlist secret + + finalize: + needs: [bump, build-test-publish-wheels] + if: | + github.repository == 'NVIDIA/Megatron-LM' + && (success() || !failure()) + && !cancelled() + uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_release_finalize.yml@v1.0.0 + with: + release-ref: ${{ inputs.release-ref || github.sha }} + release-version: ${{ needs.bump.outputs.release-version }} + library-name: Megatron Core + pypi-name: megatron-core + validate-only: ${{ github.event_name != 'workflow_dispatch' }} + dry-run: ${{ inputs.dry-run || false }} create-gh-release: ${{ inputs.create-gh-release || true }} - gh-release-use-changelog-builder: ${{ inputs.generate-changelog }} - publish-docs: ${{ inputs.publish-docs }} - gh-release-from-tag: ${{ inputs.gh-release-from-tag }} - secrets: - TWINE_PASSWORD: ${{ (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/r')) && secrets.SVC_PYPI_TOKEN || secrets.SVC_PYPI_TEST_TOKEN }} - SLACK_WEBHOOK: ${{ (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/r')) && secrets.SLACK_MAIN_CHANNEL_WEBHOOK || secrets.SLACK_CI_CHANNEL_WEBHOOK }} - PAT: ${{ secrets.PAT }} - AWS_ASSUME_ROLE_ARN: ${{ secrets.AWS_ASSUME_ROLE_ARN }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AKAMAI_HOST: ${{ secrets.AKAMAI_HOST }} - AKAMAI_CLIENT_TOKEN: ${{ secrets.AKAMAI_CLIENT_TOKEN }} - AKAMAI_CLIENT_SECRET: ${{ secrets.AKAMAI_CLIENT_SECRET }} - AKAMAI_ACCESS_TOKEN: ${{ secrets.AKAMAI_ACCESS_TOKEN }} - S3_BUCKET_NAME: ${{ secrets.S3_BUCKET_NAME }} + gh-release-tag-prefix: core_ + gh-release-use-changelog-builder: ${{ inputs.generate-changelog || false }} + gh-release-from-tag: ${{ inputs.gh-release-from-tag || '' }} + publish-docs: ${{ inputs.publish-docs || true }} + docs-target-path: megatron-core/developer-guide + publish-as-latest: true + run-on-version-tag-only: ${{ github.ref_name != 'main' }} + app-id: ${{ vars.BOT_ID }} + secrets: inherit # pragma: allowlist secret + + release-summary: + needs: [pre-flight, bump, build-test-publish-wheels, finalize] + if: github.repository == 'NVIDIA/Megatron-LM' && !cancelled() + runs-on: ubuntu-latest + steps: + - name: Result + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + FAILED_JOBS=$(gh run view $GITHUB_RUN_ID --repo ${{ github.repository }} --json jobs --jq '[.jobs[] | select(.conclusion == "failure" or .conclusion == "timed_out" or .conclusion == "action_required")] | length') + + if [ "${FAILED_JOBS:-0}" -eq 0 ]; then + echo "✅ All previous jobs completed successfully" + exit 0 + else + echo "❌ Found $FAILED_JOBS failed job(s)" + gh run view $GITHUB_RUN_ID --repo ${{ github.repository }} --json jobs --jq '.jobs[] | select(.conclusion == "failure" or .conclusion == "timed_out" or .conclusion == "action_required") | .name' + exit 1 + fi diff --git a/.github/workflows/request-nvskills-ci.yml b/.github/workflows/request-nvskills-ci.yml new file mode 100644 index 00000000000..01c9b5c7569 --- /dev/null +++ b/.github/workflows/request-nvskills-ci.yml @@ -0,0 +1,22 @@ +name: Request NVSkills CI + +on: + issue_comment: + types: [created] + push: + +jobs: + request: + if: > + (github.event_name == 'issue_comment' && + github.event.issue.pull_request && + startsWith(github.event.comment.body, '/nvskills-ci')) || + (github.event_name == 'push' && + github.actor == (vars.NVSKILLS_SIGNATURE_PUSH_ACTOR || 'nv-skills-ci[bot]') && + startsWith(github.event.head_commit.message, vars.NVSKILLS_SIGNATURE_COMMIT_TITLE || 'Attach NVSkills validation signatures')) + permissions: + contents: read + pull-requests: read + uses: NVIDIA/skills/.github/workflows/team-request.yml@main + secrets: + NVSKILLS_CI_DISPATCH_TOKEN: ${{ secrets.NVSKILLS_CI_DISPATCH_TOKEN }} diff --git a/.gitignore b/.gitignore index 5556d1d5a4a..b16e6212199 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ build .coverage_* *.egg-info *~ +*.swp slurm* logs .vscode @@ -22,4 +23,5 @@ docs/_build docs/apidocs # Git worktrees -.worktrees/ \ No newline at end of file +.worktrees/ +.claude/worktrees/ diff --git a/.gitlab/scripts/build.sh b/.gitlab/scripts/build.sh index 0f34b838384..15c926ed51f 100644 --- a/.gitlab/scripts/build.sh +++ b/.gitlab/scripts/build.sh @@ -61,6 +61,7 @@ DOCKER_BUILDKIT=1 docker build \ --builder=container \ --build-arg JET_API_VERSION=$JET_API_VERSION \ --build-arg FROM_IMAGE_NAME=$BASE_IMAGE \ + --build-arg IMAGE_TYPE=$IMAGE_TYPE \ --provenance=false \ --push \ --progress plain \ diff --git a/.gitlab/stages/01.build.yml b/.gitlab/stages/01.build.yml index aef3d64b014..f06add00fa2 100644 --- a/.gitlab/stages/01.build.yml +++ b/.gitlab/stages/01.build.yml @@ -52,13 +52,11 @@ test:pre_build_image: parallel: matrix: - IMAGE: CI_MCORE_LTS_IMAGE - FILE: Dockerfile.ci.dev - IMAGE_TYPE: lts + FILE: Dockerfile.ci.lts BASE_IMAGE: nvcr.io/nvidia/pytorch:25.09-py3 PLATFORM: amd64 - IMAGE: CI_MCORE_LTS_IMAGE - FILE: Dockerfile.ci.dev - IMAGE_TYPE: lts + FILE: Dockerfile.ci.lts BASE_IMAGE: nvcr.io/nvidia/pytorch:25.09-py3 PLATFORM: arm64 - IMAGE: CI_MCORE_DEV_IMAGE diff --git a/.gitlab/stages/04.functional-tests.yml b/.gitlab/stages/04.functional-tests.yml index 2ba48eb5ffd..75158635914 100644 --- a/.gitlab/stages/04.functional-tests.yml +++ b/.gitlab/stages/04.functional-tests.yml @@ -88,6 +88,36 @@ functional:configure: "--slurm-account ${CI_SLURM_ACCOUNT}" "--no-enable-warmup" ) + - | + SMOKE_ARGS=( + "--scope L0-smoke" + "--n-repeat 1" + "--time-limit 540" + "--enable-lightweight-mode" + "--test-cases all" + "--container-image ${UTILITY_IMAGE}" + "--container-tag ${CI_PIPELINE_ID}" + "--dependent-job functional:configure" + "--record-checkpoints false" + "--slurm-account ${CI_SLURM_ACCOUNT}" + "--no-enable-warmup" + ) + - | + export PYTHONPATH=$(pwd) + python tests/test_utils/python_scripts/generate_jet_trigger_job.py \ + ${SMOKE_ARGS[@]} \ + --environment dev \ + --platform dgx_h100 \ + --cluster $H100_CLUSTER \ + --output-path "functional-smoke-h100-job.yaml" + - | + export PYTHONPATH=$(pwd) + python tests/test_utils/python_scripts/generate_jet_trigger_job.py \ + ${SMOKE_ARGS[@]} \ + --environment dev \ + --platform dgx_gb200 \ + --cluster $GB200_CLUSTER \ + --output-path "functional-smoke-gb200-job.yaml" - | export PYTHONPATH=$(pwd) python tests/test_utils/python_scripts/generate_jet_trigger_job.py \ @@ -144,6 +174,8 @@ functional:configure: ${RELEASE_ARGS[@]} artifacts: paths: + - functional-smoke-h100-job.yaml + - functional-smoke-gb200-job.yaml - functional-test-job-lts-A100.yaml - functional-test-job-lts-H100.yaml - functional-test-job-dev-A100.yaml @@ -176,9 +208,134 @@ functional:configure: inherit: variables: true +.functional_smoke_rules: + stage: functional_tests + rules: + - if: $BUILD == "no" + when: never + - if: $FUNCTIONAL_TEST == "yes" && $FUNCTIONAL_TEST_SCOPE =~ /^(mr|nightly|weekly|release)$/ + when: on_success + - when: never + +# Reuses `.functional_run` (needs, trigger strategy, variables, inherit) and +# only overrides what is smoke-specific: the smoke rules, the generated smoke +# artifact, and the fixed H100 cluster. `.functional_smoke_rules` is listed last +# so its rules win over `.functional_run`'s inherited `.functional_tests_rules`. +functional:smoke-h100: + extends: [.functional_run, .functional_smoke_rules] + trigger: + include: + - artifact: functional-smoke-h100-job.yaml + job: functional:configure + strategy: depend + variables: + CLUSTER: H100 + +# GB200 counterpart of functional:smoke-h100. Same lightweight L0-smoke recipe set, +# generated for the dgx_gb200 platform, gating the GB200 functional runs. +functional:smoke-gb200: + extends: [.functional_run, .functional_smoke_rules] + trigger: + include: + - artifact: functional-smoke-gb200-job.yaml + job: functional:configure + strategy: depend + variables: + CLUSTER: GB200 + +# Prune artifacts older than 14 days from the JET assets/artifacts roots on +# each cluster. Gated on release/weekly scope so it does not run on every MR +# pipeline. Functional `run_*` jobs depend on the matching cleanup job via +# `optional: true`, so pipelines without a cleanup job (e.g. `mr` scope) still +# start their workloads immediately. +.functional_cleanup_rules: + stage: functional_tests + rules: + - if: $BUILD == "no" + when: never + - if: $FUNCTIONAL_TEST == "yes" && ($FUNCTIONAL_TEST_SCOPE == "release" || $FUNCTIONAL_TEST_SCOPE == "weekly") + when: on_success + - when: never + +.functional_cleanup: + needs: + - functional:configure + - test:build_image + extends: [.functional_cleanup_rules] + image: ${UTILITY_IMAGE}:${CI_PIPELINE_ID} + tags: + - arch/amd64 + - env/prod + - origin/jet-fleet + - owner/jet-core + - purpose/utility + - team/megatron + timeout: 30m + allow_failure: true + script: + - | + case "$CLEANUP_PLATFORM" in + dgx_a100) + GPU_CLUSTER=$([[ "$CLUSTER_A100" != "" ]] && echo "$CLUSTER_A100" || echo "$DEFAULT_A100_CLUSTER") + ;; + dgx_h100) + GPU_CLUSTER=$([[ "$CLUSTER_H100" != "" ]] && echo "$CLUSTER_H100" || echo "$DEFAULT_H100_CLUSTER") + ;; + dgx_gb200) + GPU_CLUSTER=$([[ "$CLUSTER_GB200" != "" ]] && echo "$CLUSTER_GB200" || echo "$DEFAULT_GB200_CLUSTER") + ;; + *) + echo "Unknown CLEANUP_PLATFORM: $CLEANUP_PLATFORM" + exit 1 + ;; + esac + - CLEANUP_CLUSTER="cpu_${GPU_CLUSTER#dgx*_}" + - echo "Cleanup target = $CLEANUP_CLUSTER (CPU partition of $GPU_CLUSTER)" + - export PYTHONPATH=$(pwd) + - export RO_API_TOKEN=${PAT} + - | + python tests/test_utils/python_scripts/launch_jet_workload.py \ + --model cleanup \ + --test-case cleanup_old_artifacts \ + --environment dev \ + --n-repeat 1 \ + --time-limit 1800 \ + --scope cleanup \ + --account ${CI_SLURM_ACCOUNT} \ + --cluster ${CLEANUP_CLUSTER} \ + --platform ${CLEANUP_PLATFORM} \ + --container-tag ${CI_PIPELINE_ID} \ + --container-image ${UTILITY_IMAGE} \ + --record-checkpoints false + +functional:cleanup_dgx_a100: + extends: [.functional_cleanup] + variables: + CLEANUP_PLATFORM: dgx_a100 + CLEANUP_PATHS: $CLEANUP_PATHS_DGX_A100 + +functional:cleanup_dgx_h100: + extends: [.functional_cleanup] + variables: + CLEANUP_PLATFORM: dgx_h100 + CLEANUP_PATHS: $CLEANUP_PATHS_DGX_H100 + +functional:cleanup_dgx_gb200: + extends: [.functional_cleanup] + variables: + CLEANUP_PLATFORM: dgx_gb200 + CLEANUP_PATHS: $CLEANUP_PATHS_DGX_GB200 + functional:run_lts_dgx_a100: extends: [.functional_run] allow_failure: true + needs: + - functional:configure + - test:build_image + - job: functional:smoke-h100 + optional: true + - job: functional:cleanup_dgx_a100 + optional: true variables: ENVIRONMENT: lts CLUSTER: A100 @@ -186,6 +343,13 @@ functional:run_lts_dgx_a100: functional:run_lts_dgx_h100: extends: [.functional_run] allow_failure: true + needs: + - functional:configure + - test:build_image + - job: functional:smoke-h100 + optional: true + - job: functional:cleanup_dgx_h100 + optional: true variables: ENVIRONMENT: lts CLUSTER: H100 @@ -193,24 +357,52 @@ functional:run_lts_dgx_h100: functional:run_lts_dgx_gb200: extends: [.functional_run] allow_failure: true + needs: + - functional:configure + - test:build_image + - job: functional:smoke-gb200 + optional: true + - job: functional:cleanup_dgx_gb200 + optional: true variables: ENVIRONMENT: lts CLUSTER: GB200 functional:run_dev_dgx_a100: extends: [.functional_run] + needs: + - functional:configure + - test:build_image + - job: functional:smoke-h100 + optional: true + - job: functional:cleanup_dgx_a100 + optional: true variables: ENVIRONMENT: dev CLUSTER: A100 functional:run_dev_dgx_h100: extends: [.functional_run] + needs: + - functional:configure + - test:build_image + - job: functional:smoke-h100 + optional: true + - job: functional:cleanup_dgx_h100 + optional: true variables: ENVIRONMENT: dev CLUSTER: H100 functional:run_dev_dgx_gb200: extends: [.functional_run] + needs: + - functional:configure + - test:build_image + - job: functional:smoke-gb200 + optional: true + - job: functional:cleanup_dgx_gb200 + optional: true variables: ENVIRONMENT: dev CLUSTER: GB200 diff --git a/AGENTS.md b/AGENTS.md index 70e8152cbf4..e747867f8b0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,8 +24,28 @@ skill keyword — infer it from the artifact you read. - All PRs must be created as **drafts**. Use `gh pr create --draft` or the GitHub UI draft option. - Never push branches directly to `https://github.com/NVIDIA/Megatron-LM`. You must push your branch to a personal fork (e.g. `https://github.com//Megatron-LM`), then open a PR from the fork's branch against `NVIDIA/Megatron-LM`. +- Commit PR changes with both `-s` and `-S`: `-s` adds the required `Signed-off-by` trailer, and `-S` signs the commit so copy-pr-bot and `/ok to test` can verify the pushed commit without manually specifying the SHA. - Read @docs/developer/contribute.md for the full contribution policy, including code style, commit message conventions, and issue guidelines. ### Code Quality - After editing imports in any Python files, always run `uv run isort` on those files to fix import order before committing. + +### Megatron Core Process Groups + +- In `megatron/core` production code, avoid adding new direct reads of global + process groups from `parallel_state` (for example, + `parallel_state.get_tensor_model_parallel_group()` or directly imported + `get_*_group()` helpers). Prefer accepting a `ProcessGroupCollection` or an + explicit `torch.distributed.ProcessGroup` from the caller and passing that + through. +- Allowed compatibility points include `megatron/core/parallel_state.py`, + `megatron/core/process_groups_config.py`, initialization/bootstrap code that + materializes a `ProcessGroupCollection` from MPU globals, tests, docs, and + migration fallbacks with an explicit comment. +- This guidance targets Megatron Core library code. Do not apply it to + `megatron/training` or other training-loop code unless the PR explicitly + opts into that migration. +- In reviews, flag new direct `parallel_state.get_*_group()` usage in + `megatron/core` unless it is one of the compatibility points above. This is + advisory guidance, not a CI gate. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6b128dce590..e0fa644cd2d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,3 +1,3 @@ # Contributing to Megatron -Visit our [contributing page](https://docs.nvidia.com/megatron-core/developer-guide/latest/developer/contribute.html). \ No newline at end of file +Visit our [contributing page](https://docs.nvidia.com/megatron-core/developer-guide/nightly/developer/contribute.html). diff --git a/SECURITY.md b/SECURITY.md index 728cdb4a1d2..0355bbc1f37 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,25 +1,25 @@ ## Security -NVIDIA is dedicated to the security and trust of our software products and services, including all source code repositories managed through our organization. +NVIDIA is dedicated to the security and trust of its software products and services, including all source code repositories managed through its organization. -If you need to report a security issue, please use the appropriate contact points outlined below. **Please do not report security vulnerabilities through GitHub.** If a potential security issue is inadvertently reported via a public issue or pull request, NVIDIA maintainers may limit public discussion and redirect the reporter to the appropriate private disclosure channels. +If you need to report a security issue, use the appropriate contact points outlined below. **Do not report security vulnerabilities through GitHub.** If someone inadvertently reports a potential security issue through a public issue or pull request, NVIDIA maintainers may limit public discussion and redirect the reporter to the appropriate private disclosure channels. ## Reporting Potential Security Vulnerability in an NVIDIA Product To report a potential security vulnerability in any NVIDIA product: - Web: [Security Vulnerability Submission Form](https://www.nvidia.com/object/submit-security-vulnerability.html) -- E-Mail: psirt@nvidia.com - - We encourage you to use the following PGP key for secure email communication: [NVIDIA public PGP Key for communication](https://www.nvidia.com/en-us/security/pgp-key) - - Please include the following information: +- Email: `psirt@nvidia.com` + - For secure email communication, use the following PGP key: [NVIDIA public PGP Key for communication](https://www.nvidia.com/en-us/security/pgp-key). + - Include the following information: - Product/Driver name and version/branch that contains the vulnerability - - Type of vulnerability (code execution, denial of service, buffer overflow, etc.) + - Type of vulnerability (including code execution, denial of service, and buffer overflow) - Instructions to reproduce the vulnerability - Proof-of-concept or exploit code - - Potential impact of the vulnerability, including how an attacker could exploit the vulnerability + - Potential impact of the vulnerability, including how an attacker could exploit it -While NVIDIA currently does not have a bug bounty program, we do offer acknowledgement when an externally reported security issue is addressed under our coordinated vulnerability disclosure policy. Please visit our [Product Security Incident Response Team (PSIRT)](https://www.nvidia.com/en-us/security/psirt-policies/) policies page for more information. +While NVIDIA does not currently have a bug bounty program, the team provides acknowledgment for externally reported security issues addressed under the coordinated vulnerability disclosure policy. Visit the [Product Security Incident Response Team (PSIRT)](https://www.nvidia.com/en-us/security/psirt-policies/) policies page for more information. ## NVIDIA Product Security -For all security-related concerns, please visit NVIDIA's Product Security portal at https://www.nvidia.com/en-us/security +For all security-related concerns, visit the [NVIDIA Product Security portal](https://www.nvidia.com/en-us/security). diff --git a/codecov.yml b/codecov.yml index aa37017f082..fb1e12b547c 100644 --- a/codecov.yml +++ b/codecov.yml @@ -12,3 +12,5 @@ coverage: if_not_found: success fixes: - "/opt/megatron-lm/::" +ignore: + - "megatron/core/fusions/*" diff --git a/docker/.ngc_version.dev b/docker/.ngc_version.dev index 2c33440d4e2..59a585ec262 100644 --- a/docker/.ngc_version.dev +++ b/docker/.ngc_version.dev @@ -1 +1 @@ -nvcr.io/nvidia/pytorch:26.02-py3 \ No newline at end of file +nvcr.io/nvidia/pytorch:26.04-py3 \ No newline at end of file diff --git a/docker/Dockerfile.ci.lts b/docker/Dockerfile.ci.lts new file mode 100644 index 00000000000..6a2042e345d --- /dev/null +++ b/docker/Dockerfile.ci.lts @@ -0,0 +1,123 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# syntax=docker/dockerfile:1.3-labs +# +# LTS CI image for Megatron-LM. +# +# The LTS image is bumped only once a year and intentionally lags behind the +# floating dev tag. Its Python dependency set is therefore pinned directly in +# this Dockerfile (rather than in pyproject.toml) so the pyproject can host +# meaningful module-level extras (inference, RL, MoE, ...) without colliding +# with the LTS pin set. +# + +ARG FROM_IMAGE_NAME +FROM ${FROM_IMAGE_NAME} as main +ENV PIP_CONSTRAINT="" +ENV DEBIAN_FRONTEND=noninteractive +ARG UV_VERSION=0.7.2 +ARG YQ_VERSION=4.44.1 +ENV PATH="/root/.local/bin:$PATH" +ARG UV_PROJECT_ENVIRONMENT=/opt/venv +ENV UV_PROJECT_ENVIRONMENT=${UV_PROJECT_ENVIRONMENT} +ENV VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT +ENV PATH="$UV_PROJECT_ENVIRONMENT/bin:$PATH" +ENV UV_LINK_MODE=copy + +RUN bash -ex <<"EOF" + apt-get update + apt-get install -y --no-install-recommends gettext python3-venv psmisc uuid-runtime + apt-get clean + python -m venv /opt/jet + ARCH=$(uname -m) + case "${ARCH}" in \ + "x86_64") YQ_ARCH=amd64 ;; \ + "aarch64") YQ_ARCH=arm64 ;; \ + "armv7l") YQ_ARCH=arm ;; \ + *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;; \ + esac + wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${YQ_ARCH} -O /usr/local/bin/yq + chmod a+x /usr/local/bin/yq + curl -LsSf https://astral.sh/uv/${UV_VERSION}/install.sh | sh +EOF + +COPY README.md pyproject.toml uv.lock /workspace/ +COPY megatron/core/__init__.py /workspace/megatron/core/ +COPY megatron/core/package_info.py /workspace/megatron/core/ +ENV NVTE_BUILD_NUM_PHILOX_ROUNDS=3 +RUN --mount=type=cache,target=/root/.cache/uv \ + bash -ex <<"EOF" + export NVTE_CUDA_ARCHS="80;90;100" + uv venv ${UV_PROJECT_ENVIRONMENT} --system-site-packages + uv sync --only-group build + uv sync --extra mlm --extra ssm --extra te --link-mode copy --locked \ + --no-install-package torch \ + --no-install-package torchvision \ + --no-install-package triton \ + --no-install-package transformer-engine-cu12 \ + --no-install-package nvidia-cublas-cu12 \ + --no-install-package nvidia-cuda-cupti-cu12 \ + --no-install-package nvidia-cuda-nvrtc-cu12 \ + --no-install-package nvidia-cuda-runtime-cu12 \ + --no-install-package nvidia-cudnn-cu12 \ + --no-install-package nvidia-cufft-cu12 \ + --no-install-package nvidia-cufile-cu12 \ + --no-install-package nvidia-curand-cu12 \ + --no-install-package nvidia-cusolver-cu12 \ + --no-install-package nvidia-cusparse-cu12 \ + --no-install-package nvidia-cusparselt-cu12 \ + --no-install-package nvidia-nccl-cu12 +EOF + +# LTS-specific Python dependencies. +# +# These used to live in `[project.optional-dependencies].lts` in pyproject.toml, +# but were moved out so pyproject.toml can host meaningful per-module +# extras. The pinned set lives in `docker/lts/requirements.txt` and is reviewed +# at LTS bump time only. +COPY docker/lts/requirements.txt /workspace/docker/lts/requirements.txt +RUN --mount=type=cache,target=/root/.cache/uv \ + bash -ex <<"EOF" + uv pip install -r /workspace/docker/lts/requirements.txt +EOF + +# Install DeepEP +COPY docker/patches/deepep.patch /workspace/deepep.patch +RUN bash -ex <<"EOF" + cd /workspace + uv pip install nvidia-nvshmem-cu13==3.4.5 + pushd /opt/venv/lib/python3.12/site-packages/nvidia/nvshmem/lib/ + ln -s libnvshmem_host.so.3 libnvshmem_host.so + popd + + git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git + pushd DeepEP + git checkout 34152ae28f80bcc3ee38d7a12cb2ad87cfd4ea72 + patch -p1 < /workspace/deepep.patch + popd + TORCH_CUDA_ARCH_LIST="9.0 10.0 12.0" uv pip install --no-build-isolation -v DeepEP/. + rm -rf DeepEP +EOF + +COPY assets/ /opt/data/ +ENV UV_PYTHON=$UV_PROJECT_ENVIRONMENT/bin/python + +##### For NVIDIANS only ##### +FROM main as jet +ARG JET_API_VERSION +ENV PATH="$PATH:/opt/jet/bin" +RUN --mount=type=secret,id=JET_INDEX_URLS bash -ex <<"EOF" + JET_INDEX_URLS=$(cat /run/secrets/JET_INDEX_URLS) + python -m venv /opt/jet + /opt/jet/bin/pip install --no-cache-dir $JET_INDEX_URLS \ + "jet-api==$JET_API_VERSION" "setuptools<82.0.0" +EOF + +RUN --mount=type=secret,id=JET_INDEX_URLS \ + --mount=type=secret,id=LOGGER_INDEX_URL bash -ex <<"EOF" + JET_INDEX_URLS=$(cat /run/secrets/JET_INDEX_URLS) + LOGGER_INDEX_URL=$(cat /run/secrets/LOGGER_INDEX_URL) + uv pip install --no-cache-dir --upgrade $LOGGER_INDEX_URL "one-logger" + uv pip install --no-cache-dir --upgrade "setuptools>=80" + uv pip install --no-cache-dir --upgrade $JET_INDEX_URLS "jet-client~=4.0" +EOF +### diff --git a/docker/Dockerfile.ci.nemo b/docker/Dockerfile.ci.nemo index b00349e101a..8f250b4374a 100644 --- a/docker/Dockerfile.ci.nemo +++ b/docker/Dockerfile.ci.nemo @@ -1,8 +1,8 @@ # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. # syntax=docker/dockerfile:1.3-labs -ARG FROM_IMAGE_NAME -FROM ${FROM_IMAGE_NAME} as main +ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:26.04-py3 +FROM ${FROM_IMAGE_NAME} AS main RUN apt-get update && \ apt-get install -y --no-install-recommends gettext && \ @@ -11,7 +11,7 @@ RUN apt-get update && \ chmod a+x /usr/local/bin/yq ##### For NVIDIANS only ##### -FROM main as jet +FROM main AS jet ARG JET_API_VERSION RUN --mount=type=secret,id=JET_INDEX_URLS \ JET_INDEX_URLS=$(cat /run/secrets/JET_INDEX_URLS) && \ diff --git a/docker/Dockerfile.linting b/docker/Dockerfile.linting index 259c0bbedcd..bf27b768374 100644 --- a/docker/Dockerfile.linting +++ b/docker/Dockerfile.linting @@ -1,7 +1,7 @@ # syntax=docker/dockerfile:experimental -ARG FROM_IMAGE_NAME -FROM $FROM_IMAGE_NAME as main +ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:26.04-py3 +FROM $FROM_IMAGE_NAME AS main ENV DEBIAN_FRONTEND=noninteractive ARG UV_VERSION=0.7.2 ARG YQ_VERSION=4.44.1 @@ -16,7 +16,7 @@ COPY megatron/core/package_info.py megatron/core/__init__.py /opt/megatron-lm/me RUN uv sync --locked --only-group linting --only-group test --only-group ci ##### For NVIDIANS only ##### -FROM main as jet +FROM main AS jet ARG JET_API_VERSION RUN --mount=type=secret,id=JET_INDEX_URLS \ JET_INDEX_URLS=$(cat /run/secrets/JET_INDEX_URLS) && \ diff --git a/docker/common/install.sh b/docker/common/install.sh index 01003c0e7aa..90561879ea8 100644 --- a/docker/common/install.sh +++ b/docker/common/install.sh @@ -55,6 +55,19 @@ if [[ "$ENVIRONMENT" != "dev" && "$ENVIRONMENT" != "lts" ]]; then exit 1 fi +# AUT-479: LTS Python dependencies were moved out of pyproject.toml into +# docker/Dockerfile.ci.lts. This script targets the floating dev stack and no +# longer builds a working LTS environment by itself. +if [[ "$ENVIRONMENT" == "lts" ]]; then + echo "Error: --environment lts is no longer supported by install.sh." + echo " LTS dependencies are pinned in docker/Dockerfile.ci.lts." + echo " Build the LTS image directly with:" + echo " docker build --target main \\" + echo " --build-arg FROM_IMAGE_NAME=\$(cat docker/.ngc_version.lts) \\" + echo " -f docker/Dockerfile.ci.lts -t megatron-lm:local-lts ." + exit 1 +fi + main() { if [[ -n "${PAT:-}" ]]; then echo -e "machine github.com\n login token\n password $PAT" >~/.netrc @@ -136,7 +149,7 @@ main() { . $UV_PROJECT_ENVIRONMENT/bin/activate pip install --pre --no-cache-dir --upgrade pip - pip install --pre --no-cache-dir torch pybind11 wheel_stub ninja wheel packaging "setuptools<80.0.0,>=77.0.0" + pip install --pre --no-cache-dir torch pybind11 wheel_stub ninja wheel packaging "setuptools>=80" pip install --pre --no-cache-dir --no-build-isolation . fi diff --git a/docker/common/install_source_wheels.sh b/docker/common/install_source_wheels.sh index eaf601c6045..7eaaef2e46f 100644 --- a/docker/common/install_source_wheels.sh +++ b/docker/common/install_source_wheels.sh @@ -50,4 +50,4 @@ fi uv pip install --no-cache-dir \ $MAMBA_WHEEL \ $CAUSALCONV1D_WHEEL \ - "setuptools<80.0.0,>=77.0.0" + "setuptools>=80" diff --git a/docker/lts/requirements.txt b/docker/lts/requirements.txt new file mode 100644 index 00000000000..17889e7d401 --- /dev/null +++ b/docker/lts/requirements.txt @@ -0,0 +1,24 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + +# LTS Python dependency pins for Megatron-LM. +# +# To bump the LTS pin set: +# 1. Edit the versions below (or regenerate with `uv pip compile` against a +# requirements.in containing the loose specs). +# 2. Rebuild `docker/Dockerfile.ci.lts` and run the LTS CI lane. + +tqdm==4.67.3 +einops==0.8.2 # was: einops~=0.8 +tensorstore==0.1.84 # was: tensorstore~=0.1,!=0.1.46,!=0.1.72 +multi-storage-client==0.49.0 # was: multi-storage-client~=0.27 +opentelemetry-api==1.33.1 # was: opentelemetry-api~=1.33.1 +megatron-energon[av_decode]==7.3.2 # was: megatron-energon[av_decode]~=6.0.1 +av==17.0.1 +flashinfer-python==0.6.11.post3 # was: flashinfer-python>=0.5.0,<0.7.0 +wget==3.2 +onnxscript==0.7.0 +fastapi==0.136.3 # was: fastapi~=0.50 (forces compat with pydantic 2.0) +datasets==4.8.5 +emerging_optimizers @ git+https://github.com/NVIDIA-NeMo/Emerging-Optimizers.git@v0.2.0 +nvidia-resiliency-ext==0.6.0 +zstandard==0.25.0 diff --git a/docs/conf.py b/docs/conf.py index 5606eaa5809..f354c1b8f69 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -107,6 +107,9 @@ # Github links are now getting rate limited from the Github Actions linkcheck_ignore = [".*github\\.com.*", ".*githubusercontent\\.com.*", "http://localhost.*"] +linkcheck_retries = 10 +linkcheck_rate_limit_timeout = 600 +linkcheck_workers = 1 # PyTorch docs use a JS-rendered frontend; anchor IDs are injected at runtime # and are not present in the static HTML that linkcheck fetches. diff --git a/docs/developer/contribute.md b/docs/developer/contribute.md index 30a39e1cbc0..1f6c461f580 100644 --- a/docs/developer/contribute.md +++ b/docs/developer/contribute.md @@ -11,60 +11,117 @@ This document outlines the processes and policies for issues and pull requests by non-NVIDIA contributors to the Megatron-LM GitHub repository. -Everyone is welcome to contribute to the project! We recently migrated from using an internal repo to doing all development directly from the GitHub repository. +Everyone is welcome to contribute. The project recently migrated from an internal repository to doing all development directly on GitHub. -When contributing it is important to ensure that changes are in line with the project direction. Small changes to fix bugs are welcomed and appreciated. **If proposing large architectural changes or changes for stylistic reasons open an issue first so we can discuss it.** +When contributing, it is important to ensure that changes are in line with the project direction. Small bug fixes are always welcome. **If proposing large architectural changes or changes for stylistic reasons, open an issue first for discussion.** -## Issue policy +## Issue Policy -Please do file any bugs you find, keeping the following in mind: +File any bugs you find, keeping the following in mind: -- If filing a bug, i.e. you have found something that doesn't work as expected, use the BUG template. -- If you've found a regression in speed or accuracy use the REGRESSION template. -- If you are requesting a new feature or modification of an existing feature use the ENHANCEMENT template. -- If opening an issue to ask a question no template is needed but please make your question as clear and concise as possible. +- If filing a bug, that is, you have found something that does not work as expected, use the `BUG` template. +- If you've found a regression in speed or accuracy, use the `REGRESSION` template. +- If you are requesting a new feature or modification of an existing feature, use the `ENHANCEMENT` template. +- If opening an issue to ask a question, you do not need a template, but make your question as clear and concise as possible. - One issue per bug. Putting multiple things in the same issue makes both discussion and completion unnecessarily complicated. -- Your bug is mostly likely to get attention from the development team quickly if we can easily reproduce it. +- Reproducible bugs get the fastest attention from the development team. - Use proper spelling, grammar, and punctuation. - Write in an authoritative and technical tone. -## Code submission policy +## Code Submission Policy ### Do - Format new code in a style that is consistent with the file being changed. Megatron-LM doesn't (yet) have a style guide or enforced formatting. -- Split your changes into separate, atomic commits i.e. A commit per feature or fix. -- Make sure your commits are rebased on the master branch. +- Split your changes into separate, atomic commits, that is, a commit per feature or fix. +- Make sure your commits are rebased on the `main` branch. - Write the commit message subject line in the imperative mood ("Change the default argument for X", not "Changed the default argument for X"). - Write your commit messages in proper English, with care and punctuation. -- Check the spelling of your code, comments and commit messages. +- Check the spelling of your code, comments, and commit messages. ### Don't -- Submit code that's incompatible with the project licence. +- Submit code that's incompatible with the project license. - Touch anything outside the stated scope of the PR. This includes formatting changes to code not relevant to the PR. - Iterate excessively on your design across multiple commits. - Include commented-out code. - Attempt large architectural changes without first opening an issue to discuss. +## Signing Your Work + +- We require that all contributors "sign-off" on their commits. This certifies that the contribution is your original work, or you have rights to submit it under the same license, or a compatible license. + + - Any contribution which contains commits that are not Signed-Off will not be accepted. + +- To sign off on a commit you simply use the `--signoff` (or `-s`) option when committing your changes: + + ```bash + git commit -s -m "Add cool feature." + ``` + + This will append the following to your commit message: + + ``` + Signed-off-by: Your Name + ``` + +- Full text of the DCO: + + ``` + Developer Certificate of Origin + Version 1.1 + + Copyright (C) 2004, 2006 The Linux Foundation and its contributors. + + Everyone is permitted to copy and distribute verbatim copies of this + license document, but changing it is not allowed. + + + Developer's Certificate of Origin 1.1 + + By making a contribution to this project, I certify that: + + (a) The contribution was created in whole or in part by me and I + have the right to submit it under the open source license + indicated in the file; or + + (b) The contribution is based upon previous work that, to the best + of my knowledge, is covered under an appropriate open source + license and I have the right under that license to submit that + work with modifications, whether created in whole or in part + by me, under the same open source license (unless I am + permitted to submit under a different license), as indicated + in the file; or + + (c) The contribution was provided directly to me by some other + person who certified (a), (b) or (c) and I have not modified + it. + + (d) I understand and agree that this project and the contribution + are public and that a record of the contribution (including all + personal information I submit with it, including my sign-off) is + maintained indefinitely and may be redistributed consistent with + this project or the open source license(s) involved. + ``` + ## Issue and Pull Request Q&A -### I've submitted an issue and PR. When can I expect to get some feedback? +### Response Time for Issues and PRs -You should receive a response within 2 business days. +You should receive a response within two business days. -### I need help, who should I ping? +### Getting Help -Use [@mcore-oncall](https://github.com/orgs/NVIDIA/teams/mcore-oncall). +Use `@NVIDIA/mcore-oncall`. -### If my issue or PR isn't getting attention, what should I do? +### Escalating Unresponsive Issues or PRs -After 2 business days, tag the user [@mcore-oncall](https://github.com/orgs/NVIDIA/teams/mcore-oncall). +After two business days, tag `@NVIDIA/mcore-oncall`. -### Is there a policy for issues and PRs that haven't been touched in X days? Should they be closed? +### Stale Issue and PR Policy -Yes, we have a bot that will mark untouched PRs as "stale" after 60 days. +A bot marks untouched PRs as "stale" after 60 days. -We have a long backlog of issues and PRs dating back years. We are trying to triage these now by working backwards. Older issues we believe may still be relevant may recieve a request to re-test them with the latest code. If there's no response they may be closed. Again, if you they should be re-opened then just respond with a comment to that effect. +A long backlog of issues and PRs dates back years, and triage works backwards through it. Older issues that may still be relevant receive a request to re-test with the latest code. Without a response, the issue may be closed. To request reopening, respond with a comment. -Thank you! \ No newline at end of file +Thank you. diff --git a/docs/discussions/README.md b/docs/discussions/README.md index aab65fc65ca..a63aefc138a 100644 --- a/docs/discussions/README.md +++ b/docs/discussions/README.md @@ -19,6 +19,13 @@ This directory contains in-depth guides, tutorials, and discussions about optimi ### Training Guides +## Previous News + +- **[2025/05]** Megatron Core v0.11.0 brings new capabilities for multi-data center LLM training ([blog](https://developer.nvidia.com/blog/turbocharge-llm-training-across-long-haul-data-center-networks-with-nvidia-nemo-framework/)). +- **[2024/07]** Megatron Core v0.7 improves scalability and training resiliency and adds support for multimodal training ([blog](https://developer.nvidia.com/blog/train-generative-ai-models-more-efficiently-with-new-nvidia-Megatron-Core-functionalities/)). +- **[2024/06]** Megatron Core added support for Mamba-based models. Review the paper [An Empirical Study of Mamba-based Language Models](https://arxiv.org/pdf/2406.07887) and [code example](https://github.com/NVIDIA/Megatron-LM/tree/ssm/examples/mamba). +- **[2024/01 Announcement]** NVIDIA has released the core capabilities in **Megatron-LM** into [**Megatron Core**](https://github.com/NVIDIA/Megatron-LM/tree/main/megatron/core) in this repository. Megatron Core expands upon Megatron-LM's GPU-optimized techniques with more cutting-edge innovations on system-level optimizations, featuring composable and modular APIs. + ## Contributing To contribute a guide or tutorial, follow this structure: diff --git a/docs/get-started/install.md b/docs/get-started/install.md index 3e60a1fbb81..f9bf3177893 100644 --- a/docs/get-started/install.md +++ b/docs/get-started/install.md @@ -9,6 +9,8 @@ # Installation +Megatron Core can be installed from PyPI, built from source, or run inside an NGC container. Choose the method that best fits your workflow. PyPI is the simplest path for most users, source installs suit active development, and the NGC container provides a fully configured environment with no manual dependency management. + ## System Requirements ### Hardware @@ -34,13 +36,13 @@ curl -LsSf https://astral.sh/uv/install.sh | sh ## Option A: Pip Install (Recommended) -Install the latest stable release from PyPI: +This is the fastest way to get started. Install the latest stable release from PyPI and begin training without building anything from source: ```bash uv pip install megatron-core ``` -To include optional training dependencies (Weights & Biases, SentencePiece, HF Transformers): +To include optional training dependencies (Weights & Biases, SentencePiece, and Hugging Face Transformers): ```bash uv pip install "megatron-core[training]" @@ -54,11 +56,11 @@ uv pip install --no-build-isolation "megatron-core[training,dev]" ``` ```{note} -`--no-build-isolation` requires build dependencies to be pre-installed in the environment. `torch` is needed because several `[dev]` packages (`mamba-ssm`, `nv-grouped-gemm`, `transformer-engine`) import it at build time to compile CUDA kernels. Expect this step to take **20+ minutes** depending on your hardware. If you prefer pre-built binaries, the [NGC Container](#option-c-ngc-container) ships with these pre-compiled. +`--no-build-isolation` requires build dependencies to be pre-installed in the environment. `torch` is needed because several `[dev]` packages (`mamba-ssm`, `nv-grouped-gemm`, and `transformer-engine`) import it at build time to compile CUDA kernels. Expect this step to take **20+ minutes** depending on your hardware. If you prefer pre-built binaries, the [NGC Container](#option-c-nvidia-gpu-cloud-ngc-container) ships with these pre-compiled. ``` ```{warning} -Building from source can consume a large amount of memory. By default the build runs one compiler job per CPU core, which can cause out-of-memory failures on machines with many cores. To limit parallel compilation jobs, set the `MAX_JOBS` environment variable before installing (for example, `MAX_JOBS=4`). +Building from source can consume a large amount of memory. By default, the build runs one compiler job per CPU core, which can cause out-of-memory failures on machines with many cores. To limit parallel compilation jobs, set the `MAX_JOBS` environment variable before installing (for example, `MAX_JOBS=4`). ``` ```{tip} @@ -74,7 +76,7 @@ git clone https://github.com/NVIDIA/Megatron-LM.git ## Option B: Install from Source -For development or to run the latest unreleased code: +Use a source install to contribute changes, run unreleased features, or step through the code during debugging. This clones the repository and installs the package in editable mode. ```bash git clone https://github.com/NVIDIA/Megatron-LM.git @@ -82,7 +84,7 @@ cd Megatron-LM uv pip install -e . ``` -To install with all development dependencies (includes Transformer Engine, requires pre-installed build deps): +To install with all development dependencies (includes Transformer Engine, requires pre-installed build dependencies): ```bash uv pip install --group build @@ -94,11 +96,11 @@ If the build runs out of memory, limit parallel compilation jobs with `MAX_JOBS= ``` -## Option C: NGC Container +## Option C: NVIDIA GPU Cloud (NGC) Container -For a pre-configured environment with all dependencies pre-installed (PyTorch, CUDA, cuDNN, NCCL, Transformer Engine), use the [PyTorch NGC Container](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/pytorch). +For a pre-configured environment with all dependencies pre-installed (PyTorch, CUDA, cuDNN, NCCL, and Transformer Engine), use the [PyTorch NGC Container](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/pytorch). -We recommend using the **previous month's** NGC container rather than the latest one to ensure compatibility with the current Megatron Core release and testing matrix. +Use the **previous month's** NGC container rather than the latest one to ensure compatibility with the current Megatron Core release and testing matrix. ```bash docker run --gpus all -it --rm \ @@ -112,7 +114,7 @@ docker run --gpus all -it --rm \ The NGC PyTorch container constrains the Python environment globally using `PIP_CONSTRAINT`. The `-e PIP_CONSTRAINT=` flag above unsets this so that Megatron Core and its dependencies install correctly. ``` -Then install Megatron Core inside the container (torch is already available in the NGC image): +Then install Megatron Core inside the container (`torch` is already available in the NGC image): ```bash pip install uv diff --git a/docs/get-started/overview.md b/docs/get-started/overview.md index 5ceddcb1f41..d228abd3292 100644 --- a/docs/get-started/overview.md +++ b/docs/get-started/overview.md @@ -9,18 +9,18 @@ # Overview -Megatron-Core and Megatron-LM are open-source tools that are typically used together to train LLMs at scale across GPUs. Megatron-Core expands the capability of Megatron-LM. Megatron Bridge connects Megatron-Core and Megatron-LM to other popular training models, such as Hugging Face. +Megatron-Core and Megatron-LM are open-source tools that typically work together to train LLMs at scale across GPUs. Megatron-Core expands the capability of Megatron-LM. Megatron Bridge connects Megatron-Core and Megatron-LM to other popular training models, such as Hugging Face. ## Megatron Core -NVIDIA Megatron Core is a library of essential building blocks for highly efficient large-scale generative AI training. It can be used to train models with high throughput at scale across thousands of GPUs. It provides an extensive set of tools for multimodal and speech AI. It expands Megatron-LM capabilities. +NVIDIA Megatron Core is a library of essential building blocks for highly efficient large-scale generative AI training. It trains models with high throughput at scale across thousands of GPUs and provides an extensive set of tools for multimodal and speech AI. -Megatron-Core contains GPU-optimized techniques featuring advanced parallelism strategies, optimizations like FP8 training, and support for the latest LLM, MoE, and multimodal architectures. It abstracts these techniques into composable and modular APIs. +Megatron-Core contains GPU-optimized techniques featuring advanced parallelism strategies, optimizations like FP8 training, and support for the latest LLM, Mixture of Experts (MoE), and multimodal architectures. It abstracts these techniques into composable and modular APIs. Megatron-Core is compatible with all NVIDIA Tensor Core GPUs and popular LLM architectures such as GPT, BERT, T5, and RETRO. -**Composable library** with GPU-optimized building blocks for custom training frameworks. +Megatron Core is a **composable library** with GPU-optimized building blocks for custom training frameworks. **Best for:** @@ -31,20 +31,20 @@ Megatron-Core is compatible with all NVIDIA Tensor Core GPUs and popular LLM arc **What you get:** - Composable transformer building blocks (attention, MLP) -- Advanced parallelism strategies (TP, PP, DP, EP, CP) +- Advanced parallelism strategies (TP, PP, DP, EP, and CP) - Pipeline schedules and distributed optimizers -- Mixed precision support (FP16, BF16, FP8) +- Mixed precision support (FP16, BF16, and FP8) - GPU-optimized kernels and memory management - High-performance dataloaders and dataset utilities -- Model architectures (LLaMA, Qwen, GPT, Mixtral, Mamba) +- Model architectures (LLaMA, Qwen, GPT, Mixtral, and Mamba) ## Megatron-LM -Megatron-LM is a reference implementation, with a lightweight large-scale LLM training framework. It offers a customizable native PyTorch training loop with fewer abstraction layers. It was designed for scaling transformer models to the multi-billion and trillion-parameter regimes under realistic memory and compute constraints. **It serves as a direct entry point for exploring Megatron-Core.** +Megatron-LM is a reference implementation, with a lightweight large-scale LLM training framework. It offers a customizable native PyTorch training loop with fewer abstraction layers. It scales transformer models to the multi-billion and trillion-parameter regimes under realistic memory and compute constraints. **It serves as a direct entry point for exploring Megatron-Core.** It uses advanced parallelization techniques including model parallelism (tensor and pipeline), to allow models with billions of parameters to fit and train across large GPU clusters. It enables breakthroughs in large-scale NLP tasks. It splits model computations across many GPUs, overcoming single-GPU memory limits for training huge models, like GPT-style transformers. -**Reference implementation** that includes Megatron Core plus everything needed to train models. +Megatron-LM is a **reference implementation** that includes Megatron Core plus everything needed to train models. **Best for:** @@ -55,7 +55,7 @@ It uses advanced parallelization techniques including model parallelism (tensor **What you get:** -- Pre-configured training scripts for GPT, LLaMA, DeepSeek, Qwen, and more. +- Pre-configured training scripts for GPT, LLaMA, DeepSeek, Qwen, and more - End-to-end examples from data prep to evaluation - Research-focused tools and utilities @@ -63,9 +63,9 @@ It uses advanced parallelization techniques including model parallelism (tensor ## Megatron Bridge -Megatron Bridge provides out-of-the-box bridges and training recipes for models built on top of base model architectures from Megatron Core. +Megatron Bridge provides out-of-the-box training recipes for models built on top of base model architectures from Megatron Core. -Megatron Bridge provides a parallelism-aware pathway to convert models and checkpoints. This bidirectional converter performs on-the-fly, model-parallel-aware, per-parameter conversion, and full in-memory loading. +It includes a parallelism-aware pathway to convert models and checkpoints. This bidirectional converter performs on-the-fly, model-parallel-aware, per-parameter conversion, and full in-memory loading. After training or modifying a Megatron model, you can convert it again for deployment or sharing. Refer to the [Megatron Bridge repository](https://github.com/NVIDIA-NeMo/Megatron-Bridge) for the code and training recipes. @@ -73,16 +73,16 @@ After training or modifying a Megatron model, you can convert it again for deplo **Libraries used by Megatron Core:** -- **[Megatron Energon](https://github.com/NVIDIA/Megatron-Energon)** - Multi-modal data loader (text, images, video, audio) with distributed loading and dataset blending +- **[Megatron Energon](https://github.com/NVIDIA/Megatron-Energon)** - Multi-modal data loader (text, images, video, and audio) with distributed loading and dataset blending - **[Transformer Engine](https://github.com/NVIDIA/TransformerEngine)** - Optimized kernels and FP8 mixed precision support - **[Resiliency Extension (NVRx)](https://github.com/NVIDIA/nvidia-resiliency-ext)** - Fault tolerant training with failure detection and recovery **Libraries using Megatron Core:** - **[Megatron Bridge](https://github.com/NVIDIA-NeMo/Megatron-Bridge)** - Training library with bidirectional checkpoint conversion between Hugging Face and Megatron, customizable training loops, and production-ready recipes -- **[NeMo RL](https://github.com/NVIDIA-NeMo/RL)** - Scalable toolkit for efficient reinforcement learning with RLHF, DPO, and other post-training methods +- **[NeMo RL](https://github.com/NVIDIA-NeMo/RL)** - Scalable toolkit for efficient reinforcement learning with Reinforcement Learning from Human Feedback (RLHF), Direct Preference Optimization (DPO), and other post-training methods - **[NeMo Framework](https://docs.nvidia.com/nemo-framework/user-guide/latest/overview.html)** - Enterprise framework with cloud-native support and end-to-end examples -- **[Model Optimizer (ModelOpt)](https://github.com/NVIDIA/Model-Optimizer)** - Model optimization toolkit for quantization, pruning, distillation, speculative decoding, and more. Check out end-to-end examples in [examples/post_training/modelopt](https://github.com/NVIDIA/Megatron-LM/tree/main/examples/post_training/modelopt). +- **[Model Optimizer (ModelOpt)](https://github.com/NVIDIA/Model-Optimizer)** - Model optimization toolkit for quantization, pruning, distillation, speculative decoding, and more, with end-to-end examples in [examples/post_training/modelopt](https://github.com/NVIDIA/Megatron-LM/tree/main/examples/post_training/modelopt). -**Compatible with:** [Hugging Face Accelerate](https://github.com/huggingface/accelerate), [Colossal-AI](https://github.com/hpcaitech/ColossalAI), [DeepSpeed](https://github.com/microsoft/DeepSpeed) +**Compatible with:** [Hugging Face Accelerate](https://github.com/huggingface/accelerate), [Colossal-AI](https://github.com/hpcaitech/ColossalAI), and [DeepSpeed](https://github.com/microsoft/DeepSpeed) diff --git a/docs/get-started/quickstart.md b/docs/get-started/quickstart.md index 9d68016ece7..66b3bc100d6 100644 --- a/docs/get-started/quickstart.md +++ b/docs/get-started/quickstart.md @@ -9,11 +9,11 @@ # Your First Training Run -This guide walks you through running your first training jobs with Megatron Core. Make sure you have completed [installation](install.md) before proceeding. +This guide walks you through two training examples and then covers data preparation for your own datasets. You start with a minimal distributed loop to validate your environment, then run a full LLaMA-3 training job. Make sure you have completed [installation](install.md) before proceeding. ## Minimal Training Example -Run a minimal distributed training loop with mock data on 2 GPUs: +Start with the simplest possible setup, a distributed training loop using mock data on two GPUs. This verifies that your environment is configured correctly before moving to real models. ```bash torchrun --nproc_per_node=2 examples/run_simple_mcore_train_loop.py @@ -21,7 +21,7 @@ torchrun --nproc_per_node=2 examples/run_simple_mcore_train_loop.py ## LLaMA-3 Training Example -Train an LLaMA-3 8B model with FP8 precision on 8 GPUs using mock data: +With the environment validated, run a production-scale example. The following script trains a LLaMA-3 8B model with FP8 mixed precision on eight GPUs using mock data, demonstrating tensor parallelism and optimized kernels. ```bash ./examples/llama/train_llama3_8b_h100_fp8.sh @@ -42,6 +42,8 @@ Each line should contain a `text` field: ### 2. Preprocess the Data +Run the preprocessing script to tokenize and convert your data into binary format: + ```bash python tools/preprocess_data.py \ --input data.jsonl \ @@ -55,7 +57,7 @@ python tools/preprocess_data.py \ ### Key Arguments - `--input`: Path to input JSON/JSONL file -- `--output-prefix`: Prefix for output binary files (.bin and .idx) +- `--output-prefix`: Prefix for output binary files (`.bin` and `.idx`) - `--tokenizer-type`: Tokenizer type (`HuggingFaceTokenizer`, `GPT2BPETokenizer`, and so on) - `--tokenizer-model`: Path to tokenizer model file - `--workers`: Number of parallel workers for processing @@ -65,4 +67,4 @@ python tools/preprocess_data.py \ - Explore [Parallelism Strategies](../user-guide/parallelism-guide.md) to scale your training - Learn about [Data Preparation](../user-guide/data-preparation.md) best practices -- Check out [Advanced Features](../user-guide/features/index.md) +- Explore [Advanced Features](../user-guide/features/index.md) for FP8 training, context parallelism, and more diff --git a/docs/get-started/releasenotes.md b/docs/get-started/releasenotes.md index e624de19f15..b296bdfb254 100644 --- a/docs/get-started/releasenotes.md +++ b/docs/get-started/releasenotes.md @@ -12,8 +12,8 @@ ## Roadmaps -Stay up-to-date with our development roadmaps and planned features: +Stay up-to-date with the development roadmaps and planned features: -- **[MoE Q3-Q4 2025 Roadmap](https://github.com/NVIDIA/Megatron-LM/issues/1729)** - Comprehensive MoE feature development including DeepSeek-V3, Qwen3, advanced parallelism, FP8 optimizations, and Blackwell enhancements -- **[GPT-OSS Implementation Tracker](https://github.com/NVIDIA/Megatron-LM/issues/1739)** - Advanced features including YaRN RoPE scaling, attention sinks, and custom activation functions +- **[MoE Q3-Q4 2025 Roadmap](https://github.com/NVIDIA/Megatron-LM/issues/1729)** - Comprehensive MoE feature development including DeepSeek-V3, Qwen3, advanced parallelism, FP8 optimizations, and Blackwell enhancements. +- **[GPT-OSS Implementation Tracker](https://github.com/NVIDIA/Megatron-LM/issues/1739)** - Advanced features including YaRN RoPE scaling, attention sinks, and custom activation functions. diff --git a/docs/llama_mistral.md b/docs/llama_mistral.md index 2754405610c..6f084084e81 100644 --- a/docs/llama_mistral.md +++ b/docs/llama_mistral.md @@ -37,12 +37,10 @@ Architecturally Llama-2, Llama-3 and Mistral-7b are very similar. As such Megatr - [Download Huggingface checkpoints](#download-huggingface-checkpoints) - [Convert checkpoint format](#convert-checkpoint-format) - [Huggingface format](#huggingface-format) - - [(Optional) Validate checkpoints](#optional-validate-checkpoints) - [Launch model](#launch-model) - [Mistral-7b](#mistral-7b) - [Download Huggingface checkpoints](#download-huggingface-checkpoints) - [Convert checkpoint format](#convert-checkpoint-format) - - [(Optional) Validate checkpoints](#optional-validate-checkpoints) - [Launch model](#launch-model) - [Other Llama-like model support](#other-llama-like-model-support) - [Known numerical differences](#known-numerical-differences) @@ -210,14 +208,6 @@ python Megatron-Bridge/examples/conversion/convert_checkpoints.py import \ After this conversion, we are ready to load the checkpoints into a Megatron GPT model. -## (Optional) Validate checkpoints - -A Megatron-LM text generation server for Llama3 can be launched using the script `examples/inference/llama_mistral/run_text_generation_llama3.sh `. For Llama3.1, please use `examples/inference/llama_mistral/run_text_generation_llama3.1.sh`. - -Once running, query the server with `curl 'http://:5000/api' -X 'PUT' -H 'Content-Type: application/json; charset=UTF-8' -d '{"prompts":[""], "tokens_to_generate":100, "top_k":1}'`. - -A reference generation for comparison can be obtained from the Huggingface transformers library by running `python examples/llama_mistral/huggingface_reference.py --model_path --prompt `. - ## Launch model If loading for either inference or finetuning, use the following arguments for Llama 3.0: @@ -314,14 +304,6 @@ python Megatron-Bridge/examples/conversion/convert_checkpoints.py import \ After this conversion, we are ready to load the checkpoints into a Megatron GPT model. -## (Optional) Validate checkpoints - -A Megatron-LM text generation server for Mistral-7B can be launched using the script `examples/inference/llama_mistral/run_text_generation_mistral.sh `. - -Once running, query the server with `curl 'http://:5000/api' -X 'PUT' -H 'Content-Type: application/json; charset=UTF-8' -d '{"prompts":[""], "tokens_to_generate":100, "top_k":1}'`. - -A reference generation for comparison can be obtained from the Huggingface transformers library by running `python examples/inference/llama_mistral/huggingface_reference.py --model_path --prompt `. - ## Launch model If loading for either inference or finetuning, use the following arguments: diff --git a/docs/user-guide/features/fine_grained_activation_offloading.md b/docs/user-guide/features/fine_grained_activation_offloading.md index 6e8ee3040c7..f83645d7ec4 100644 --- a/docs/user-guide/features/fine_grained_activation_offloading.md +++ b/docs/user-guide/features/fine_grained_activation_offloading.md @@ -13,7 +13,7 @@ Contributed in collaboration with RedNote. Memory is often the limiting factor for very large sparse MoE models such as DeepSeek-V3 and Qwen3-235B. Fine-grained recomputation lowers activation memory at the cost of extra compute. Offloading can use host-device bandwidth so that reload overlaps compute and keeps overhead small in many setups. Fine-grained activation offloading moves activations at module granularity so you can tune how much activation memory leaves the device and adjust training throughput. -Supported offloading modules are `"attn_norm"`, `"core_attn"`, `"attn_proj"`, `"mlp_norm"`, `"expert_fc1"`, and `"moe_act"`. They can be combined with fine-grained recomputation to free almost all activations for a transformer layer on the device. +Supported offloading modules are `"attn_norm"`, `"qkv_linear"`, `"core_attn"`, `"attn_proj"`, `"mlp_norm"`, `"expert_fc1"`, `"moe_act"`, and `"fused_group_mlp"`. They can be combined with fine-grained recomputation to free almost all activations for a transformer layer on the device. `fused_group_mlp` requires `--use-transformer-engine-op-fuser` and offloads the whole fused grouped MLP, so it cannot be combined with `expert_fc1` or `moe_act`. ## Features @@ -33,7 +33,7 @@ Supported offloading modules are `"attn_norm"`, `"core_attn"`, `"attn_proj"`, `" --fine-grained-activation-offloading # Modules whose inputs are offloaded (refer to your training script for list or delimiter syntax). -# Choices: "attn_norm", "core_attn", "attn_proj", "mlp_norm", "expert_fc1", "moe_act". +# Choices: "attn_norm", "qkv_linear", "core_attn", "attn_proj", "mlp_norm", "expert_fc1", "moe_act", "fused_group_mlp". --offload-modules expert_fc1 ``` @@ -43,7 +43,7 @@ Supported offloading modules are `"attn_norm"`, `"core_attn"`, `"attn_proj"`, `" # Optional: cap inflight D2H offloads per offload group to N (omit or None in most setups). # Required as a non-None non-negative integer when fine-grained activation offloading is used with # full-iteration CUDA graphs (--cuda-graph-impl full_iteration); see prose below. ---fine_grained_offloading_max_inflight_offloads +--fine-grained-offloading-max-inflight-offloads ``` TransformerConfig.fine_grained_offloading_max_inflight_offloads caps, per offload group (for example `moe_act`, `qkv_linear`), how many D2H copies may be in flight before a main-stream wait_event. 0 waits after each offload; larger values allow more overlap; None skips these joins. diff --git a/docs/user-guide/features/paged_stash.md b/docs/user-guide/features/paged_stash.md index 4b7d807ace2..b5b97144905 100644 --- a/docs/user-guide/features/paged_stash.md +++ b/docs/user-guide/features/paged_stash.md @@ -21,7 +21,7 @@ Whenever `moe_expert_rank_capacity_factor` is set, a **runner** wraps forward-ba ## Prerequisites -HybridEP + TE fused grouped experts are required whenever `moe_expert_rank_capacity_factor` is set. With `moe_paged_stash` enabled: capacity factor must be set; no `cpu_offloading`; `offload_modules` must not include `expert_fc1` or `moe_act`. The runner is active whenever capacity factor is set (even without `--moe-paged-stash`) for over-budget reruns; stash overflow is checked only when paged stashing is on. +HybridEP + TE fused grouped experts are required whenever `moe_expert_rank_capacity_factor` is set. With `moe_paged_stash` enabled: capacity factor must be set; no `cpu_offloading`; `offload_modules` must not include `expert_fc1`, `moe_act`, or `fused_group_mlp`. The runner is active whenever capacity factor is set (even without `--moe-paged-stash`) for over-budget reruns; stash overflow is checked only when paged stashing is on. ## Configuration diff --git a/pretrain_bert.py b/examples/bert/pretrain_bert.py similarity index 100% rename from pretrain_bert.py rename to examples/bert/pretrain_bert.py diff --git a/examples/dynamic_context_parallel/README.md b/examples/dynamic_context_parallel/README.md new file mode 100644 index 00000000000..70864e42293 --- /dev/null +++ b/examples/dynamic_context_parallel/README.md @@ -0,0 +1,161 @@ +# Dynamic Context Parallel Benchmark + +This example compares regular DP-balanced packed-sequence training against +Dynamic Context Parallelism (DCP) on the same variable-length mock workload. + +The script reuses the normal Megatron-LM training stack: + +- `pretrain_gpt.py` builds and trains the GPT model. +- `MockVarlenDataset` creates THD-format variable-length samples. +- `DefaultDynamicCPScheduler` is enabled only for the DCP run. + +No model class or custom dataset class is introduced by this example. + +## Run + +From the Megatron-LM repository root: + +```bash +GPUS_PER_NODE=8 bash examples/dynamic_context_parallel/benchmark_dcp.sh +``` + +The default topology is `TP=1, CP=4, PP=1`, so the default run expects eight +GPUs and uses data parallel size two. By default the script sets +`NUM_MICROBATCHES=8` and +`GLOBAL_BATCH_SIZE=MICRO_BATCH_SIZE * DP_SIZE * NUM_MICROBATCHES`. Keeping more +than one microbatch per global batch gives the DCP scheduler enough +variable-length samples to assign smaller local CP groups instead of expanding +every sample back to the full fixed CP size. This mock THD workload also keeps +`MICRO_BATCH_SIZE=1`; increase effective batch size with `NUM_MICROBATCHES` and +data-parallel ranks. + +The script runs two jobs: + +1. Baseline packed sequence training: + + ```text + --sequence-packing-scheduler dp_balanced + --context-parallel-size 4 + ``` + +2. Dynamic CP training: + + ```text + --dynamic-context-parallel + --sequence-packing-scheduler default_dynamic_cp + --context-parallel-size 4 + ``` + +Both runs use the same model, batch size, sequence distribution, and +`--max-seqlen-per-dp-cp-rank`. The script also sets +`--moe-token-dispatcher-type alltoall`, which Megatron-Core currently requires +when sequence packing is enabled. It sets `--num-workers 0` by default to keep +mock collation in the main process for this variable-length THD workload. + +## What Makes DCP Useful Here + +The mock dataset draws sequence lengths from a lognormal distribution: + +```json +{"mode":"distribution","type":"lognormal","format":"thd","min_seq_len":128,"max_seq_len":8192,"mean_seq_len":1024,"lognormal_sigma":1.5} +``` + +With the default `--max-seqlen-per-dp-cp-rank 2048`, DCP can assign different +local CP sizes: + +- Short samples up to 2048 tokens can use one rank. +- Medium samples up to 4096 tokens can use two ranks. +- Long samples up to 8192 tokens can use four ranks. + +The baseline keeps the full fixed CP size for the packed workload. DCP can +spread short samples over the DPxCP domain instead of making every sample occupy +the full CP group. + +## Output + +At the end, the script prints: + +```text +=== Dynamic CP benchmark summary === +Iteration-time statistics exclude the first 10 logged iterations. +Baseline dp_balanced average: ... +Dynamic CP average: ... +Average speedup: ... +Baseline dp_balanced median: ... +Dynamic CP median: ... +Median speedup: ... +Baseline dp_balanced 10% trimmed avg: ... +Dynamic CP 10% trimmed avg: ... +10% trimmed mean speedup: ... +``` + +It parses Megatron-LM's regular training log line: + +```text +elapsed time per iteration (ms): ... +``` + +Logs and TensorBoard output are written under `dcp_benchmark_output/` by +default. + +## Slurm Benchmark Results + +The following single-run measurements were collected on June 9, 2026 on a +Slurm cluster using four GPUs per node and +`/home/tolong/nvidian+nemo+26.02.rc5.sqsh`. +The run used the default benchmark shape (`TP=1`, `CP=4`, `PP=1`, +`TRAIN_ITERS=30`, `WARMUP_ITERS=10`, `NUM_MICROBATCHES=8`, +`MAX_SEQLEN_PER_DP_CP_RANK=2048`) and the default lognormal mock VarlenDataset +distribution shown above. Statistics exclude the first 10 logged iterations. + +The 10% trimmed mean is the primary comparison because short DCP runs can have +large first-use spikes when dynamic groups are exercised. Arithmetic means are +included to show that variance. + +| Nodes | GPUs | Slurm job | Baseline trimmed mean (ms) | DCP trimmed mean (ms) | Trimmed speedup | Baseline avg (ms) | DCP avg (ms) | +| ----- | ---- | --------- | -------------------------- | --------------------- | --------------- | ----------------- | ------------ | +| 1 | 4 | 3239751 | 195.269 | 151.906 | 1.285x | 216.125 | 155.150 | +| 2 | 8 | 3239752 | 220.119 | 155.850 | 1.412x | 226.515 | 157.905 | +| 4 | 16 | 3239753 | 226.775 | 187.125 | 1.212x | 231.620 | 189.470 | +| 8 | 32 | 3239754 | 286.206 | 208.088 | 1.375x | 287.780 | 225.875 | +| 16 | 64 | 3239955 | 271.988 | 181.912 | 1.495x | 281.480 | 217.620 | + +## Useful Overrides + +Use a larger model: + +```bash +NUM_LAYERS=32 HIDDEN_SIZE=4096 FFN_HIDDEN_SIZE=16384 NUM_ATTENTION_HEADS=32 \ +GPUS_PER_NODE=8 bash examples/dynamic_context_parallel/benchmark_dcp.sh +``` + +Use a more skewed long/short distribution: + +```bash +VARLEN_DATASET_JSON='{"mode":"distribution","type":"lognormal","format":"thd","min_seq_len":128,"max_seq_len":8192,"mean_seq_len":768,"lognormal_sigma":1.8}' \ +GPUS_PER_NODE=8 bash examples/dynamic_context_parallel/benchmark_dcp.sh +``` + +Use exact sequence lengths from a CSV: + +```bash +VARLEN_DATASET_JSON='{"mode":"file","format":"thd","path":"/path/to/lengths.csv"}' \ +GPUS_PER_NODE=8 bash examples/dynamic_context_parallel/benchmark_dcp.sh +``` + +Reduce the runtime: + +```bash +TRAIN_ITERS=12 WARMUP_ITERS=3 GPUS_PER_NODE=8 \ +bash examples/dynamic_context_parallel/benchmark_dcp.sh +``` + +Use a specific Python interpreter: + +```bash +PYTHON=/opt/venv/bin/python GPUS_PER_NODE=8 \ +bash examples/dynamic_context_parallel/benchmark_dcp.sh +``` + +For stable numbers, keep the same GPU allocation, run more iterations, and +avoid checkpointing or evaluation during the measured window. diff --git a/examples/dynamic_context_parallel/benchmark_dcp.sh b/examples/dynamic_context_parallel/benchmark_dcp.sh new file mode 100755 index 00000000000..1643c4799f9 --- /dev/null +++ b/examples/dynamic_context_parallel/benchmark_dcp.sh @@ -0,0 +1,268 @@ +#!/bin/bash + +set -euo pipefail + +export CUDA_DEVICE_MAX_CONNECTIONS=${CUDA_DEVICE_MAX_CONNECTIONS:-1} +export NVTE_ALLOW_NONDETERMINISTIC_ALGO=${NVTE_ALLOW_NONDETERMINISTIC_ALGO:-1} + +GPUS_PER_NODE=${GPUS_PER_NODE:-8} +NUM_NODES=${NUM_NODES:-1} +NODE_RANK=${NODE_RANK:-0} +MASTER_ADDR=${MASTER_ADDR:-localhost} +MASTER_PORT=${MASTER_PORT:-6000} +PYTHON=${PYTHON:-python} + +TP_SIZE=${TP_SIZE:-1} +CP_SIZE=${CP_SIZE:-4} +PP_SIZE=${PP_SIZE:-1} +MOE_TOKEN_DISPATCHER_TYPE=${MOE_TOKEN_DISPATCHER_TYPE:-alltoall} +WORLD_SIZE=$((GPUS_PER_NODE * NUM_NODES)) +MODEL_PARALLEL_SIZE=$((TP_SIZE * CP_SIZE * PP_SIZE)) + +NUM_LAYERS=${NUM_LAYERS:-12} +HIDDEN_SIZE=${HIDDEN_SIZE:-2048} +FFN_HIDDEN_SIZE=${FFN_HIDDEN_SIZE:-8192} +NUM_ATTENTION_HEADS=${NUM_ATTENTION_HEADS:-16} +SEQ_LENGTH=${SEQ_LENGTH:-8192} +MAX_POSITION_EMBEDDINGS=${MAX_POSITION_EMBEDDINGS:-$SEQ_LENGTH} + +MICRO_BATCH_SIZE=${MICRO_BATCH_SIZE:-1} +GLOBAL_BATCH_SIZE=${GLOBAL_BATCH_SIZE:-} +NUM_MICROBATCHES=${NUM_MICROBATCHES:-8} +TRAIN_ITERS=${TRAIN_ITERS:-30} +WARMUP_ITERS=${WARMUP_ITERS:-10} +LOG_INTERVAL=${LOG_INTERVAL:-1} +NUM_WORKERS=${NUM_WORKERS:-0} + +MAX_SEQLEN_PER_DP_CP_RANK=${MAX_SEQLEN_PER_DP_CP_RANK:-2048} +MIN_DYNAMIC_CONTEXT_PARALLEL_SIZE=${MIN_DYNAMIC_CONTEXT_PARALLEL_SIZE:-1} + +VOCAB_SIZE=${VOCAB_SIZE:-131072} +NULL_TOKENIZER_PAD_ID=${NULL_TOKENIZER_PAD_ID:-0} + +OUTPUT_DIR=${OUTPUT_DIR:-"${PWD}/dcp_benchmark_output"} +DATA_CACHE_PATH=${DATA_CACHE_PATH:-"${OUTPUT_DIR}/data_cache"} +DEFAULT_VARLEN_DATASET_JSON='{"mode":"distribution","type":"lognormal","format":"thd","min_seq_len":128,"max_seq_len":8192,"mean_seq_len":1024,"lognormal_sigma":1.5}' +VARLEN_DATASET_JSON=${VARLEN_DATASET_JSON:-$DEFAULT_VARLEN_DATASET_JSON} + +PRETRAIN_SCRIPT_PATH="pretrain_gpt.py" + +if [[ ! -f "$PRETRAIN_SCRIPT_PATH" ]]; then + echo "Error: $PRETRAIN_SCRIPT_PATH not found. Run this script from the Megatron-LM repo root." + exit 1 +fi + +if (( WORLD_SIZE < MODEL_PARALLEL_SIZE )); then + echo "Error: need at least TP_SIZE * CP_SIZE * PP_SIZE GPUs." + echo "Got GPUS_PER_NODE=${GPUS_PER_NODE}, NUM_NODES=${NUM_NODES}, TP_SIZE=${TP_SIZE}, CP_SIZE=${CP_SIZE}, PP_SIZE=${PP_SIZE}." + exit 1 +fi + +if (( WORLD_SIZE % MODEL_PARALLEL_SIZE != 0 )); then + echo "Error: total GPUs must be divisible by TP_SIZE * CP_SIZE * PP_SIZE." + echo "Got WORLD_SIZE=${WORLD_SIZE}, TP_SIZE=${TP_SIZE}, CP_SIZE=${CP_SIZE}, PP_SIZE=${PP_SIZE}." + exit 1 +fi + +DP_SIZE=$((WORLD_SIZE / MODEL_PARALLEL_SIZE)) +MICRO_BATCH_TIMES_DP=$((MICRO_BATCH_SIZE * DP_SIZE)) +GLOBAL_BATCH_SIZE=${GLOBAL_BATCH_SIZE:-$((MICRO_BATCH_TIMES_DP * NUM_MICROBATCHES))} + +if (( MICRO_BATCH_SIZE != 1 )); then + echo "Error: this variable-length THD mock benchmark expects MICRO_BATCH_SIZE=1." + echo "Increase effective batch size with NUM_MICROBATCHES and data-parallel ranks." + exit 1 +fi + +if (( GLOBAL_BATCH_SIZE % MICRO_BATCH_TIMES_DP != 0 )); then + echo "Error: GLOBAL_BATCH_SIZE must be divisible by MICRO_BATCH_SIZE * DP_SIZE." + echo "Got GLOBAL_BATCH_SIZE=${GLOBAL_BATCH_SIZE}, MICRO_BATCH_SIZE=${MICRO_BATCH_SIZE}, DP_SIZE=${DP_SIZE}." + exit 1 +fi +EFFECTIVE_NUM_MICROBATCHES=$((GLOBAL_BATCH_SIZE / MICRO_BATCH_TIMES_DP)) + +if (( TRAIN_ITERS <= WARMUP_ITERS )); then + echo "Error: TRAIN_ITERS must be greater than WARMUP_ITERS." + exit 1 +fi + +mkdir -p "$OUTPUT_DIR" "$DATA_CACHE_PATH" + +echo "WORLD_SIZE=${WORLD_SIZE} DP_SIZE=${DP_SIZE} CP_SIZE=${CP_SIZE} TP_SIZE=${TP_SIZE} PP_SIZE=${PP_SIZE}" +echo "MICRO_BATCH_SIZE=${MICRO_BATCH_SIZE} GLOBAL_BATCH_SIZE=${GLOBAL_BATCH_SIZE} NUM_MICROBATCHES=${EFFECTIVE_NUM_MICROBATCHES}" + +DISTRIBUTED_ARGS=( + --nproc_per_node "$GPUS_PER_NODE" + --nnodes "$NUM_NODES" + --node_rank "$NODE_RANK" + --master_addr "$MASTER_ADDR" + --master_port "$MASTER_PORT" +) + +MODEL_ARGS=( + --use-mcore-models + --num-layers "$NUM_LAYERS" + --hidden-size "$HIDDEN_SIZE" + --ffn-hidden-size "$FFN_HIDDEN_SIZE" + --num-attention-heads "$NUM_ATTENTION_HEADS" + --seq-length "$SEQ_LENGTH" + --max-position-embeddings "$MAX_POSITION_EMBEDDINGS" + --attention-dropout 0.0 + --hidden-dropout 0.0 + --transformer-impl transformer_engine + --attention-backend flash + --moe-token-dispatcher-type "$MOE_TOKEN_DISPATCHER_TYPE" +) + +TRAINING_ARGS=( + --micro-batch-size "$MICRO_BATCH_SIZE" + --global-batch-size "$GLOBAL_BATCH_SIZE" + --train-iters "$TRAIN_ITERS" + --lr-decay-iters "$TRAIN_ITERS" + --lr 1.5e-4 + --min-lr 1.0e-5 + --lr-decay-style cosine + --lr-warmup-iters 0 + --weight-decay 1.0e-2 + --clip-grad 1.0 + --bf16 + --calculate-per-token-loss + --no-gradient-accumulation-fusion +) + +PARALLEL_ARGS=( + --tensor-model-parallel-size "$TP_SIZE" + --pipeline-model-parallel-size "$PP_SIZE" + --context-parallel-size "$CP_SIZE" +) + +if (( TP_SIZE > 1 )); then + PARALLEL_ARGS+=(--sequence-parallel) +fi + +DATA_ARGS=( + --use-varlen-dataset + --mock-data + --varlen-mock-dataset-config-json "$VARLEN_DATASET_JSON" + --tokenizer-type NullTokenizer + --vocab-size "$VOCAB_SIZE" + --null-tokenizer-pad-id "$NULL_TOKENIZER_PAD_ID" + --split 99,1,0 + --data-cache-path "$DATA_CACHE_PATH" + --dataloader-type single + --num-workers "$NUM_WORKERS" +) + +LOGGING_ARGS=( + --log-interval "$LOG_INTERVAL" + --log-throughput + --timing-log-level 0 + --eval-interval 1000000 + --eval-iters 1 + --save-interval 1000000 + --distributed-backend nccl + --distributed-timeout-minutes 60 +) + +extract_iteration_stats_ms() { + local log_file=$1 + "$PYTHON" - "$log_file" "$WARMUP_ITERS" <<'PY' +import re +import statistics +import sys + +log_file = sys.argv[1] +warmup = int(sys.argv[2]) +pattern = re.compile(r"elapsed time per iteration \(ms\): ([0-9.]+)") + +values = [] +with open(log_file, errors="ignore") as f: + for line in f: + match = pattern.search(line) + if match: + values.append(float(match.group(1))) + +values = values[warmup:] +if not values: + print("nan nan nan") + raise SystemExit + +average = sum(values) / len(values) +median = statistics.median(values) +trim_count = int(len(values) * 0.1) +trimmed = sorted(values) +if len(trimmed) - 2 * trim_count > 0: + trimmed = trimmed[trim_count : len(trimmed) - trim_count] +trimmed_average = sum(trimmed) / len(trimmed) + +print(f"{average:.3f} {median:.3f} {trimmed_average:.3f}") +PY +} + +calc_speedup() { + local base=$1 + local dcp=$2 + awk -v base="$base" -v dcp="$dcp" 'BEGIN { + if (base > 0 && dcp > 0) { + printf "%.2f", base / dcp + } else { + printf "nan" + } + }' +} + +run_case() { + local name=$1 + shift + + local tensorboard_dir="${OUTPUT_DIR}/tensorboard_${name}" + local log_file="${OUTPUT_DIR}/${name}.log" + rm -rf "$tensorboard_dir" + mkdir -p "$tensorboard_dir" + + echo + echo "=== Running ${name} ===" + echo "Log: ${log_file}" + + "$PYTHON" -m torch.distributed.run "${DISTRIBUTED_ARGS[@]}" \ + "$PRETRAIN_SCRIPT_PATH" \ + "${MODEL_ARGS[@]}" \ + "${TRAINING_ARGS[@]}" \ + "${PARALLEL_ARGS[@]}" \ + "${DATA_ARGS[@]}" \ + "${LOGGING_ARGS[@]}" \ + --tensorboard-dir "$tensorboard_dir" \ + "$@" 2>&1 | tee "$log_file" +} + +run_case baseline \ + --sequence-packing-scheduler dp_balanced \ + --max-seqlen-per-dp-cp-rank "$MAX_SEQLEN_PER_DP_CP_RANK" + +run_case dcp \ + --dynamic-context-parallel \ + --sequence-packing-scheduler default_dynamic_cp \ + --min-dynamic-context-parallel-size "$MIN_DYNAMIC_CONTEXT_PARALLEL_SIZE" \ + --max-seqlen-per-dp-cp-rank "$MAX_SEQLEN_PER_DP_CP_RANK" + +read -r baseline_avg_ms baseline_median_ms baseline_trimmed_ms < <(extract_iteration_stats_ms "${OUTPUT_DIR}/baseline.log") +read -r dcp_avg_ms dcp_median_ms dcp_trimmed_ms < <(extract_iteration_stats_ms "${OUTPUT_DIR}/dcp.log") + +avg_speedup=$(calc_speedup "$baseline_avg_ms" "$dcp_avg_ms") +median_speedup=$(calc_speedup "$baseline_median_ms" "$dcp_median_ms") +trimmed_speedup=$(calc_speedup "$baseline_trimmed_ms" "$dcp_trimmed_ms") + +echo +echo "=== Dynamic CP benchmark summary ===" +echo "Iteration-time statistics exclude the first ${WARMUP_ITERS} logged iterations." +echo "Baseline dp_balanced average: ${baseline_avg_ms} ms" +echo "Dynamic CP average: ${dcp_avg_ms} ms" +echo "Average speedup: ${avg_speedup}x" +echo "Baseline dp_balanced median: ${baseline_median_ms} ms" +echo "Dynamic CP median: ${dcp_median_ms} ms" +echo "Median speedup: ${median_speedup}x" +echo "Baseline dp_balanced 10% trimmed avg: ${baseline_trimmed_ms} ms" +echo "Dynamic CP 10% trimmed avg: ${dcp_trimmed_ms} ms" +echo "10% trimmed mean speedup: ${trimmed_speedup}x" +echo +echo "Logs and TensorBoard data are under ${OUTPUT_DIR}" diff --git a/examples/inference/README.md b/examples/inference/README.md index 3259bf7f943..a2b10ab6b26 100644 --- a/examples/inference/README.md +++ b/examples/inference/README.md @@ -1,288 +1,111 @@ ### Megatron Core Inference Documentation -This guide provides an example for Megatron Core for running model inference. +This guide provides an example for Megatron Core for running model inference. ### Contents -- [Megatron Core Inference Documentation](#megatron-core-inference-documentation) -- [Contents](#contents) - - [1. Quick Start](#1-quick-start) - - [1.1 Understanding The Code](#11-understanding-the-code) - - [1.2 Running The Code](#12-running-the-code) - - [2. Flow of Control In MCore Backend](#2-flow-of-control-in-mcore-backend) - - [3. Customizing The Inference Pipeline](#3-customizing-the-inference-pipeline) - - [3.1. Create Your Own Inference Backend](#31-create-your-own-inference-backend) - - [3.2. Create Your Own Text Generation Controller](#32-create-your-own-text-generation-controller) - - [3.3. Support Other Models](#33-support-other-models) - - [3.3. Modify Inference Parameters](#33-modify-inference-parameters) - - [4. Future work](#4-future-work) - -
- -#### 1. Quickstart -This example runs statically-batched inference on a model trained using Megatron Core. The entrypoint is [gpt_static_inference.py](./gpt/gpt_static_inference.py). A similar workflow can be adapted for [gpt_dynamic_inference.py](./gpt/gpt_dynamic_inference.py). - -
- -##### 1.1 Code Walkthrough -***STEP 1 - Initialize model parallel and other default arguments*** -The micro batch size defaults to 1. It is not used in tensor-parallelism only, and for pipeline-parallel models it is calculated at runtime. -```python -# Initialize Megatron model using the same model provider from training. - initialize_megatron( - args_defaults={'no_load_rng': True, 'no_load_optim': True, 'micro_batch_size': 1} - ) -``` - -***STEP 2 - Load the model using the model_provider_function*** -The model provider function supports both MCore and Legacy models. - -```python - # Load the model checkpoint - model = get_model(model_provider, wrap_with_ddp=False) - load_checkpoint(model, None, None) - model.eval() - model = model[0] -``` - -***STEP 3 - Choose an engine*** -Text generation requires an inference engine, which includes a scheduler. The default engine is the [Megatron Core engine](../../megatron/core/inference/engine/mcore_engine.py) with a [text generation controller](../../megatron/core/inference/text_generation_controllers/text_generation_controller.py). TRTLLMEngine will be supported in the future. -```python - # Create an inference wrapper to setup the model. - inference_wrapped_model = GPTInferenceWrapper(model, args) - - # Define a sampling loop. - text_generation_controller = TextGenerationController( - inference_wrapped_model=inference_wrapped_model, - tokenizer=tokenizer - ) - - # Create a static or dynamic inference engine. - inference_engine = StaticInferenceEngine( - text_generation_controller=text_generation_controller, - max_batch_size=args.max_batch_size -) -``` - -***STEP 4 - Run text generation*** -The [SamplingParams](../../megatron/core/inference/sampling_params.py) class uses suggested defaults. Customize this to change top_p, top_k, number of tokens to generate, etc. The result is returned as a list of [InferenceRequests](../../megatron/core/inference/inference_request.py). -```python - results: List[InferenceRequest] = inference_engine.generate( - prompts=args.prompts, sampling_params=sampling_params - ) - - if torch.distributed.get_rank() == 0: - for idx, result in enumerate(results): - print(f' ------------- RESULT FOR PROMPT {idx} --------------- ') - result = { - 'id': result.request_id, - 'input_prompt': result.prompt, - 'generated_text': result.generated_text, - 'generated_tokens' : result.generated_tokens - } - print(result) -``` - -
- -##### 1.2 Running The Code -An example Slurm script is shown below. Set the tokenizer paths, inference params, and other settings appropriately. - -For a recap on sampling parameters, refer to [this blog](https://ivibudh.medium.com/a-guide-to-controlling-llm-model-output-exploring-top-k-top-p-and-temperature-parameters-ed6a31313910). +- [What's in here](#whats-in-here) +- [Offline inference](#offline-inference) +- [OpenAI-compatible inference server](#openai-compatible-inference-server) +- [Advanced examples](#advanced-examples) +- [See also](#see-also) + +### What's in here + +These examples drive the high-level inference API in `megatron/core/inference/apis/` +(`MegatronLLM` for sync, `MegatronAsyncLLM` for async + HTTP serving). For +the API surface and mental model see +[`megatron/core/inference/README.md`](../../megatron/core/inference/README.md). + +The two top-level Python entrypoints cover all common workflows: + +- **`offline_inference.py`** — batched offline generation. Supports the + 3 mode combinations (sync+direct, sync+coordinator, async+coordinator) via CLI flags. + Replaces the `gpt_dynamic_inference.py` and + `gpt_dynamic_inference_with_coordinator.py` paths. +- **`launch_inference_server.py`** — OpenAI-compatible HTTP server using + `MegatronAsyncLLM.serve(...)`. Replaces the + `tools/run_dynamic_text_generation_server.py` path. + +`utils.py` holds shared helpers (`Request`, `build_requests`, +`build_dynamic_engine_setup_prefix`, output formatting, JSON dump) used by +both new examples and by the `advanced/` scripts. + +### Offline inference + +`offline_inference.py` runs synthetic-load inference on a Megatron model and +prints a setup-prefix line, a "Unique prompts + outputs" table, and a +throughput summary. Optional JSON dump for regression testing via +`--output-path`. + +The shell wrapper `run_offline_inference.sh` packages the typical Qwen +2.5-1.5B configuration. Required CLI args: `--hf-token`, `--checkpoint`. +Optional: `--mode sync|async` (default `sync`), `--use-coordinator` (default +off, i.e. direct mode), `--nproc ` (default `8`). Currently async + direct is not supported. + +```bash +# sync + direct (defaults) +bash examples/inference/run_offline_inference.sh \ + --hf-token --checkpoint /path/to/qwen-1.5b + +# sync + coordinator +bash examples/inference/run_offline_inference.sh \ + --hf-token --checkpoint /path/to/qwen-1.5b --use-coordinator + +# async + coordinator +bash examples/inference/run_offline_inference.sh \ + --hf-token --checkpoint /path/to/qwen-1.5b --mode async --use-coordinator ``` -# Slurm cluster settings -ACCOUNT= -MLM_PATH=/path/to/megatron-lm -GPT_CKPT=/path/to/gpt/ckpt -VOCAB_MERGE_FILE_PATH=/path/to/vocab/and/merge/file -CONTAINER_IMAGE=nvcr.io/ea-bignlp/ga-participants/nemofw-training:23.11 - -srun --account $ACCOUNT \ ---job-name=$ACCOUNT:inference \ ---partition=batch \ ---time=01:00:00 \ ---container-image $CONTAINER_IMAGE \ ---container-mounts $MLM_PATH:/workspace/megatron-lm/,$GPT_CKPT:/workspace/mcore_gpt_ckpt,$VOCAB_MERGE_FILE_PATH:/workspace/tokenizer \ ---no-container-mount-home \ ---pty /bin/bash \ - -# Inside the container run the following. - -cd megatron-lm/ -export CUDA_DEVICE_MAX_CONNECTIONS=1 - -TOKENIZER_ARGS=( - --vocab-file /workspace/tokenizer/gpt2-vocab.json - --merge-file /workspace/tokenizer/gpt2-merges.txt - --tokenizer-type GPT2BPETokenizer -) - -MODEL_ARGS=( - --use-checkpoint-args - --use-mcore-models - --load /workspace/mcore_gpt_ckpt -) - -INFERENCE_SPECIFIC_ARGS=( - --attention-dropout 0.0 - --hidden-dropout 0.0 - --num-tokens-to-generate 20 - --max-batch-size 4 -) - -torchrun --nproc-per-node=4 examples/inference/gpt/gpt_static_inference.py \ - ${TOKENIZER_ARGS[@]} \ - ${MODEL_ARGS[@]} \ - ${INFERENCE_SPECIFIC_ARGS[@]} \ - --prompts "prompt one " "sample prompt two" "sample prompt 3" - -NOTE: Other parameters which can be customized for inference: ---temperature (Sampling temperature) ---top_k (top_k sampling) ---top_p (top_p sampling) ---num-tokens-to-generate (Number of tokens to generate for each prompt) ---inference-batch-times-seqlen-threshold (During inference, if batch-size times sequence-length is smaller than this threshold then we will not use microbatched pipelining.') ---use-dist-ckpt (If using dist checkpoint format for the model) - -``` - -
+All four modes produce numerically identical generated text. The high-level +API rejects `--use-coordinator` with `--inference-repeat-n > 1` (engine +reset is unsafe in coordinator mode — see +[`megatron/core/inference/README.md`](../../megatron/core/inference/README.md)). +### OpenAI-compatible inference server -#### 2. Control Flow in the MCore Backend -An example of inference with static batching is provided in [gpt_static_inference.py](./gpt/gpt_static_inference.py). -* [mcore_engine](../../megatron/core/inference/engines/mcore_engine.py) **generate()** function is called with the input prompts. -* The `Scheduler` in the engine will add these prompts to the [active requests] pool (../../megatron/core/inference/inference_request.py) until max batch size is hit. Remaining requests will be added to the waiting requests pool. -* The engine will run until all requests (waiting + active) are completed. - * The active requests are passed into **generate_all_output_tokens_static_batch()** of the text generation controller . - * This function uses the **prep_model_for_inference()** method of the [model_inference_wrappers](../../megatron/core/inference/model_inference_wrappers/abstract_model_inference_wrapper.py) and runs an autoregressive sampling loop - * In the autoregressive loop, the **get_batch_for_context_window()** method of the inference wrapper is called to slice out the input tokens and masks - * Input tokens and masks are passed it into the **run_one_forward_step()** method, which calls the model `.forward()` method to get the output logits - * Output logits are synchronized across all pipeline parallel ranks - * The text generation controller obtains the log probabilities and samples tokens based on the strategy defined in the sampling parameters. - * The sampled tokens are then appended to the input prompt tokens for the next iteration - * The **update_generation_status()** method of the text generation controller checks which prompts have finished generating or hit a stop condition - * After the inference loop, the result is detokenized and stored as an attribute of the InferenceRequest. These requests are marked as completed. - * The **update_requests_pool()** method of the scheduler moves completed requests into the completed request pool and waiting requests into the active request pool +`launch_inference_server.py` uses `MegatronAsyncLLM.serve(blocking=True)` +on a coordinator-backed engine. The HTTP frontend exposes +`/v1/completions` and `/v1/chat/completions` on global rank 0. -
+The shell wrapper `run_inference_server.sh` packages the Nemotron-6 3B +hybrid MoE configuration (TP 2, EP 8, PP 1). Required CLI args: +`--hf-token`, `--hf-home`, `--checkpoint`. Optional: `--nproc ` (default +`8`). -#### 3. Customizing The Inference Pipeline - -The inference pipeline supports three levels of customization: - -* **Inference engine** - The MCore Engine supports static and dynamic batching. Modify this to add a new backend. -* **Text generation controller** - The main sampling loop. Customize this to support alternative tokenization or implement a new sampling strategy. -* **Inference Wrapped Model** - Change this to support a new model. -* **Modify Inference Parameters** - Change this to update top_p, top_k, number of tokens to be generated, temperature, and other sampling parameters. - -
- -##### 3.1. Create Your Own Inference Backend -The [abstract_engine.py](./../../megatron/core/inference/engine/abstract_engine.py) file contains a `generate` method that can be extended to support a new backend. - -```python -class AbstractEngine(ABC): - @staticmethod - def generate(self) -> dict: - """The abstract backend's generate function. - - To define a new backend, implement this method and return the outputs as a dictionary. +```bash +bash examples/inference/run_inference_server.sh \ + --hf-token \ + --hf-home /path/to/hf_home \ + --checkpoint /path/to/nemotron-3b-hybrid-moe ``` -
- -##### 3.2. Implement a new Sampling Loop - -The [TextGenerationController](../../megatron/core/inference/text_generation_controllers/text_generation_controller.py) contains the main sampling loop and can be modified to support new tokenization, detokenization, or sampling strategies. - -``` python -class TextGenerationController: - - def tokenize_prompt(self, prompt: str) -> Tuple[torch.Tensor, torch.Tensor]: - """Utility to tokenize the input prompts""" - - def sample_from_logits( - self, - last_token_logits: torch.Tensor, - sampling_params: SamplingParams, - vocab_size: int, - generation_started : Optional[torch.Tensor] = None, - top_n_logprobs_dict: Dict[int, List[Dict[str, float]]] = None, - ) -> torch.Tensor: - """Samples the logits to generate outputs - - Given the logits of the last token, this function samples according to the parameters defined in sampling_params and returns the sampled tokens. If sampling_params.top_n_logprobs > 0 - at each step it also updates the top_n_logprobs_dict. - """ - - def update_generation_status( - self, - updated_prompts_tokens: torch.Tensor, - generation_started: torch.Tensor, - current_context_end_position: int, - is_generation_done_tensor: torch.Tensor, - generated_sequence_lengths: torch.Tensor, - ) -> torch.Tensor: - """Function to check which prompts have reached an end condition +When the server is ready you'll see the readiness banner (~2 minutes after +launch on Nemotron-6 3B): - We check which prompts have reached an end condition and set the corresponding flags of the is_generation_done_tensor to True . The generated sequence lengths increases as we keep generating, until that prompts hits an eod condition. The generation started status tensor helps us determine which prompts have started generating - """ - - def generate_all_output_tokens_static_batch( - self, active_requests: OrderedDict[int, InferenceRequest], - ) -> OrderedDict[int, InferenceRequest]: - """Utility to generate all the output tokens and probabilities for the prompts . - - This utility generates the output tokens for a static batch. It runs the forward steps till all prompts complete generation, updates the status of these requests to completed, adds the generated result and returns these requests - """ - - def detokenize_generations(self, prompt_tokens_with_generated_tokens: torch.Tensor) -> str: - """Detokenize the output generations""" ``` - -
- -##### 3.3. Support Other Models -Extend [abstract_model_inference_wrapper.py](./../../megatron/core/inference/model_inference_wrappers/abstract_model_inference_wrapper.py) to support other models. The abstract model wrapper implements: -* Forward method which calls the model `forward` method depending on model parallel settings -* Initializes the model and puts it in `.eval()` mode -* Setup for the input parameters (max batch size, max seq length) - -The following methods should be implemented: -```python -class AbstractModelInferenceWrapper: - def prep_model_for_inference(self, prompts_tokens: torch.Tensor): - """A utility function for preparing model for inference - - The function gets called once before the auto regressive inference loop. It puts the model in eval mode , and gets some model and inference data parameters. Extend this to build position ids ,attention mask etc, so that required slices can be extracted during the forward pass - """ - - @abc.abstractclassmethod - def get_batch_for_context_window(self) -> List: - """Returns the input data for inference - - This function gets called iteratively in the inference loop. It can be used to extract relevant input from the prompt tokens, attention mask etc. required for each step in inference. +INFO:root:Inference co-ordinator is ready to receive requests! +INFO:hypercorn.error:Running on http://0.0.0.0:5000 (CTRL + C to quit) ``` -Refer to [gpt_inference_wrapper.py](../../megatron/core/inference/model_inference_wrappers/gpt/gpt_inference_wrapper.py) for an example of implementing this for GPTModel. - -
+Send requests with any OpenAI-compatible client. The dynamic server +currently returns `"model": "EMPTY"` and does not validate the request +`model` field — pass anything you like. -##### 3.3. Modify Inference Parameters -We use [common inference params](../../megatron/core/inference/sampling_params.py) for text generation. Customize this to change `top_p`, `top_k`, number of tokens to generate etc. Other attributes can be added for the inference loop as shown below. +### Advanced examples -``` -from megatron.core.inference.sampling_params import SamplingParams - -c = SamplingParams(temperature=0.5) -c.add_attributes({'min_length':4, 'eod_id':153}) -``` +`advanced/` contains scripts that drive the lower-level +`megatron.core.inference` APIs directly — manual `add_request` / +`step_modern` stepping, explicit coordinator / `InferenceClient` +lifecycle, the static engine, and T5 inference. Use these when you need +step-level scheduling control, custom forward-step / sampling +integration, or are migrating existing pipelines. For typical workflows, +prefer `offline_inference.py` and `launch_inference_server.py`. CI +recipes under `tests/test_utils/recipes/h100/{gpt,moe,mamba}-*-inference.yaml` +still target these scripts. -
+### See also -#### 4. Future work -The following features are planned for future releases. -* TRTLLM Engine support -* Continuous batching optimizations -* Speculative decoding \ No newline at end of file +- API reference: [`megatron/core/inference/README.md`](../../megatron/core/inference/README.md) +- Low-level engine: [`megatron/core/inference/`](../../megatron/core/inference/) +- Functional tests: `tests/functional_tests/test_cases/gpt/gpt_offline_inference_*` + `gpt_inference_server_smoke_*` +- Unit tests: `tests/unit_tests/inference/high_level_api/` diff --git a/examples/inference/gpt/gpt_dynamic_inference.py b/examples/inference/advanced/gpt_dynamic_inference.py similarity index 99% rename from examples/inference/gpt/gpt_dynamic_inference.py rename to examples/inference/advanced/gpt_dynamic_inference.py index e5df38fe856..9bb5b0cf08f 100644 --- a/examples/inference/gpt/gpt_dynamic_inference.py +++ b/examples/inference/advanced/gpt_dynamic_inference.py @@ -20,7 +20,7 @@ os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir)) ) -from examples.inference.gpt.utils import ( +from examples.inference.utils import ( Request, build_dynamic_engine_setup_prefix, build_requests, @@ -517,7 +517,7 @@ def escape_str(s): print( f"{setup_prefix} … " f"throughput: {throughput:.3f} tok/s … ", f"total time: {total_time:.3f}s … " - f"mem {peak_alloc_gb:.1f}/{peak_resvd_gb:.1f} GB … " + f"mem {peak_alloc_gb:.1f} allocated/{peak_resvd_gb:.1f} reserved GB … " f"steps: {engine.context.step_count:d} … " f"capture {capture_str}", ) diff --git a/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py b/examples/inference/advanced/gpt_dynamic_inference_with_coordinator.py similarity index 99% rename from examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py rename to examples/inference/advanced/gpt_dynamic_inference_with_coordinator.py index 8214363192b..34380e86c6f 100644 --- a/examples/inference/gpt/gpt_dynamic_inference_with_coordinator.py +++ b/examples/inference/advanced/gpt_dynamic_inference_with_coordinator.py @@ -12,7 +12,7 @@ import torch import torch.distributed as dist -from examples.inference.gpt.utils import Request, build_dynamic_engine_setup_prefix, build_requests +from examples.inference.utils import Request, build_dynamic_engine_setup_prefix, build_requests from megatron.core.inference.engines import DynamicInferenceEngine from megatron.core.inference.engines.dynamic_engine import EngineState from megatron.core.inference.inference_client import InferenceClient diff --git a/examples/inference/gpt/gpt_static_inference.py b/examples/inference/advanced/gpt_static_inference.py similarity index 99% rename from examples/inference/gpt/gpt_static_inference.py rename to examples/inference/advanced/gpt_static_inference.py index 9c748fdf795..b616eefc723 100644 --- a/examples/inference/gpt/gpt_static_inference.py +++ b/examples/inference/advanced/gpt_static_inference.py @@ -29,7 +29,7 @@ import json from typing import List -from examples.inference.gpt.utils import build_requests +from examples.inference.utils import build_requests from megatron.inference.utils import add_inference_args, get_model_for_inference from megatron.training import get_args, get_tokenizer, print_rank_0 from megatron.training.initialize import initialize_megatron diff --git a/examples/inference/t5/simple_t5_batch_inference.py b/examples/inference/advanced/simple_t5_batch_inference.py similarity index 100% rename from examples/inference/t5/simple_t5_batch_inference.py rename to examples/inference/advanced/simple_t5_batch_inference.py index 1aca74b3176..3591936a2ae 100644 --- a/examples/inference/t5/simple_t5_batch_inference.py +++ b/examples/inference/advanced/simple_t5_batch_inference.py @@ -4,9 +4,10 @@ import sys from argparse import Namespace +import pretrain_t5 import torch +from pretrain_t5 import model_provider -import pretrain_t5 from megatron.core.inference.engines import AbstractEngine, StaticInferenceEngine from megatron.core.inference.inference_request import InferenceRequest from megatron.core.inference.model_inference_wrappers.inference_wrapper_config import ( @@ -21,7 +22,6 @@ ) from megatron.core.tokenizers.utils.build_tokenizer import build_tokenizer from megatron.core.transformer.module import MegatronModule -from pretrain_t5 import model_provider sys.path.append( os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir)) diff --git a/examples/inference/gpt/gpt_dynamic_inference_12b.sh b/examples/inference/gpt/gpt_dynamic_inference_12b.sh deleted file mode 100644 index ca21bb170a5..00000000000 --- a/examples/inference/gpt/gpt_dynamic_inference_12b.sh +++ /dev/null @@ -1,127 +0,0 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. - -# Run dynamic batching inference on the 12B GPT model. - -set -u - -# Libraries. -pip install simpy -pip install sentencepiece -pip install tiktoken - -# Environment variables. -export CUDA_DEVICE_MAX_CONNECTIONS=1 - -# Checkpoint. -: ${CHECKPOINT_DIR:?"CHECKPOINT_DIR is not set"} -: ${TOKENIZER_MODEL:?"TOKENIZER_MODEL is not set"} - -# Prompts. -: ${NUM_TOKENS_TO_PROMPT="8 32"} -: ${NUM_TOKENS_TO_GENERATE=256} -: ${INCOMING_REQUESTS_DURATION=10.} -: ${INCOMING_REQUESTS_PER_SEC=100.} - -# Dynamic context. -: ${BUFFER_SIZE_GB=50.} - -# Cuda graphs. -: ${NUM_CUDA_GRAPHS=16} - -# Miscellaneous. -: ${USE_COORDINATOR=0} -: ${ENGINE=dynamic} -: ${EXTRA_ARGS=""} -# NSIGHT_PREFIX=/path/to/nsight/profile - -# Arguments. -ARGS=" \ - --no-persist-layer-norm \ - --apply-layernorm-1p \ - --no-position-embedding \ - --group-query-attention \ - --num-query-groups 8 \ - --load ${CHECKPOINT_DIR} \ - --use-checkpoint-args \ - --untie-embeddings-and-output-weights \ - --disable-bias-linear \ - --use-rotary-position-embeddings \ - --position-embedding-type rope \ - --rotary-base 1000000 \ - --rotary-percent 1.0 \ - --swiglu \ - --normalization RMSNorm \ - --attention-dropout 0.0 \ - --hidden-dropout 0.0 \ - --exit-duration-in-mins 5740 \ - --tensor-model-parallel-size 1 \ - --pipeline-model-parallel-size 1 \ - --num-layers 40 \ - --hidden-size 5120 \ - --ffn-hidden-size 14336 \ - --num-attention-heads 32 \ - --kv-channels 128 \ - --seq-length 1024 \ - --max-position-embeddings 1024 \ - --micro-batch-size 64 \ - --bf16 \ - --tokenizer-type TikTokenizer \ - --tiktoken-pattern v2 \ - --tokenizer-model ${TOKENIZER_MODEL} \ - --distributed-timeout-minutes 2400 \ - --use-flash-attn \ - --inference-rng-tracker \ - \ - --inference-dynamic-batching \ - --inference-dynamic-batching-buffer-size-gb ${BUFFER_SIZE_GB} \ - \ - ${EXTRA_ARGS} \ -" - -# Cuda graphs. -if [ "${NUM_CUDA_GRAPHS}" != "0" ]; then - ARGS+=" \ - --cuda-graph-impl local \ - --inference-dynamic-batching-num-cuda-graphs ${NUM_CUDA_GRAPHS} \ - " -else - ARGS+=" \ - --cuda-graph-impl none \ - " -fi - -# Prompts. -if [[ -v PROMPTS ]]; then - ARGS+=" \ - --prompts ${PROMPTS} \ - --num-tokens-to-generate ${NUM_TOKENS_TO_GENERATE} \ - " -elif [[ -v PROMPT_FILE ]]; then - ARGS+=" \ - --prompt-file ${PROMPT_FILE} \ - --num-tokens-to-generate ${NUM_TOKENS_TO_GENERATE} \ - " -else - ARGS+=" \ - --num-tokens-to-prompt ${NUM_TOKENS_TO_PROMPT} \ - --num-tokens-to-generate ${NUM_TOKENS_TO_GENERATE} \ - --incoming-requests-duration ${INCOMING_REQUESTS_DURATION} \ - --incoming-requests-per-sec ${INCOMING_REQUESTS_PER_SEC} \ - " -fi - -# Command. -if [[ "${USE_COORDINATOR}" == "0" ]]; then - CMD="python -m examples.inference.gpt.gpt_${ENGINE}_inference ${ARGS}" -else - CMD="python -um examples.inference.gpt.gpt_${ENGINE}_inference_with_coordinator ${ARGS}" -fi - -if [[ -v NSIGHT_PREFIX ]]; then - CMD="nsys profile -s none -t nvtx,cuda --cudabacktrace=all --cuda-graph-trace=node --python-backtrace=cuda --wait all -o ${NSIGHT_PREFIX} --force-overwrite true --capture-range=cudaProfilerApi --capture-range-end=stop ${CMD}" -fi - -echo "~~~" -echo "CMD ... ${CMD}." -echo "~~~" -eval ${CMD} diff --git a/examples/inference/gpt/gpt_dynamic_inference_357m.sh b/examples/inference/gpt/gpt_dynamic_inference_357m.sh deleted file mode 100644 index cc99bdddec1..00000000000 --- a/examples/inference/gpt/gpt_dynamic_inference_357m.sh +++ /dev/null @@ -1,115 +0,0 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. - -# Run dynamic batching inference on the 357M GPT model. - -set -u - -# Libraries. -pip install simpy -pip install sentencepiece -pip install tiktoken - -# Environment variables. -export CUDA_DEVICE_MAX_CONNECTIONS=1 - -# Checkpoint. -: ${CHECKPOINT_DIR:?"CHECKPOINT_DIR is not set"} -: ${VOCAB_FILE:?"VOCAB_FILE is not set"} -: ${MERGE_FILE:?"MERGE_FILE is not set"} - -# Prompts. -: ${NUM_TOKENS_TO_PROMPT="8 32"} -: ${NUM_TOKENS_TO_GENERATE=256} -: ${INCOMING_REQUESTS_DURATION=10.} -: ${INCOMING_REQUESTS_PER_SEC=100.} - -# Dynamic context. -: ${BUFFER_SIZE_GB=50.} - -# Cuda graphs. -: ${NUM_CUDA_GRAPHS=16} - -# Miscellaneous. -: ${USE_COORDINATOR=0} -: ${ENGINE=dynamic} -: ${NPROC_PER_NODE=1} -: ${EXTRA_ARGS=""} -# NSIGHT_PREFIX=/path/to/nsight/profile - -# Arguments. -ARGS=" \ - --exit-on-missing-checkpoint \ - --transformer-impl local \ - --load ${CHECKPOINT_DIR} \ - --tokenizer-type GPT2BPETokenizer \ - --vocab-file ${VOCAB_FILE} \ - --merge-file ${MERGE_FILE} \ - --exit-on-missing-checkpoint \ - --max-position-embeddings 2048 \ - --seq-length 2048 \ - --tensor-model-parallel-size 1 \ - --pipeline-model-parallel-size 1 \ - --num-layers 24 \ - --num-attention-heads 16 \ - --hidden-size 1024 \ - --bf16 \ - --micro-batch-size 1 \ - --attention-dropout 0.0 \ - --hidden-dropout 0.0 \ - --seed 42 \ - --use-flash-attn \ - --inference-rng-tracker \ - \ - --inference-dynamic-batching \ - --inference-dynamic-batching-buffer-size-gb ${BUFFER_SIZE_GB} \ - \ - ${EXTRA_ARGS} \ -" - -# Cuda graphs. -if [ "${NUM_CUDA_GRAPHS}" != "0" ]; then - ARGS+=" \ - --cuda-graph-impl local \ - --inference-dynamic-batching-num-cuda-graphs ${NUM_CUDA_GRAPHS} \ - " -else - ARGS+=" \ - --cuda-graph-impl none \ - " -fi - -# Prompts. -if [[ -v PROMPTS ]]; then - ARGS+=" \ - --prompts ${PROMPTS} \ - --num-tokens-to-generate ${NUM_TOKENS_TO_GENERATE} \ - " -elif [[ -v PROMPT_FILE ]]; then - ARGS+=" \ - --prompt-file ${PROMPT_FILE} \ - --num-tokens-to-generate ${NUM_TOKENS_TO_GENERATE} \ - " -else - ARGS+=" \ - --num-tokens-to-prompt ${NUM_TOKENS_TO_PROMPT} \ - --num-tokens-to-generate ${NUM_TOKENS_TO_GENERATE} \ - --incoming-requests-duration ${INCOMING_REQUESTS_DURATION} \ - --incoming-requests-per-sec ${INCOMING_REQUESTS_PER_SEC} \ - " -fi - -# Command. -if [[ "${USE_COORDINATOR}" == "0" ]]; then - CMD="python -m examples.inference.gpt.gpt_${ENGINE}_inference ${ARGS}" -else - CMD="python -m torch.distributed.run --nproc-per-node ${NPROC_PER_NODE} -m examples.inference.gpt.gpt_${ENGINE}_inference_with_coordinator ${ARGS}" -fi - -if [[ -v NSIGHT_PREFIX ]]; then - CMD="nsys profile -s none -t nvtx,cuda --cudabacktrace=all --cuda-graph-trace=node --python-backtrace=cuda --wait all -o ${NSIGHT_PREFIX} --force-overwrite true --capture-range=cudaProfilerApi --capture-range-end=stop ${CMD}" -fi - -echo "~~~" -echo "CMD ... ${CMD}." -echo "~~~" -eval ${CMD} diff --git a/examples/inference/launch_inference_server.py b/examples/inference/launch_inference_server.py new file mode 100644 index 00000000000..c980e142c6d --- /dev/null +++ b/examples/inference/launch_inference_server.py @@ -0,0 +1,101 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +"""OpenAI-compatible inference server using the Megatron high-level API. + +Mirrors tools/run_dynamic_text_generation_server.py but drives the +``DynamicInferenceEngine`` through ``MegatronAsyncLLM.serve(...)`` instead +of building the coordinator/engine pipeline manually. Coordinator mode is +required (HTTP serving uses the coordinator path); ``use_coordinator=True`` +is hardcoded in the script. +""" + +import asyncio +import os +import sys +from argparse import ArgumentParser + +import torch + +sys.path.append( + os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir)) +) + +from megatron.core.inference.apis import MegatronAsyncLLM, ServeConfig +from megatron.core.tokenizers.utils.build_tokenizer import build_tokenizer +from megatron.core.utils import configure_nvtx_profiling +from megatron.inference.utils import ( + add_inference_args, + get_inference_config_from_model_and_args, + get_model_for_inference, +) +from megatron.training import get_args, initialize_megatron +from megatron.training.arguments import parse_and_validate_args + + +def add_serve_args(parser: ArgumentParser) -> ArgumentParser: + parser = add_inference_args(parser) + group = parser.add_argument_group(title='High-level inference server') + group.add_argument("--coordinator-host", type=str, default=None) + group.add_argument("--coordinator-port", type=int, default=None) + group.add_argument("--host", type=str, default="0.0.0.0", help="HTTP bind host") + group.add_argument("--port", type=int, default=5000, help="HTTP bind port") + group.add_argument("--parsers", type=str, nargs="+", default=[], help="Response parser names") + group.add_argument( + "--verbose", action="store_true", default=False, help="Per-request HTTP logging" + ) + group.add_argument( + "--frontend-replicas", + type=int, + default=4, + help="Number of HTTP frontend processes spawned on the primary rank.", + ) + return parser + + +async def _serve(args, model, tokenizer, inference_config): + async with MegatronAsyncLLM( + model=model, + tokenizer=tokenizer, + inference_config=inference_config, + use_coordinator=True, + coordinator_host=args.coordinator_host, + coordinator_port=args.coordinator_port, + ) as llm: + serve_config = ServeConfig( + host=args.host, + port=args.port, + parsers=args.parsers, + verbose=args.verbose, + frontend_replicas=args.frontend_replicas, + ) + await llm.serve(serve_config, blocking=True) + + +def main(): + parse_and_validate_args( + extra_args_provider=add_serve_args, + args_defaults={'no_load_rng': True, 'no_load_optim': True}, + ) + initialize_megatron() + + args = get_args() + + # Match the legacy tool's NVTX gating. + if args.profile and args.nvtx_ranges: + configure_nvtx_profiling(True) + + tokenizer = build_tokenizer(args) + model = get_model_for_inference() + inference_config = get_inference_config_from_model_and_args(model, args) + + try: + asyncio.run(_serve(args, model, tokenizer, inference_config)) + except KeyboardInterrupt: + print("Server process interrupted by user.") + finally: + if torch.distributed.is_initialized(): + torch.distributed.destroy_process_group() + + +if __name__ == "__main__": + main() diff --git a/examples/inference/llama_mistral/huggingface_reference.py b/examples/inference/llama_mistral/huggingface_reference.py deleted file mode 100644 index 9d8f4465f65..00000000000 --- a/examples/inference/llama_mistral/huggingface_reference.py +++ /dev/null @@ -1,25 +0,0 @@ -import argparse -from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer - -# Set up argument parsing -parser = argparse.ArgumentParser(description="Script for text generation with a specific model and prompt.") -parser.add_argument('--prompt', type=str, required=True, help="Prompt text to use for text generation") -parser.add_argument('--model-path', type=str, required=True, help="Path to the Huggingface model checkpoint") - -# Parse command-line arguments -args = parser.parse_args() - -model_path = args.model_path -prompt = args.prompt - -config = AutoConfig.from_pretrained(model_path) -tokenizer = AutoTokenizer.from_pretrained(model_path, config=config) -model = AutoModelForCausalLM.from_pretrained(model_path, config=config).cuda() - -inputs = tokenizer(prompt, return_tensors="pt") -for key in inputs: - inputs[key] = inputs[key].cuda() -# top_k, top_p and do_sample are set for greedy argmax based sampling - -outputs = model.generate(**inputs, max_length=100, do_sample=False, top_p=0, top_k=0, temperature=1.0) -print(tokenizer.decode(outputs[0], skip_special_tokens=True)) \ No newline at end of file diff --git a/examples/inference/llama_mistral/run_static_inference_llama4_scout.sh b/examples/inference/llama_mistral/run_static_inference_llama4_scout.sh deleted file mode 100755 index cc8cfac5e69..00000000000 --- a/examples/inference/llama_mistral/run_static_inference_llama4_scout.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/bin/bash -export CUDA_DEVICE_MAX_CONNECTIONS=1 -export NVTE_APPLY_QK_LAYER_SCALING=0 - -DISTRIBUTED_ARGS="--nproc_per_node 8 \ - --nnodes 1 \ - --node_rank 0 \ - --master_addr 0.0.0.0 \ - --master_port 6000" - -# Fill in checkpoint path to Llama 4 Scout to run -CHECKPOINT= -PROMPTS="What is the capital of France?" -TOKENS_TO_GENERATE=4 -MAX_BATCH_SIZE=2 - -MODEL_ARGS=" \ - --micro-batch-size 1 \ - --bf16 \ - --no-masked-softmax-fusion \ - --disable-bias-linear \ - --untie-embeddings-and-output-weights \ - --position-embedding-type rope \ - --no-rope-fusion \ - --normalization RMSNorm \ - --swiglu \ - --num-layers 48 \ - --hidden-size 5120 \ - --ffn-hidden-size 16384 \ - --num-attention-heads 40 \ - --group-query-attention \ - --num-query-groups 8 \ - --qk-layernorm \ - --num-experts 16 \ - --moe-ffn-hidden-size 8192 \ - --moe-router-score-function sigmoid \ - --moe-router-topk 1 \ - --moe-router-topk-scaling-factor 1.0 \ - --moe-shared-expert-intermediate-size 8192 \ - --moe-aux-loss-coeff 1e-3 \ - --moe-token-dispatcher-type alltoall \ - --moe-token-drop-policy probs \ - --moe-router-load-balancing-type seq_aux_loss \ - --seq-length 4096 \ - --max-position-embeddings 4096 \ - --tokenizer-type HuggingFaceTokenizer \ - --make-vocab-size-divisible-by 128 \ - --use-mcore-models \ - --rotary-interleaved \ - --rotary-percent 1.0 \ - --rotary-base 500000 \ - --rope-scaling-factor 8.0 \ - --use-rope-scaling \ - --no-bias-swiglu-fusion \ - --qk-l2-norm \ - --moe-apply-probs-on-input \ - --moe-router-dtype fp64 \ -" - -torchrun $DISTRIBUTED_ARGS -m examples.inference.gpt.gpt_static_inference \ - --load ${CHECKPOINT} \ - --tokenizer-model unsloth/Llama-4-Scout-17B-16E-Instruct \ - --dist-ckpt-strictness log_unexpected \ - --tensor-model-parallel-size 8 \ - --prompts ${PROMPTS} \ - --num-tokens-to-generate ${TOKENS_TO_GENERATE} \ - --max-batch-size ${MAX_BATCH_SIZE} \ - ${MODEL_ARGS} diff --git a/examples/inference/llama_mistral/run_text_generation_llama3.1.sh b/examples/inference/llama_mistral/run_text_generation_llama3.1.sh deleted file mode 100755 index 06584f0917d..00000000000 --- a/examples/inference/llama_mistral/run_text_generation_llama3.1.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash -# This example will start serving the Llama3.1-8B model -export NCCL_IB_SL=1 -export CUDA_DEVICE_MAX_CONNECTIONS=1 -export NVTE_APPLY_QK_LAYER_SCALING=0 - -DISTRIBUTED_ARGS="--nproc_per_node 1 \ - --nnodes 1 \ - --node_rank 0 \ - --master_addr 0.0.0.0 \ - --master_port 6000" - -# Ensure CHECKPOINT and TOKENIZER_MODEL are provided -if [ -z "$1" ] || [ -z "$2" ]; then - echo "Error: You must provide CHECKPOINT and TOKENIZER_MODEL as command-line arguments." - echo "Usage: $0 /path/to/checkpoint /path/to/tokenizer_model" - exit 1 -fi - -# Assign command-line arguments to variables -CHECKPOINT=$1 -TOKENIZER_MODEL=$2 - -pip install flask-restful - -torchrun $DISTRIBUTED_ARGS tools/run_text_generation_server.py \ - --use-checkpoint-args \ - --disable-bias-linear \ - --tokenizer-type HuggingFaceTokenizer \ - --tokenizer-model ${TOKENIZER_MODEL} \ - --transformer-impl transformer_engine \ - --normalization RMSNorm \ - --group-query-attention \ - --num-query-groups 8 \ - --no-masked-softmax-fusion \ - --attention-softmax-in-fp32 \ - --attention-dropout 0.0 \ - --hidden-dropout 0.0 \ - --untie-embeddings-and-output-weights \ - --position-embedding-type rope \ - --rotary-percent 1.0 \ - --rotary-base 500000 \ - --use-rope-scaling \ - --use-rotary-position-embeddings \ - --swiglu \ - --tensor-model-parallel-size 1 \ - --pipeline-model-parallel-size 1 \ - --num-layers 32 \ - --hidden-size 4096 \ - --ffn-hidden-size 14336 \ - --load ${CHECKPOINT} \ - --num-attention-heads 32 \ - --max-position-embeddings 131072 \ - --bf16 \ - --micro-batch-size 1 \ - --seq-length 8192 diff --git a/examples/inference/llama_mistral/run_text_generation_llama3.sh b/examples/inference/llama_mistral/run_text_generation_llama3.sh deleted file mode 100755 index c5fc4103ab5..00000000000 --- a/examples/inference/llama_mistral/run_text_generation_llama3.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash -# This example will start serving the Llama3-8B model -export NCCL_IB_SL=1 -export CUDA_DEVICE_MAX_CONNECTIONS=1 -export NVTE_APPLY_QK_LAYER_SCALING=0 - -DISTRIBUTED_ARGS="--nproc_per_node 1 \ - --nnodes 1 \ - --node_rank 0 \ - --master_addr 0.0.0.0 \ - --master_port 6000" - -# Ensure CHECKPOINT and TOKENIZER_MODEL are provided -if [ -z "$1" ] || [ -z "$2" ]; then - echo "Error: You must provide CHECKPOINT and TOKENIZER_MODEL as command-line arguments." - echo "Usage: $0 /path/to/checkpoint /path/to/tokenizer_model" - exit 1 -fi - -# Assign command-line arguments to variables -CHECKPOINT=$1 -TOKENIZER_MODEL=$2 - -pip install flask-restful - -torchrun $DISTRIBUTED_ARGS tools/run_text_generation_server.py \ - --use-checkpoint-args \ - --disable-bias-linear \ - --tokenizer-type HuggingFaceTokenizer \ - --tokenizer-model ${TOKENIZER_MODEL} \ - --transformer-impl transformer_engine \ - --normalization RMSNorm \ - --group-query-attention \ - --num-query-groups 8 \ - --no-masked-softmax-fusion \ - --attention-softmax-in-fp32 \ - --attention-dropout 0.0 \ - --hidden-dropout 0.0 \ - --untie-embeddings-and-output-weights \ - --position-embedding-type rope \ - --rotary-percent 1.0 \ - --rotary-base 500000 \ - --use-rotary-position-embeddings \ - --swiglu \ - --tensor-model-parallel-size 1 \ - --pipeline-model-parallel-size 1 \ - --num-layers 32 \ - --hidden-size 4096 \ - --ffn-hidden-size 14336 \ - --load ${CHECKPOINT} \ - --num-attention-heads 32 \ - --max-position-embeddings 8192 \ - --bf16 \ - --micro-batch-size 1 \ - --seq-length 8192 diff --git a/examples/inference/llama_mistral/run_text_generation_mistral.sh b/examples/inference/llama_mistral/run_text_generation_mistral.sh deleted file mode 100755 index 4358fd494c7..00000000000 --- a/examples/inference/llama_mistral/run_text_generation_mistral.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash -# This example will start serving the Mistral-7B-v0.3 model -export NCCL_IB_SL=1 -export CUDA_DEVICE_MAX_CONNECTIONS=1 - -DISTRIBUTED_ARGS="--nproc_per_node 1 \ - --nnodes 1 \ - --node_rank 0 \ - --master_addr 0.0.0.0 \ - --master_port 6000" - -# Ensure CHECKPOINT and TOKENIZER_MODEL are provided -if [ -z "$1" ] || [ -z "$2" ]; then - echo "Error: You must provide CHECKPOINT and TOKENIZER_MODEL as command-line arguments." - echo "Usage: $0 /path/to/checkpoint /path/to/tokenizer_model" - exit 1 -fi - -# Assign command-line arguments to variables -CHECKPOINT=$1 -TOKENIZER_MODEL=$2 - -pip install flask-restful - -torchrun $DISTRIBUTED_ARGS tools/run_text_generation_server.py \ - --tokenizer-type HuggingFaceTokenizer \ - --tokenizer-model ${TOKENIZER_MODEL} \ - --use-checkpoint-args \ - --apply-layernorm-1p \ - --transformer-impl transformer_engine \ - --normalization RMSNorm \ - --group-query-attention \ - --num-query-groups 8 \ - --no-masked-softmax-fusion \ - --use-flash-attn \ - --untie-embeddings-and-output-weights \ - --disable-bias-linear \ - --position-embedding-type rope \ - --rotary-percent 1.0 \ - --rotary-base 1000000 \ - --swiglu \ - --ffn-hidden-size 14336 \ - --tensor-model-parallel-size 1 \ - --pipeline-model-parallel-size 1 \ - --num-layers 32 \ - --hidden-size 4096 \ - --load ${CHECKPOINT} \ - --num-attention-heads 32 \ - --max-position-embeddings 4096 \ - --bf16 \ - --micro-batch-size 1 \ - --seq-length 4096 \ - --seed 101 diff --git a/examples/inference/offline_inference.py b/examples/inference/offline_inference.py new file mode 100644 index 00000000000..167ad1084d1 --- /dev/null +++ b/examples/inference/offline_inference.py @@ -0,0 +1,293 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +"""Offline inference example using the Megatron high-level API. + +Mirrors examples/inference/advanced/gpt_dynamic_inference.py but drives the +``DynamicInferenceEngine`` through ``MegatronLLM`` (sync) or +``MegatronAsyncLLM`` (async, via ``--async-mode``) instead of the manual +add_request/step_modern loop. Output format (setup prefix, unique prompt +blocks, throughput line, optional JSON dump) matches that script. + +Run modes are selected at the CLI: + + # sync, direct (default) + python -m examples.inference.offline_inference --load ... + + # sync, coordinator + python -m examples.inference.offline_inference --load --use-coordinator ... + + # async (with or without --use-coordinator) + python -m examples.inference.offline_inference --load --async-mode ... +""" + +import asyncio +import logging +import os +import sys +from argparse import ArgumentParser + +import torch +import torch.distributed as dist + +sys.path.append( + os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir)) +) + +from examples.inference.utils import ( + build_dynamic_engine_setup_prefix, + build_requests, + dump_inference_results_to_json, + get_curr_time, + get_global_peak_memory_stats_bytes, + print_unique_prompts_and_outputs, +) +from megatron.core.inference.apis import MegatronAsyncLLM, MegatronLLM +from megatron.core.inference.sampling_params import SamplingParams +from megatron.core.tokenizers.utils.build_tokenizer import build_tokenizer +from megatron.core.utils import configure_nvtx_profiling +from megatron.inference.utils import ( + add_inference_args, + get_inference_config_from_model_and_args, + get_model_for_inference, +) +from megatron.training import initialize_megatron +from megatron.training.arguments import parse_and_validate_args + + +def add_offline_inference_args(parser: ArgumentParser) -> ArgumentParser: + parser = add_inference_args(parser) + group = parser.add_argument_group(title='Offline inference (high-level API)') + group.add_argument("--use-coordinator", action="store_true", default=False) + group.add_argument("--coordinator-host", type=str, default=None) + group.add_argument("--coordinator-port", type=int, default=None) + group.add_argument( + "--async-mode", + action="store_true", + default=False, + help="Drive MegatronAsyncLLM via asyncio.run instead of MegatronLLM.", + ) + return parser + + +def _validate_high_level_api_args(args): + # engine.reset() between trials races the runtime engine loop in + # coordinator mode (engine_loop_task runs on the runtime thread). + if args.use_coordinator and args.inference_repeat_n > 1: + raise ValueError( + "--use-coordinator with --inference-repeat-n > 1 is not supported: " + "engine.reset() races the runtime engine loop in coordinator mode." + ) + # The high-level API takes one sampling_params per generate() call. + if args.prompt_file and getattr(args, "num_tokens_from_file", False): + raise ValueError( + "--prompt-file with --num-tokens-from-file produces per-request " + "num_tokens_to_generate, but the high-level API takes one " + "sampling_params per generate() call. Use a uniform " + "--num-tokens-to-generate instead." + ) + + +def _validate_prompt_lengths(args, llm, requests): + # Validate prompt lengths against the resolved max_tokens (default + # is filled in by DynamicInferenceContext during construction). + if args.enable_chunked_prefill: + return + invalid = { + idx: len(r.prompt_tokens) + for idx, r in enumerate(requests) + if len(r.prompt_tokens) > llm.context.max_tokens + } + assert not invalid, "request idxs with prompts longer than context.max_tokens: " ", ".join( + f"{k}({v})" for k, v in invalid.items() + ) + + +def _capture_engine_stats(llm) -> dict: + return { + "step_count": llm.engine.context.step_count, + "lifetime_prefill_token_count": llm.engine.context.lifetime_prefill_token_count, + "capture_stats": llm.engine.capture_stats, + } + + +def _print_setup_prefix(setup_prefix: str) -> None: + if dist.get_rank() == 0: + print("~~~") + print(setup_prefix) + print("~~~") + + +def _report_results(args, setup_prefix, results, throughputs, total_time, peak_mem_stats, captured): + if dist.get_rank() != 0: + return + + print_unique_prompts_and_outputs(results) + dump_inference_results_to_json( + args, + results, + throughputs, + peak_mem_stats, + captured["step_count"], + captured["lifetime_prefill_token_count"], + ) + + stats = torch.cuda.memory_stats() + peak_alloc_gb = stats["allocated_bytes.all.peak"] / 1024**3 + peak_resvd_gb = stats["reserved_bytes.all.peak"] / 1024**3 + throughput = throughputs[-1] if throughputs else 0.0 + capture_str = ( + f"{captured['capture_stats']['time']:.2f} sec" if captured["capture_stats"] else "--" + ) + print("~~~") + print( + f"{setup_prefix} … " f"throughput: {throughput:.3f} tok/s … ", + f"total time: {total_time:.3f}s … " + f"mem {peak_alloc_gb:.1f}/{peak_resvd_gb:.1f} GB … " + f"steps: {captured['step_count']:d} … " + f"capture {capture_str}", + ) + print("~~~") + + +def _run_sync(args, model, tokenizer, inference_config, requests, prompts_list, sampling_params): + results = [] + throughputs = [] + total_time = 0.0 + captured = {"step_count": 0, "lifetime_prefill_token_count": 0, "capture_stats": None} + setup_prefix = "" + + with MegatronLLM( + model=model, + tokenizer=tokenizer, + inference_config=inference_config, + use_coordinator=args.use_coordinator, + coordinator_host=args.coordinator_host, + coordinator_port=args.coordinator_port, + ) as llm: + setup_prefix = build_dynamic_engine_setup_prefix(args, model, llm.context, requests) + _validate_prompt_lengths(args, llm, requests) + + # Coordinator mode: only the primary rank submits work; worker ranks + # fall through and block in __exit__ until shutdown propagates STOP. + if llm.is_primary_rank: + _print_setup_prefix(setup_prefix) + for trial_idx in range(args.inference_repeat_n): + # Skip first-trial reset; the engine is fresh post-construction. + if trial_idx > 0: + llm.engine.reset() + torch.cuda.reset_peak_memory_stats() + + t = get_curr_time(do_broadcast=not args.use_coordinator) + results = llm.generate(prompts_list, sampling_params) + torch.cuda.synchronize() + total_time = get_curr_time(do_broadcast=not args.use_coordinator) - t + + total_output_tokens = sum(len(r.generated_tokens) for r in results) + throughputs.append(total_output_tokens / total_time) + captured = _capture_engine_stats(llm) + + # Engine is shut down on all ranks; safe to all-reduce peak-memory now. + peak_mem_stats = get_global_peak_memory_stats_bytes() + _report_results(args, setup_prefix, results, throughputs, total_time, peak_mem_stats, captured) + + +async def _run_async( + args, model, tokenizer, inference_config, requests, prompts_list, sampling_params +): + results = [] + throughputs = [] + total_time = 0.0 + captured = {"step_count": 0, "lifetime_prefill_token_count": 0, "capture_stats": None} + setup_prefix = "" + + async with MegatronAsyncLLM( + model=model, + tokenizer=tokenizer, + inference_config=inference_config, + use_coordinator=args.use_coordinator, + coordinator_host=args.coordinator_host, + coordinator_port=args.coordinator_port, + ) as llm: + setup_prefix = build_dynamic_engine_setup_prefix(args, model, llm.context, requests) + _validate_prompt_lengths(args, llm, requests) + + if llm.is_primary_rank: + _print_setup_prefix(setup_prefix) + for trial_idx in range(args.inference_repeat_n): + if trial_idx > 0: + llm.engine.reset() + torch.cuda.reset_peak_memory_stats() + + t = get_curr_time(do_broadcast=not args.use_coordinator) + results = await llm.generate(prompts_list, sampling_params) + torch.cuda.synchronize() + total_time = get_curr_time(do_broadcast=not args.use_coordinator) - t + + total_output_tokens = sum(len(r.generated_tokens) for r in results) + throughputs.append(total_output_tokens / total_time) + captured = _capture_engine_stats(llm) + + peak_mem_stats = get_global_peak_memory_stats_bytes() + _report_results(args, setup_prefix, results, throughputs, total_time, peak_mem_stats, captured) + + +def main(): + args = parse_and_validate_args( + extra_args_provider=add_offline_inference_args, + args_defaults={'no_load_rng': True, 'no_load_optim': True}, + ) + initialize_megatron() + _validate_high_level_api_args(args) + + if os.environ.get("NSIGHT_PREFIX"): + torch.cuda.cudart().cudaProfilerStart() + + level = getattr(logging, os.getenv("LOG_LEVEL", "INFO").upper(), logging.INFO) + logging.basicConfig(level=level, force=True) + configure_nvtx_profiling(True) + + tokenizer = build_tokenizer(args) + torch.cuda.reset_peak_memory_stats() + + sampling_params = SamplingParams( + temperature=args.temperature, + top_k=args.top_k, + top_p=args.top_p, + skip_prompt_log_probs=args.skip_prompt_log_probs, + return_log_probs=args.return_log_probs, + num_tokens_to_generate=args.num_tokens_to_generate, + termination_id=args.termination_id if args.termination_id is not None else tokenizer.eod, + top_n_logprobs=args.top_n_logprobs, + stop_words=args.stop_words, + ) + + model = get_model_for_inference() + inference_config = get_inference_config_from_model_and_args(model, args) + requests = build_requests(args, tokenizer, sampling_params) + + max_gen_length = sampling_params.num_tokens_to_generate + max_context_length = max(len(r.prompt_tokens) for r in requests) + inference_config.max_sequence_length = max_context_length + max_gen_length + + prompts_list = [r.prompt_text for r in requests] + + runner_args = ( + args, + model, + tokenizer, + inference_config, + requests, + prompts_list, + sampling_params, + ) + if args.async_mode: + asyncio.run(_run_async(*runner_args)) + else: + _run_sync(*runner_args) + + if os.environ.get("NSIGHT_PREFIX"): + torch.cuda.cudart().cudaProfilerStop() + + +if __name__ == "__main__": + main() diff --git a/examples/inference/run_inference_server.sh b/examples/inference/run_inference_server.sh new file mode 100644 index 00000000000..1faf482fd6a --- /dev/null +++ b/examples/inference/run_inference_server.sh @@ -0,0 +1,99 @@ +#!/bin/bash +# OpenAI-compatible inference server launcher for the Megatron high-level API. +# +# Required CLI args: +# --hf-token Hugging Face token for tokenizer downloads. +# --hf-home Hugging Face cache directory. +# --checkpoint Path to the Megatron checkpoint passed as --load. +# +# Optional CLI args: +# --nproc Number of processes (default: 8). +# +# Example: +# bash run_inference_server.sh \ +# --hf-token hf_xxx \ +# --hf-home /path/to/hf_home \ +# --checkpoint /path/to/ckpt + +HF_TOKEN="" +HF_HOME="" +CHECKPOINT="" +NPROC=8 + +while [[ $# -gt 0 ]]; do + case "$1" in + --hf-token) + HF_TOKEN="$2" + shift 2 + ;; + --hf-home) + HF_HOME="$2" + shift 2 + ;; + --checkpoint) + CHECKPOINT="$2" + shift 2 + ;; + --nproc) + NPROC="$2" + shift 2 + ;; + -h|--help) + sed -n '2,16p' "$0" + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + echo "Run with -h for usage." >&2 + exit 1 + ;; + esac +done + +if [[ -z "$HF_TOKEN" ]]; then + echo "Error: --hf-token is required" >&2 + exit 1 +fi +if [[ -z "$HF_HOME" ]]; then + echo "Error: --hf-home is required" >&2 + exit 1 +fi +if [[ -z "$CHECKPOINT" ]]; then + echo "Error: --checkpoint is required" >&2 + exit 1 +fi + +export HF_TOKEN +export HF_HOME +# Required by Megatron when using tensor or context parallelism. +export CUDA_DEVICE_MAX_CONNECTIONS=1 + +torchrun --nproc-per-node "$NPROC" \ + -m examples.inference.launch_inference_server \ + --tensor-model-parallel-size 2 \ + --expert-tensor-parallel-size 1 \ + --expert-model-parallel-size 8 \ + --sequence-parallel \ + --pipeline-model-parallel-size 1 \ + --inference-max-seq-length 4096 \ + --load "$CHECKPOINT" \ + --micro-batch-size 1 \ + --moe-router-dtype fp32 \ + --moe-token-dispatcher-type alltoall \ + --use-checkpoint-args \ + --bf16 \ + --attention-backend flash \ + --transformer-impl inference_optimized \ + --te-rng-tracker \ + --inference-rng-tracker \ + --cuda-graph-impl "local" \ + --dist-ckpt-strictness log_unexpected \ + --inference-dynamic-batching-buffer-size-gb 20 \ + --model-provider hybrid \ + --inference-dynamic-batching-max-tokens 2048 \ + --enable-chunked-prefill \ + --inference-logging-step-interval 50 \ + --inference-dynamic-batching-num-cuda-graphs -1 \ + --cuda-graph-scope full_iteration_inference \ + --inference-dynamic-batching-max-requests 256 \ + --return-log-probs diff --git a/examples/inference/run_offline_inference.sh b/examples/inference/run_offline_inference.sh new file mode 100644 index 00000000000..a833f81514a --- /dev/null +++ b/examples/inference/run_offline_inference.sh @@ -0,0 +1,114 @@ +#!/bin/bash +# Offline inference launcher for the Megatron high-level API examples. +# +# Requires `simpy` (used by examples/inference/utils.py for synthetic request +# arrival simulation). If it is not already installed: +# pip install simpy +# +# Required CLI args: +# --hf-token Hugging Face token for tokenizer downloads. +# --checkpoint Path to the Megatron checkpoint passed as --load. +# +# Optional CLI args: +# --mode sync|async Selects MegatronLLM vs MegatronAsyncLLM (default: sync). +# --use-coordinator Run in coordinator mode (default: direct). +# --nproc Number of processes (default: 8). +# +# Examples: +# sync + direct (defaults): +# bash run_offline_inference.sh --hf-token hf_xxx --checkpoint /path/to/ckpt +# sync + coordinator: +# bash run_offline_inference.sh --hf-token hf_xxx --checkpoint /path/to/ckpt --use-coordinator +# async + coordinator: +# bash run_offline_inference.sh --hf-token hf_xxx --checkpoint /path/to/ckpt --mode async --use-coordinator + +HF_TOKEN="" +CHECKPOINT="" +MODE="sync" +USE_COORDINATOR=0 +NPROC=8 + +while [[ $# -gt 0 ]]; do + case "$1" in + --hf-token) + HF_TOKEN="$2" + shift 2 + ;; + --checkpoint) + CHECKPOINT="$2" + shift 2 + ;; + --mode) + MODE="$2" + shift 2 + ;; + --use-coordinator) + USE_COORDINATOR=1 + shift + ;; + --nproc) + NPROC="$2" + shift 2 + ;; + -h|--help) + sed -n '2,26p' "$0" + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + echo "Run with -h for usage." >&2 + exit 1 + ;; + esac +done + +if [[ -z "$HF_TOKEN" ]]; then + echo "Error: --hf-token is required" >&2 + exit 1 +fi +if [[ -z "$CHECKPOINT" ]]; then + echo "Error: --checkpoint is required" >&2 + exit 1 +fi +if [[ "$MODE" != "sync" && "$MODE" != "async" ]]; then + echo "Invalid --mode='$MODE'; expected 'sync' or 'async'." >&2 + exit 1 +fi + +export HF_TOKEN + +EXTRA_ARGS="" +if [[ "$USE_COORDINATOR" == "1" ]]; then + EXTRA_ARGS="$EXTRA_ARGS --use-coordinator" +fi +if [[ "$MODE" == "async" ]]; then + EXTRA_ARGS="$EXTRA_ARGS --async-mode" +fi + +torchrun --nproc-per-node "$NPROC" \ + -m examples.inference.offline_inference $EXTRA_ARGS \ + --load "$CHECKPOINT" \ + --bf16 \ + --tensor-model-parallel-size 1 \ + --micro-batch-size 64 \ + --dist-ckpt-strictness log_unexpected \ + --inference-rng-tracker \ + --cuda-graph-impl local \ + --decode-only-cuda-graphs \ + --tokenizer-type HuggingFaceTokenizer \ + --tokenizer-model Qwen/Qwen2.5-1.5B \ + --no-use-tokenizer-model-from-checkpoint-args \ + --num-layers 28 \ + --hidden-size 1536 \ + --num-attention-heads 12 \ + --max-position-embeddings 32768 \ + --num-query-groups 2 \ + --group-query-attention \ + --swiglu \ + --normalization RMSNorm \ + --disable-bias-linear \ + --position-embedding-type rope \ + --rotary-percent 1.0 \ + --rotary-base 1000000 \ + --seq-length 32768 \ + --ffn-hidden-size 8960 diff --git a/examples/inference/run_text_generation_server_345M.sh b/examples/inference/run_text_generation_server_345M.sh deleted file mode 100755 index e8e61adb163..00000000000 --- a/examples/inference/run_text_generation_server_345M.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -# This example will start serving the 345M model. -DISTRIBUTED_ARGS="--nproc_per_node 1 \ - --nnodes 1 \ - --node_rank 0 \ - --master_addr localhost \ - --master_port 6000" - -CHECKPOINT= -VOCAB_FILE= -MERGE_FILE= - -export CUDA_DEVICE_MAX_CONNECTIONS=1 - -pip install flask-restful - -torchrun $DISTRIBUTED_ARGS tools/run_text_generation_server.py \ - --tensor-model-parallel-size 1 \ - --pipeline-model-parallel-size 1 \ - --num-layers 24 \ - --hidden-size 1024 \ - --load ${CHECKPOINT} \ - --num-attention-heads 16 \ - --max-position-embeddings 1024 \ - --tokenizer-type GPT2BPETokenizer \ - --fp16 \ - --micro-batch-size 1 \ - --seq-length 1024 \ - --vocab-file $VOCAB_FILE \ - --merge-file $MERGE_FILE \ - --seed 42 diff --git a/examples/inference/run_text_generation_server_345M_8_tensor_parallel.sh b/examples/inference/run_text_generation_server_345M_8_tensor_parallel.sh deleted file mode 100755 index 368cec3b312..00000000000 --- a/examples/inference/run_text_generation_server_345M_8_tensor_parallel.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -# This example will start serving the 345M model that is partitioned 8 way tensor parallel -DISTRIBUTED_ARGS="--nproc_per_node 8 \ - --nnodes 1 \ - --node_rank 0 \ - --master_addr localhost \ - --master_port 6000" - -CHECKPOINT= -VOCAB_FILE= -MERGE_FILE= - -pip install flask-restful - -python -m torch.distributed.launch $DISTRIBUTED_ARGS tools/run_text_generation_server.py \ - --tensor-model-parallel-size 8 \ - --pipeline-model-parallel-size 1 \ - --num-layers 24 \ - --hidden-size 1024 \ - --load ${CHECKPOINT} \ - --num-attention-heads 16 \ - --max-position-embeddings 1024 \ - --tokenizer-type GPT2BPETokenizer \ - --fp16 \ - --micro-batch-size 1 \ - --seq-length 1024 \ - --vocab-file $VOCAB_FILE \ - --merge-file $MERGE_FILE \ - --seed 42 diff --git a/examples/inference/gpt/utils.py b/examples/inference/utils.py similarity index 69% rename from examples/inference/gpt/utils.py rename to examples/inference/utils.py index c9b1c05c544..234d8c7c5eb 100644 --- a/examples/inference/gpt/utils.py +++ b/examples/inference/utils.py @@ -1,11 +1,13 @@ # Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. import copy +import hashlib import itertools import json import random import time from argparse import ArgumentParser, Namespace +from collections import defaultdict from functools import partial from typing import Any, List, Optional @@ -31,10 +33,10 @@ def get_default_sampling_params(termination_id: int = None): ) -def get_curr_time() -> float: +def get_curr_time(do_broadcast: bool = True) -> float: """Get synchronized time across ranks.""" curr_time = torch.cuda.LongTensor([time.time_ns()]) - if torch.distributed.is_initialized(): + if torch.distributed.is_initialized() and do_broadcast: torch.distributed.broadcast(curr_time, src=0) return curr_time.item() / 10**9 @@ -324,3 +326,109 @@ def get_global_peak_memory_stats_bytes() -> dict: torch.distributed.all_reduce(t, op=torch.distributed.ReduceOp.MAX) peak_alloc = int(t[0].item()) return {"mem-max-allocated-bytes": peak_alloc} + + +def escape_str(s: str) -> str: + return s.replace("\n", "\\n") + + +def print_unique_prompts_and_outputs(results: List["DynamicInferenceRequest"]) -> None: + """Print unique prompts and their outputs in gpt_dynamic_inference.py format. + + Reads from the high-level API's ``DynamicInferenceRequest`` records returned + by ``MegatronLLM.generate`` / ``MegatronAsyncLLM.generate``. + """ + print("~~~~ Unique prompts + outputs. ~~~~") + + unique_prompt_map = defaultdict(list) + for idx, req in enumerate(results): + unique_prompt_map[req.prompt].append(idx) + + for unique_idx, (prompt_text, request_idxs) in enumerate(unique_prompt_map.items()): + prompt_len = len(results[request_idxs[0]].prompt_tokens) + print( + f"\n{unique_idx+1}/{len(unique_prompt_map)}" + f"[n {len(request_idxs)}, l {prompt_len}] {escape_str(prompt_text)}" + ) + + output_map = defaultdict(list) + for idx in request_idxs: + output_map[results[idx].generated_text].append(idx) + + for output_text, output_request_idxs in output_map.items(): + evicted = any( + event.type.name == "EVICT" + for idx in output_request_idxs + for event in results[idx].events + ) + if output_text is not None: + o_hash = hashlib.sha256((prompt_text + output_text).encode()).hexdigest()[:6] + o_len = len(results[output_request_idxs[0]].generated_tokens) + escaped_output_text = escape_str(output_text) + else: + o_hash = "--" + o_len = 0 + escaped_output_text = "--" + print( + f" >>>> [n {len(output_request_idxs)}, {o_len} tokens, hash {o_hash}" + f"{', ' if evicted else ''}] {escaped_output_text}" + ) + + +def dump_inference_results_to_json( + args: Namespace, + results: List["DynamicInferenceRequest"], + throughputs: List[float], + peak_mem_stats: dict, + step_count: int, + lifetime_prefill_token_count: int, +) -> None: + """JSON dump of per-request results matching legacy gpt_dynamic_inference.py shape. + + Reads from the high-level API's ``DynamicInferenceRequest`` records. + Note: ``latency`` is currently always ``None`` in direct mode because the + low-level engine doesn't populate it on ``DynamicInferenceRequest.merge()``; + will be populated once that field is wired up upstream. + """ + if not args.output_path: + return + + json_results = {} + for i, req in enumerate(results): + if i % args.output_every_n_results == 0 or i == len(results) - 1: + # cuda_graph_request_count_map is only populated by the legacy + # add_request/step_modern loop and is not surfaced through the + # high-level API; omitting it here. + result_dict = { + "input_prompt": req.prompt, + "generated_text": req.generated_text, + "generated_tokens": req.generated_tokens, + "latency": req.latency, + "ttft": req.ttft, + "step_count": step_count, + "top_n_logprobs": getattr(req, 'generated_top_n_logprobs', None), + "prompt_top_n_logprobs": getattr(req, 'prompt_top_n_logprobs', None), + } + if req.sampling_params.return_log_probs: + prompt_lp = getattr(req, 'prompt_log_probs', None) + generated_lp = getattr(req, 'generated_log_probs', None) + result_dict["prompt_logprobs"] = prompt_lp + result_dict["generated_logprobs"] = generated_lp + # Synthesize the legacy "logprobs" field as the concatenation, + # since DynamicInferenceRequest doesn't carry a single combined list. + if prompt_lp is not None or generated_lp is not None: + result_dict["logprobs"] = (prompt_lp or []) + (generated_lp or []) + else: + result_dict["logprobs"] = None + if args.output_request_events: + result_dict["events"] = [e.serialize() for e in req.events] + json_results[req.request_id] = result_dict + + if args.record_throughput: + json_results["throughput"] = throughputs + json_results.update(peak_mem_stats) + json_results["lifetime_prefill_token_count"] = lifetime_prefill_token_count + + print(f' Saving results to {args.output_path}') + with open(args.output_path, "w") as fp: + json.dump(json_results, fp, indent=1) diff --git a/examples/megatron_fsdp/README.md b/examples/megatron_fsdp/README.md index c8bf9efc9bf..0acbb166d49 100644 --- a/examples/megatron_fsdp/README.md +++ b/examples/megatron_fsdp/README.md @@ -65,22 +65,32 @@ bash examples/megatron_fsdp/train_llama3_8b_fsdp_h100_fp8.sh # With real data bash examples/megatron_fsdp/train_llama3_8b_fsdp_h100_fp8.sh \ checkpoints/llama3_8b_fsdp_fp8 \ - tensorboard_logs/llama3_8b_fsdp_fp8 \ + /path/to/data_prefix \ /path/to/tokenizer \ - /path/to/data_prefix + nsys_profiles/llama3_8b_fsdp_fp8 \ + tensorboard_logs/llama3_8b_fsdp_fp8 + +# With Nsight Systems profiling (steps 4–6 on rank 0) +NSYS_PROFILE=1 bash examples/megatron_fsdp/train_llama3_8b_fsdp_h100_fp8.sh + +# Without uv (use the ambient `python`) +USE_UV=0 bash examples/megatron_fsdp/train_llama3_8b_fsdp_h100_fp8.sh ``` | Positional Argument | Default | Description | |---------------------|---------|-------------| -| `$1` — Checkpoint path | `checkpoints/llama3_8b_fsdp_fp8` | Directory for saving and loading checkpoints. | -| `$2` — TensorBoard path | `tensorboard_logs/llama3_8b_fsdp_fp8` | Directory for TensorBoard logs. | +| `$1` — Checkpoint Path | `checkpoints/llama3_8b_fsdp_fp8` | Directory for saving and loading checkpoints. | +| `$2` — Data Path | `MOCK` | Data prefix for training data, or `MOCK` for mock data. | | `$3` — Tokenizer | `MOCK` | Path to a tokenizer model, or `MOCK` for `NullTokenizer`. | -| `$4` — Data path | `MOCK` | Data prefix for training data, or `MOCK` for mock data. | +| `$4` — NSight Profiling Path | `nsys_profiles/llama3_8b_fsdp_fp8` | Output path (without extension) for the `.nsys-rep` file when `NSYS_PROFILE=1`. | +| `$5` — TensorBoard Path | `tensorboard_logs/llama3_8b_fsdp_fp8` | Directory for TensorBoard logs. | #### Environment Variables | Variable | Default | Description | |----------|---------|-------------| +| `USE_UV` | `1` | Set to `1` to launch via `uv run` (project venv). Set to `0` to use the ambient `python`. | +| `NSYS_PROFILE` | `0` | Set to `1` to wrap the launch in `nsys profile`. Captures steps 4–6 on rank 0 via `--capture-range=cudaProfilerApi`, with CUDA graph node tracing and CUDA memory usage enabled. Output goes to the path in `$4`. | | `USE_MEGATRON_FSDP` | `1` | Set to `1` to enable Megatron-FSDP. Set to `0` to train with standard DDP. | | `SHARDING_STRATEGY` | `optim_grads_params` | FSDP sharding strategy (ZeRO-3). Options: `no_shard`, `optim`, `optim_grads`, `optim_grads_params`. | | `OUTER_SHARDING_STRATEGY` | `no_shard` | DP-Outer sharding strategy for HSDP/HFSDP. Options: `no_shard`, `optim`. | @@ -95,6 +105,7 @@ bash examples/megatron_fsdp/train_llama3_8b_fsdp_h100_fp8.sh \ - **Precision**: FP8 (hybrid format) with BF16 training and BF16 gradient reduction - **Batch size**: micro-batch=1, global-batch=128, sequence length=8192 - **Optimizations**: NCCL user buffers, FSDP double buffering, manual registration, meta-device initialization, per-token loss, overlapped grad-reduce and param-gather +- **Launch**: `[uv run] [nsys profile ...] python -m torch.distributed.run ... pretrain_gpt.py ...` — the `uv run` and `nsys profile` prefixes are toggled by `USE_UV` and `NSYS_PROFILE` respectively. --- diff --git a/examples/megatron_fsdp/train_llama3_8b_fsdp_h100_fp8.sh b/examples/megatron_fsdp/train_llama3_8b_fsdp_h100_fp8.sh old mode 100644 new mode 100755 index ddd3f160fa7..b45efd1bca1 --- a/examples/megatron_fsdp/train_llama3_8b_fsdp_h100_fp8.sh +++ b/examples/megatron_fsdp/train_llama3_8b_fsdp_h100_fp8.sh @@ -1,12 +1,14 @@ #!/bin/bash CHECKPOINT_PATH=${1:-"checkpoints/llama3_8b_fsdp_fp8"} -TENSORBOARD_LOGS_PATH=${2:-"tensorboard_logs/llama3_8b_fsdp_fp8"} -TOKENIZER_ARG=${3:-"MOCK"} # Path to tokenizer model, or "MOCK" -DATA_ARG=${4:-"MOCK"} # Data prefix, or "MOCK" +DATA_ARG=${2:-"MOCK"} # Data prefix, or "MOCK" +TOKENIZER_ARG=${3:-"MOCK"} # Path to tokenizer model, or "MOCK" +NSYS_PROFILE_PATH=${4:-"nsys_profiles/llama3_8b_fsdp_fp8"} +TENSORBOARD_LOGS_PATH=${5:-"tensorboard_logs/llama3_8b_fsdp_fp8"} # Create directories if they don't exist mkdir -p "$(dirname "$CHECKPOINT_PATH")" +mkdir -p "$(dirname "$NSYS_PROFILE_PATH")" mkdir -p "$(dirname "$TENSORBOARD_LOGS_PATH")" # Distributed training setup @@ -21,6 +23,18 @@ WORLD_SIZE=$(($GPUS_PER_NODE*$NUM_NODES)) # is run from the root of the Megatron-LM repository. PRETRAIN_SCRIPT_PATH="pretrain_gpt.py" +# NSight Profiling +NSYS_PROFILE=${NSYS_PROFILE:-0} + +# Optional `uv run` venv prefix. With uv, nsys (and its child workers) all +# inherit the project venv. Without uv, fall back to the ambient `python`. +USE_UV=${USE_UV:-1} +if [ "${USE_UV}" = 1 ]; then + VENV_PREFIX="uv run" +else + VENV_PREFIX="" +fi + # Model & Training Parameters USE_MEGATRON_FSDP=${USE_MEGATRON_FSDP:-1} SHARDING_STRATEGY=${SHARDING_STRATEGY:-"optim_grads_params"} @@ -89,6 +103,7 @@ TRAINING_ARGS=( --adam-beta2 0.95 --bf16 --cross-entropy-loss-fusion + --no-check-for-nan-in-loss-and-grad --manual-gc --empty-unused-memory-level 1 --exit-duration-in-mins 235 @@ -104,7 +119,7 @@ if [ "${USE_MEGATRON_FSDP}" = 1 ]; then --calculate-per-token-loss --init-model-with-meta-device --ckpt-format fsdp_dtensor - --grad-reduce-in-bf16 + --grad-reduce-in-bf16 # Will be deprecated soon! --use-nccl-ub --fsdp-double-buffer --fsdp-manual-registration @@ -116,6 +131,10 @@ if [ "${USE_MEGATRON_FSDP}" = 1 ]; then # --megatron-fsdp-main-params-dtype fp32 # --megatron-fsdp-main-grads-dtype auto # --megatron-fsdp-grad-comm-dtype auto + # To use decoupled (mixed-precision) gradients... + # --use-precision-aware-optimizer + # To use full-iteration CUDA graphs with Megatron-FSDP... + # --cuda-graph-impl full_iteration ) fi @@ -183,15 +202,32 @@ EVAL_AND_LOGGING_ARGS=( --eval-interval 100 --save-interval 1000 --log-throughput - --profile - --profile-step-start 4 - --profile-step-end 6 --distributed-timeout-minutes 60 --save "$CHECKPOINT_PATH" --load "$CHECKPOINT_PATH" --tensorboard-dir "$TENSORBOARD_LOGS_PATH" ) +# Profiling (NSYS_PROFILE=1 bash ...) +if [ "${NSYS_PROFILE}" = 1 ]; then + TRAINING_ARGS+=( + --profile + --profile-step-start 8 + --profile-step-end 12 + --profile-ranks 0 + ) + PROFILE_CMD=( + nsys profile + --sample=none --cpuctxsw=none + --trace=cuda,nvtx,cublas,cudnn + --capture-range=cudaProfilerApi --capture-range-end=stop + --cuda-graph-trace=node --cuda-memory-usage=true + -f true -x true -o "$NSYS_PROFILE_PATH" + ) +else + PROFILE_CMD=() +fi + # Ensure pretrain_gpt.py is found if [ ! -f "$PRETRAIN_SCRIPT_PATH" ]; then echo "Error: pretrain_gpt.py not found at $PRETRAIN_SCRIPT_PATH" @@ -199,8 +235,8 @@ if [ ! -f "$PRETRAIN_SCRIPT_PATH" ]; then exit 1 fi -# Run the training command -torchrun ${DISTRIBUTED_ARGS[@]} \ +# Run the training command. +$VENV_PREFIX "${PROFILE_CMD[@]}" python -m torch.distributed.run ${DISTRIBUTED_ARGS[@]} \ "$PRETRAIN_SCRIPT_PATH" \ ${MODEL_ARGS[@]} \ ${TRAINING_ARGS[@]} \ diff --git a/examples/mimo/training/__init__.py b/examples/mimo/training/__init__.py new file mode 100644 index 00000000000..26496bfed70 --- /dev/null +++ b/examples/mimo/training/__init__.py @@ -0,0 +1 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. diff --git a/examples/mimo/training/distributed.py b/examples/mimo/training/distributed.py new file mode 100644 index 00000000000..6c7b56b5622 --- /dev/null +++ b/examples/mimo/training/distributed.py @@ -0,0 +1,61 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Bring up torch.distributed and the global memory buffer for hetero MIMO without initializing MPU.""" + +from __future__ import annotations + +import os + +import torch +import torch.distributed as dist + +from megatron.core import parallel_state +from megatron.training.utils import print_rank_0 + +__all__ = ["initialize_distributed", "print_rank_0", "shutdown_distributed"] + + +def initialize_distributed() -> None: + """Bring up torch.distributed + the global memory buffer without MPU init.""" + local_rank = int(os.environ.get("LOCAL_RANK", "0")) + torch.cuda.set_device(local_rank) + if not dist.is_initialized(): + dist.init_process_group(backend="nccl", device_id=torch.device(f"cuda:{local_rank}")) + assert_parallel_state_uninitialized() + try: + parallel_state.get_global_memory_buffer() + except AssertionError: + parallel_state._set_global_memory_buffer() + dist.barrier() + + +def assert_parallel_state_uninitialized() -> None: + """Assert no Megatron model-parallel globals are set so the hetero path owns setup.""" + # Defensive parallel_state reads (allowed compatibility point); check each global MIMO + # constructs, since model_parallel_is_initialized() omits CP/embedding. + checks = ( + ("data_parallel", parallel_state.is_initialized), + ("tensor_model_parallel", _grp(parallel_state.get_tensor_model_parallel_group)), + ("pipeline_model_parallel", _grp(parallel_state.get_pipeline_model_parallel_group)), + ("context_parallel", _grp(parallel_state.get_context_parallel_group)), + ("embedding", _grp(parallel_state.get_embedding_group)), + ("position_embedding", _grp(parallel_state.get_position_embedding_group)), + ) + initialized = [label for label, predicate in checks if predicate()] + if initialized: + raise RuntimeError( + "Hetero MIMO bootstrap expects Megatron parallel_state process groups to be " + f"uninitialized, but found: {', '.join(initialized)}" + ) + + +def _grp(getter): + """Turn a group getter into a no-arg "is it set?" predicate.""" + return lambda: getter(check_initialized=False) is not None + + +def shutdown_distributed() -> None: + """Tear down the global memory buffer and torch.distributed (thin stock teardown).""" + parallel_state.destroy_global_memory_buffer() + if dist.is_initialized(): + dist.destroy_process_group() diff --git a/examples/mimo/training/runtime.py b/examples/mimo/training/runtime.py new file mode 100644 index 00000000000..ee7119c1501 --- /dev/null +++ b/examples/mimo/training/runtime.py @@ -0,0 +1,154 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Per-rank runtime setup (RNG seeding, freezing, DDP wrapping) for hetero MIMO training.""" + +from __future__ import annotations + +import argparse + +import torch + +from examples.mimo.training.topology import HeteroTopology +from megatron.core.distributed import DistributedDataParallel, DistributedDataParallelConfig +from megatron.core.models.mimo.config.role import MIMO_LANGUAGE_MODULE_KEY +from megatron.core.models.mimo.model.base import MimoModel +from megatron.core.process_groups_config import ProcessGroupCollection +from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed +from megatron.core.transformer.module import Float16Module +from megatron.core.utils import get_pg_rank, get_pg_size +from megatron.training.training import resolve_ddp_bucket_size, wrap_model_chunks_with_ddp +from megatron.training.utils import print_rank_0 + + +class _EncoderFloat16Module(Float16Module): + """Float16Module that keeps encoder outputs in model precision for the bridge.""" + + def forward(self, *inputs, fp32_output=False, **kwargs): # noqa: D102 + return super().forward(*inputs, fp32_output=fp32_output, **kwargs) + + +def configure_module_rng( + args: argparse.Namespace, pg_collection: ProcessGroupCollection, role_seed_offset: int +) -> None: + """Seed the CUDA RNG tracker for one module role from its tp/pp coordinates plus the offset. + + The seed is shared across a module's DP/CP replicas but distinct across PP stages and roles, + so disjoint modules (and stages) get independent RNG state. Caller invokes once per active + module on this rank. + """ + for _required in ("pp", "tp", "ep", "expt_tp"): + assert ( + getattr(pg_collection, _required, None) is not None + ), f"pg_collection passed to configure_module_rng must define {_required}" + pp_rank = get_pg_rank(pg_collection.pp) + tp_rank = get_pg_rank(pg_collection.tp) + ep_rank = get_pg_rank(pg_collection.ep) + expt_tp_rank = get_pg_rank(pg_collection.expt_tp) + seed = args.seed + role_seed_offset + (100 * pp_rank) + torch.manual_seed(seed) + model_parallel_cuda_manual_seed( + seed, tp_rank=tp_rank, ep_rank=ep_rank, etp_rank=expt_tp_rank, force_reset_rng=True + ) + + +def _freeze_modality_submodule(submodule: torch.nn.Module, args: argparse.Namespace) -> None: + """Freeze the encoder backbone (--freeze-vit) and/or projector (--freeze-projection).""" + if getattr(args, "freeze_vit", False): + submodule.encoders.requires_grad_(False) + if getattr(args, "freeze_projection", False): + submodule.input_projections.requires_grad_(False) + submodule.output_projections.requires_grad_(False) + + +def _module_config(module: torch.nn.Module): + """Return the module's own config, else the first descendant config (e.g. an encoder).""" + config = getattr(module, "config", None) + if config is not None: + return config + for child in module.modules(): + config = getattr(child, "config", None) + if config is not None: + return config + raise ValueError("Cannot resolve a config for DDP wrapping from module") + + +def _maybe_float16_wrap(module: torch.nn.Module, config, is_encoder: bool) -> torch.nn.Module: + """Wrap a submodule in Float16Module when fp16/bf16 is enabled; encoders keep bf16 outputs.""" + if not (getattr(config, "fp16", False) or getattr(config, "bf16", False)): + return module + cls = _EncoderFloat16Module if is_encoder else Float16Module + return cls(config, module) + + +def wrap_active_modules_with_ddp( + args: argparse.Namespace, mimo_model: MimoModel, topology: HeteroTopology +) -> None: + """Freeze (per --freeze-* flags), Float16Module-wrap, and DDP-wrap each active module.""" + pad_buckets = getattr(args, "ddp_pad_buckets_for_high_nccl_busbw", False) + grad_reduce_in_fp32 = getattr(args, "accumulate_allreduce_grads_in_fp32", True) + + ddp_stream = torch.cuda.Stream() + ddp_stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(ddp_stream): + if mimo_model.language_model is not None: + if getattr(args, "freeze_lm", False): + mimo_model.language_model.requires_grad_(False) + overlap = getattr(args, "overlap_grad_reduce", False) + ddp_config = DistributedDataParallelConfig( + overlap_grad_reduce=overlap, + overlap_param_gather=getattr(args, "overlap_param_gather", False), + num_buckets=getattr(args, "ddp_num_buckets", None), + bucket_size=getattr(args, "ddp_bucket_size", None), + pad_buckets_for_high_nccl_busbw=pad_buckets, + use_distributed_optimizer=True, + grad_reduce_in_fp32=grad_reduce_in_fp32, + ) + # Resolve the absolute bucket size on the real config, as get_model does. + ddp_config.bucket_size = resolve_ddp_bucket_size( + ddp_config, + topology.module_pgs[MIMO_LANGUAGE_MODULE_KEY].dp_cp, + overlap, + sum(p.numel() for p in mimo_model.language_model.parameters()), + ) + lm_config = _module_config(mimo_model.language_model) + lm_module = _maybe_float16_wrap(mimo_model.language_model, lm_config, is_encoder=False) + print_rank_0("wrapping language model in DDP") + mimo_model.language_model = wrap_model_chunks_with_ddp( + [lm_module], + lm_config, + ddp_config, + DP=DistributedDataParallel, + pg_collection=topology.module_pgs[MIMO_LANGUAGE_MODULE_KEY], + )[0] + + for name, submodule in mimo_model.modality_submodules.items(): + if submodule is None or name not in topology.module_pgs: + continue + _freeze_modality_submodule(submodule, args) + ddp_config = DistributedDataParallelConfig( + overlap_grad_reduce=False, + overlap_param_gather=False, + num_buckets=getattr(args, "ddp_num_buckets", None), + bucket_size=getattr(args, "ddp_bucket_size", None), + pad_buckets_for_high_nccl_busbw=pad_buckets, + use_distributed_optimizer=True, + grad_reduce_in_fp32=grad_reduce_in_fp32, + ) + # Encoders keep overlap off; resolve_ddp_bucket_size returns None there. + ddp_config.bucket_size = resolve_ddp_bucket_size( + ddp_config, + topology.module_pgs[name].dp_cp, + False, + sum(p.numel() for p in submodule.parameters()), + ) + enc_config = _module_config(submodule) + enc_module = _maybe_float16_wrap(submodule, enc_config, is_encoder=True) + print_rank_0(f"wrapping modality submodule {name!r} in DDP") + mimo_model.modality_submodules[name] = wrap_model_chunks_with_ddp( + [enc_module], + enc_config, + ddp_config, + DP=DistributedDataParallel, + pg_collection=topology.module_pgs[name], + )[0] + torch.cuda.current_stream().wait_stream(ddp_stream) diff --git a/examples/mimo/training/topology.py b/examples/mimo/training/topology.py new file mode 100644 index 00000000000..cf22de86627 --- /dev/null +++ b/examples/mimo/training/topology.py @@ -0,0 +1,247 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Per-module ``HyperCommGrid`` topology and process-group ownership for hetero MIMO training.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Optional + +import torch.distributed as dist + +from megatron.core.hyper_comm_grid import HyperCommGrid +from megatron.core.models.mimo.config.role import MIMO_LANGUAGE_MODULE_KEY, ModuleLayout, RankRole +from megatron.core.parallel_state import default_embedding_ranks, default_position_embedding_ranks +from megatron.core.process_groups_config import ( + MultiModuleProcessGroupCollection, + ProcessGroupCollection, +) + +_EXPERT_VIEW = "expert" + + +@dataclass +class ModuleGridSpec: + """One module's grid factorization and placement. + + ``num_ranks`` is the ground truth; ``dp`` and ``expt_dp`` are derived in ``__post_init__``. + """ + + name: str + num_ranks: int + tp: int = 1 + cp: int = 1 + pp: int = 1 + ep: int = 1 + rank_offset: int = 0 + # Experts default to TP=1 (set explicitly for MoE); intentionally not Megatron's etp=tp default. + expt_tp: int = 1 + dp: int = field(init=False) + expt_dp: int = field(init=False) + + def __post_init__(self) -> None: + dense = self.tp * self.cp * self.pp + if self.num_ranks % dense != 0: + raise ValueError( + f"num_ranks ({self.num_ranks}) must be divisible by tp*cp*pp ({dense})" + ) + self.dp = self.num_ranks // dense + expert = self.expt_tp * self.ep * self.pp + if self.num_ranks % expert != 0: + raise ValueError( + f"num_ranks ({self.num_ranks}) must be divisible by expt_tp*ep*pp ({expert})" + ) + self.expt_dp = self.num_ranks // expert + + @property + def size(self) -> int: + """Total ranks spanned by this module's grid.""" + return self.num_ranks + + +@dataclass +class HeteroTopology: + """Process groups and rank topology for one hetero MIMO run.""" + + grids: dict[str, HyperCommGrid] + module_pgs: dict[str, ProcessGroupCollection] + schedule_pg_collection: MultiModuleProcessGroupCollection + + def destroy(self) -> None: + """Destroy every process group owned by this topology.""" + destroyed: set[int] = set() + for pgc in self.module_pgs.values(): + for pg in (pgc.embd, pgc.pos_embd): + if pg is None or id(pg) in destroyed or not _is_process_group_member(pg): + continue + dist.destroy_process_group(pg) + destroyed.add(id(pg)) + for grid in self.grids.values(): + grid.destroy() + + +def create_topology(specs: list[ModuleGridSpec]) -> HeteroTopology: + """Build every module's grid, PGC, and embedding groups. + + Exactly one spec must be named ``MIMO_LANGUAGE_MODULE_KEY`` (the language module); specs must + tile ``[0, world_size)`` with no gaps (validated by :func:`_validate_grid_layout`). + """ + if not specs: + raise ValueError("create_topology requires at least one ModuleGridSpec") + language_specs = [spec for spec in specs if spec.name == MIMO_LANGUAGE_MODULE_KEY] + if len(language_specs) != 1: + raise ValueError( + f"create_topology requires exactly one spec named {MIMO_LANGUAGE_MODULE_KEY!r} " + f"(the language module), got {len(language_specs)}" + ) + language_name = MIMO_LANGUAGE_MODULE_KEY + + grids: dict[str, HyperCommGrid] = {} + module_pgs: dict[str, ProcessGroupCollection] = {} + try: + for spec in specs: + grids[spec.name] = _build_grid(spec) + _validate_grid_layout(grids) + + for name, grid in grids.items(): + module_pgs[name] = pg_collection_from_grid(grid, is_language=(name == language_name)) + + schedule_pg_collection = build_schedule_pg_collection(grids, module_pgs, language_name) + return HeteroTopology( + grids=grids, module_pgs=module_pgs, schedule_pg_collection=schedule_pg_collection + ) + except Exception: + HeteroTopology(grids=grids, module_pgs=module_pgs, schedule_pg_collection=None).destroy() + raise + + +def _build_grid(spec: ModuleGridSpec) -> HyperCommGrid: + """Create a dense grid plus its expert view and the process groups MIMO needs.""" + grid = HyperCommGrid( + shape=[spec.tp, spec.cp, spec.dp, spec.pp], + dim_names=["tp", "cp", "dp", "pp"], + rank_offset=spec.rank_offset, + backend="nccl", + ) + # Expert factorization over the same rank span; pp is shared with the base view. + grid.register_view( + _EXPERT_VIEW, + shape=[spec.expt_tp, spec.ep, spec.expt_dp, spec.pp], + dim_names=["expt_tp", "ep", "expt_dp", "pp"], + shared_dims=["pp"], + ) + + try: + for dims in (["tp"], ["cp"], ["pp"], ["dp"], ["dp", "cp"], ["tp", "cp"], ["tp", "pp"]): + grid.create_pg(dims) + for dims in (["ep"], ["expt_tp"], ["expt_dp"], ["expt_tp", "ep"], ["expt_tp", "ep", "pp"]): + grid.create_pg(dims, view=_EXPERT_VIEW) + except Exception: + grid.destroy() + raise + return grid + + +def _validate_grid_layout(grids: dict[str, HyperCommGrid]) -> None: + """Assert grids tile the world disjointly (non-colocated) XOR fully share ranks (colocated), + with no gaps. Colocated-vs-not is decided via the core ``RankRole.build`` path. + """ + spans = {name: (g.rank_offset, g.rank_offset + g.size) for name, g in grids.items()} + names = list(spans) + all_same = all(spans[n] == spans[names[0]] for n in names) + pairwise_disjoint = all( + spans[a][1] <= spans[b][0] or spans[b][1] <= spans[a][0] + for i, a in enumerate(names) + for b in names[i + 1 :] + ) + if not (all_same or pairwise_disjoint): + raise ValueError( + f"Module grids must either fully share ranks or be pairwise disjoint, got {spans}" + ) + + # Disjoint spans must also leave no rank uncovered (their union == [0, world_size)). + world_size = dist.get_world_size() + covered_ranks: set[int] = set() + for start, end in spans.values(): + covered_ranks.update(range(start, end)) + if covered_ranks != set(range(world_size)): + raise ValueError( + f"Module grids must partition the world [0, {world_size}) with no gaps, got {spans}" + ) + + modality_names = [n for n in names if n != MIMO_LANGUAGE_MODULE_KEY] + role = RankRole.build(modality_names, grids) + expected = ModuleLayout.COLOCATED if all_same else ModuleLayout.NON_COLOCATED + if role.mode is not expected: + raise ValueError(f"RankRole reported {role.mode} but rank spans imply {expected}: {spans}") + + +def pg_collection_from_grid( + grid: HyperCommGrid, is_language: bool = False +) -> ProcessGroupCollection: + """Adapt a populated ``HyperCommGrid`` into a ``ProcessGroupCollection``. + + Only the language module gets embedding groups; others leave ``embd``/``pos_embd`` as ``None``. + """ + pgc = ProcessGroupCollection() + pgc.tp = grid.get_pg("tp") + pgc.cp = grid.get_pg("cp") + pgc.pp = grid.get_pg("pp") + pgc.dp = grid.get_pg("dp") + pgc.dp_cp = grid.get_pg(["dp", "cp"]) + pgc.intra_dp_cp = pgc.dp_cp + pgc.tp_cp = grid.get_pg(["tp", "cp"]) + pgc.mp = grid.get_pg(["tp", "pp"]) + pgc.ep = grid.get_pg("ep", view=_EXPERT_VIEW) + pgc.expt_tp = grid.get_pg("expt_tp", view=_EXPERT_VIEW) + pgc.expt_dp = grid.get_pg("expt_dp", view=_EXPERT_VIEW) + pgc.intra_expt_dp = pgc.expt_dp + pgc.tp_ep = grid.get_pg(["expt_tp", "ep"], view=_EXPERT_VIEW) + pgc.tp_ep_pp = grid.get_pg(["expt_tp", "ep", "pp"], view=_EXPERT_VIEW) + pgc.embd = None + pgc.pos_embd = None + if is_language: + _build_language_embedding_groups(grid, pgc) + return pgc + + +def _build_language_embedding_groups(grid: HyperCommGrid, pgc: ProcessGroupCollection) -> None: + """Set this rank's word/position embedding groups, mirroring parallel_state. + + Creation is collective: every grid rank calls ``new_group`` for each PP tuple. + """ + own_pp_ranks = tuple(dist.get_process_group_ranks(pgc.pp)) if pgc.pp is not None else () + for pp_ranks in grid.get_rank_enum("pp"): + emb_group = dist.new_group(ranks=default_embedding_ranks(list(pp_ranks))) + pos_group = dist.new_group(ranks=default_position_embedding_ranks(list(pp_ranks))) + if tuple(pp_ranks) == own_pp_ranks: + if _is_process_group_member(emb_group): + pgc.embd = emb_group + if _is_process_group_member(pos_group): + pgc.pos_embd = pos_group + + +def build_schedule_pg_collection( + grids: dict[str, HyperCommGrid], + module_pgs: dict[str, ProcessGroupCollection], + language_name: str, +) -> MultiModuleProcessGroupCollection: + """Build the schedule-facing collection of the modules this rank participates in.""" + rank_modules = {} + rank_language_name = None + for name, grid in grids.items(): + # Include only modules this rank belongs to (colocated -> all; non-colocated -> its own), + # so language_model_module_name is set only when this rank is in the language module. + if not grid.is_current_rank_in_grid(): + continue + rank_modules[name] = module_pgs[name] + if name == language_name: + rank_language_name = name + return MultiModuleProcessGroupCollection( + module_pgs=rank_modules, language_model_module_name=rank_language_name + ) + + +def _is_process_group_member(pg: Optional[dist.ProcessGroup]) -> bool: + """Whether the current rank belongs to ``pg`` (get_rank returns -1 for non-members, no raise).""" + return pg is not None and dist.get_rank(group=pg) >= 0 diff --git a/examples/moe_recipes/README.md b/examples/moe_recipes/README.md index f77f6fad115..ffd8e4bf8c4 100644 --- a/examples/moe_recipes/README.md +++ b/examples/moe_recipes/README.md @@ -17,16 +17,43 @@ This directory contains self-contained MoE training recipes. Each YAML file incl TP/PP/EP/CP/ETP MBS/GBS/SL Features + Median TFLOP/s/GPU + + DeepSeek-V4-Flash + GB200 MXFP8 + 128 + 1/1/64/1/1 + 1/2048/4096 + BSHD; paged stash; full CG; HybridEP + 646.4 + DeepSeek-V3 + H100 BF16 + 1024 + 1/16/64/1/1 + 1/8192/4096 + BF16 baseline + + + + H100 FP8 + 1024 + 2/8/64/1/1 + 1/8192/4096 + DeepEP; EP overlap + + + B200 MXFP8 256 1/8/32/1/1 1/2048/4096 DeepEP; EP overlap + 776.7 GB200 MXFP8 @@ -34,6 +61,7 @@ This directory contains self-contained MoE training recipes. Each YAML file incl 1/4/64/1/1 1/8192/4096 HybridEP; partial CG; EP overlap; offload + 1127.6 GB300 MXFP8 @@ -41,28 +69,40 @@ This directory contains self-contained MoE training recipes. Each YAML file incl 1/4/64/1/1 1/8192/4096 HybridEP; partial CG; EP overlap + 1357.7 - H100 BF16 - 1024 - 1/16/64/1/1 - 1/8192/4096 - BF16 baseline + Qwen3-235B-A22B + H100 BF16 + 256 + 2/8/32/1/1 + 1/2048/4096 + router/preprocess CG; HybridEP; EP overlap + 214.3 - H100 FP8 - 1024 - 2/8/64/1/1 - 1/8192/4096 - DeepEP; EP overlap + B200 MXFP8 + 128 + 2/4/16/1/1 + 3/3072/4096 + DeepEP; EP overlap; VPP12 + 633.9 + + + B300 MXFP8 partial CG + 128 + 2/1/16/1/1 + 3/9216/4096 + partial CG; HybridEP; EP overlap + 922.6 - Qwen3-235B-A22B GB200 MXFP8 full CG 128 1/1/64/1/1 1/8192/4096 paged stash; full CG; HybridEP; EP overlap + 1159.5 GB200 MXFP8 partial CG @@ -70,6 +110,7 @@ This directory contains self-contained MoE training recipes. Each YAML file incl 1/1/64/1/1 1/8192/4096 partial CG; HybridEP; EP overlap + 1018 GB300 MXFP8 full CG @@ -77,13 +118,48 @@ This directory contains self-contained MoE training recipes. Each YAML file incl 1/1/64/1/1 1/8192/4096 paged stash; full CG; HybridEP; EP overlap + 1323.1 - H100 BF16 - 256 - 2/8/32/1/1 - 1/2048/4096 - router/preprocess CG; HybridEP; EP overlap + Qwen3-30B-A3B + H100 FP8 + 32 + 1/1/8/1/1 + 1/256/4096 + FP8 blockwise; router/preprocess CG + 232.4 + + + H100 BF16 + 32 + 1/1/8/1/1 + 1/256/4096 + BF16 baseline + 234.1 + + + GB200 BF16 + 16 + 1/1/16/1/1 + 4/512/4096 + BF16 baseline + 528 + + + GB200 MXFP8 partial CG + 16 + 1/1/16/1/1 + 4/512/4096 + MXFP8; partial CG + 514 + + + GB200 MXFP8 paged stash + 16 + 1/1/16/1/1 + 4/512/4096 + MXFP8; paged stash; full CG + 646.8 diff --git a/examples/moe_recipes/deepseek_v4_flash/gb200/mxfp8_SL4K_128GPU_TP1PP1EP64.yaml b/examples/moe_recipes/deepseek_v4_flash/gb200/mxfp8_SL4K_128GPU_TP1PP1EP64.yaml new file mode 100644 index 00000000000..9fe34ff9ca3 --- /dev/null +++ b/examples/moe_recipes/deepseek_v4_flash/gb200/mxfp8_SL4K_128GPU_TP1PP1EP64.yaml @@ -0,0 +1,242 @@ +DEPENDENCIES: + pytorch_base_image: nvcr.io/nvidia/pytorch:26.04-py3 + dockerfile: | + # Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + # IMAGE_NAME: dsv4-gb200-torch2604 + # + # DeepSeek-V4 training container for GB200 (arm64). + # + + FROM nvcr.io/nvidia/pytorch:26.04-py3 AS base + + ENV SHELL=/bin/bash + + # System packages + yq + RUN bash -ex <<"EOF" + rm -rf /opt/megatron-lm + apt-get update + apt-get install -y --no-install-recommends \ + sudo gdb bash-builtins git zsh autojump tmux curl gettext libfabric-dev + wget https://github.com/mikefarah/yq/releases/download/v4.27.5/yq_linux_arm64 -O /usr/bin/yq + chmod +x /usr/bin/yq + apt-get clean + rm -rf /var/lib/apt/lists/* + EOF + + # Python deps (mcore + dev + test + one-logger + cutlass-dsl pin) + RUN unset PIP_CONSTRAINT && pip install --no-cache-dir \ + debugpy dm-tree torch_tb_profiler einops wandb \ + sentencepiece tokenizers transformers==4.57.1 torchvision ftfy modelcards datasets tqdm pydantic omegaconf \ + nvidia-pytriton py-spy yapf darker \ + tiktoken flask-restful \ + nltk wrapt pytest pytest_asyncio pytest-cov pytest_mock pytest-random-order \ + black==24.4.2 isort==5.13.2 flake8==7.1.0 pylint==3.2.6 coverage mypy \ + one-logger --index-url https://sc-hw-artf.nvidia.com/artifactory/api/pypi/hwinf-mlwfo-pypi/simple \ + setuptools==69.5.1 nvidia-cutlass-dsl==4.5.2 + + # TransformerEngine pinned to release_v2.9-based commit with CPU/quantization fixes + ARG TE_COMMIT="3bca93857a9103ee7869d57c464936547573860a" + RUN pip install --no-cache-dir flash-attn-4==4.0.0b4 nvidia-mathdx==25.1.1 && \ + unset PIP_CONSTRAINT && \ + NVTE_CUDA_ARCHS="100a;103a" NVTE_BUILD_THREADS_PER_JOB=8 NVTE_FRAMEWORK=pytorch \ + pip install --no-build-isolation --no-cache-dir \ + "git+https://github.com/NVIDIA/TransformerEngine.git@${TE_COMMIT}" + + # HybridEP + WORKDIR /home/ + RUN git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git && \ + cd DeepEP && git checkout 1b8f467965bb818bf2f6511e06993f5607e1721f && \ + TORCH_CUDA_ARCH_LIST="10.0" pip install --no-build-isolation . + + # Fast Hadamard Transform (used by DSA indexer) + WORKDIR /home/ + RUN git clone https://github.com/Dao-AILab/fast-hadamard-transform.git && \ + cd fast-hadamard-transform && \ + pip install --no-build-isolation . + + # Emerging-Optimizers (Muon) + WORKDIR /home/ + RUN git clone https://github.com/NVIDIA-NeMo/Emerging-Optimizers.git && \ + cd Emerging-Optimizers && \ + pip install --no-build-isolation . + + # FlashMLA (DSA kernels) + WORKDIR /opt/ + RUN git clone --branch nv_dev https://github.com/deepseek-ai/FlashMLA.git && \ + cd FlashMLA && \ + FLASH_MLA_DISABLE_SM90=1 \ + NVCC_THREADS=16 \ + CFLAGS="-I/usr/local/cuda/include/cccl" \ + CXXFLAGS="-I/usr/local/cuda/include/cccl" \ + pip install --no-build-isolation . + + # cudnn_frontend + RUN pip install apache-tvm-ffi && \ + pip install --force-reinstall --no-deps --no-build-isolation git+https://github.com/NVIDIA/cudnn-frontend.git && \ + pip install --force-reinstall 'nvidia-cutlass-dsl[cu13]==4.5.2' + + RUN unset PIP_CONSTRAINT && \ + pip install --no-cache-dir nvidia-resiliency-ext==0.6.0 + + # Cleanup + RUN rm -rf /root/.cache /tmp/* + WORKDIR /home/ + +ENV_VARS: + TORCH_NCCL_AVOID_RECORD_STREAMS: '0' + NVTE_ALLOW_NONDETERMINISTIC_ALGO: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True,graph_capture_record_stream_reuse:True + NCCL_NVLS_ENABLE: '0' + NVTE_FUSED_ATTN: '1' + NVTE_NORM_FWD_USE_CUDNN: '1' + NVTE_NORM_BWD_USE_CUDNN: '1' + PYTHONWARNINGS: ignore + NCCL_DEBUG: VERSION + NCCL_GRAPH_REGISTER: '0' + NVTE_CUTEDSL_FUSED_GROUPED_MLP: '1' + NVTE_CPU_OFFLOAD_V1: '1' + NUM_OF_TOKENS_PER_CHUNK_COMBINE_API: '128' + NUM_OF_STAGES_DISPATCH_API: '10' + NUM_OF_IN_FLIGHT_S2G_DISPATCH_API: '8' +ARGS: + tokenizer_type: HuggingFaceTokenizer + tokenizer_model: unsloth/DeepSeek-V3 + num_layers: 43 + hidden_size: 4096 + num_attention_heads: 64 + kv_channels: 512 + max_position_embeddings: 4096 + normalization: RMSNorm + norm_epsilon: 1e-6 + swiglu: true + disable_bias_linear: true + untie_embeddings_and_output_weights: true + position_embedding_type: rope + rotary_base: 10000 + make_vocab_size_divisible_by: 3232 + multi_latent_attention: true + q_lora_rank: 1024 + qk_pos_emb_head_dim: 64 + v_head_dim: 512 + rotary_scaling_factor: 4 + mscale: 1.0 + mscale_all_dim: 1.0 + qk_layernorm: true + o_groups: 8 + o_lora_rank: 1024 + original_max_position_embeddings: 65536 + experimental_attention_variant: dsv4_hybrid + csa_window_size: 128 + csa_compress_ratios: ([0,0,4]+[128,4]*20+[0]) + csa_compress_rotary_base: 40000 + dsa_indexer_n_heads: 64 + dsa_indexer_head_dim: 128 + dsa_indexer_topk: 512 + dsa_indexer_loss_coeff: 1e-2 + dsa_indexer_use_sparse_loss: true + num_experts: 256 + moe_n_hash_layers: 3 + moe_ffn_hidden_size: 2048 + moe_shared_expert_intermediate_size: 2048 + moe_router_load_balancing_type: seq_aux_loss + moe_router_topk: 6 + moe_aux_loss_coeff: 1e-4 + moe_router_topk_scaling_factor: 1.5 + moe_router_score_function: sqrtsoftplus + moe_router_enable_expert_bias: true + moe_router_bias_update_rate: 1e-3 + activation_func_clamp_value: 10.0 + enable_hyper_connections: true + num_residual_streams: 4 + mhc_sinkhorn_iterations: 20 + use_fused_mhc: true + mtp_num_layers: 1 + mtp_loss_scaling_factor: 0.1 + attention_dropout: 0.0 + hidden_dropout: 0.0 + mock_data: true + seq_length: 4096 + moe_router_force_load_balancing: true + tensor_model_parallel_size: 1 + pipeline_model_parallel_size: 1 + expert_model_parallel_size: 64 + context_parallel_size: 1 + expert_tensor_parallel_size: 1 + use_distributed_optimizer: true + sequence_parallel: true + overlap_grad_reduce: true + overlap_param_gather: true + moe_token_dispatcher_type: flex + moe_flex_dispatcher_backend: hybridep + moe_hybridep_num_sms: 32 + moe_grouped_gemm: true + moe_permute_fusion: true + moe_router_fusion: true + moe_router_dtype: fp32 + recompute_granularity: selective + recompute_modules: + - mla_up_proj + use_transformer_engine_op_fuser: true + moe_mlp_glu_interleave_size: 32 + moe_paged_stash: true + moe_paged_stash_page_size: 64 + moe_paged_stash_buffer_size_factor_cuda: 1.2 + moe_paged_stash_buffer_size_factor_cpu: 0.0 + moe_pad_experts_for_cuda_graph_inference: true + moe_expert_rank_capacity_factor: 1.5 + cuda_graph_impl: local + cuda_graph_scope: full_iteration + use_mcore_models: true + use_flash_attn: true + transformer_impl: transformer_engine + micro_batch_size: 1 + global_batch_size: 2048 + train_samples: 585937500 + exit_duration_in_mins: 220 + no_save_optim: true + no_check_for_nan_in_loss_and_grad: true + cross_entropy_loss_fusion: true + cross_entropy_fusion_impl: native + no_create_attention_mask_in_dataloader: true + manual_gc: true + manual_gc_interval: 10 + lr: 3.9e-06 + min_lr: 3.9e-07 + lr_warmup_init: 3.9e-07 + lr_decay_style: cosine + lr_decay_samples: 584765624 + lr_warmup_samples: 1536000 + weight_decay: 0.1 + clip_grad: 1.0 + adam_beta1: 0.9 + adam_beta2: 0.95 + bf16: true + fp8_recipe: mxfp8 + fp8_format: e4m3 + fp8_param_gather: true + reuse_grad_buf_for_mxfp8_param_ag: true + use_precision_aware_optimizer: true + main_grads_dtype: fp32 + main_params_dtype: fp32 + exp_avg_dtype: bf16 + exp_avg_sq_dtype: bf16 + moe_router_padding_for_quantization: true + init_method_std: 0.02 + eval_iters: 32 + eval_interval: 200 + finetune: false + no_load_optim: true + no_load_rng: true + auto_detect_ckpt_format: true + load: ${LOAD_PATH} + save_interval: 500 + dist_ckpt_strictness: log_all + log_throughput: true + log_interval: 1 + logging_level: 20 + log_timers_to_tensorboard: true + log_memory_to_tensorboard: true + log_validation_ppl_to_tensorboard: true + tensorboard_dir: ${OUTPUT_PATH}/tensorboard + wandb_exp_name: DeepSeek-V4-Flash-GB200-MXFP8-TP1PP1EP64-GBS2048SEQLEN4096 + enable_experimental: true diff --git a/examples/moe_recipes/qwen3_235b/b200/mxfp8_128GPU_TP2PP4EP16.yaml b/examples/moe_recipes/qwen3_235b/b200/mxfp8_128GPU_TP2PP4EP16.yaml new file mode 100644 index 00000000000..f3aed1dbe86 --- /dev/null +++ b/examples/moe_recipes/qwen3_235b/b200/mxfp8_128GPU_TP2PP4EP16.yaml @@ -0,0 +1,197 @@ +DEPENDENCIES: + pytorch_base_image: nvcr.io/nvidia/pytorch:26.03-py3 + dockerfile: | + # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + # IMAGE_NAME: b300-torch2603 + + ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:26.03-py3 + FROM ${FROM_IMAGE_NAME} AS base + + ENV SHELL=/bin/bash + ENV DEBIAN_FRONTEND=noninteractive + + ARG YQ_VERSION=4.27.5 + ARG TE_TAG=release_v2.14 + ARG NVTE_CUDA_ARCHS="100a;103a" + + # Install system dependencies + RUN bash -ex <<"EOF" + rm -rf /opt/megatron-lm + apt-get update + apt-get install -y --no-install-recommends git curl gettext sudo + ARCH=$(uname -m) + case "${ARCH}" in + "x86_64") YQ_ARCH=amd64 ;; + "aarch64") YQ_ARCH=arm64 ;; + *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;; + esac + wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${YQ_ARCH} -O /usr/bin/yq + chmod +x /usr/bin/yq + apt-get clean + rm -rf /var/lib/apt/lists/* + EOF + + # Install Megatron-Core dependencies (mlm + dev + test groups from pyproject.toml) + RUN bash -ex <<"EOF" + unset PIP_CONSTRAINT + pip install \ + sentencepiece tiktoken transformers accelerate \ + wandb einops tqdm datasets nvtx \ + flask-restful flask[async] fastapi hypercorn \ + nltk wrapt pydantic pyyaml omegaconf \ + pytest==8.3.5 pytest-mock pytest-cov pytest-random-order pytest-asyncio \ + coverage tensorboard + EOF + + # Install TransformerEngine + RUN unset PIP_CONSTRAINT && \ + NVTE_CUDA_ARCHS="${NVTE_CUDA_ARCHS}" NVTE_FRAMEWORK=pytorch \ + pip install --no-build-isolation --no-cache-dir \ + "git+https://github.com/nvidia/TransformerEngine.git@${TE_TAG}" + + # Install Flash Attention 4, CUTLASS DSL, cuDNN frontend, and FLA for + # Qwen3.5 Gated Delta Net support. + # Keep cudnn-frontend at 1.22.1: 1.23.0 removes the c_dtype kwarg from + # grouped_gemm_quant_wrapper_sm100, which TE 2.14 still passes. + RUN pip install --no-cache-dir flash-attn-4==4.0.0b4 "nvidia-cutlass-dsl[cu13]==4.4.2" && \ + pip install --no-cache-dir --no-deps "nvidia-cudnn-frontend==1.22.1" && \ + pip install --no-cache-dir fla-core==0.4.2 flash-linear-attention==0.4.2 + + # ========================= + # Option 1: HybridEP + # ========================= + FROM base AS hybridep + + RUN bash -ex <<"EOF" + cd /workspace + git clone https://github.com/linux-rdma/rdma-core.git + cd rdma-core && git checkout tags/v60.0 && sh build.sh + apt-get update + apt-get install -y --no-install-recommends libnvidia-ml-dev + git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git + cd DeepEP && git checkout 7febc6e25660af0f54d95dd781ecdcd62265ecca + RDMA_CORE_HOME=/workspace/rdma-core/build HYBRID_EP_MULTINODE=1 \ + TORCH_CUDA_ARCH_LIST="10.0" MAX_JOBS=8 \ + pip install --no-build-isolation . + apt-get purge -y libnvidia-ml-dev + apt-get autoremove -y + rm -rf /root/.cache /tmp/* /var/lib/apt/lists/* + EOF + + WORKDIR /workspace/ +ENV_VARS: + NVTE_ALLOW_NONDETERMINISTIC_ALGO: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NCCL_NVLS_ENABLE: '0' + NVTE_FUSED_ATTN: '1' + NVTE_NORM_FWD_USE_CUDNN: '1' + NVTE_NORM_BWD_USE_CUDNN: '1' + CUDA_DEVICE_MAX_CONNECTIONS: '32' +ARGS: + tokenizer_type: HuggingFaceTokenizer + tokenizer_model: Qwen/Qwen3-235B-A22B + num_layers: 94 + hidden_size: 4096 + ffn_hidden_size: 12288 + num_attention_heads: 64 + kv_channels: 128 + max_position_embeddings: 4096 + group_query_attention: true + num_query_groups: 4 + qk_layernorm: true + normalization: RMSNorm + norm_epsilon: 1e-6 + swiglu: true + disable_bias_linear: true + untie_embeddings_and_output_weights: true + position_embedding_type: rope + rotary_percent: 1.0 + rotary_base: 1000000 + make_vocab_size_divisible_by: 1187 + num_experts: 128 + moe_ffn_hidden_size: 1536 + moe_router_load_balancing_type: aux_loss + moe_router_topk: 8 + moe_router_pre_softmax: false + moe_aux_loss_coeff: 1e-3 + attention_dropout: 0.0 + hidden_dropout: 0.0 + mock_data: true + seq_length: 4096 + moe_router_force_load_balancing: true + tensor_model_parallel_size: 2 + pipeline_model_parallel_size: 4 + expert_model_parallel_size: 16 + context_parallel_size: 1 + expert_tensor_parallel_size: 1 + num_layers_per_virtual_pipeline_stage: 2 + use_distributed_optimizer: true + sequence_parallel: true + account_for_embedding_in_pipeline_split: true + account_for_loss_in_pipeline_split: true + delay_wgrad_compute: true + overlap_moe_expert_parallel_comm: true + overlap_p2p_comm: true + moe_token_dispatcher_type: flex + moe_flex_dispatcher_backend: deepep + moe_grouped_gemm: true + moe_permute_fusion: true + moe_router_fusion: true + moe_router_dtype: fp32 + moe_router_padding_for_quantization: true + moe_deepep_num_sms: 20 + use_mcore_models: true + use_flash_attn: true + transformer_impl: transformer_engine + micro_batch_size: 3 + global_batch_size: 3072 + train_samples: 268554688 + exit_duration_in_mins: 230 + no_create_attention_mask_in_dataloader: true + cross_entropy_loss_fusion: true + cross_entropy_fusion_impl: te + manual_gc: true + manual_gc_interval: 5 + lr: 3.9e-06 + min_lr: 3.9e-07 + lr_warmup_init: 3.9e-07 + lr_decay_style: cosine + lr_decay_samples: 584765624 + lr_warmup_samples: 1536000 + weight_decay: 0.1 + clip_grad: 1.0 + adam_beta1: 0.9 + adam_beta2: 0.95 + bf16: true + fp8_format: e4m3 + fp8_recipe: mxfp8 + fp8_param_gather: true + reuse_grad_buf_for_mxfp8_param_ag: true + overlap_grad_reduce: true + overlap_param_gather: true + use_precision_aware_optimizer: true + main_grads_dtype: fp32 + main_params_dtype: fp32 + exp_avg_dtype: bf16 + exp_avg_sq_dtype: bf16 + init_method_std: 0.02 + eval_iters: 32 + eval_interval: 500 + finetune: true + auto_detect_ckpt_format: true + no_load_rng: true + no_load_optim: true + load: ${LOAD_PATH} + save_interval: 100 + dist_ckpt_strictness: log_all + log_throughput: true + log_interval: 1 + log_timers_to_tensorboard: true + log_memory_to_tensorboard: true + log_num_zeros_in_grad: true + log_params_norm: true + log_validation_ppl_to_tensorboard: true + logging_level: 40 + tensorboard_dir: ${OUTPUT_PATH}/tensorboard + wandb_exp_name: Qwen3-235B-A22B-B200-MXFP8-TP2PP4EP16-MBS3GBS3072 + enable_experimental: true diff --git a/examples/moe_recipes/qwen3_235b/b300/mxfp8_128GPU_TP2PP1EP16_partial_cg.yaml b/examples/moe_recipes/qwen3_235b/b300/mxfp8_128GPU_TP2PP1EP16_partial_cg.yaml new file mode 100644 index 00000000000..f7567f21def --- /dev/null +++ b/examples/moe_recipes/qwen3_235b/b300/mxfp8_128GPU_TP2PP1EP16_partial_cg.yaml @@ -0,0 +1,201 @@ +DEPENDENCIES: + pytorch_base_image: nvcr.io/nvidia/pytorch:26.03-py3 + dockerfile: | + # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + # IMAGE_NAME: b300-torch2603 + + ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:26.03-py3 + FROM ${FROM_IMAGE_NAME} AS base + + ENV SHELL=/bin/bash + ENV DEBIAN_FRONTEND=noninteractive + + ARG YQ_VERSION=4.27.5 + ARG TE_TAG=release_v2.14 + ARG NVTE_CUDA_ARCHS="100a;103a" + + # Install system dependencies + RUN bash -ex <<"EOF" + rm -rf /opt/megatron-lm + apt-get update + apt-get install -y --no-install-recommends git curl gettext sudo + ARCH=$(uname -m) + case "${ARCH}" in + "x86_64") YQ_ARCH=amd64 ;; + "aarch64") YQ_ARCH=arm64 ;; + *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;; + esac + wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${YQ_ARCH} -O /usr/bin/yq + chmod +x /usr/bin/yq + apt-get clean + rm -rf /var/lib/apt/lists/* + EOF + + # Install Megatron-Core dependencies (mlm + dev + test groups from pyproject.toml) + RUN bash -ex <<"EOF" + unset PIP_CONSTRAINT + pip install \ + sentencepiece tiktoken transformers accelerate \ + wandb einops tqdm datasets nvtx \ + flask-restful flask[async] fastapi hypercorn \ + nltk wrapt pydantic pyyaml omegaconf \ + pytest==8.3.5 pytest-mock pytest-cov pytest-random-order pytest-asyncio \ + coverage tensorboard + EOF + + # Install TransformerEngine + RUN unset PIP_CONSTRAINT && \ + NVTE_CUDA_ARCHS="${NVTE_CUDA_ARCHS}" NVTE_FRAMEWORK=pytorch \ + pip install --no-build-isolation --no-cache-dir \ + "git+https://github.com/nvidia/TransformerEngine.git@${TE_TAG}" + + # Install Flash Attention 4, CUTLASS DSL, cuDNN frontend, and FLA for + # Qwen3.5 Gated Delta Net support. + # Keep cudnn-frontend at 1.22.1: 1.23.0 removes the c_dtype kwarg from + # grouped_gemm_quant_wrapper_sm100, which TE 2.14 still passes. + RUN pip install --no-cache-dir flash-attn-4==4.0.0b4 "nvidia-cutlass-dsl[cu13]==4.4.2" && \ + pip install --no-cache-dir --no-deps "nvidia-cudnn-frontend==1.22.1" && \ + pip install --no-cache-dir fla-core==0.4.2 flash-linear-attention==0.4.2 + + # ========================= + # Option 1: HybridEP + # ========================= + FROM base AS hybridep + + RUN bash -ex <<"EOF" + cd /workspace + git clone https://github.com/linux-rdma/rdma-core.git + cd rdma-core && git checkout tags/v60.0 && sh build.sh + apt-get update + apt-get install -y --no-install-recommends libnvidia-ml-dev + git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git + cd DeepEP && git checkout 7febc6e25660af0f54d95dd781ecdcd62265ecca + RDMA_CORE_HOME=/workspace/rdma-core/build HYBRID_EP_MULTINODE=1 \ + TORCH_CUDA_ARCH_LIST="10.0" MAX_JOBS=8 \ + pip install --no-build-isolation . + apt-get purge -y libnvidia-ml-dev + apt-get autoremove -y + rm -rf /root/.cache /tmp/* /var/lib/apt/lists/* + EOF + + WORKDIR /workspace/ +ENV_VARS: + NVTE_ALLOW_NONDETERMINISTIC_ALGO: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NCCL_NVLS_ENABLE: '0' + NVTE_FUSED_ATTN: '1' + NVTE_NORM_FWD_USE_CUDNN: '1' + NVTE_NORM_BWD_USE_CUDNN: '1' + CUDA_DEVICE_MAX_CONNECTIONS: '32' +ARGS: + tokenizer_type: HuggingFaceTokenizer + tokenizer_model: Qwen/Qwen3-235B-A22B + num_layers: 94 + hidden_size: 4096 + ffn_hidden_size: 12288 + num_attention_heads: 64 + kv_channels: 128 + max_position_embeddings: 4096 + group_query_attention: true + num_query_groups: 4 + qk_layernorm: true + normalization: RMSNorm + norm_epsilon: 1e-6 + swiglu: true + disable_bias_linear: true + untie_embeddings_and_output_weights: true + position_embedding_type: rope + rotary_percent: 1.0 + rotary_base: 1000000 + make_vocab_size_divisible_by: 1187 + num_experts: 128 + moe_ffn_hidden_size: 1536 + moe_router_load_balancing_type: aux_loss + moe_router_topk: 8 + moe_router_pre_softmax: false + moe_aux_loss_coeff: 1e-3 + attention_dropout: 0.0 + hidden_dropout: 0.0 + mock_data: true + seq_length: 4096 + moe_router_force_load_balancing: true + tensor_model_parallel_size: 2 + pipeline_model_parallel_size: 1 + expert_model_parallel_size: 16 + context_parallel_size: 1 + expert_tensor_parallel_size: 1 + use_distributed_optimizer: true + sequence_parallel: true + account_for_embedding_in_pipeline_split: true + account_for_loss_in_pipeline_split: true + delay_wgrad_compute: true + overlap_moe_expert_parallel_comm: true + moe_token_dispatcher_type: flex + moe_flex_dispatcher_backend: hybridep + moe_grouped_gemm: true + moe_permute_fusion: true + moe_router_fusion: true + moe_router_dtype: fp32 + moe_router_padding_for_quantization: true + moe_hybridep_num_sms: 32 + cuda_graph_impl: transformer_engine + cuda_graph_scope: + - attn + - moe_router + cuda_graph_warmup_steps: 2 + te_rng_tracker: true + use_mcore_models: true + use_flash_attn: true + transformer_impl: transformer_engine + micro_batch_size: 3 + global_batch_size: 9216 + train_samples: 268554688 + exit_duration_in_mins: 230 + no_create_attention_mask_in_dataloader: true + cross_entropy_loss_fusion: true + cross_entropy_fusion_impl: te + manual_gc: true + manual_gc_interval: 5 + lr: 3.9e-06 + min_lr: 3.9e-07 + lr_warmup_init: 3.9e-07 + lr_decay_style: cosine + lr_decay_samples: 584765624 + lr_warmup_samples: 1536000 + weight_decay: 0.1 + clip_grad: 1.0 + adam_beta1: 0.9 + adam_beta2: 0.95 + bf16: true + fp8_format: e4m3 + fp8_recipe: mxfp8 + fp8_param_gather: true + reuse_grad_buf_for_mxfp8_param_ag: true + overlap_grad_reduce: true + overlap_param_gather: true + use_precision_aware_optimizer: true + main_grads_dtype: fp32 + main_params_dtype: fp32 + exp_avg_dtype: bf16 + exp_avg_sq_dtype: bf16 + init_method_std: 0.02 + eval_iters: 32 + eval_interval: 500 + finetune: true + auto_detect_ckpt_format: true + no_load_rng: true + no_load_optim: true + load: ${LOAD_PATH} + save_interval: 100 + dist_ckpt_strictness: log_all + log_throughput: true + log_interval: 1 + log_timers_to_tensorboard: true + log_memory_to_tensorboard: true + log_num_zeros_in_grad: true + log_params_norm: true + log_validation_ppl_to_tensorboard: true + logging_level: 40 + tensorboard_dir: ${OUTPUT_PATH}/tensorboard + wandb_exp_name: Qwen3-235B-A22B-B300-MXFP8-TP2PP1EP16-MBS3GBS9216 + enable_experimental: true diff --git a/examples/moe_recipes/qwen3_30b/gb200/bf16_16GPU_TP1PP1EP16.yaml b/examples/moe_recipes/qwen3_30b/gb200/bf16_16GPU_TP1PP1EP16.yaml new file mode 100644 index 00000000000..b4a678b9223 --- /dev/null +++ b/examples/moe_recipes/qwen3_30b/gb200/bf16_16GPU_TP1PP1EP16.yaml @@ -0,0 +1,181 @@ +DEPENDENCIES: + pytorch_base_image: nvcr.io/nvidia/pytorch:26.03-py3 + dockerfile: | + # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + # IMAGE_NAME: gb200-torch2603 + + ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:26.03-py3 + FROM ${FROM_IMAGE_NAME} AS base + + ENV SHELL=/bin/bash + ENV DEBIAN_FRONTEND=noninteractive + + ARG YQ_VERSION=4.27.5 + ARG TE_TAG=release_v2.14 + ARG NVTE_CUDA_ARCHS="100a" + + # Install system dependencies + RUN bash -ex <<"EOF" + rm -rf /opt/megatron-lm + apt-get update + apt-get install -y --no-install-recommends git curl gettext sudo + ARCH=$(uname -m) + case "${ARCH}" in + "x86_64") YQ_ARCH=amd64 ;; + "aarch64") YQ_ARCH=arm64 ;; + *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;; + esac + wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${YQ_ARCH} -O /usr/bin/yq + chmod +x /usr/bin/yq + apt-get clean + rm -rf /var/lib/apt/lists/* + EOF + + # Install Megatron-Core dependencies (mlm + dev + test groups from pyproject.toml) + RUN bash -ex <<"EOF" + unset PIP_CONSTRAINT + pip install \ + sentencepiece tiktoken transformers accelerate \ + wandb einops tqdm datasets nvtx \ + flask-restful flask[async] fastapi hypercorn \ + nltk wrapt pydantic pyyaml omegaconf \ + pytest==8.3.5 pytest-mock pytest-cov pytest-random-order pytest-asyncio \ + coverage tensorboard + EOF + + # Install TransformerEngine + RUN unset PIP_CONSTRAINT && \ + NVTE_CUDA_ARCHS="${NVTE_CUDA_ARCHS}" NVTE_FRAMEWORK=pytorch \ + pip install --no-build-isolation --no-cache-dir \ + "git+https://github.com/nvidia/TransformerEngine.git@${TE_TAG}" + + # Install Flash Attention 4, CUTLASS DSL, and cuDNN frontend with CuTe DSL support. + # Keep cudnn-frontend at 1.22.1: 1.23.0 removes the c_dtype kwarg from + # grouped_gemm_quant_wrapper_sm100, which TE 2.14 still passes. + RUN pip install --no-cache-dir flash-attn-4==4.0.0b4 "nvidia-cutlass-dsl[cu13]==4.4.2" && \ + pip install --no-cache-dir --no-deps "nvidia-cudnn-frontend==1.22.1" + + # Install Flash Linear Attention for gated-delta-net Qwen3.5-VL recipes. + RUN pip install --no-cache-dir --no-deps \ + fla-core==0.4.2 \ + flash-linear-attention==0.4.2 + + # ========================= + # Option 1: HybridEP + # ========================= + FROM base AS hybridep + + RUN bash -ex <<"EOF" + cd /workspace + git clone https://github.com/linux-rdma/rdma-core.git + cd rdma-core && git checkout tags/v60.0 && sh build.sh + apt-get update + apt-get install -y --no-install-recommends libnvidia-ml-dev + git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git + cd DeepEP && git checkout 7febc6e25660af0f54d95dd781ecdcd62265ecca + RDMA_CORE_HOME=/workspace/rdma-core/build HYBRID_EP_MULTINODE=1 \ + TORCH_CUDA_ARCH_LIST="10.0" MAX_JOBS=8 \ + pip install --no-build-isolation . + apt-get purge -y libnvidia-ml-dev + apt-get autoremove -y + rm -rf /root/.cache /tmp/* /var/lib/apt/lists/* + EOF + + WORKDIR /workspace/ +ENV_VARS: + NVTE_ALLOW_NONDETERMINISTIC_ALGO: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NCCL_NVLS_ENABLE: '0' + HF_HUB_OFFLINE: '1' + NVTE_FUSED_ATTN: '1' + NVTE_NORM_FWD_USE_CUDNN: '1' + NVTE_NORM_BWD_USE_CUDNN: '1' + CUDA_DEVICE_MAX_CONNECTIONS: '1' +ARGS: + tokenizer_type: HuggingFaceTokenizer + tokenizer_model: Qwen/Qwen3-30B-A3B + num_layers: 48 + hidden_size: 2048 + ffn_hidden_size: 6144 + num_attention_heads: 32 + kv_channels: 128 + max_position_embeddings: 40960 + group_query_attention: true + num_query_groups: 4 + qk_layernorm: true + normalization: RMSNorm + norm_epsilon: 1e-6 + swiglu: true + disable_bias_linear: true + untie_embeddings_and_output_weights: true + position_embedding_type: rope + rotary_percent: 1.0 + rotary_base: 1000000 + make_vocab_size_divisible_by: 1187 + num_experts: 128 + moe_ffn_hidden_size: 768 + moe_router_load_balancing_type: aux_loss + moe_router_topk: 8 + moe_router_pre_softmax: false + moe_aux_loss_coeff: 1e-3 + attention_dropout: 0.0 + hidden_dropout: 0.0 + mock_data: true + seq_length: 4096 + moe_router_force_load_balancing: true + tensor_model_parallel_size: 1 + pipeline_model_parallel_size: 1 + expert_model_parallel_size: 16 + context_parallel_size: 1 + expert_tensor_parallel_size: 1 + use_distributed_optimizer: true + sequence_parallel: true + moe_token_dispatcher_type: flex + moe_flex_dispatcher_backend: hybridep + moe_grouped_gemm: true + moe_permute_fusion: true + moe_router_fusion: true + moe_router_dtype: fp32 + use_mcore_models: true + use_flash_attn: true + transformer_impl: transformer_engine + micro_batch_size: 4 + global_batch_size: 512 + train_samples: 268554688 + exit_duration_in_mins: 230 + no_create_attention_mask_in_dataloader: true + cross_entropy_loss_fusion: true + cross_entropy_fusion_impl: te + manual_gc: true + manual_gc_interval: 5 + lr: 0.00012 + min_lr: 1.2e-05 + lr_decay_style: cosine + lr_decay_samples: 255126953 + lr_warmup_samples: 162761 + weight_decay: 0.1 + clip_grad: 1.0 + adam_beta1: 0.9 + adam_beta2: 0.95 + bf16: true + init_method_std: 0.02 + eval_iters: 32 + eval_interval: 500 + finetune: true + auto_detect_ckpt_format: true + no_load_rng: true + no_load_optim: true + load: ${LOAD_PATH} + save_interval: 500 + dist_ckpt_strictness: log_all + log_throughput: true + log_interval: 1 + log_timers_to_tensorboard: true + log_memory_to_tensorboard: true + log_num_zeros_in_grad: true + log_params_norm: true + log_validation_ppl_to_tensorboard: true + logging_level: 40 + tensorboard_dir: ${OUTPUT_PATH}/tensorboard + wandb_exp_name: Qwen3-30B-GB200-BF16-TP1PP1EP16-GBS512 + enable_experimental: true diff --git a/examples/moe_recipes/qwen3_30b/gb200/mxfp8_16GPU_TP1PP1EP16_paged_stash.yaml b/examples/moe_recipes/qwen3_30b/gb200/mxfp8_16GPU_TP1PP1EP16_paged_stash.yaml new file mode 100644 index 00000000000..694373bbaad --- /dev/null +++ b/examples/moe_recipes/qwen3_30b/gb200/mxfp8_16GPU_TP1PP1EP16_paged_stash.yaml @@ -0,0 +1,195 @@ +DEPENDENCIES: + pytorch_base_image: nvcr.io/nvidia/pytorch:26.03-py3 + dockerfile: | + # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + # IMAGE_NAME: gb200-torch2603 + + ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:26.03-py3 + FROM ${FROM_IMAGE_NAME} AS base + + ENV SHELL=/bin/bash + ENV DEBIAN_FRONTEND=noninteractive + + ARG YQ_VERSION=4.27.5 + ARG TE_TAG=release_v2.14 + ARG NVTE_CUDA_ARCHS="100a" + + # Install system dependencies + RUN bash -ex <<"EOF" + rm -rf /opt/megatron-lm + apt-get update + apt-get install -y --no-install-recommends git curl gettext sudo + ARCH=$(uname -m) + case "${ARCH}" in + "x86_64") YQ_ARCH=amd64 ;; + "aarch64") YQ_ARCH=arm64 ;; + *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;; + esac + wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${YQ_ARCH} -O /usr/bin/yq + chmod +x /usr/bin/yq + apt-get clean + rm -rf /var/lib/apt/lists/* + EOF + + # Install Megatron-Core dependencies (mlm + dev + test groups from pyproject.toml) + RUN bash -ex <<"EOF" + unset PIP_CONSTRAINT + pip install \ + sentencepiece tiktoken transformers accelerate \ + wandb einops tqdm datasets nvtx \ + flask-restful flask[async] fastapi hypercorn \ + nltk wrapt pydantic pyyaml omegaconf \ + pytest==8.3.5 pytest-mock pytest-cov pytest-random-order pytest-asyncio \ + coverage tensorboard + EOF + + # Install TransformerEngine + RUN unset PIP_CONSTRAINT && \ + NVTE_CUDA_ARCHS="${NVTE_CUDA_ARCHS}" NVTE_FRAMEWORK=pytorch \ + pip install --no-build-isolation --no-cache-dir \ + "git+https://github.com/nvidia/TransformerEngine.git@${TE_TAG}" + + # Install Flash Attention 4, CUTLASS DSL, and cuDNN frontend with CuTe DSL support. + # Keep cudnn-frontend at 1.22.1: 1.23.0 removes the c_dtype kwarg from + # grouped_gemm_quant_wrapper_sm100, which TE 2.14 still passes. + RUN pip install --no-cache-dir flash-attn-4==4.0.0b4 "nvidia-cutlass-dsl[cu13]==4.4.2" && \ + pip install --no-cache-dir --no-deps "nvidia-cudnn-frontend==1.22.1" + + # Install Flash Linear Attention for gated-delta-net Qwen3.5-VL recipes. + RUN pip install --no-cache-dir --no-deps \ + fla-core==0.4.2 \ + flash-linear-attention==0.4.2 + + # ========================= + # Option 1: HybridEP + # ========================= + FROM base AS hybridep + + RUN bash -ex <<"EOF" + cd /workspace + git clone https://github.com/linux-rdma/rdma-core.git + cd rdma-core && git checkout tags/v60.0 && sh build.sh + apt-get update + apt-get install -y --no-install-recommends libnvidia-ml-dev + git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git + cd DeepEP && git checkout 7febc6e25660af0f54d95dd781ecdcd62265ecca + RDMA_CORE_HOME=/workspace/rdma-core/build HYBRID_EP_MULTINODE=1 \ + TORCH_CUDA_ARCH_LIST="10.0" MAX_JOBS=8 \ + pip install --no-build-isolation . + apt-get purge -y libnvidia-ml-dev + apt-get autoremove -y + rm -rf /root/.cache /tmp/* /var/lib/apt/lists/* + EOF + + WORKDIR /workspace/ +ENV_VARS: + NVTE_ALLOW_NONDETERMINISTIC_ALGO: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True,graph_capture_record_stream_reuse:True + NCCL_NVLS_ENABLE: '0' + NCCL_GRAPH_REGISTER: 0 + HF_HUB_OFFLINE: '1' + NVTE_FUSED_ATTN: '1' + NVTE_NORM_FWD_USE_CUDNN: '1' + NVTE_NORM_BWD_USE_CUDNN: '1' + CUDA_DEVICE_MAX_CONNECTIONS: '1' + NVTE_CUTEDSL_FUSED_GROUPED_MLP: '1' +ARGS: + tokenizer_type: HuggingFaceTokenizer + tokenizer_model: Qwen/Qwen3-30B-A3B + num_layers: 48 + hidden_size: 2048 + ffn_hidden_size: 6144 + num_attention_heads: 32 + kv_channels: 128 + max_position_embeddings: 40960 + group_query_attention: true + num_query_groups: 4 + qk_layernorm: true + normalization: RMSNorm + norm_epsilon: 1e-6 + swiglu: true + disable_bias_linear: true + untie_embeddings_and_output_weights: true + position_embedding_type: rope + rotary_percent: 1.0 + rotary_base: 1000000 + make_vocab_size_divisible_by: 1187 + num_experts: 128 + moe_ffn_hidden_size: 768 + moe_router_load_balancing_type: aux_loss + moe_router_topk: 8 + moe_router_pre_softmax: false + moe_aux_loss_coeff: 1e-3 + attention_dropout: 0.0 + hidden_dropout: 0.0 + mock_data: true + seq_length: 4096 + moe_router_force_load_balancing: true + tensor_model_parallel_size: 1 + pipeline_model_parallel_size: 1 + expert_model_parallel_size: 16 + context_parallel_size: 1 + expert_tensor_parallel_size: 1 + use_distributed_optimizer: true + sequence_parallel: true + moe_token_dispatcher_type: flex + moe_flex_dispatcher_backend: hybridep + moe_grouped_gemm: true + moe_permute_fusion: true + moe_router_fusion: true + moe_router_dtype: fp32 + use_transformer_engine_op_fuser: true + moe_paged_stash: true + moe_expert_rank_capacity_factor: 1.5 + moe_paged_stash_page_size: 64 + moe_paged_stash_buffer_size_factor_cuda: 1.1 + cuda_graph_impl: local + cuda_graph_scope: full_iteration + moe_pad_experts_for_cuda_graph_inference: true + moe_mlp_glu_interleave_size: 32 + use_mcore_models: true + use_flash_attn: true + transformer_impl: transformer_engine + micro_batch_size: 4 + global_batch_size: 512 + train_samples: 268554688 + exit_duration_in_mins: 230 + no_create_attention_mask_in_dataloader: true + no_check_for_nan_in_loss_and_grad: true + cross_entropy_loss_fusion: true + cross_entropy_fusion_impl: te + manual_gc: true + manual_gc_interval: 5 + lr: 0.00012 + min_lr: 1.2e-05 + lr_decay_style: cosine + lr_decay_samples: 255126953 + lr_warmup_samples: 162761 + weight_decay: 0.1 + clip_grad: 1.0 + adam_beta1: 0.9 + adam_beta2: 0.95 + bf16: true + fp8_format: e4m3 + fp8_recipe: mxfp8 + init_method_std: 0.02 + eval_iters: 32 + eval_interval: 500 + finetune: true + auto_detect_ckpt_format: true + no_load_rng: true + no_load_optim: true + load: ${LOAD_PATH} + save_interval: 500 + dist_ckpt_strictness: log_all + log_throughput: true + log_interval: 1 + log_timers_to_tensorboard: true + log_memory_to_tensorboard: true + log_num_zeros_in_grad: true + log_params_norm: true + log_validation_ppl_to_tensorboard: true + logging_level: 20 + tensorboard_dir: ${OUTPUT_PATH}/tensorboard + wandb_exp_name: Qwen3-30B-GB200-MXFP8-PagedStash-TP1PP1EP16-GBS512 + enable_experimental: true diff --git a/examples/moe_recipes/qwen3_30b/gb200/mxfp8_16GPU_TP1PP1EP16_partial_cg.yaml b/examples/moe_recipes/qwen3_30b/gb200/mxfp8_16GPU_TP1PP1EP16_partial_cg.yaml new file mode 100644 index 00000000000..9a03735e782 --- /dev/null +++ b/examples/moe_recipes/qwen3_30b/gb200/mxfp8_16GPU_TP1PP1EP16_partial_cg.yaml @@ -0,0 +1,190 @@ +DEPENDENCIES: + pytorch_base_image: nvcr.io/nvidia/pytorch:26.03-py3 + dockerfile: | + # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + # IMAGE_NAME: gb200-torch2603 + + ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:26.03-py3 + FROM ${FROM_IMAGE_NAME} AS base + + ENV SHELL=/bin/bash + ENV DEBIAN_FRONTEND=noninteractive + + ARG YQ_VERSION=4.27.5 + ARG TE_TAG=release_v2.14 + ARG NVTE_CUDA_ARCHS="100a" + + # Install system dependencies + RUN bash -ex <<"EOF" + rm -rf /opt/megatron-lm + apt-get update + apt-get install -y --no-install-recommends git curl gettext sudo + ARCH=$(uname -m) + case "${ARCH}" in + "x86_64") YQ_ARCH=amd64 ;; + "aarch64") YQ_ARCH=arm64 ;; + *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;; + esac + wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${YQ_ARCH} -O /usr/bin/yq + chmod +x /usr/bin/yq + apt-get clean + rm -rf /var/lib/apt/lists/* + EOF + + # Install Megatron-Core dependencies (mlm + dev + test groups from pyproject.toml) + RUN bash -ex <<"EOF" + unset PIP_CONSTRAINT + pip install \ + sentencepiece tiktoken transformers accelerate \ + wandb einops tqdm datasets nvtx \ + flask-restful flask[async] fastapi hypercorn \ + nltk wrapt pydantic pyyaml omegaconf \ + pytest==8.3.5 pytest-mock pytest-cov pytest-random-order pytest-asyncio \ + coverage tensorboard + EOF + + # Install TransformerEngine + RUN unset PIP_CONSTRAINT && \ + NVTE_CUDA_ARCHS="${NVTE_CUDA_ARCHS}" NVTE_FRAMEWORK=pytorch \ + pip install --no-build-isolation --no-cache-dir \ + "git+https://github.com/nvidia/TransformerEngine.git@${TE_TAG}" + + # Install Flash Attention 4, CUTLASS DSL, and cuDNN frontend with CuTe DSL support. + # Keep cudnn-frontend at 1.22.1: 1.23.0 removes the c_dtype kwarg from + # grouped_gemm_quant_wrapper_sm100, which TE 2.14 still passes. + RUN pip install --no-cache-dir flash-attn-4==4.0.0b4 "nvidia-cutlass-dsl[cu13]==4.4.2" && \ + pip install --no-cache-dir --no-deps "nvidia-cudnn-frontend==1.22.1" + + # Install Flash Linear Attention for gated-delta-net Qwen3.5-VL recipes. + RUN pip install --no-cache-dir --no-deps \ + fla-core==0.4.2 \ + flash-linear-attention==0.4.2 + + # ========================= + # Option 1: HybridEP + # ========================= + FROM base AS hybridep + + RUN bash -ex <<"EOF" + cd /workspace + git clone https://github.com/linux-rdma/rdma-core.git + cd rdma-core && git checkout tags/v60.0 && sh build.sh + apt-get update + apt-get install -y --no-install-recommends libnvidia-ml-dev + git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git + cd DeepEP && git checkout 7febc6e25660af0f54d95dd781ecdcd62265ecca + RDMA_CORE_HOME=/workspace/rdma-core/build HYBRID_EP_MULTINODE=1 \ + TORCH_CUDA_ARCH_LIST="10.0" MAX_JOBS=8 \ + pip install --no-build-isolation . + apt-get purge -y libnvidia-ml-dev + apt-get autoremove -y + rm -rf /root/.cache /tmp/* /var/lib/apt/lists/* + EOF + + WORKDIR /workspace/ +ENV_VARS: + NVTE_ALLOW_NONDETERMINISTIC_ALGO: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NCCL_NVLS_ENABLE: '0' + HF_HUB_OFFLINE: '1' + NVTE_FUSED_ATTN: '1' + NVTE_NORM_FWD_USE_CUDNN: '1' + NVTE_NORM_BWD_USE_CUDNN: '1' + CUDA_DEVICE_MAX_CONNECTIONS: '1' + NCCL_GRAPH_REGISTER: '0' +ARGS: + tokenizer_type: HuggingFaceTokenizer + tokenizer_model: Qwen/Qwen3-30B-A3B + num_layers: 48 + hidden_size: 2048 + ffn_hidden_size: 6144 + num_attention_heads: 32 + kv_channels: 128 + max_position_embeddings: 40960 + group_query_attention: true + num_query_groups: 4 + qk_layernorm: true + normalization: RMSNorm + norm_epsilon: 1e-6 + swiglu: true + disable_bias_linear: true + untie_embeddings_and_output_weights: true + position_embedding_type: rope + rotary_percent: 1.0 + rotary_base: 1000000 + make_vocab_size_divisible_by: 1187 + num_experts: 128 + moe_ffn_hidden_size: 768 + moe_router_load_balancing_type: aux_loss + moe_router_topk: 8 + moe_router_pre_softmax: false + moe_aux_loss_coeff: 1e-3 + attention_dropout: 0.0 + hidden_dropout: 0.0 + mock_data: true + seq_length: 4096 + moe_router_force_load_balancing: true + tensor_model_parallel_size: 1 + pipeline_model_parallel_size: 1 + expert_model_parallel_size: 16 + context_parallel_size: 1 + expert_tensor_parallel_size: 1 + use_distributed_optimizer: true + sequence_parallel: true + moe_token_dispatcher_type: flex + moe_flex_dispatcher_backend: hybridep + moe_grouped_gemm: true + moe_permute_fusion: true + moe_router_fusion: true + moe_router_dtype: fp32 + external_cuda_graph: true + cuda_graph_scope: + - attn + - moe_router + - moe_preprocess + te_rng_tracker: true + use_mcore_models: true + use_flash_attn: true + transformer_impl: transformer_engine + micro_batch_size: 4 + global_batch_size: 512 + train_samples: 268554688 + exit_duration_in_mins: 230 + no_create_attention_mask_in_dataloader: true + cross_entropy_loss_fusion: true + cross_entropy_fusion_impl: te + manual_gc: true + manual_gc_interval: 5 + lr: 0.00012 + min_lr: 1.2e-05 + lr_decay_style: cosine + lr_decay_samples: 255126953 + lr_warmup_samples: 162761 + weight_decay: 0.1 + clip_grad: 1.0 + adam_beta1: 0.9 + adam_beta2: 0.95 + bf16: true + fp8_format: e4m3 + fp8_recipe: mxfp8 + init_method_std: 0.02 + eval_iters: 32 + eval_interval: 500 + finetune: true + auto_detect_ckpt_format: true + no_load_rng: true + no_load_optim: true + load: ${LOAD_PATH} + save_interval: 500 + dist_ckpt_strictness: log_all + log_throughput: true + log_interval: 1 + log_timers_to_tensorboard: true + log_memory_to_tensorboard: true + log_num_zeros_in_grad: true + log_params_norm: true + log_validation_ppl_to_tensorboard: true + logging_level: 40 + tensorboard_dir: ${OUTPUT_PATH}/tensorboard + wandb_exp_name: Qwen3-30B-GB200-MXFP8-PartialCG-TP1PP1EP16-GBS512 + enable_experimental: true diff --git a/examples/moe_recipes/qwen3_30b/h100/bf16_32GPU_TP1PP1EP8.yaml b/examples/moe_recipes/qwen3_30b/h100/bf16_32GPU_TP1PP1EP8.yaml new file mode 100644 index 00000000000..a55ac6434c5 --- /dev/null +++ b/examples/moe_recipes/qwen3_30b/h100/bf16_32GPU_TP1PP1EP8.yaml @@ -0,0 +1,176 @@ +DEPENDENCIES: + pytorch_base_image: nvcr.io/nvidia/pytorch:26.03-py3 + dockerfile: | + # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + # IMAGE_NAME: h100-torch2603 + + ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:26.03-py3 + FROM ${FROM_IMAGE_NAME} AS base + + ENV SHELL=/bin/bash + ENV DEBIAN_FRONTEND=noninteractive + + ARG YQ_VERSION=4.27.5 + ARG TE_TAG=release_v2.14 + ARG NVTE_CUDA_ARCHS="90a" + + # Install system dependencies + RUN bash -ex <<"EOF" + rm -rf /opt/megatron-lm + apt-get update + apt-get install -y --no-install-recommends git curl gettext sudo + ARCH=$(uname -m) + case "${ARCH}" in + "x86_64") YQ_ARCH=amd64 ;; + "aarch64") YQ_ARCH=arm64 ;; + *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;; + esac + wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${YQ_ARCH} -O /usr/bin/yq + chmod +x /usr/bin/yq + apt-get clean + rm -rf /var/lib/apt/lists/* + EOF + + # Install Megatron-Core dependencies (mlm + dev + test groups from pyproject.toml) + RUN bash -ex <<"EOF" + unset PIP_CONSTRAINT + pip install \ + sentencepiece tiktoken transformers accelerate \ + wandb einops tqdm datasets nvtx \ + flask-restful flask[async] fastapi hypercorn \ + nltk wrapt pydantic pyyaml omegaconf \ + pytest==8.3.5 pytest-mock pytest-cov pytest-random-order pytest-asyncio \ + coverage tensorboard + EOF + + # Install TransformerEngine + RUN unset PIP_CONSTRAINT && \ + NVTE_CUDA_ARCHS="${NVTE_CUDA_ARCHS}" NVTE_FRAMEWORK=pytorch \ + pip install --no-build-isolation --no-cache-dir \ + "git+https://github.com/nvidia/TransformerEngine.git@${TE_TAG}" + + # Install Flash Attention (standard, not FA4 which requires SM100+) + RUN pip install --no-cache-dir flash-attn + + # Install Flash Linear Attention for gated-delta-net Qwen3.5-VL recipes. + RUN pip install --no-cache-dir --no-deps \ + fla-core==0.4.2 \ + flash-linear-attention==0.4.2 + + # ========================= + # Option 1: HybridEP + # ========================= + FROM base AS hybridep + + RUN bash -ex <<"EOF" + cd /workspace + git clone https://github.com/linux-rdma/rdma-core.git + cd rdma-core && git checkout tags/v60.0 && sh build.sh + apt-get update + apt-get install -y --no-install-recommends libnvidia-ml-dev + git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git + cd DeepEP && git checkout cf78085241ebfdd809da8f169b41fa08e589b316 + RDMA_CORE_HOME=/workspace/rdma-core/build HYBRID_EP_MULTINODE=1 \ + TORCH_CUDA_ARCH_LIST="9.0" MAX_JOBS=8 \ + pip install --no-build-isolation . + apt-get purge -y libnvidia-ml-dev + apt-get autoremove -y + rm -rf /root/.cache /tmp/* /var/lib/apt/lists/* + EOF + + WORKDIR /workspace/ +ENV_VARS: + NVTE_ALLOW_NONDETERMINISTIC_ALGO: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NCCL_NVLS_ENABLE: '0' + HF_HUB_OFFLINE: '1' + NVTE_FUSED_ATTN: '1' + NVTE_NORM_FWD_USE_CUDNN: '1' + NVTE_NORM_BWD_USE_CUDNN: '1' + CUDA_DEVICE_MAX_CONNECTIONS: '1' +ARGS: + tokenizer_type: HuggingFaceTokenizer + tokenizer_model: Qwen/Qwen3-30B-A3B + num_layers: 48 + hidden_size: 2048 + ffn_hidden_size: 6144 + num_attention_heads: 32 + kv_channels: 128 + max_position_embeddings: 40960 + group_query_attention: true + num_query_groups: 4 + qk_layernorm: true + normalization: RMSNorm + norm_epsilon: 1e-6 + swiglu: true + disable_bias_linear: true + untie_embeddings_and_output_weights: true + position_embedding_type: rope + rotary_percent: 1.0 + rotary_base: 1000000 + make_vocab_size_divisible_by: 1187 + num_experts: 128 + moe_ffn_hidden_size: 768 + moe_router_load_balancing_type: aux_loss + moe_router_topk: 8 + moe_router_pre_softmax: false + moe_aux_loss_coeff: 1e-3 + attention_dropout: 0.0 + hidden_dropout: 0.0 + mock_data: true + seq_length: 4096 + moe_router_force_load_balancing: true + tensor_model_parallel_size: 1 + pipeline_model_parallel_size: 1 + expert_model_parallel_size: 8 + context_parallel_size: 1 + expert_tensor_parallel_size: 1 + use_distributed_optimizer: true + sequence_parallel: true + overlap_grad_reduce: true + overlap_param_gather: true + moe_token_dispatcher_type: flex + moe_flex_dispatcher_backend: hybridep + moe_grouped_gemm: true + moe_permute_fusion: true + moe_router_fusion: true + moe_router_dtype: fp32 + use_mcore_models: true + transformer_impl: transformer_engine + micro_batch_size: 1 + global_batch_size: 256 + train_samples: 268554688 + exit_duration_in_mins: 230 + no_create_attention_mask_in_dataloader: true + cross_entropy_loss_fusion: true + cross_entropy_fusion_impl: te + manual_gc: true + manual_gc_interval: 5 + lr: 0.00012 + min_lr: 1.2e-05 + lr_decay_style: cosine + lr_decay_samples: 255126953 + lr_warmup_samples: 162761 + weight_decay: 0.1 + clip_grad: 1.0 + adam_beta1: 0.9 + adam_beta2: 0.95 + bf16: true + init_method_std: 0.02 + eval_iters: 32 + eval_interval: 500 + finetune: true + auto_detect_ckpt_format: true + load: ${LOAD_PATH} + save_interval: 500 + dist_ckpt_strictness: log_all + log_throughput: true + log_interval: 1 + log_timers_to_tensorboard: true + log_memory_to_tensorboard: true + log_num_zeros_in_grad: true + log_params_norm: true + log_validation_ppl_to_tensorboard: true + tensorboard_dir: ${OUTPUT_PATH}/tensorboard + wandb_exp_name: Qwen3-30B-TP1PP1EP8-GBS256 + enable_experimental: true diff --git a/examples/moe_recipes/qwen3_30b/h100/fp8_32GPU_TP1PP1EP8.yaml b/examples/moe_recipes/qwen3_30b/h100/fp8_32GPU_TP1PP1EP8.yaml new file mode 100644 index 00000000000..2954da5387f --- /dev/null +++ b/examples/moe_recipes/qwen3_30b/h100/fp8_32GPU_TP1PP1EP8.yaml @@ -0,0 +1,190 @@ +DEPENDENCIES: + pytorch_base_image: nvcr.io/nvidia/pytorch:26.03-py3 + dockerfile: | + # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + # IMAGE_NAME: h100-torch2603 + + ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:26.03-py3 + FROM ${FROM_IMAGE_NAME} AS base + + ENV SHELL=/bin/bash + ENV DEBIAN_FRONTEND=noninteractive + + ARG YQ_VERSION=4.27.5 + ARG TE_TAG=release_v2.14 + ARG NVTE_CUDA_ARCHS="90a" + + # Install system dependencies + RUN bash -ex <<"EOF" + rm -rf /opt/megatron-lm + apt-get update + apt-get install -y --no-install-recommends git curl gettext sudo + ARCH=$(uname -m) + case "${ARCH}" in + "x86_64") YQ_ARCH=amd64 ;; + "aarch64") YQ_ARCH=arm64 ;; + *) echo "Unsupported architecture: ${ARCH}" && exit 1 ;; + esac + wget https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${YQ_ARCH} -O /usr/bin/yq + chmod +x /usr/bin/yq + apt-get clean + rm -rf /var/lib/apt/lists/* + EOF + + # Install Megatron-Core dependencies (mlm + dev + test groups from pyproject.toml) + RUN bash -ex <<"EOF" + unset PIP_CONSTRAINT + pip install \ + sentencepiece tiktoken transformers accelerate \ + wandb einops tqdm datasets nvtx \ + flask-restful flask[async] fastapi hypercorn \ + nltk wrapt pydantic pyyaml omegaconf \ + pytest==8.3.5 pytest-mock pytest-cov pytest-random-order pytest-asyncio \ + coverage tensorboard + EOF + + # Install TransformerEngine + RUN unset PIP_CONSTRAINT && \ + NVTE_CUDA_ARCHS="${NVTE_CUDA_ARCHS}" NVTE_FRAMEWORK=pytorch \ + pip install --no-build-isolation --no-cache-dir \ + "git+https://github.com/nvidia/TransformerEngine.git@${TE_TAG}" + + # Install Flash Attention (standard, not FA4 which requires SM100+) + RUN pip install --no-cache-dir flash-attn + + # Install Flash Linear Attention for gated-delta-net Qwen3.5-VL recipes. + RUN pip install --no-cache-dir --no-deps \ + fla-core==0.4.2 \ + flash-linear-attention==0.4.2 + + # ========================= + # Option 1: HybridEP + # ========================= + FROM base AS hybridep + + RUN bash -ex <<"EOF" + cd /workspace + git clone https://github.com/linux-rdma/rdma-core.git + cd rdma-core && git checkout tags/v60.0 && sh build.sh + apt-get update + apt-get install -y --no-install-recommends libnvidia-ml-dev + git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git + cd DeepEP && git checkout cf78085241ebfdd809da8f169b41fa08e589b316 + RDMA_CORE_HOME=/workspace/rdma-core/build HYBRID_EP_MULTINODE=1 \ + TORCH_CUDA_ARCH_LIST="9.0" MAX_JOBS=8 \ + pip install --no-build-isolation . + apt-get purge -y libnvidia-ml-dev + apt-get autoremove -y + rm -rf /root/.cache /tmp/* /var/lib/apt/lists/* + EOF + + WORKDIR /workspace/ +ENV_VARS: + NVTE_ALLOW_NONDETERMINISTIC_ALGO: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NCCL_NVLS_ENABLE: '0' + HF_HUB_OFFLINE: '1' + NVTE_FUSED_ATTN: '1' + NVTE_NORM_FWD_USE_CUDNN: '1' + NVTE_NORM_BWD_USE_CUDNN: '1' + CUDA_DEVICE_MAX_CONNECTIONS: '1' + NCCL_GRAPH_REGISTER: '0' +ARGS: + tokenizer_type: HuggingFaceTokenizer + tokenizer_model: Qwen/Qwen3-30B-A3B + num_layers: 48 + hidden_size: 2048 + ffn_hidden_size: 6144 + num_attention_heads: 32 + kv_channels: 128 + max_position_embeddings: 40960 + group_query_attention: true + num_query_groups: 4 + qk_layernorm: true + normalization: RMSNorm + norm_epsilon: 1e-6 + swiglu: true + disable_bias_linear: true + untie_embeddings_and_output_weights: true + position_embedding_type: rope + rotary_percent: 1.0 + rotary_base: 1000000 + make_vocab_size_divisible_by: 1187 + num_experts: 128 + moe_ffn_hidden_size: 768 + moe_router_load_balancing_type: aux_loss + moe_router_topk: 8 + moe_router_pre_softmax: false + moe_aux_loss_coeff: 1e-3 + attention_dropout: 0.0 + hidden_dropout: 0.0 + mock_data: true + seq_length: 4096 + moe_router_force_load_balancing: true + tensor_model_parallel_size: 1 + pipeline_model_parallel_size: 1 + expert_model_parallel_size: 8 + context_parallel_size: 1 + expert_tensor_parallel_size: 1 + use_distributed_optimizer: true + sequence_parallel: true + overlap_grad_reduce: true + overlap_param_gather: true + moe_token_dispatcher_type: flex + moe_flex_dispatcher_backend: hybridep + moe_grouped_gemm: true + moe_permute_fusion: true + moe_router_fusion: true + moe_router_dtype: fp32 + cuda_graph_impl: transformer_engine + cuda_graph_scope: + - attn + - moe_router + - moe_preprocess + use_mcore_models: true + transformer_impl: transformer_engine + micro_batch_size: 1 + global_batch_size: 256 + train_samples: 268554688 + exit_duration_in_mins: 230 + no_create_attention_mask_in_dataloader: true + cross_entropy_loss_fusion: true + cross_entropy_fusion_impl: te + manual_gc: true + manual_gc_interval: 5 + lr: 0.00012 + min_lr: 1.2e-05 + lr_decay_style: cosine + lr_decay_samples: 255126953 + lr_warmup_samples: 162761 + weight_decay: 0.1 + clip_grad: 1.0 + adam_beta1: 0.9 + adam_beta2: 0.95 + bf16: true + fp8_recipe: blockwise + fp8_format: e4m3 + fp8_param_gather: true + use_precision_aware_optimizer: true + main_grads_dtype: fp32 + main_params_dtype: fp32 + exp_avg_dtype: bf16 + exp_avg_sq_dtype: bf16 + init_method_std: 0.02 + eval_iters: 32 + eval_interval: 500 + finetune: true + auto_detect_ckpt_format: true + load: ${LOAD_PATH} + save_interval: 500 + dist_ckpt_strictness: log_all + log_throughput: true + log_interval: 1 + log_timers_to_tensorboard: true + log_memory_to_tensorboard: true + log_num_zeros_in_grad: true + log_params_norm: true + log_validation_ppl_to_tensorboard: true + tensorboard_dir: ${OUTPUT_PATH}/tensorboard + wandb_exp_name: Qwen3-30B-FP8-PartialCG-TP1PP1EP8-GBS256 + enable_experimental: true diff --git a/examples/multimodal/Dockerfile b/examples/multimodal/Dockerfile index d7c4fd41af5..46db1319566 100644 --- a/examples/multimodal/Dockerfile +++ b/examples/multimodal/Dockerfile @@ -37,7 +37,7 @@ RUN uv pip install --system --no-cache --break-system-packages \ flask-restful \ wandb \ bitstring \ - filetype + filetype \ setuptools # Install CLIP from GitHub diff --git a/examples/multimodal/layer_specs.py b/examples/multimodal/layer_specs.py index c51fb69f496..caff5ac7e0b 100644 --- a/examples/multimodal/layer_specs.py +++ b/examples/multimodal/layer_specs.py @@ -1,4 +1,6 @@ # Copyright (c) 2024-2026, NVIDIA CORPORATION. All rights reserved. +from functools import partial + import torch from megatron.core.extensions.transformer_engine import HAVE_TE @@ -112,7 +114,7 @@ def get_layer_spec_te(is_vit=False, padding=False) -> ModuleSpec: submodules=SelfAttentionSubmodules( linear_qkv=not_none(TELayerNormColumnParallelLinear), core_attention=not_none(TEDotProductAttention), - linear_proj=TERowParallelLinear, + linear_proj=not_none(TERowParallelLinear), q_layernorm=IdentityOp, k_layernorm=IdentityOp, ), @@ -158,7 +160,7 @@ def get_hybrid_layer_spec_te(padding=False) -> ModuleSpec: submodules=SelfAttentionSubmodules( linear_qkv=not_none(TELayerNormColumnParallelLinear), core_attention=not_none(TEDotProductAttention), - linear_proj=TERowParallelLinear, + linear_proj=not_none(TERowParallelLinear), ), ), self_attn_bda=get_bias_dropout_add, @@ -170,8 +172,8 @@ def get_hybrid_layer_spec_te(padding=False) -> ModuleSpec: mlp_layer=ModuleSpec( module=MLPLayer, submodules=TransformerLayerSubmodules( - mlp=ModuleSpec( - module=MLP, + mlp=partial( + MLP.as_mlp_submodule, submodules=MLPSubmodules( linear_fc1=not_none(TELayerNormColumnParallelLinear), linear_fc2=not_none(TERowParallelLinear), @@ -184,10 +186,10 @@ def get_hybrid_layer_spec_te(padding=False) -> ModuleSpec: ) -def get_mlp_module_spec(use_te: bool = True) -> ModuleSpec: +def get_mlp_module_spec(use_te: bool = True): # Dense MLP w/ or w/o TE modules. - return ModuleSpec( - module=MLP, + return partial( + MLP.as_mlp_submodule, submodules=MLPSubmodules( linear_fc1=not_none(TEColumnParallelLinear) if use_te else ColumnParallelLinear, linear_fc2=not_none(TERowParallelLinear) if use_te else RowParallelLinear, @@ -195,9 +197,9 @@ def get_mlp_module_spec(use_te: bool = True) -> ModuleSpec: ) -def get_norm_mlp_module_spec_te() -> ModuleSpec: - return ModuleSpec( - module=MLP, +def get_norm_mlp_module_spec_te(): + return partial( + MLP.as_mlp_submodule, submodules=MLPSubmodules( linear_fc1=not_none(TELayerNormColumnParallelLinear), linear_fc2=not_none(TERowParallelLinear), diff --git a/examples/multimodal/nvlm/internvit.py b/examples/multimodal/nvlm/internvit.py index 0018bb5ccb9..d38ac64c16b 100644 --- a/examples/multimodal/nvlm/internvit.py +++ b/examples/multimodal/nvlm/internvit.py @@ -160,10 +160,10 @@ def sharded_state_dict(self, prefix='', sharded_offsets=(), metadata={}): return super().sharded_state_dict(prefix, sharded_offsets, metadata) -def get_mlp_module_spec(use_te: bool = True) -> ModuleSpec: +def get_mlp_module_spec(use_te: bool = True): # Dense MLP w/ or w/o TE modules. - return ModuleSpec( - module=MLP, + return partial( + MLP.as_mlp_submodule, submodules=MLPSubmodules( linear_fc1=TEColumnParallelLinear if use_te else ColumnParallelLinear, linear_fc2=TERowParallelLinear if use_te else RowParallelLinear, diff --git a/examples/multimodal/radio/radio_g.py b/examples/multimodal/radio/radio_g.py index 9883d58db61..a3d0317b03b 100644 --- a/examples/multimodal/radio/radio_g.py +++ b/examples/multimodal/radio/radio_g.py @@ -1,12 +1,11 @@ # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. from functools import partial -import torch - from examples.multimodal.layer_scaling import ( LayerScalingTransformerLayer, get_bias_dropout_add_layer_scaling, ) +from megatron.core.extensions.transformer_engine import HAVE_TE from megatron.core.tensor_parallel.layers import ColumnParallelLinear, RowParallelLinear from megatron.core.transformer.attention import SelfAttention, SelfAttentionSubmodules from megatron.core.transformer.dot_product_attention import DotProductAttention @@ -14,9 +13,8 @@ from megatron.core.transformer.identity_op import IdentityOp from megatron.core.transformer.mlp import MLP, MLPSubmodules from megatron.core.transformer.spec_utils import ModuleSpec -from megatron.core.transformer.transformer_layer import TransformerLayer, TransformerLayerSubmodules +from megatron.core.transformer.transformer_layer import TransformerLayerSubmodules from megatron.core.typed_torch import not_none -from megatron.core.extensions.transformer_engine import HAVE_TE if HAVE_TE: from megatron.core.extensions.transformer_engine import ( @@ -51,10 +49,10 @@ LNImpl = WrappedTorchNorm -def get_mlp_module_spec(use_te: bool = True) -> ModuleSpec: +def get_mlp_module_spec(use_te: bool = True): # Dense MLP w/ or w/o TE modules. - return ModuleSpec( - module=MLP, + return partial( + MLP.as_mlp_submodule, submodules=MLPSubmodules( linear_fc1=not_none(TEColumnParallelLinear) if use_te else ColumnParallelLinear, linear_fc2=not_none(TERowParallelLinear) if use_te else RowParallelLinear, @@ -62,9 +60,9 @@ def get_mlp_module_spec(use_te: bool = True) -> ModuleSpec: ) -def get_norm_mlp_module_spec_te() -> ModuleSpec: - return ModuleSpec( - module=MLP, +def get_norm_mlp_module_spec_te(): + return partial( + MLP.as_mlp_submodule, submodules=MLPSubmodules( linear_fc1=not_none(TELayerNormColumnParallelLinear), linear_fc2=not_none(TERowParallelLinear), @@ -125,7 +123,7 @@ def get_radio_g_layer_spec_te() -> ModuleSpec: submodules=SelfAttentionSubmodules( linear_qkv=not_none(TELayerNormColumnParallelLinear), core_attention=not_none(TEDotProductAttention), - linear_proj=TERowParallelLinear, + linear_proj=not_none(TERowParallelLinear), q_layernorm=IdentityOp, k_layernorm=IdentityOp, ), diff --git a/examples/multimodal/train.py b/examples/multimodal/train.py index 98536f72d1e..82927c61793 100644 --- a/examples/multimodal/train.py +++ b/examples/multimodal/train.py @@ -26,9 +26,9 @@ get_tensor_model_parallel_rank, is_pipeline_last_stage, ) -from megatron.core.utils import nvtx_range_pop, nvtx_range_push +from megatron.core.utils import get_batch_on_this_cp_rank, nvtx_range_pop, nvtx_range_push from megatron.training import get_args, get_timers, get_tokenizer, pretrain -from megatron.training.utils import get_batch_on_this_cp_rank, is_last_rank +from megatron.training.utils import is_last_rank def get_batch(data_iterator, image_token_index, img_seq_len): diff --git a/examples/multimodal_dev/README.md b/examples/multimodal_dev/README.md new file mode 100644 index 00000000000..e4e6c53ceb4 --- /dev/null +++ b/examples/multimodal_dev/README.md @@ -0,0 +1,222 @@ +# multimodal_dev — Standalone Multimodal Training + +Standalone, model-agnostic training entry point for multimodal +vision-language models built on Megatron-Core (FSDP + EP). + +## Directory Structure + +``` +multimodal_dev/ +├── pretrain_multimodal.py # Training entry point (model-agnostic) +├── forward_step.py # Forward step, TP broadcast, loss computation +├── arguments.py # Multimodal CLI arguments +├── data/ +│ └── mock.py # Mock dataset for end-to-end testing +├── models/ +│ ├── __init__.py # MODEL_REGISTRY — central model registry +│ ├── base.py # MultimodalModel base class (vision encoder + GPTModel) +│ └── qwen35_vl/ # Qwen3.5-VL architecture +│ ├── factory.py # Factory functions for pretrain entry point +│ ├── model.py # Qwen35VLModel (MRoPE, vision encoder wiring) +│ ├── configuration.py # TransformerConfig builders and constants +│ ├── specs.py # Layer spec builders (hybrid attention, ViT) +│ ├── mrope.py # 3D MRoPE position ID computation +│ └── vision_encoder.py# ViT encoder (patch embed, merger, RoPE) +└── scripts/ # Launch scripts (torchrun, Slurm) +``` + +## Quick Start + +```bash +torchrun --nproc_per_node=8 multimodal_dev/pretrain_multimodal.py \ + --model-arch qwen35_vl \ + --dataset-provider mock \ + ... # other Megatron args (--num-layers, --hidden-size, etc.) +``` + +## Checkpoint Conversion (HF → Megatron-FSDP DTensor) + +Convert a HuggingFace release to a Megatron-FSDP DTensor checkpoint via +[Megatron-Bridge](https://github.com/NVIDIA-NeMo/Megatron-Bridge) before +pretraining from pretrained weights. + +### Setup + +Clone Bridge and pin its `3rdparty/Megatron-LM` submodule to this branch: + +```bash +git clone --recurse-submodules https://github.com/NVIDIA-NeMo/Megatron-Bridge.git +cd Megatron-Bridge/3rdparty/Megatron-LM +git remote add wplf https://github.com/wplf/Megatron-LM.git +git fetch wplf feat/qwen35-vl-example +git checkout feat/qwen35-vl-example +cd ../.. +``` + +### Convert + +Single 8×GPU node, EP=8 / TP=CP=1; substitute any Qwen3.5 variant for +`--hf-model`: + +```bash +PYTHONPATH=./src:./3rdparty/Megatron-LM/ \ + torchrun --nproc_per_node=8 \ + examples/conversion/mfsdp/convert_checkpoints_fsdp.py import \ + --hf-model Qwen/Qwen3.5-35B-A3B \ + --megatron-path ${WORKSPACE}/models/Qwen/Qwen3.5-35B-A3B-fsdp \ + --ckpt-format fsdp_dtensor \ + --ep 8 +``` + +HF weights are auto-fetched on first run via `huggingface_hub`. Adjust +`--tp` / `--cp` / `--ep` to match the training topology (must satisfy +`WORLD_SIZE % (TP*CP*EP) == 0`). + +### Output + +``` +${WORKSPACE}/models/Qwen/Qwen3.5-35B-A3B-fsdp/ +├── iter_0000000/ +│ ├── __0_0.distcp .. __7_0.distcp # FSDP DTensor shards, one per rank (~18 GB each for 35B-A3B) +│ ├── .metadata +│ ├── run_config.yaml +│ └── train_state.pt +├── latest_checkpointed_iteration.txt +└── latest_train_state.pt +``` + +### Bridge dependency + +Requires +[NVIDIA-NeMo/Megatron-Bridge#3987](https://github.com/NVIDIA-NeMo/Megatron-Bridge/pull/3987) +(skip tokenizer save). Without that fix the checkpoint is still written +correctly but the script exits non-zero after save with +`AttributeError: 'TokenizerConfig' object has no attribute 'make_vocab_size_divisible_by'` +against this branch's `megatron.core.tokenizers.utils.build_tokenizer`. + +## Architecture + +`pretrain_multimodal.py` is **model-agnostic**. All model-specific logic +is delegated to factory functions registered in `MODEL_REGISTRY` +(`models/__init__.py`). The entry point handles only generic concerns: + +- Building `language_config` from Megatron CLI args +- Constructing `vision_config` via the registry +- Applying vision recompute and dtype propagation +- Routing to model and dataset factories + +The `forward_step` is also model-agnostic — it uses the model's +`compute_position_ids()` method polymorphically and passes a standard +batch dict. + +## Adding a New Model Architecture + +Adding a new model (e.g. `llava_next`) requires **no changes** to +`pretrain_multimodal.py` or `forward_step.py`. Follow these steps: + +### Step 1 — Create the model package + +``` +multimodal_dev/models/llava_next/ +├── __init__.py +├── factory.py # Required: factory functions +├── configuration.py # Vision/language TransformerConfig builders +├── model.py # Model class (subclass MultimodalModel) +├── specs.py # Layer spec builders +└── vision_encoder.py # Vision encoder (if custom) +``` + +### Step 2 — Implement factory functions + +Create `factory.py` with up to three functions: + +```python +# models/llava_next/factory.py + +def post_language_config(language_config, args): + """(Optional) Mutate language_config with model-specific fields.""" + # e.g. language_config.some_field = value + pass + +def set_vision_flops_metadata(args, language_config, vision_config): + """(Optional) Set vision FLOPs metadata on args.""" + args.count_vision_model_flops = True + args.vision_flops_variant = "llava_next" + # ... set dimension fields for FLOPs calculation + +def build_model(args, language_config, vision_config, **kwargs): + """(Required) Build and return the complete model instance.""" + from .model import LlavaNextModel + from .specs import get_llava_next_language_spec + + language_spec = get_llava_next_language_spec( + config=language_config, + vp_stage=kwargs.get("vp_stage", None), + pp_rank=None, + ) + return LlavaNextModel( + language_config=language_config, + language_spec=language_spec, + vision_config=vision_config, + # ... model-specific args + ) +``` + +### Step 3 — Register in `MODEL_REGISTRY` + +Add an entry in `models/__init__.py`: + +```python +from multimodal_dev.models.llava_next.configuration import ( + get_llava_next_vision_config, +) +from multimodal_dev.models.llava_next.factory import ( + build_model as _build_llava_next_model, + post_language_config as _llava_next_post_language_config, + set_vision_flops_metadata as _llava_next_vision_flops, +) + +MODEL_REGISTRY["llava_next"] = { + "model_factory_fn": _build_llava_next_model, # required + "vision_config_fn": get_llava_next_vision_config, # required + "post_language_config_fn": _llava_next_post_language_config, # optional + "vision_flops_fn": _llava_next_vision_flops, # optional + "dataset_providers": { # optional + "mock": "multimodal_dev.data.llava_mock.train_valid_test_datasets_provider", + }, +} +``` + +### Step 4 — (Optional) Add a dataset provider + +Create a dataset module under `data/` if the model needs custom data +preprocessing. The provider function signature is: + +```python +def train_valid_test_datasets_provider(train_val_test_num_samples): + """Return (train_dataset, val_dataset, test_dataset).""" + ... +``` + +Register it in the `dataset_providers` dict of the registry entry. +Providers can be either direct callables or dotted import path strings +(resolved lazily at runtime). + +### Step 5 — Launch + +```bash +torchrun --nproc_per_node=8 multimodal_dev/pretrain_multimodal.py \ + --model-arch llava_next \ + --dataset-provider mock \ + ... +``` + +## Registry Entry Reference + +| Field | Required | Signature | +|-------|----------|-----------| +| `model_factory_fn` | Yes | `(args, language_config, vision_config, **kwargs) -> MegatronModule` | +| `vision_config_fn` | Yes | `(num_layers_override=None) -> TransformerConfig` | +| `post_language_config_fn` | No | `(language_config, args) -> None` | +| `vision_flops_fn` | No | `(args, language_config, vision_config) -> None` | +| `dataset_providers` | No | `Dict[str, str \| callable]` | diff --git a/examples/multimodal_dev/__init__.py b/examples/multimodal_dev/__init__.py new file mode 100644 index 00000000000..26496bfed70 --- /dev/null +++ b/examples/multimodal_dev/__init__.py @@ -0,0 +1 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. diff --git a/examples/multimodal_dev/arguments.py b/examples/multimodal_dev/arguments.py new file mode 100644 index 00000000000..35655831bfb --- /dev/null +++ b/examples/multimodal_dev/arguments.py @@ -0,0 +1,100 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Extra CLI arguments for multimodal_dev standalone training.""" + + +def add_multimodal_args(parser): + """Add multimodal-specific arguments to the Megatron argument parser.""" + group = parser.add_argument_group( + "Multimodal", "Multimodal model arguments", + ) + + group.add_argument( + "--model-arch", + type=str, + default="qwen35_vl", + help="Model architecture. Available: qwen35_vl", + ) + group.add_argument( + "--model-variant", + type=str, + default="proxy", + help="Model variant (size). E.g. proxy, 9b, 397b_a17b", + ) + group.add_argument( + "--dataset-provider", + type=str, + default="mock", + help="Dataset provider: mock", + ) + group.add_argument( + "--image-token-id", + type=int, + default=248056, + help="Token ID for image placeholder tokens", + ) + group.add_argument( + "--image-size", + type=int, + default=224, + help="Image size (height and width) for mock data", + ) + group.add_argument( + "--total-seq-length", + type=int, + default=1024, + help="Total sequence length for mock data", + ) + group.add_argument( + "--image-seq-length", + type=int, + default=256, + help="Number of image tokens in mock data", + ) + group.add_argument( + "--vision-num-layers", + type=int, + default=None, + help=( + "Override for vision backbone depth. " + "Useful for proxy perf runs." + ), + ) + group.add_argument( + "--hf-processor-path", + type=str, + default=None, + help=( + "HuggingFace processor path for real VLM datasets " + "(e.g. Qwen/Qwen2.5-VL-7B-Instruct)" + ), + ) + group.add_argument( + "--recompute-vision", + action="store_true", + default=False, + help=( + "Enable full activation recomputation for vision encoder layers. " + "Uses uniform method and recomputes every layer. " + "Independent of the decoder --recompute-* flags." + ), + ) + group.add_argument( + "--use-packed-sequence", + action="store_true", + default=False, + help=( + "Pack variable-length sequences into THD format to eliminate " + "padding waste." + ), + ) + group.add_argument( + "--use-vanilla-collate-fn", + action="store_true", + default=False, + help=( + "Use vanilla collate function to collate the data." + ), + ) + + return parser diff --git a/examples/multimodal_dev/data/__init__.py b/examples/multimodal_dev/data/__init__.py new file mode 100644 index 00000000000..26496bfed70 --- /dev/null +++ b/examples/multimodal_dev/data/__init__.py @@ -0,0 +1 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. diff --git a/examples/multimodal_dev/data/cord_v2.py b/examples/multimodal_dev/data/cord_v2.py new file mode 100644 index 00000000000..69fd4c13ec4 --- /dev/null +++ b/examples/multimodal_dev/data/cord_v2.py @@ -0,0 +1,397 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""CORD-V2 VLM dataset for multimodal_dev training. + +Single-turn image-text dataset using a HuggingFace ``AutoProcessor`` for +tokenization and image preprocessing. This module is the reference +implementation for the CORD-V2 receipt-OCR dataset. No multi-turn support — +each sample is one image + question → answer pair. + +Each image is preprocessed via ``qwen_vl_utils.process_vision_info`` and +fed to the processor with Qwen-VL's recommended ``min_pixels`` / +``max_pixels`` budget, so the per-sample patch grid varies with aspect +ratio. The run script must therefore pass ``--use-vanilla-collate-fn`` +(which the example launcher does) so the dataloader does not try to stack +variable-shape tensors. + +Usage:: + + torchrun ... pretrain_multimodal.py \\ + --model-arch qwen35_vl --dataset-provider cord_v2 \\ + --hf-processor-path Qwen/Qwen3.5-397B-A17B \\ + --total-seq-length 4096 --use-vanilla-collate-fn + +Adding another VLM dataset +-------------------------- + +The dataset layer mirrors the model layer's registry pattern: each dataset +ships its own module and a ``train_valid_test_datasets_provider`` factory, +and the model's registry entry maps a ``--dataset-provider`` name to that +factory's dotted path. To add a new dataset (e.g. NLVR2): + +1. Create ``examples/multimodal_dev/data/.py`` with:: + + def train_valid_test_datasets_provider(train_val_test_num_samples): + ... # build datasets using args from get_args() + return train_ds, val_ds, test_ds + +2. Register it under the relevant model in + ``examples/multimodal_dev/models/__init__.py``:: + + MODEL_REGISTRY["qwen35_vl"]["dataset_providers"][""] = ( + "examples.multimodal_dev.data." + ".train_valid_test_datasets_provider" + ) + +3. Launch with ``--dataset-provider ``. + +No edits to ``pretrain_multimodal.py`` or ``forward_step.py`` are required. +""" + +import json +import logging +import random +from typing import Dict, List, Optional + +import torch +from torch.utils.data import Dataset + +try: + from qwen_vl_utils import process_vision_info + HAVE_QWEN_VL_UTILS = True +except ImportError: + HAVE_QWEN_VL_UTILS = False + +logger = logging.getLogger(__name__) + +# Qwen-VL recommended pixel-budget range; lets the processor pick a +# per-image patch grid that respects aspect ratio. +_QWEN_VL_MIN_PIXELS = 256 * 28 * 28 # 200_704 +_QWEN_VL_MAX_PIXELS = 1280 * 28 * 28 # 1_003_520 + + +# --------------------------------------------------------------------------- +# CORD-V2 helpers +# --------------------------------------------------------------------------- + +def _json2token(obj, sort_json_key=True): + """Convert a JSON object to a token-sequence string (Donut format).""" + if isinstance(obj, dict): + if len(obj) == 1 and "text_sequence" in obj: + return obj["text_sequence"] + output = "" + keys = sorted(obj.keys(), reverse=True) if sort_json_key else obj.keys() + for k in keys: + output += f"" + _json2token(obj[k], sort_json_key) + f"" + return output + if isinstance(obj, list): + return "".join(_json2token(item, sort_json_key) for item in obj) + return str(obj) + + +def load_cord_v2(split="train"): + """Load CORD-V2 and return a list of ``{image, question, answer}`` dicts.""" + from datasets import load_dataset + + ds = load_dataset("naver-clova-ix/cord-v2", split=split) + rng = random.Random(42) + examples = [] + for ex in ds: + gt = json.loads(ex["ground_truth"]) + gt_jsons = gt.get("gt_parses") or [gt["gt_parse"]] + text = rng.choice( + [_json2token(g, sort_json_key=True) for g in gt_jsons] + ) + examples.append( + {"image": ex["image"], "question": "Describe this image.", "answer": text} + ) + return examples + + +# --------------------------------------------------------------------------- +# Dataset +# --------------------------------------------------------------------------- + +class CordV2VLMDataset(Dataset): + """Single-turn VLM dataset backed by CORD-V2. + + Each sample is tokenized by the HF ``AutoProcessor`` and the image is + handed to the processor with Qwen-VL's dynamic-resolution budget + (``min_pixels`` / ``max_pixels``); the per-image patch grid varies with + aspect ratio. + + Args: + examples: Output of :func:`load_cord_v2`. + processor: ``AutoProcessor`` instance. + seq_length: End-truncate ``input_ids`` at this length. + image_token_id: Token ID for image placeholders. + target_length: Virtual dataset length (repeats examples if needed). + + NOTE: + For the Qwen3.5-VL processor, the temporal patch dimension is + always 2 (the processor duplicates a single frame so the 3D conv + behaves like a 2D conv on one image) — ``image_grid_thw`` therefore + has shape ``[num_images, 3]`` with ``T=2`` per image. + ``pixel_values`` has shape ``[total_patches, 3 * T * P * P]`` where + ``P`` is the processor's patch size. + """ + + def __init__( + self, + examples: List[Dict], + processor, + seq_length: int = 2048, + image_token_id: Optional[int] = None, + target_length: Optional[int] = None, + ): + if not HAVE_QWEN_VL_UTILS: + raise ImportError( + "qwen_vl_utils is required for Qwen3.5-VL preprocessing. " + "Install with `pip install qwen-vl-utils`.", + ) + self.examples = examples + self.processor = processor + self.seq_length = seq_length + self._length = target_length if target_length else len(examples) + tok = processor.tokenizer + # Falling back to 0 is unsafe: token 0 is a real vocab token in many + # tokenizers (incl. Qwen) and would be silently masked. Prefer EOS, + # and require at least one of pad/eos to be set. + if tok.pad_token_id is not None: + self.pad_token_id = int(tok.pad_token_id) + elif tok.eos_token_id is not None: + self.pad_token_id = int(tok.eos_token_id) + else: + raise ValueError( + "Tokenizer has neither pad_token_id nor eos_token_id; " + "cannot derive a safe pad id for loss masking.", + ) + + # Resolve image token ID. Vision embeddings are scattered into + # positions equal to this id by the model, so a wrong id silently + # breaks training — fail loudly rather than return None. + if image_token_id is not None: + self.image_token_id = int(image_token_id) + else: + vocab = tok.get_vocab() + for candidate in ("<|image_pad|>", "<|placeholder|>"): + if candidate in vocab: + self.image_token_id = int(vocab[candidate]) + break + else: + raise ValueError( + "Could not resolve image token id from tokenizer " + f"({type(tok).__name__}); pass --image-token-id " + "explicitly.", + ) + + # Structural tokens that must never appear as a loss target: + # pad, image, plus everything the tokenizer registered as special + # (im_start/im_end, vision_start/vision_end, video_pad, endoftext...). + # Mirrors megatron-bridge's extract_skipped_token_ids convention. + skipped: set = set(int(x) for x in (tok.all_special_ids or [])) + skipped.add(self.pad_token_id) + skipped.add(self.image_token_id) + self.skipped_token_ids = torch.tensor( + sorted(skipped), dtype=torch.long, + ) + + def __len__(self) -> int: + return self._length + + def _mark_assistant_span( + self, + input_ids_list: List[int], + asst_text: str, + loss_mask: torch.Tensor, + ) -> bool: + """Find ``asst_text`` as a contiguous token span in ``input_ids_list`` + and set ``loss_mask`` to 1 over those positions. + + Substring tokenization is sensitive to surrounding whitespace and + BPE merge boundaries, so we try a few common variants. Returns + True if a span was found. + """ + tokenizer = self.processor.tokenizer + n = len(input_ids_list) + variants = ( + asst_text, + asst_text + "\n", + asst_text.strip(), + asst_text.strip() + "\n", + ) + for variant in variants: + span_tokens = tokenizer( + variant, add_special_tokens=False, + )["input_ids"] + m = len(span_tokens) + if m == 0 or m > n: + continue + # Backward search: rightmost match = the actual assistant turn. + for start in range(n - m, -1, -1): + if input_ids_list[start : start + m] == span_tokens: + loss_mask[start : start + m] = 1.0 + return True + return False + + def __getitem__(self, idx: int) -> Dict[str, torch.Tensor]: + example = self.examples[idx % len(self.examples)] + + # Conversation schema must include the actual image object inside + # the content so the chat template + process_vision_info can extract + # it (matches megatron-bridge's qwen2_5_collate_fn convention; also + # used by the Qwen3-VL processor). + conversation = [ + { + "role": "user", + "content": [ + {"type": "image", "image": example["image"]}, + {"type": "text", "text": example["question"]}, + ], + }, + { + "role": "assistant", + "content": [{"type": "text", "text": example["answer"]}], + }, + ] + + text = self.processor.apply_chat_template( + conversation, tokenize=False, add_generation_prompt=False, + ) + images, _ = process_vision_info(conversation) + batch = self.processor( + text=[text], + images=images, + return_tensors="pt", + min_pixels=_QWEN_VL_MIN_PIXELS, + max_pixels=_QWEN_VL_MAX_PIXELS, + ) + + input_ids = batch["input_ids"].squeeze(0) + pixel_values = batch["pixel_values"].to(torch.bfloat16) + image_grid_thw = batch["image_grid_thw"] # [num_images, 3] + + # End-truncate so the model never sees more than seq_length tokens. + # Qwen-VL chat template puts the image at the user-turn start and the + # assistant answer trails at the end, so end-truncation preserves the + # image_pad block for normal-sized images; if the image grid alone + # already exceeds seq_length, the model will fail loudly at the + # masked_scatter step. + if input_ids.shape[0] > self.seq_length: + logger.warning( + "Sample idx=%d has %d tokens > seq_length=%d; truncating.", + idx, input_ids.shape[0], self.seq_length, + ) + input_ids = input_ids[: self.seq_length] + + # SFT loss mask: start fully masked, then unmask only the assistant + # answer span found via substring token search (mirrors + # megatron-bridge's create_multiturn_loss_mask_by_search). The user + # turn, chat-template tags, and image tokens stay masked. + loss_mask = torch.zeros_like(input_ids, dtype=torch.float32) + found = self._mark_assistant_span( + input_ids.tolist(), example["answer"], loss_mask, + ) + if not found: + logger.warning( + "Assistant span not located for example idx=%d; " + "loss_mask will be all-zero for this sample.", + idx, + ) + + # Shifted next-token labels: labels[i] is the target for position i. + labels = input_ids.clone() + labels[:-1] = input_ids[1:] + labels[-1] = -100 + + # Mask structural tokens on the *labels* (the prediction targets), + # not on input_ids — matches the next-token timeline. + labels[torch.isin(labels, self.skipped_token_ids)] = -100 + + # Shift loss_mask left by one so position i decides whether to learn + # input_ids[i] -> labels[i] (== input_ids[i+1]). Last position is + # never trained (no next token to predict). + loss_mask = torch.cat( + [loss_mask[1:], torch.zeros(1, dtype=loss_mask.dtype)], + ) + + # Enforce label = -100 wherever we won't compute loss. + labels[loss_mask == 0] = -100 + + return { + "input_ids": input_ids, + "labels": labels, + "loss_mask": loss_mask, + "pixel_values": pixel_values, + "image_grid_thw": image_grid_thw, + } + + +# --------------------------------------------------------------------------- +# Megatron dataset provider interface +# --------------------------------------------------------------------------- + +def train_valid_test_datasets_provider(train_val_test_num_samples): + """Provide CORD-V2 train / val / test datasets. + + Requires ``--hf-processor-path`` to point to a HuggingFace VL model + (e.g. ``Qwen/Qwen3.5-397B-A17B``) whose processor handles tokenization + and image preprocessing. + """ + from transformers import AutoProcessor + + from megatron.training import get_args + + args = get_args() + + processor_path = getattr(args, "hf_processor_path", None) + if processor_path is None: + raise ValueError( + "cord_v2 dataset requires --hf-processor-path " + "(e.g. Qwen/Qwen3.5-397B-A17B)" + ) + processor = AutoProcessor.from_pretrained( + processor_path, trust_remote_code=True, + ) + + seq_length = ( + getattr(args, "total_seq_length", None) + or getattr(args, "seq_length", 2048) + ) + image_token_id = getattr(args, "image_token_id", None) + + # Load real data + train_examples = load_cord_v2(split="train") + val_examples = load_cord_v2(split="validation") + test_examples = load_cord_v2(split="test") + + def _make(examples, num_samples): + return CordV2VLMDataset( + examples=examples, + processor=processor, + seq_length=seq_length, + image_token_id=image_token_id, + target_length=num_samples, + ) + + # MegatronPretrainingSampler asserts total_samples > 0, so val/test + # datasets must have non-zero length even when eval is disabled. + train_ds = _make(train_examples, train_val_test_num_samples[0]) + val_ds = _make(val_examples, max(train_val_test_num_samples[1], 1)) + test_ds = _make(test_examples, max(train_val_test_num_samples[2], 1)) + + return train_ds, val_ds, test_ds + + +if __name__ == "__main__": + from transformers import AutoProcessor + processor = AutoProcessor.from_pretrained( + "Qwen/Qwen3.5-397B-A17B", trust_remote_code=True, + ) + examples = load_cord_v2(split="train") + dataset = CordV2VLMDataset( + examples=examples, + processor=processor, + image_token_id=248056, + ) + print(dataset[0]) diff --git a/examples/multimodal_dev/data/mock.py b/examples/multimodal_dev/data/mock.py new file mode 100644 index 00000000000..0975b132013 --- /dev/null +++ b/examples/multimodal_dev/data/mock.py @@ -0,0 +1,192 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Mock dataset for multimodal_dev end-to-end testing. + +Generates synthetic image + text data. Each sample has random text +tokens with image-token placeholders, random pixel values sized for the +vision encoder, 3D MRoPE position IDs, and shifted labels. +""" + +import torch +from torch.utils.data import Dataset + +from examples.multimodal_dev.models.qwen35_vl.configuration import ( + QWEN35_VL_IMAGE_TOKEN_ID, + QWEN35_VL_VIDEO_TOKEN_ID, + QWEN35_VL_VISION_START_TOKEN_ID, +) +from examples.multimodal_dev.models.qwen35_vl.mrope import get_rope_index + + +class MockQwen35VLDataset(Dataset): + """Synthetic Qwen3.5-VL training samples. + + Args: + num_samples: Number of samples. + seq_length: Total sequence length (text + image tokens). + image_seq_length: Number of image tokens per sample. + vocab_size: Vocabulary size for random text tokens. + image_token_id: Token ID for image placeholders. + video_token_id: Token ID for video placeholders. + vision_start_token_id: Token ID marking start of a vision region. + image_size: Image height and width in pixels. + patch_size: Spatial patch size. + temporal_patch_size: Temporal patch size. + spatial_merge_size: Spatial merge factor. + """ + + def __init__( + self, + num_samples: int = 1000, + seq_length: int = 1024, + image_seq_length: int = 256, + vocab_size: int = 248320, + image_token_id: int = QWEN35_VL_IMAGE_TOKEN_ID, + video_token_id: int = QWEN35_VL_VIDEO_TOKEN_ID, + vision_start_token_id: int = QWEN35_VL_VISION_START_TOKEN_ID, + image_size: int = 224, + patch_size: int = 16, + temporal_patch_size: int = 2, + spatial_merge_size: int = 2, + ): + self.num_samples = num_samples + self.seq_length = seq_length + self.vocab_size = vocab_size + self.image_token_id = image_token_id + self.video_token_id = video_token_id + self.vision_start_token_id = vision_start_token_id + self.image_size = image_size + self.patch_size = patch_size + self.temporal_patch_size = temporal_patch_size + self.spatial_merge_size = spatial_merge_size + + h_patches = image_size // patch_size + w_patches = image_size // patch_size + t_patches = temporal_patch_size + self.grid_thw = torch.tensor([[t_patches, h_patches, w_patches]]) + + self.num_merged_tokens = ( + t_patches + * (h_patches // spatial_merge_size) + * (w_patches // spatial_merge_size) + ) + self.image_seq_length = min( + image_seq_length, self.num_merged_tokens, + ) + self.total_patches = t_patches * h_patches * w_patches + + def __len__(self): + return self.num_samples + + def __getitem__(self, idx): + # Reserve 1 slot for the vision_start sentinel before image tokens. + text_length = self.seq_length - self.image_seq_length - 1 + text_tokens = torch.randint( + 1, self.vocab_size, (text_length,), dtype=torch.long, + ) + special_ids = { + self.image_token_id, + self.video_token_id, + self.vision_start_token_id, + } + for sid in special_ids: + text_tokens[text_tokens == sid] = 1 + + prefix_len = text_length // 2 + suffix_len = text_length - prefix_len + input_ids = torch.cat([ + text_tokens[:prefix_len], + torch.tensor( + [self.vision_start_token_id], dtype=torch.long, + ), + torch.full( + (self.image_seq_length,), + self.image_token_id, + dtype=torch.long, + ), + text_tokens[prefix_len: prefix_len + suffix_len], + ]) + + labels = input_ids.clone() + labels[:-1] = input_ids[1:] + labels[-1] = 0 + + loss_mask = (input_ids != self.image_token_id).float() + loss_mask[-1] = 0 + + pixel_dim = ( + 3 + * self.temporal_patch_size + * self.patch_size + * self.patch_size + ) + pixel_values = torch.randn(self.total_patches, pixel_dim) + + image_grid_thw = self.grid_thw.clone() + + position_ids, _ = get_rope_index( + spatial_merge_size=self.spatial_merge_size, + image_token_id=self.image_token_id, + video_token_id=self.video_token_id, + vision_start_token_id=self.vision_start_token_id, + input_ids=input_ids.unsqueeze(0), + image_grid_thw=image_grid_thw, + ) + position_ids = position_ids.squeeze(1) + + return { + "input_ids": input_ids, + "labels": labels, + "loss_mask": loss_mask, + "cu_seqlens": torch.tensor([0, self.seq_length], dtype=torch.int32), + "cu_seqlens_padded": torch.tensor( + [0, self.seq_length], dtype=torch.int32, + ), + "max_seqlen": torch.tensor(self.seq_length, dtype=torch.int32), + "position_ids": position_ids, + "pixel_values": pixel_values, + "image_grid_thw": image_grid_thw, + } + + +def mock_collate_fn(batch): + """Collate: handles position_ids ``[3, S]`` stacking.""" + result = {} + keys = batch[0].keys() + for key in keys: + tensors = [sample[key] for sample in batch] + if key == "position_ids": + result[key] = torch.stack(tensors, dim=1) + elif key == "image_grid_thw": + result[key] = torch.cat(tensors, dim=0) + elif key == "pixel_values": + result[key] = torch.cat(tensors, dim=0) + else: + result[key] = torch.stack(tensors, dim=0) + return result + + +def train_valid_test_datasets_provider(train_val_test_num_samples): + """Provide mock train / val / test datasets.""" + from megatron.training import get_args + + args = get_args() + kwargs = dict( + seq_length=getattr(args, "total_seq_length", 1024), + image_seq_length=getattr(args, "image_seq_length", 256), + vocab_size=getattr(args, "padded_vocab_size", 248320), + image_token_id=getattr(args, "image_token_id", 248056), + image_size=getattr(args, "image_size", 224), + ) + + train_ds = MockQwen35VLDataset( + num_samples=train_val_test_num_samples[0], **kwargs, + ) + val_ds = MockQwen35VLDataset( + num_samples=train_val_test_num_samples[1], **kwargs, + ) + test_ds = MockQwen35VLDataset( + num_samples=train_val_test_num_samples[2], **kwargs, + ) + + return train_ds, val_ds, test_ds diff --git a/examples/multimodal_dev/forward_step.py b/examples/multimodal_dev/forward_step.py new file mode 100644 index 00000000000..ce01cea6c30 --- /dev/null +++ b/examples/multimodal_dev/forward_step.py @@ -0,0 +1,432 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Forward step, TP broadcast, and loss for multimodal_dev training.""" + +import math +from functools import partial +from itertools import accumulate +from typing import Any, Dict, Iterator, Optional + +import torch +import torch.nn.functional as F + +from megatron.core import mpu +from megatron.core.packed_seq_params import PackedSeqParams +from megatron.core.parallel_state import ( + get_tensor_model_parallel_group, + get_tensor_model_parallel_rank, + get_tensor_model_parallel_src_rank, +) +from megatron.training import get_args + +# ------------------------------------------------------------------- +# dtype <-> int mapping for cross-rank broadcast +# ------------------------------------------------------------------- + +_DTYPE_MAP = { + torch.float32: 0, + torch.float16: 1, + torch.bfloat16: 2, + torch.int64: 3, + torch.int32: 4, + torch.bool: 5, +} +_ID_MAP = {v: k for k, v in _DTYPE_MAP.items()} + + +def _dtype_to_id(dtype): + return _DTYPE_MAP.get(dtype, 0) + + +def _id_to_dtype(id_val): + return _ID_MAP.get(id_val, torch.float32) + + +# ------------------------------------------------------------------- +# Tensor broadcast helper +# ------------------------------------------------------------------- + + +def _broadcast_tensor(tensor, src, group, device): + """Broadcast a single tensor from *src* to all ranks in *group*.""" + ndim = torch.tensor( + [len(tensor.shape) if tensor is not None else 0], dtype=torch.long, device=device + ) + torch.distributed.broadcast(ndim, src, group=group) + + if ndim.item() == 0: + return None + + if tensor is not None: + shape_tensor = torch.tensor(list(tensor.shape), dtype=torch.long, device=device) + dtype_id = torch.tensor([_dtype_to_id(tensor.dtype)], dtype=torch.long, device=device) + else: + shape_tensor = torch.zeros(ndim.item(), dtype=torch.long, device=device) + dtype_id = torch.zeros(1, dtype=torch.long, device=device) + + torch.distributed.broadcast(shape_tensor, src, group=group) + torch.distributed.broadcast(dtype_id, src, group=group) + + dtype = _id_to_dtype(dtype_id.item()) + shape = tuple(shape_tensor.tolist()) + + if tensor is None: + tensor = torch.empty(shape, dtype=dtype, device=device) + torch.distributed.broadcast(tensor, src, group=group) + return tensor + + +# ------------------------------------------------------------------- +# Batch broadcast across TP ranks +# ------------------------------------------------------------------- + + +def broadcast_data_batch(data, device="cuda"): + """Broadcast a data-batch dict from TP rank 0 to all TP ranks.""" + src = get_tensor_model_parallel_src_rank() + group = get_tensor_model_parallel_group() + + if data is None: + data = {} + + if get_tensor_model_parallel_rank() == 0: + keys = list(data.keys()) + key_str = ",".join(keys) + key_bytes = key_str.encode("utf-8") + key_len = torch.tensor([len(key_bytes)], dtype=torch.long, device=device) + else: + key_len = torch.zeros(1, dtype=torch.long, device=device) + keys = [] + + torch.distributed.broadcast(key_len, src, group=group) + + if get_tensor_model_parallel_rank() == 0: + key_tensor = torch.tensor(list(key_bytes), dtype=torch.uint8, device=device) + else: + key_tensor = torch.zeros(key_len.item(), dtype=torch.uint8, device=device) + + torch.distributed.broadcast(key_tensor, src, group=group) + + if get_tensor_model_parallel_rank() != 0: + key_str = bytes(key_tensor.cpu().tolist()).decode("utf-8") + keys = key_str.split(",") if key_str else [] + + result = {} + for key in keys: + tensor = data.get(key, None) if data else None + if tensor is not None and isinstance(tensor, torch.Tensor): + tensor = tensor.to(device) + result[key] = _broadcast_tensor( + tensor if isinstance(tensor, torch.Tensor) else None, src, group, device + ) + + return result + + +# ------------------------------------------------------------------- +# THD (packed sequence) helpers +# ------------------------------------------------------------------- + + +def _build_packed_seq_params(seq_lengths: torch.Tensor, device: torch.device) -> PackedSeqParams: + """Build ``PackedSeqParams`` from per-sample valid sequence lengths. + + Args: + seq_lengths: ``[B]`` valid token counts per sample. + device: Target device for cu_seqlens tensors. + + Returns: + A ``PackedSeqParams`` instance with ``qkv_format='thd'``. + """ + if not isinstance(seq_lengths, torch.Tensor): + seq_lengths = torch.tensor(seq_lengths) + lengths_t = seq_lengths.to(device=device, dtype=torch.int32) + cu_seqlens = torch.zeros(lengths_t.numel() + 1, dtype=torch.int32, device=device) + torch.cumsum(lengths_t, dim=0, out=cu_seqlens[1:]) + max_seqlen = int(lengths_t.max().item()) + return _build_packed_seq_params_from_cu_seqlens(cu_seqlens=cu_seqlens, max_seqlen=max_seqlen) + + +def _build_packed_seq_params_from_cu_seqlens( + cu_seqlens: torch.Tensor, max_seqlen: int +) -> PackedSeqParams: + """Build ``PackedSeqParams`` from packed cumulative sequence lengths. + + ``cu_seqlens`` must already be on the target compute device. + """ + cs = cu_seqlens.to(dtype=torch.int32) + total_tokens = int(cs[-1].item()) + return PackedSeqParams( + cu_seqlens_q=cs, + cu_seqlens_kv=cs, + cu_seqlens_q_padded=cs, + cu_seqlens_kv_padded=cs, + max_seqlen_q=max_seqlen, + max_seqlen_kv=max_seqlen, + qkv_format='thd', + total_tokens=total_tokens, + ) + + +def pack_or_pad_batch( + batch: Optional[list[Dict[str, Any]]], + use_packed_sequence: bool = False, + seq_length: Optional[int] = None, + device="cuda", +) -> Dict[str, Any]: + """Pack or pad a ``[B, S]`` batch into ``[1, T]`` THD or ``[B, S]`` BSHD. + + Must be invoked on every TP rank. On the TP source rank ``batch`` is + the per-sample dict list from the dataset; on other TP ranks ``batch`` + may be ``None`` (the function relies on the trailing TP broadcast to + distribute results). All metadata needed to reconstruct + ``PackedSeqParams`` (``cu_seqlens``, ``cu_seqlens_padded``, + ``max_seqlen``, ``total_tokens``) is broadcast alongside the data, so + every rank can build an identical ``PackedSeqParams`` on its own. + """ + tp_size = mpu.get_tensor_model_parallel_world_size() + cp_size = mpu.get_context_parallel_world_size() + is_src = mpu.get_tensor_model_parallel_rank() == 0 + + # SP is an explicit runtime option; TP>1 does not imply SP is enabled. + # get_args() itself raises in test contexts where megatron globals are + # not initialised. + try: + has_sp = bool(getattr(get_args(), "sequence_parallel", False)) + except AssertionError: + has_sp = False + + if cp_size > 1: + divisible_by = (tp_size * cp_size * 2) if has_sp else (cp_size * 2) + else: + divisible_by = tp_size if has_sp else 1 + + if use_packed_sequence: + packed_batch: Dict[str, Any] = {} + + if is_src: + assert batch is not None, "source TP rank must provide a batch" + input_ids_list, labels_list, loss_mask_list = [], [], [] + pixel_values_list, image_grid_thw_list = [], [] + seqlens_list, seqlens_padded_list = [], [] + + for sample in batch: + seqlen = sample["input_ids"].shape[0] + assert ( + sample["labels"].shape == sample["input_ids"].shape == sample["loss_mask"].shape + ), "labels, input_ids, and loss_mask must have the same shape" + target_len = math.ceil(seqlen / divisible_by) * divisible_by + input_ids_list.append(F.pad(sample["input_ids"], (0, target_len - seqlen), value=0)) + labels_list.append(F.pad(sample["labels"], (0, target_len - seqlen), value=-100)) + loss_mask_list.append(F.pad(sample["loss_mask"], (0, target_len - seqlen), value=0)) + seqlens_list.append(seqlen) + seqlens_padded_list.append(target_len) + pixel_values_list.append(sample["pixel_values"]) + image_grid_thw_list.append(sample["image_grid_thw"]) + + cu_seqlens = list(accumulate(seqlens_list, initial=0)) + cu_seqlens_padded = list(accumulate(seqlens_padded_list, initial=0)) + + # padding_mask: True at collate-padded positions within each packed + # sample. Real tokens occupy [cu_seqlens_padded[i], +seqlens_list[i]); + # the tail up to cu_seqlens_padded[i+1] is padding. Consumed by MoE + # routing in megatron.core to exclude padded tokens from aux loss, + # z-loss, and expert-bias accumulation. + total_tokens_padded = cu_seqlens_padded[-1] + padding_mask_thd = torch.zeros(total_tokens_padded, dtype=torch.bool) + for i, real_seqlen in enumerate(seqlens_list): + pad_start = cu_seqlens_padded[i] + real_seqlen + pad_end = cu_seqlens_padded[i + 1] + if pad_end > pad_start: + padding_mask_thd[pad_start:pad_end] = True + + packed_batch["input_ids"] = torch.concat(input_ids_list, dim=0).unsqueeze(0) + packed_batch["labels"] = torch.concat(labels_list, dim=0).unsqueeze(0) + packed_batch["loss_mask"] = torch.concat(loss_mask_list, dim=0).unsqueeze(0) + packed_batch["padding_mask"] = padding_mask_thd.unsqueeze(0) + packed_batch["pixel_values"] = torch.concat(pixel_values_list) + packed_batch["image_grid_thw"] = torch.concat(image_grid_thw_list) + # cu_seqlens / cu_seqlens_padded need to reach non-source TP ranks + # so each rank can build an identical PackedSeqParams. + packed_batch["cu_seqlens"] = torch.tensor(cu_seqlens, dtype=torch.int32, device=device) + packed_batch["cu_seqlens_padded"] = torch.tensor( + cu_seqlens_padded, dtype=torch.int32, device=device + ) + + packed_batch = broadcast_data_batch(packed_batch, device=device) + + cu_seqlens_t = packed_batch.pop("cu_seqlens") + cu_seqlens_padded_t = packed_batch.pop("cu_seqlens_padded") + # Derive max_seqlen / total_tokens from the (broadcast) cu_seqlens — + # no extra collective needed. + max_seqlen_q = int((cu_seqlens_padded_t[1:] - cu_seqlens_padded_t[:-1]).max().item()) + total_tokens = int(cu_seqlens_padded_t[-1].item()) + + packed_batch["packed_seq_params"] = PackedSeqParams( + qkv_format="thd", + cu_seqlens_q=cu_seqlens_t, + cu_seqlens_kv=cu_seqlens_t, + cu_seqlens_q_padded=cu_seqlens_padded_t, + cu_seqlens_kv_padded=cu_seqlens_padded_t, + max_seqlen_q=max_seqlen_q, + max_seqlen_kv=max_seqlen_q, + total_tokens=total_tokens, + ) + return packed_batch + + # ---------- padded (BSHD) branch ---------- + assert seq_length is not None, "seq_length must be provided when use_packed_sequence is False" + padded_batch: Dict[str, Any] = {} + + if is_src: + assert batch is not None, "source TP rank must provide a batch" + max_seqlens = max(x["input_ids"].shape[0] for x in batch) + target_seqlens = min(max_seqlens, seq_length) + # Round target seqlen up to the parallelism alignment factor so the + # batched tensor is divisible for CP (+SP) splitting downstream. + if divisible_by > 1: + target_seqlens = math.ceil(target_seqlens / divisible_by) * divisible_by + + # Capture real lengths before in-place padding so we can build a + # padding_mask for MoE routing (True at collate-padded positions). + real_seqlens = [s["input_ids"].shape[0] for s in batch] + + for sample in batch: + sample["input_ids"] = F.pad( + sample["input_ids"], (0, target_seqlens - sample["input_ids"].shape[0]), value=0 + ) + sample["labels"] = F.pad( + sample["labels"], (0, target_seqlens - sample["labels"].shape[0]), value=-100 + ) + sample["loss_mask"] = F.pad( + sample["loss_mask"], (0, target_seqlens - sample["loss_mask"].shape[0]), value=0 + ) + + padded_batch["input_ids"] = torch.concat( + [x["input_ids"].unsqueeze(0) for x in batch], dim=0 + ) + padded_batch["labels"] = torch.concat([x["labels"].unsqueeze(0) for x in batch], dim=0) + padded_batch["loss_mask"] = torch.concat( + [x["loss_mask"].unsqueeze(0) for x in batch], dim=0 + ) + positions = torch.arange(target_seqlens).unsqueeze(0) + padded_batch["padding_mask"] = positions >= torch.tensor(real_seqlens).unsqueeze(1) + padded_batch["pixel_values"] = torch.concat([x["pixel_values"] for x in batch]) + padded_batch["image_grid_thw"] = torch.concat([x["image_grid_thw"] for x in batch]) + + return broadcast_data_batch(padded_batch, device=device) + + +# ------------------------------------------------------------------- +# get_batch +# ------------------------------------------------------------------- + + +def get_batch(data_iterator: Iterator[list[Dict[str, Any]]]): + """Get a batch from *data_iterator* and broadcast across TP ranks.""" + device = "cuda" + args = get_args() + + if get_tensor_model_parallel_rank() == 0: + try: + data = next(data_iterator) + has_data = torch.tensor([1], dtype=torch.uint8, device=device) + except StopIteration: + has_data = torch.tensor([0], dtype=torch.uint8, device=device) + data = None + else: + has_data = torch.empty(1, dtype=torch.uint8, device=device) + data = None + + src = get_tensor_model_parallel_src_rank() + group = get_tensor_model_parallel_group() + torch.distributed.broadcast(has_data, src, group=group) + + if has_data.item() == 0: + return None + + # Because broadcast will not broadcast packed_seq_params, we move it into pack_or_pad_batch + batch = pack_or_pad_batch(data, args.use_packed_sequence, args.seq_length, device=device) + + # Fix shapes produced by default_collate. + if "position_ids" in batch and batch["position_ids"] is not None: + p = batch["position_ids"] + if p.dim() == 3 and p.shape[1] == 3: + batch["position_ids"] = p.permute(1, 0, 2).contiguous() + + if "pixel_values" in batch and batch["pixel_values"] is not None: + pv = batch["pixel_values"] + if pv.dim() == 3: + B, P, D = pv.shape + batch["pixel_values"] = pv.reshape(B * P, D) + + if "image_grid_thw" in batch and batch["image_grid_thw"] is not None: + g = batch["image_grid_thw"] + if g.dim() == 3: + batch["image_grid_thw"] = g.squeeze(1) + + return batch + + +# ------------------------------------------------------------------- +# Loss +# ------------------------------------------------------------------- + + +def loss_func(loss_mask, output_tensor): + """Compute masked language model loss.""" + losses = output_tensor.float() + loss_mask = loss_mask.contiguous().view(-1).float() + + total_tokens = loss_mask.sum().clone().detach().to(torch.int) + total_loss = torch.sum(losses.view(-1) * loss_mask) + reporting_loss = torch.cat([total_loss.clone().detach().view(1), total_tokens.view(1)]) + + return (total_loss, total_tokens, {"lm loss": reporting_loss}) + + +# ------------------------------------------------------------------- +# Forward step +# ------------------------------------------------------------------- + + +def forward_step(data_iterator, model): + """Forward step for multimodal_dev training.""" + batch = get_batch(data_iterator) + + if batch is None: + return None, None + + pixel_values = batch.get("pixel_values", None) + if ( + pixel_values is not None + and pixel_values.is_floating_point() + and pixel_values.dtype == torch.float32 + ): + pixel_values = pixel_values.bfloat16() + + # We don't provide position_ids, now. Let model handle it itself. + output_tensor = model( + input_ids=batch["input_ids"], + position_ids=batch.get("position_ids"), + attention_mask=batch.get("attention_mask", None), + labels=batch.get("labels", None), + loss_mask=batch.get("loss_mask", None), + padding_mask=batch.get("padding_mask", None), + pixel_values=pixel_values, + image_grid_thw=batch.get("image_grid_thw", None), + packed_seq_params=batch.get("packed_seq_params", None), + ) + + loss_mask = batch.get("loss_mask", None) + if loss_mask is None: + loss_mask = torch.ones_like(batch["input_ids"], dtype=torch.float) + + # Slice loss_mask the same way the model sliced its inputs, so the + # mask aligns with the CP-shard output. Delegated to MultimodalModel + # so the slicing rule lives in one place. + from examples.multimodal_dev.models.base import MultimodalModel + + loss_mask = MultimodalModel.cp_split_loss_mask(loss_mask, batch.get("packed_seq_params", None)) + + return output_tensor, partial(loss_func, loss_mask) diff --git a/examples/multimodal_dev/models/__init__.py b/examples/multimodal_dev/models/__init__.py new file mode 100644 index 00000000000..225414055e6 --- /dev/null +++ b/examples/multimodal_dev/models/__init__.py @@ -0,0 +1,62 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Model registry for multimodal_dev training. + +Maps ``--model-arch`` to a set of factory functions that fully encapsulate +model-specific logic. The training entry point (``pretrain_multimodal.py``) +remains model-agnostic — adding a new architecture only requires a new +registry entry (and its backing module) without touching the entry point. + +Registry entry fields +--------------------- +``model_factory_fn`` *(required)* + ``(args, language_config, vision_config, **kwargs) -> MegatronModule`` + Builds and returns the complete model instance. + +``vision_config_fn`` *(required)* + ``(num_layers_override=None, variant=None) -> TransformerConfig`` + Returns the vision encoder TransformerConfig. + +``post_language_config_fn`` *(optional)* + ``(language_config, args) -> None`` + Mutates the language TransformerConfig in-place with model-specific + fields (e.g. ``mrope_section``). + +``vision_flops_fn`` *(optional)* + ``(args, language_config, vision_config) -> None`` + Sets vision FLOPs metadata on ``args`` for training throughput logging. + +``dataset_providers`` *(optional)* + ``Dict[str, str | callable]`` + Maps ``--dataset-provider`` names to callables (or dotted import paths + resolved lazily) with signature + ``(train_val_test_num_samples) -> (train_ds, val_ds, test_ds)``. +""" + +from examples.multimodal_dev.models.qwen35_vl.configuration import get_qwen35_vl_vision_config +from examples.multimodal_dev.models.qwen35_vl.factory import build_model as _build_qwen35_vl_model +from examples.multimodal_dev.models.qwen35_vl.factory import ( + post_language_config as _qwen35_vl_post_language_config, +) +from examples.multimodal_dev.models.qwen35_vl.factory import ( + set_vision_flops_metadata as _qwen35_vl_vision_flops, +) + +MODEL_REGISTRY = { + "qwen35_vl": { + "model_factory_fn": _build_qwen35_vl_model, + "vision_config_fn": get_qwen35_vl_vision_config, + "post_language_config_fn": _qwen35_vl_post_language_config, + "vision_flops_fn": _qwen35_vl_vision_flops, + "dataset_providers": { + "mock": ( + "examples.multimodal_dev.data.mock" + ".train_valid_test_datasets_provider" + ), + "cord_v2": ( + "examples.multimodal_dev.data.cord_v2" + ".train_valid_test_datasets_provider" + ), + }, + }, +} diff --git a/examples/multimodal_dev/models/base.py b/examples/multimodal_dev/models/base.py new file mode 100644 index 00000000000..a85f36c468f --- /dev/null +++ b/examples/multimodal_dev/models/base.py @@ -0,0 +1,388 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Base multimodal model for FSDP + EP training. + +Composes a vision encoder and a ``GPTModel`` language decoder. Designed +for FSDP + EP: always builds the **full** model on every rank (no PP +flags). PP support is only available through the MIMO ``MimoModel`` +assembly path. + +Subclasses override ``compute_position_ids()`` for model-specific +position encoding (e.g. MRoPE for Qwen3.5-VL). +""" + +import contextlib +from typing import Optional + +import torch +from torch import Tensor + +from megatron.core import parallel_state, tensor_parallel +from megatron.core.models.gpt import GPTModel +from megatron.core.transformer.module import MegatronModule +from megatron.core.transformer.spec_utils import ModuleSpec +from megatron.core.transformer.transformer_config import TransformerConfig + + +def _cp_split_tensor(tensor, seq_dim, cp_size, cp_rank): + """Zigzag-split *tensor* along *seq_dim* for context parallelism (BSHD). + + Splits the sequence into ``2 * cp_size`` equal chunks, then selects + chunks ``[cp_rank, 2*cp_size - cp_rank - 1]`` and concatenates them. + This mirrors ``megatron.core.utils.get_batch_on_this_cp_rank``. + """ + S = tensor.shape[seq_dim] + assert S % (2 * cp_size) == 0, f"seq_len {S} not divisible by 2*cp_size={2 * cp_size}" + tensor = tensor.view( + *tensor.shape[:seq_dim], 2 * cp_size, S // (2 * cp_size), *tensor.shape[seq_dim + 1 :] + ) + index = torch.zeros(2, dtype=torch.int64, device=tensor.device) + index[0] = cp_rank + index[1] = 2 * cp_size - cp_rank - 1 + tensor = tensor.index_select(seq_dim, index) + tensor = tensor.view(*tensor.shape[:seq_dim], -1, *tensor.shape[seq_dim + 2 :]) + return tensor + + +class _NoCPGroup: + """Dummy size-1 process group used to bypass MRoPE's BSHD-style + zigzag of pre-computed THD freqs (Megatron-Core gap: + ``MultimodalRotaryEmbedding.forward`` lacks the ``not packed_seq`` + skip that plain ``RotaryEmbedding`` has). + """ + + def size(self): + """Pretend this group has exactly one rank.""" + return 1 + + def rank(self): + """This rank's id within the fake group is always 0.""" + return 0 + + +_NO_CP_GROUP = _NoCPGroup() + +# Note: reported ``mtp_1 loss`` drifts ~1.3% from the CP=1 baseline under +# THD+CP. Megatron-Core's logging averages per-rank pre-divided ratios +# with op=AVG, and per-rank num_tokens are unequal after MTP rolling. +# Gradients are correct; only the *logged* value drifts. + + +def _thd_cp_partition_index(cu_seqlens_padded, total_tokens, cp_size, cp_rank): + """Per-rank token index for THD + CP via TE's + ``thd_get_partitioned_indices``. Cast to int64 so the result can be + used directly with ``index_select`` regardless of TE's return dtype. + """ + from transformer_engine.pytorch import cpp_extensions as tex + + idx = tex.thd_get_partitioned_indices(cu_seqlens_padded, total_tokens, cp_size, cp_rank) + return idx.long() + + +class MultimodalModel(MegatronModule): + """Base class for multimodal vision-language models. + + Composes a pre-constructed vision encoder and a ``GPTModel`` language + decoder. Designed for FSDP + EP; always builds the full model on + every rank. + + Args: + language_config: ``TransformerConfig`` for the language decoder. + language_spec: ``ModuleSpec`` for decoder transformer layers. + vision_encoder: Pre-constructed vision encoder module. + vocab_size: Language model vocabulary size. + max_sequence_length: Maximum sequence length. + image_token_id: Token ID for image placeholder tokens. + position_embedding_type: Position embedding type for the decoder. + rotary_percent: Fraction of hidden dim for RoPE. + rotary_base: Base frequency for RoPE. + mrope_section: MRoPE channel sections. + mtp_block_spec: Optional MTP block spec. + parallel_output: Keep outputs split across TP ranks. + share_embeddings_and_output_weights: Tie input/output embeddings. + """ + + def __init__( + self, + language_config: TransformerConfig, + language_spec: ModuleSpec, + vision_encoder: MegatronModule, + vocab_size: int, + max_sequence_length: int, + image_token_id: int, + position_embedding_type: str = "rope", + rotary_percent: float = 1.0, + rotary_base: int = 10000, + mrope_section: Optional[list] = None, + mtp_block_spec: Optional[ModuleSpec] = None, + parallel_output: bool = True, + share_embeddings_and_output_weights: bool = False, + ): + super().__init__(config=language_config) + + self.image_token_id = image_token_id + + self.vision_model = vision_encoder + self.language_model = GPTModel( + config=language_config, + transformer_layer_spec=language_spec, + vocab_size=vocab_size, + max_sequence_length=max_sequence_length, + pre_process=True, + post_process=True, + parallel_output=parallel_output, + share_embeddings_and_output_weights=(share_embeddings_and_output_weights), + position_embedding_type=position_embedding_type, + rotary_percent=rotary_percent, + rotary_base=rotary_base, + mtp_block_spec=mtp_block_spec, + ) + + def set_input_tensor(self, input_tensor): + """Route input tensors (simplified, no PP routing).""" + if not isinstance(input_tensor, list): + input_tensor = [input_tensor] + assert len(input_tensor) == 1 + self.language_model.set_input_tensor(input_tensor[0]) + + def _scatter_vision_embeddings( + self, input_ids: Tensor, text_embeddings: Tensor, vision_embeddings: Tensor + ) -> Tensor: + """Replace image-token positions with vision embeddings. + + Handles sequence parallelism (gather → scatter → re-scatter). + + Args: + input_ids: ``[B, S]`` token IDs. + text_embeddings: ``[S, B, D]`` (or ``[S/TP, B, D]`` with SP). + vision_embeddings: ``[num_visual_tokens, D]``. + + Returns: + Combined embeddings, same shape as *text_embeddings*. + """ + sp = ( + self.config.sequence_parallel + and parallel_state.get_tensor_model_parallel_world_size() > 1 + ) + + if sp: + text_embeddings = tensor_parallel.gather_from_sequence_parallel_region( + text_embeddings, tensor_parallel_output_grad=False + ) + + combined = text_embeddings.transpose(0, 1).contiguous() + image_mask = input_ids == self.image_token_id + mask_expanded = image_mask.unsqueeze(-1).expand_as(combined) + combined = combined.masked_scatter(mask_expanded, vision_embeddings) + combined = combined.transpose(0, 1).contiguous() + + if sp: + combined = tensor_parallel.scatter_to_sequence_parallel_region(combined) + + return combined + + def compute_position_ids( + self, input_ids: Tensor, image_grid_thw: Optional[Tensor] = None, packed_seq_params=None + ) -> Tensor: + """Compute position IDs. Override for MRoPE etc. + + Default: simple sequential positions. ``packed_seq_params`` is + accepted for subclass compatibility (e.g. MRoPE in THD mode). + """ + B, S = input_ids.shape + return torch.arange(S, device=input_ids.device).unsqueeze(0).expand(B, -1) + + def _cp_split_for_forward( + self, + *, + decoder_input, + input_ids, + labels, + loss_mask, + attention_mask, + position_ids, + packed_seq_params, + padding_mask=None, + ): + """Apply CP split to model-forward inputs. + + BSHD path zigzag-splits each tensor along its seq dim. THD path + partitions per-sample via ``tex.thd_get_partitioned_indices`` so + chunks line up with ``cu_seqlens_q_padded`` boundaries. + ``position_ids`` and ``attention_mask`` are NOT split in THD — + MRoPE returns full freqs and TE attention's + ``_apply_rotary_pos_emb_thd`` does the per-sample CP zigzag + itself via ``_get_thd_freqs_on_this_cp_rank``. + """ + cp_size = parallel_state.get_context_parallel_world_size() + if cp_size <= 1: + return ( + decoder_input, input_ids, labels, loss_mask, + attention_mask, position_ids, padding_mask, + ) + cp_rank = parallel_state.get_context_parallel_rank() + + if packed_seq_params is not None: + total_tokens = ( + decoder_input.shape[0] if decoder_input is not None else input_ids.shape[1] + ) + idx = _thd_cp_partition_index( + packed_seq_params.cu_seqlens_q_padded, total_tokens, cp_size, cp_rank + ) + if decoder_input is not None: + decoder_input = decoder_input.index_select(0, idx) + if input_ids is not None: + input_ids = input_ids.index_select(1, idx) + if labels is not None: + labels = labels.index_select(1, idx) + if loss_mask is not None: + loss_mask = loss_mask.index_select(1, idx) + if padding_mask is not None: + padding_mask = padding_mask.index_select(1, idx) + else: + + def _split(t, seq_dim): + return ( + None + if t is None + else _cp_split_tensor(t, seq_dim=seq_dim, cp_size=cp_size, cp_rank=cp_rank) + ) + + decoder_input = _split(decoder_input, 0) + input_ids = _split(input_ids, 1) + labels = _split(labels, 1) + loss_mask = _split(loss_mask, 1) + attention_mask = _split(attention_mask, 1) + padding_mask = _split(padding_mask, 1) + + return ( + decoder_input, input_ids, labels, loss_mask, + attention_mask, position_ids, padding_mask, + ) + + @staticmethod + def cp_split_loss_mask(loss_mask, packed_seq_params): + """Slice ``loss_mask`` the same way the model slices its inputs. + + Mirrors the slicing done inside :meth:`_cp_split_for_forward` so + the loss computation outside the model can index a mask aligned + with the model's CP-shard output. Returns ``loss_mask`` unchanged + when ``CP <= 1``. + """ + cp_size = parallel_state.get_context_parallel_world_size() + if cp_size <= 1 or loss_mask is None: + return loss_mask + cp_rank = parallel_state.get_context_parallel_rank() + if packed_seq_params is not None: + idx = _thd_cp_partition_index( + packed_seq_params.cu_seqlens_q_padded, loss_mask.shape[1], cp_size, cp_rank + ) + return loss_mask.index_select(1, idx) + return _cp_split_tensor(loss_mask, seq_dim=1, cp_size=cp_size, cp_rank=cp_rank) + + @contextlib.contextmanager + def _thd_mrope_no_cp_override(self, packed_seq_params): + """Force ``rotary_pos_emb.cp_group`` to size 1 for the wrapped + forward call so MRoPE returns full-length freqs in THD mode. + Attention then applies per-sample CP zigzag itself via + ``_apply_rotary_pos_emb_thd``. Done by direct mutation rather + than via ``packed_seq_params.cp_group`` so MTP's CP-aware roll + (which reads that field) still sees the real CP group. + """ + mrope = ( + getattr(self.language_model, "rotary_pos_emb", None) + if packed_seq_params is not None + and parallel_state.get_context_parallel_world_size() > 1 + else None + ) + saved = getattr(mrope, "cp_group", None) if mrope is not None else None + if mrope is not None: + mrope.cp_group = _NO_CP_GROUP + try: + yield + finally: + if mrope is not None: + mrope.cp_group = saved + + def forward( + self, + input_ids: Tensor, + position_ids: Tensor, + attention_mask: Tensor = None, + labels: Tensor = None, + loss_mask: Tensor = None, + padding_mask: Tensor = None, + pixel_values: Tensor = None, + image_grid_thw: Tensor = None, + decoder_input: Tensor = None, + packed_seq_params=None, + **kwargs, + ): + """Forward pass. + + Args: + input_ids: ``[B, S]`` token IDs (or ``[1, T]`` in THD mode). + position_ids: ``[3, B, S]`` for MRoPE or ``[B, S]`` + (``[3, 1, T]`` / ``[1, T]`` in THD mode). + attention_mask: ``[B, S]`` attention mask (None in THD). + labels: ``[B, S]`` target token IDs (``[1, T]`` in THD). + loss_mask: ``[B, S]`` mask for loss (``[1, T]`` in THD). + padding_mask: ``[B, S]`` bool mask, True at collate-padded + positions (``[1, T]`` in THD). Forwarded to the language + decoder so MoE routing excludes padded tokens from aux + loss / z-loss / expert-bias accumulation. Distinct from + ``loss_mask``: only true padding, not SFT prompt tokens. + pixel_values: Preprocessed image pixels. + image_grid_thw: ``[num_images, 3]`` grid dimensions. + decoder_input: Pre-computed decoder input (skip embed). + packed_seq_params: ``PackedSeqParams`` for THD attention. + + Returns: + Loss tensor (post_process=True) or hidden states. + """ + if position_ids is None: + position_ids = self.compute_position_ids( + input_ids=input_ids, + image_grid_thw=image_grid_thw, + packed_seq_params=packed_seq_params, + ) + + vision_embeddings = None + if self.vision_model is not None and pixel_values is not None: + vision_embeddings = self.vision_model(pixel_values, image_grid_thw) + + if decoder_input is None and self.language_model is not None: + text_embeddings = self.language_model.embedding(input_ids=input_ids, position_ids=None) + + if vision_embeddings is not None: + decoder_input = self._scatter_vision_embeddings( + input_ids, text_embeddings, vision_embeddings + ) + else: + decoder_input = text_embeddings + + ( + decoder_input, input_ids, labels, loss_mask, + attention_mask, position_ids, padding_mask, + ) = self._cp_split_for_forward( + decoder_input=decoder_input, + input_ids=input_ids, + labels=labels, + loss_mask=loss_mask, + attention_mask=attention_mask, + position_ids=position_ids, + packed_seq_params=packed_seq_params, + padding_mask=padding_mask, + ) + + with self._thd_mrope_no_cp_override(packed_seq_params): + return self.language_model( + input_ids=input_ids, + position_ids=position_ids, + attention_mask=attention_mask, + decoder_input=decoder_input, + labels=labels, + loss_mask=loss_mask, + padding_mask=padding_mask, + packed_seq_params=packed_seq_params, + ) diff --git a/examples/multimodal_dev/models/qwen35_vl/__init__.py b/examples/multimodal_dev/models/qwen35_vl/__init__.py new file mode 100644 index 00000000000..1a0bad8b219 --- /dev/null +++ b/examples/multimodal_dev/models/qwen35_vl/__init__.py @@ -0,0 +1,70 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Qwen3.5-VL model components — the single source of truth. + +Both the standalone ``multimodal_dev`` training path and the MIMO path +import from here. +""" + +from examples.multimodal_dev.models.qwen35_vl.configuration import ( + MROPE_SECTION, + QWEN35_VL_IMAGE_TOKEN_ID, + QWEN35_VL_VIDEO_TOKEN_ID, + QWEN35_VL_VISION_END_TOKEN_ID, + QWEN35_VL_VISION_START_TOKEN_ID, + QWEN35_VL_VOCAB_SIZE, + ROTARY_BASE, + ROTARY_PERCENT, + VISION_KWARGS, + get_qwen35_vl_language_config, + get_qwen35_vl_vision_config, +) +from examples.multimodal_dev.models.qwen35_vl.factory import ( + build_model, + post_language_config, + set_vision_flops_metadata, +) +from examples.multimodal_dev.models.qwen35_vl.model import Qwen35VLModel +from examples.multimodal_dev.models.qwen35_vl.mrope import get_rope_index +from examples.multimodal_dev.models.qwen35_vl.specs import ( + get_qwen35_vl_language_spec, + get_qwen35_vl_vision_spec, +) +from examples.multimodal_dev.models.qwen35_vl.vision_encoder import ( + Qwen35VLPatchEmbed, + Qwen35VLPatchMerger, + Qwen35VLVisionEncoder, + Qwen35VLVisionRotaryEmbedding, +) + +__all__ = [ + # Model class + "Qwen35VLModel", + # Factory functions + "build_model", + "post_language_config", + "set_vision_flops_metadata", + # Vision encoder + "Qwen35VLVisionEncoder", + "Qwen35VLPatchEmbed", + "Qwen35VLPatchMerger", + "Qwen35VLVisionRotaryEmbedding", + # Config helpers + "get_qwen35_vl_vision_config", + "get_qwen35_vl_language_config", + # Spec helpers + "get_qwen35_vl_language_spec", + "get_qwen35_vl_vision_spec", + # MRoPE + "get_rope_index", + # Constants + "QWEN35_VL_IMAGE_TOKEN_ID", + "QWEN35_VL_VIDEO_TOKEN_ID", + "QWEN35_VL_VISION_START_TOKEN_ID", + "QWEN35_VL_VISION_END_TOKEN_ID", + "QWEN35_VL_VOCAB_SIZE", + "ROTARY_BASE", + "ROTARY_PERCENT", + "MROPE_SECTION", + "VISION_KWARGS", +] diff --git a/examples/multimodal_dev/models/qwen35_vl/configuration.py b/examples/multimodal_dev/models/qwen35_vl/configuration.py new file mode 100644 index 00000000000..81f73148314 --- /dev/null +++ b/examples/multimodal_dev/models/qwen35_vl/configuration.py @@ -0,0 +1,355 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Configuration helpers for Qwen3.5-VL vision-language model. + +Provides TransformerConfig builders for the vision encoder and all language +decoder variants. Both the standalone ``multimodal_dev`` training path and the +MIMO path import from here — this is the single source of truth. + +Supported language variants (HuggingFace Qwen3.5 series): + ``0.8b`` Dense 0.8B + ``2b`` Dense 2B + ``4b`` Dense 4B + ``9b`` Dense 9B + ``27b`` Dense 27B + ``35b_a3b`` MoE 35B-A3B (256 experts, top-8) + ``122b_a10b`` MoE 122B-A10B (256 experts, top-8) + ``397b_a17b`` MoE 397B-A17B (512 experts, top-10) + ``35b_a3b_light`` Reduced 35B-A3B for testing + ``proxy`` Reduced proxy based on 397B for single-node testing +""" + +from typing import Optional + +import torch + +from megatron.core.transformer.transformer_config import TransformerConfig + +# --------------------------------------------------------------------------- +# Public constants +# --------------------------------------------------------------------------- + +QWEN35_VL_IMAGE_TOKEN_ID: int = 248056 +QWEN35_VL_VIDEO_TOKEN_ID: int = 248057 +QWEN35_VL_VISION_START_TOKEN_ID: int = 248053 +QWEN35_VL_VISION_END_TOKEN_ID: int = 248054 +QWEN35_VL_VOCAB_SIZE: int = 248320 + +ROTARY_BASE: int = 10_000_000 +ROTARY_PERCENT: float = 0.25 +MROPE_SECTION: list = [11, 11, 10] + +# --------------------------------------------------------------------------- +# Vision config +# --------------------------------------------------------------------------- + +VISION_KWARGS = { + "in_channels": 3, + "patch_size": 16, + "temporal_patch_size": 2, + "spatial_merge_size": 2, + "out_hidden_size": 3584, + "max_num_positions": 2304, +} + +# Three distinct vision encoder architectures in the Qwen3.5 family. +_VISION_SMALL = { + "num_layers": 12, "hidden_size": 768, "num_attention_heads": 12, + "kv_channels": 64, "ffn_hidden_size": 3072, +} +_VISION_MEDIUM = { + "num_layers": 24, "hidden_size": 1024, "num_attention_heads": 16, + "kv_channels": 64, "ffn_hidden_size": 4096, +} +_VISION_LARGE = { + "num_layers": 27, "hidden_size": 1152, "num_attention_heads": 16, + "kv_channels": 72, "ffn_hidden_size": 4304, +} + +# Per-variant vision config. ``out_hidden_size`` equals the language model's +# hidden_size and controls the merger projection output dimension. +_VISION_VARIANT_CONFIGS = { + "0.8b": {**_VISION_SMALL, "out_hidden_size": 1024}, + "2b": {**_VISION_MEDIUM, "out_hidden_size": 2048}, + "4b": {**_VISION_MEDIUM, "out_hidden_size": 2560}, + "9b": {**_VISION_LARGE, "out_hidden_size": 4096}, + "27b": {**_VISION_LARGE, "out_hidden_size": 5120}, + "35b_a3b": {**_VISION_LARGE, "out_hidden_size": 2048}, + "122b_a10b": {**_VISION_LARGE, "out_hidden_size": 3072}, + "397b_a17b": {**_VISION_LARGE, "out_hidden_size": 4096}, +} + +# Fallback for proxy/unknown variants (large ViT, generic out_hidden_size). +_VISION_DEFAULT = {**_VISION_LARGE, "out_hidden_size": 3584} + + +def get_qwen35_vl_vision_config( + num_layers_override: Optional[int] = None, + variant: Optional[str] = None, +) -> TransformerConfig: + """TransformerConfig for the Qwen3.5-VL vision encoder. + + Three ViT architectures are used across the family: + - Small (0.8b): depth 12, 768-dim, 12 heads + - Medium (2b, 4b): depth 24, 1024-dim, 16 heads + - Large (9b, 27b, MoE variants): depth 27, 1152-dim, 16 heads + + Args: + num_layers_override: Override vision backbone depth for proxy runs. + variant: Language model variant name. When set, selects the + matching vision config from ``_VISION_VARIANT_CONFIGS`` if one + exists; otherwise the default large-ViT config is used. + """ + vcfg = _VISION_VARIANT_CONFIGS.get(variant, _VISION_DEFAULT) + num_layers = vcfg["num_layers"] + if num_layers_override is not None: + num_layers = num_layers_override + + return TransformerConfig( + num_layers=num_layers, + hidden_size=vcfg["hidden_size"], + num_attention_heads=vcfg["num_attention_heads"], + kv_channels=vcfg["kv_channels"], + ffn_hidden_size=vcfg["ffn_hidden_size"], + hidden_dropout=0.0, + attention_dropout=0.0, + layernorm_epsilon=1e-6, + normalization="LayerNorm", + gated_linear_unit=False, + activation_func=lambda x: torch.nn.functional.gelu(x, approximate="tanh"), + bias_activation_fusion=False, + apply_query_key_layer_scaling=False, + apply_rope_fusion=False, + bf16=False, + ) + + +# --------------------------------------------------------------------------- +# Language config variants +# --------------------------------------------------------------------------- + +_VARIANT_CONFIGS = { + "0.8b": { + "num_layers": 24, + "hidden_size": 1024, + "ffn_hidden_size": 3584, + "num_attention_heads": 8, + "num_query_groups": 2, + "kv_channels": 256, + "linear_num_value_heads": 16, + "num_moe_experts": None, + "moe_router_topk": None, + "moe_ffn_hidden_size": None, + "moe_shared_expert_intermediate_size": None, + }, + "2b": { + "num_layers": 24, + "hidden_size": 2048, + "ffn_hidden_size": 6144, + "num_attention_heads": 8, + "num_query_groups": 2, + "kv_channels": 256, + "linear_num_value_heads": 16, + "num_moe_experts": None, + "moe_router_topk": None, + "moe_ffn_hidden_size": None, + "moe_shared_expert_intermediate_size": None, + }, + "4b": { + "num_layers": 32, + "hidden_size": 2560, + "ffn_hidden_size": 9216, + "num_attention_heads": 16, + "num_query_groups": 4, + "kv_channels": 256, + "linear_num_value_heads": 32, + "num_moe_experts": None, + "moe_router_topk": None, + "moe_ffn_hidden_size": None, + "moe_shared_expert_intermediate_size": None, + }, + "9b": { + "num_layers": 32, + "hidden_size": 4096, + "ffn_hidden_size": 12288, + "num_attention_heads": 16, + "num_query_groups": 4, + "kv_channels": 256, + "linear_num_value_heads": 32, + "num_moe_experts": None, + "moe_router_topk": None, + "moe_ffn_hidden_size": None, + "moe_shared_expert_intermediate_size": None, + }, + "27b": { + "num_layers": 64, + "hidden_size": 5120, + "ffn_hidden_size": 17408, + "num_attention_heads": 24, + "num_query_groups": 4, + "kv_channels": 256, + "linear_num_value_heads": 48, + "num_moe_experts": None, + "moe_router_topk": None, + "moe_ffn_hidden_size": None, + "moe_shared_expert_intermediate_size": None, + }, + "35b_a3b": { + "num_layers": 40, + "hidden_size": 2048, + "ffn_hidden_size": 4096, + "num_attention_heads": 16, + "num_query_groups": 2, + "kv_channels": 256, + "linear_num_value_heads": 32, + "num_moe_experts": 256, + "moe_router_topk": 8, + "moe_ffn_hidden_size": 512, + "moe_shared_expert_intermediate_size": 512, + }, + "35b_a3b_light": { + "num_layers": 20, + "hidden_size": 2048, + "ffn_hidden_size": 4096, + "num_attention_heads": 16, + "num_query_groups": 2, + "kv_channels": 256, + "linear_num_value_heads": 32, + "num_moe_experts": 256, + "moe_router_topk": 8, + "moe_ffn_hidden_size": 512, + "moe_shared_expert_intermediate_size": 512, + }, + "122b_a10b": { + "num_layers": 48, + "hidden_size": 3072, + "ffn_hidden_size": 8192, + "num_attention_heads": 32, + "num_query_groups": 2, + "kv_channels": 256, + "linear_num_value_heads": 64, + "num_moe_experts": 256, + "moe_router_topk": 8, + "moe_ffn_hidden_size": 1024, + "moe_shared_expert_intermediate_size": 1024, + }, + "397b_a17b": { + "num_layers": 60, + "hidden_size": 4096, + "ffn_hidden_size": 10240, + "num_attention_heads": 32, + "num_query_groups": 2, + "kv_channels": 256, + "linear_num_value_heads": 64, + "num_moe_experts": 512, + "moe_router_topk": 10, + "moe_ffn_hidden_size": 1024, + "moe_shared_expert_intermediate_size": 1024, + }, + "proxy": { + "num_layers": 4, + "hidden_size": 4096, + "ffn_hidden_size": 10240, + "num_attention_heads": 32, + "num_query_groups": 2, + "kv_channels": 256, + "linear_num_value_heads": 64, + "num_moe_experts": 16, + "moe_router_topk": 2, + "moe_ffn_hidden_size": 1024, + "moe_shared_expert_intermediate_size": 1024, + }, +} + + +def get_qwen35_vl_language_config( + variant: str = "proxy", + **overrides, +) -> TransformerConfig: + """TransformerConfig for the Qwen3.5-VL language decoder. + + The ``397b_a17b`` variant reproduces the MIMO + ``get_qwen35_language_model_config()`` output exactly. + + Args: + variant: One of ``0.8b``, ``2b``, ``4b``, ``9b``, ``27b``, + ``35b_a3b``, ``122b_a10b``, ``397b_a17b``, + ``35b_a3b_light``, ``proxy``. + **overrides: Override any TransformerConfig field. + + Returns: + Fully-populated TransformerConfig. + """ + if variant not in _VARIANT_CONFIGS: + raise ValueError( + f"Unknown variant '{variant}'. " + f"Choose from {list(_VARIANT_CONFIGS.keys())}" + ) + + v = _VARIANT_CONFIGS[variant] + + kwargs = dict( + # Architecture + num_layers=v["num_layers"], + hidden_size=v["hidden_size"], + ffn_hidden_size=v["ffn_hidden_size"], + num_attention_heads=v["num_attention_heads"], + num_query_groups=v["num_query_groups"], + kv_channels=v["kv_channels"], + # Normalization & activation + normalization="RMSNorm", + layernorm_epsilon=1e-6, + layernorm_zero_centered_gamma=True, + apply_residual_connection_post_layernorm=False, + gated_linear_unit=True, + activation_func=torch.nn.functional.silu, + # MRoPE section (interleaved T/H/W layout, Qwen3.5-VL style) + mrope_section=list(MROPE_SECTION), + mrope_interleaved=True, + rotary_interleaved=False, + # Attention + qk_layernorm=True, + attention_output_gate=True, + attention_dropout=0.0, + hidden_dropout=0.0, + add_bias_linear=False, + # Hybrid attention (GatedDeltaNet) + experimental_attention_variant="gated_delta_net", + linear_attention_freq=4, + linear_conv_kernel_dim=4, + linear_key_head_dim=128, + linear_value_head_dim=128, + linear_num_key_heads=16, + linear_num_value_heads=v["linear_num_value_heads"], + # Kernel / TE fusions + bias_activation_fusion=True, + masked_softmax_fusion=True, + persist_layer_norm=True, + bias_dropout_fusion=True, + apply_rope_fusion=False, + # Precision + bf16=True, + ) + + # MoE config (only for MoE variants) + if v["num_moe_experts"] is not None: + kwargs.update( + num_moe_experts=v["num_moe_experts"], + moe_router_topk=v["moe_router_topk"], + moe_ffn_hidden_size=v["moe_ffn_hidden_size"], + moe_shared_expert_intermediate_size=v[ + "moe_shared_expert_intermediate_size" + ], + moe_shared_expert_gate=True, + moe_layer_freq=1, + moe_router_pre_softmax=False, + moe_router_load_balancing_type="global_aux_loss", + moe_permute_fusion=True, + moe_aux_loss_coeff=1e-3, + moe_grouped_gemm=True, + moe_token_dispatcher_type="alltoall", + moe_router_dtype="fp32", + ) + + kwargs.update(overrides) + return TransformerConfig(**kwargs) diff --git a/examples/multimodal_dev/models/qwen35_vl/factory.py b/examples/multimodal_dev/models/qwen35_vl/factory.py new file mode 100644 index 00000000000..3064bc5b7f4 --- /dev/null +++ b/examples/multimodal_dev/models/qwen35_vl/factory.py @@ -0,0 +1,101 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Factory functions for Qwen3.5-VL model construction. + +Encapsulates all Qwen3.5-VL-specific logic needed by ``pretrain_multimodal.py`` +so that the training entry point remains model-agnostic. +""" + +from examples.multimodal_dev.models.qwen35_vl.configuration import ( + MROPE_SECTION, + VISION_KWARGS, +) + + +def post_language_config(language_config, args): + """Apply Qwen3.5-VL-specific settings to the language TransformerConfig. + + Called after ``core_transformer_config_from_args`` to inject model-specific + fields that cannot be expressed via CLI args alone. + """ + language_config.mrope_section = list(MROPE_SECTION) + language_config.mrope_interleaved = True + + +def set_vision_flops_metadata(args, language_config, vision_config): + """Expose Qwen3.5-VL vision-model dimensions for FLOPs estimation.""" + args.count_vision_model_flops = True + args.vision_flops_variant = "qwen35_vl_v2" + args.vision_num_layers = vision_config.num_layers + args.vision_hidden_size = vision_config.hidden_size + args.vision_ffn_hidden_size = vision_config.ffn_hidden_size + args.vision_num_attention_heads = vision_config.num_attention_heads + args.vision_kv_channels = vision_config.kv_channels + args.vision_in_channels = VISION_KWARGS["in_channels"] + args.vision_patch_size = VISION_KWARGS["patch_size"] + args.vision_temporal_patch_size = VISION_KWARGS["temporal_patch_size"] + args.vision_spatial_merge_size = VISION_KWARGS["spatial_merge_size"] + args.vision_out_hidden_size = language_config.hidden_size + + +def build_model(args, language_config, vision_config, **kwargs): + """Build a complete Qwen3.5-VL model instance. + + Handles language spec construction, optional MTP block spec, and + model instantiation with Qwen3.5-VL-specific parameters. + + Args: + args: Megatron parsed arguments. + language_config: ``TransformerConfig`` for the language decoder + (already post-processed by :func:`post_language_config`). + vision_config: ``TransformerConfig`` for the vision encoder. + **kwargs: Extra keyword arguments (e.g. ``vp_stage``). + + Returns: + A :class:`Qwen35VLModel` instance. + """ + from megatron.core.models.gpt.gpt_layer_specs import ( + get_gpt_mtp_block_spec, + ) + + from examples.multimodal_dev.models.qwen35_vl.model import Qwen35VLModel + from examples.multimodal_dev.models.qwen35_vl.specs import ( + get_qwen35_vl_language_spec, + ) + + language_spec = get_qwen35_vl_language_spec( + config=language_config, + vp_stage=kwargs.get("vp_stage", None), + pp_rank=None, + ) + + mtp_block_spec = None + if getattr(args, "mtp_num_layers", None): + mtp_block_spec = get_gpt_mtp_block_spec( + config=language_config, + spec=language_spec, + use_transformer_engine=( + args.transformer_impl == "transformer_engine" + ), + vp_stage=kwargs.get("vp_stage", None), + pp_rank=None, + ) + + # When --untie-embeddings-and-output-weights is NOT passed, Megatron + # defaults to tied embeddings (share_embeddings_and_output_weights=True). + # The 0.8B variant uses tied embeddings, while larger variants untie them. + share_embeddings = not getattr( + args, "untie_embeddings_and_output_weights", False + ) + + return Qwen35VLModel( + language_config=language_config, + language_spec=language_spec, + vision_config=vision_config, + vocab_size=args.padded_vocab_size, + max_sequence_length=args.max_position_embeddings, + image_token_id=getattr(args, "image_token_id", 248056), + mtp_block_spec=mtp_block_spec, + parallel_output=True, + share_embeddings_and_output_weights=share_embeddings, + ) diff --git a/examples/multimodal_dev/models/qwen35_vl/model.py b/examples/multimodal_dev/models/qwen35_vl/model.py new file mode 100644 index 00000000000..a8fdaf67d33 --- /dev/null +++ b/examples/multimodal_dev/models/qwen35_vl/model.py @@ -0,0 +1,129 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Qwen3.5-VL multimodal model for standalone FSDP + EP training. + +Composes a Megatron-native Qwen3.5 vision encoder with a ``GPTModel`` +language decoder using MRoPE and hybrid GatedDeltaNet / full-attention +layers. +""" + +from typing import Optional + +from torch import Tensor + +from examples.multimodal_dev.models.base import MultimodalModel +from examples.multimodal_dev.models.qwen35_vl.configuration import ( + QWEN35_VL_IMAGE_TOKEN_ID, + QWEN35_VL_VIDEO_TOKEN_ID, + QWEN35_VL_VISION_START_TOKEN_ID, + QWEN35_VL_VOCAB_SIZE, + ROTARY_BASE, + ROTARY_PERCENT, + VISION_KWARGS, +) +from examples.multimodal_dev.models.qwen35_vl.mrope import get_rope_index +from examples.multimodal_dev.models.qwen35_vl.specs import get_qwen35_vl_vision_spec +from examples.multimodal_dev.models.qwen35_vl.vision_encoder import Qwen35VLVisionEncoder +from megatron.core.transformer.spec_utils import ModuleSpec +from megatron.core.transformer.transformer_config import TransformerConfig + + +class Qwen35VLModel(MultimodalModel): + """Qwen3.5-VL multimodal model. + + Args: + language_config: ``TransformerConfig`` for the language decoder. + language_spec: ``ModuleSpec`` for language decoder layers. + vision_config: ``TransformerConfig`` for the vision encoder. + vision_spec: ``ModuleSpec`` for vision encoder layers. + vocab_size: Vocabulary size. + max_sequence_length: Maximum sequence length. + image_token_id: Token ID for image placeholders. + spatial_merge_size: Vision encoder spatial merge factor. + mtp_block_spec: Optional MTP block spec. + parallel_output: Keep outputs split across TP. + share_embeddings_and_output_weights: Tie embeddings. + """ + + def __init__( + self, + language_config: TransformerConfig, + language_spec: ModuleSpec, + vision_config: TransformerConfig, + vision_spec: ModuleSpec = None, + vocab_size: int = QWEN35_VL_VOCAB_SIZE, + max_sequence_length: int = 262144, + image_token_id: int = QWEN35_VL_IMAGE_TOKEN_ID, + video_token_id: int = QWEN35_VL_VIDEO_TOKEN_ID, + vision_start_token_id: int = QWEN35_VL_VISION_START_TOKEN_ID, + spatial_merge_size: int = 2, + mtp_block_spec: ModuleSpec = None, + parallel_output: bool = True, + share_embeddings_and_output_weights: bool = False, + ): + if vision_spec is None: + vision_spec = get_qwen35_vl_vision_spec() + + self.video_token_id = video_token_id + self.vision_start_token_id = vision_start_token_id + self.spatial_merge_size = spatial_merge_size + + vkw = dict(VISION_KWARGS) + vkw["spatial_merge_size"] = spatial_merge_size + vkw["out_hidden_size"] = language_config.hidden_size + + vision_encoder = Qwen35VLVisionEncoder( + config=vision_config, + transformer_layer_spec=vision_spec, + in_channels=vkw["in_channels"], + patch_size=vkw["patch_size"], + temporal_patch_size=vkw["temporal_patch_size"], + spatial_merge_size=vkw["spatial_merge_size"], + out_hidden_size=vkw["out_hidden_size"], + max_num_positions=vkw["max_num_positions"], + ) + + super().__init__( + language_config=language_config, + language_spec=language_spec, + vision_encoder=vision_encoder, + vocab_size=vocab_size, + max_sequence_length=max_sequence_length, + image_token_id=image_token_id, + position_embedding_type="mrope", + rotary_percent=ROTARY_PERCENT, + rotary_base=ROTARY_BASE, + mrope_section=language_config.mrope_section, + mtp_block_spec=mtp_block_spec, + parallel_output=parallel_output, + share_embeddings_and_output_weights=( + share_embeddings_and_output_weights + ), + ) + + def compute_position_ids( + self, + input_ids: Tensor, + image_grid_thw: Optional[Tensor] = None, + packed_seq_params=None, + ) -> Tensor: + """Compute 3D MRoPE position IDs for Qwen3.5-VL. + + In THD mode ``input_ids`` is ``[1, T]`` and ``packed_seq_params`` + supplies per-segment boundaries; positions restart at 0 per + segment. In BSHD mode ``input_ids`` is ``[B, S]`` and + ``packed_seq_params`` should be ``None``. + + Returns: + ``[3, B, S]`` position IDs for MRoPE (``[3, 1, T]`` in THD). + """ + position_ids, _ = get_rope_index( + spatial_merge_size=self.spatial_merge_size, + image_token_id=self.image_token_id, + video_token_id=self.video_token_id, + vision_start_token_id=self.vision_start_token_id, + input_ids=input_ids, + image_grid_thw=image_grid_thw, + packed_seq_params=packed_seq_params, + ) + return position_ids diff --git a/examples/multimodal_dev/models/qwen35_vl/mrope.py b/examples/multimodal_dev/models/qwen35_vl/mrope.py new file mode 100644 index 00000000000..9e0e98b1a35 --- /dev/null +++ b/examples/multimodal_dev/models/qwen35_vl/mrope.py @@ -0,0 +1,379 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""MRoPE (Multimodal Rotary Position Embedding) position ID computation. + +Computes 3D position IDs for Qwen3.5-VL: for text tokens all three +dimensions share sequential positions; for image/video tokens the three +dimensions encode (temporal, height, width) in the merged spatial grid. + +Supports two input layouts: + +* **BSHD** — ``input_ids`` is ``[B, S]``; each row is an independent + sample (possibly padded) and ``attention_mask`` marks valid tokens. +* **THD** — ``input_ids`` is ``[1, T]``, a concatenation of ``N`` + sub-sequences. ``packed_seq_params.cu_seqlens_q_padded`` gives the + physical segment boundaries in the packed tensor and + ``cu_seqlens_q`` gives the valid (unpadded) token count inside each + segment. Position IDs restart at 0 at every segment boundary; image + / video grid rows are consumed in packed order across segments. + +Ported from Megatron-Bridge ``get_rope_index`` (which itself is adapted +from HF ``Qwen3VLForConditionalGeneration.get_rope_index``). The inner +loop iterates over vision occurrences, not individual tokens. +""" + +from typing import Optional + +import torch +from torch import Tensor + +from megatron.core.packed_seq_params import PackedSeqParams + + +def _build_sample_mrope_positions( + sample_input_ids: Tensor, + image_grid_thw: Optional[Tensor], + video_grid_thw: Optional[Tensor], + image_index: int, + video_index: int, + spatial_merge_size: int, + image_token_id: int, + video_token_id: int, + vision_start_token_id: int, +) -> tuple[Tensor, int, int]: + """Compute MRoPE position IDs for a single sub-sequence. + + Walks vision occurrences in ``sample_input_ids`` and produces a + ``[3, L]`` position tensor whose values start at 0. Advances + ``image_index`` / ``video_index`` through ``image_grid_thw`` / + ``video_grid_thw`` so callers can keep a running cursor across + multiple sub-sequences. + """ + vision_start_indices = torch.argwhere( + sample_input_ids == vision_start_token_id, + ).squeeze(1) + vision_tokens = sample_input_ids[vision_start_indices + 1] + image_nums = int((vision_tokens == image_token_id).sum()) + video_nums = int((vision_tokens == video_token_id).sum()) + # TODO: fuse into a kernel to drop the per-iter GPU<->CPU sync. + input_tokens = sample_input_ids.tolist() + llm_pos_ids_list: list = [] + st = 0 + remain_images, remain_videos = image_nums, video_nums + + for _ in range(image_nums + video_nums): + if image_token_id in input_tokens and remain_images > 0: + ed_image = input_tokens.index(image_token_id, st) + else: + ed_image = len(input_tokens) + 1 + if video_token_id in input_tokens and remain_videos > 0: + ed_video = input_tokens.index(video_token_id, st) + else: + ed_video = len(input_tokens) + 1 + + if ed_image < ed_video: + t, h, w = ( + image_grid_thw[image_index][0], + image_grid_thw[image_index][1], + image_grid_thw[image_index][2], + ) + image_index += 1 + remain_images -= 1 + ed = ed_image + else: + t, h, w = ( + video_grid_thw[video_index][0], + video_grid_thw[video_index][1], + video_grid_thw[video_index][2], + ) + video_index += 1 + remain_videos -= 1 + ed = ed_video + + llm_grid_t, llm_grid_h, llm_grid_w = ( + t.item(), + h.item() // spatial_merge_size, + w.item() // spatial_merge_size, + ) + text_len = ed - st + + st_idx = ( + llm_pos_ids_list[-1].max() + 1 + if llm_pos_ids_list + else 0 + ) + llm_pos_ids_list.append( + torch.arange(text_len).view(1, -1).expand(3, -1) + + st_idx + ) + + t_index = ( + torch.arange(llm_grid_t) + .view(-1, 1) + .expand(-1, llm_grid_h * llm_grid_w) + .flatten() + ) + h_index = ( + torch.arange(llm_grid_h) + .view(1, -1, 1) + .expand(llm_grid_t, -1, llm_grid_w) + .flatten() + ) + w_index = ( + torch.arange(llm_grid_w) + .view(1, 1, -1) + .expand(llm_grid_t, llm_grid_h, -1) + .flatten() + ) + llm_pos_ids_list.append( + torch.stack([t_index, h_index, w_index]) + + text_len + + st_idx + ) + st = ed + llm_grid_t * llm_grid_h * llm_grid_w + + if st < len(input_tokens): + st_idx = ( + llm_pos_ids_list[-1].max() + 1 + if llm_pos_ids_list + else 0 + ) + text_len = len(input_tokens) - st + llm_pos_ids_list.append( + torch.arange(text_len).view(1, -1).expand(3, -1) + + st_idx + ) + + if llm_pos_ids_list: + positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1) + else: + positions = torch.zeros( + 3, 0, + dtype=sample_input_ids.dtype, + device=sample_input_ids.device, + ) + return positions, image_index, video_index + + +def get_rope_index( + spatial_merge_size: int, + image_token_id: int, + video_token_id: int, + vision_start_token_id: int, + input_ids: Optional[Tensor] = None, + image_grid_thw: Optional[Tensor] = None, + video_grid_thw: Optional[Tensor] = None, + attention_mask: Optional[Tensor] = None, + packed_seq_params: Optional[PackedSeqParams] = None, +) -> tuple[Tensor, Tensor]: + """Compute 3D MRoPE position IDs for Qwen3-VL / Qwen3.5-VL. + + Qwen3-VL uses timestamps rather than absolute time position IDs. + + For text tokens all three dimensions share sequential positions. + For vision tokens the three dimensions encode (temporal, height, + width) in the merged spatial grid. + + Args: + spatial_merge_size: Merge factor for spatial dimensions. + image_token_id: Token ID for image placeholders. + video_token_id: Token ID for video placeholders. + vision_start_token_id: Token ID marking start of a vision region. + input_ids: ``[B, S]`` in BSHD or ``[1, T]`` in THD. + image_grid_thw: ``[num_images, 3]`` per-image + ``(temporal, height, width)`` in patch-grid units. Rows are + consumed in the order their image tokens appear in + ``input_ids`` (packed order across segments in THD). + video_grid_thw: ``[num_videos, 3]`` per-video grid dimensions. + attention_mask: ``[B, S]`` mask (1 = keep, 0 = pad). BSHD only. + packed_seq_params: When provided, selects the THD branch and + supplies segment boundaries via ``cu_seqlens_q`` (valid + lengths) and ``cu_seqlens_q_padded`` (packed layout). + + Returns: + ``(position_ids, mrope_position_deltas)`` where *position_ids* + has shape ``[3, B, S]`` (``[3, 1, T]`` in THD). + """ + if video_grid_thw is not None: + video_grid_thw = torch.repeat_interleave( + video_grid_thw, video_grid_thw[:, 0], dim=0, + ) + video_grid_thw[:, 0] = 1 + + # ----------------------------------------------------------------- + # THD (packed) branch + # ----------------------------------------------------------------- + if packed_seq_params is not None and input_ids is not None: + cu_seqlens = packed_seq_params.cu_seqlens_q + cu_seqlens_padded = getattr( + packed_seq_params, "cu_seqlens_q_padded", None, + ) + if cu_seqlens_padded is None: + cu_seqlens_padded = cu_seqlens + + assert ( + input_ids.dim() == 2 and input_ids.shape[0] == 1 + ), "THD get_rope_index expects input_ids shape [1, T]" + + total_tokens = input_ids.shape[1] + device = input_ids.device + + # Padding slots default to 1 (matches BSHD convention where + # masked positions get filled with 1). + position_ids = torch.ones( + 3, 1, total_tokens, + dtype=input_ids.dtype, device=device, + ) + deltas: list = [] + image_index = 0 + video_index = 0 + num_segs = cu_seqlens.numel() - 1 + + for k in range(num_segs): + seg_start = int(cu_seqlens_padded[k].item()) + valid_len = int( + cu_seqlens[k + 1].item() - cu_seqlens[k].item() + ) + valid_end = seg_start + valid_len + + if valid_len == 0: + deltas.append(0) + continue + + sample_input_ids = input_ids[0, seg_start:valid_end] + + if ( + image_grid_thw is not None + or video_grid_thw is not None + ): + ( + positions, + image_index, + video_index, + ) = _build_sample_mrope_positions( + sample_input_ids=sample_input_ids, + image_grid_thw=image_grid_thw, + video_grid_thw=video_grid_thw, + image_index=image_index, + video_index=video_index, + spatial_merge_size=spatial_merge_size, + image_token_id=image_token_id, + video_token_id=video_token_id, + vision_start_token_id=vision_start_token_id, + ) + else: + positions = ( + torch.arange(valid_len, device=device) + .view(1, -1) + .expand(3, -1) + ) + + position_ids[:, 0, seg_start:valid_end] = positions.to( + device=device, dtype=position_ids.dtype, + ) + + if positions.numel() > 0: + deltas.append( + int(positions.max().item()) + 1 - valid_len + ) + else: + deltas.append(0) + + mrope_position_deltas = torch.tensor( + deltas, device=device, + ).unsqueeze(1) + return position_ids, mrope_position_deltas + + # ----------------------------------------------------------------- + # BSHD branch with vision + # ----------------------------------------------------------------- + if input_ids is not None and ( + image_grid_thw is not None or video_grid_thw is not None + ): + total_input_ids = input_ids + if attention_mask is None: + attention_mask = torch.ones_like(total_input_ids) + elif attention_mask.dim() > 2: + attention_mask = attention_mask.any(dim=-1) + if attention_mask.dim() == 3: + attention_mask = attention_mask.squeeze(1) + attention_mask = attention_mask.to(dtype=total_input_ids.dtype) + + position_ids = torch.ones( + 3, + input_ids.shape[0], + input_ids.shape[1], + dtype=input_ids.dtype, + device=input_ids.device, + ) + mrope_position_deltas = [] + image_index, video_index = 0, 0 + attention_mask = attention_mask.to(total_input_ids.device) + + for i, sample_input_ids in enumerate(total_input_ids): + sample_input_ids = sample_input_ids[attention_mask[i] == 1] + ( + llm_positions, + image_index, + video_index, + ) = _build_sample_mrope_positions( + sample_input_ids=sample_input_ids, + image_grid_thw=image_grid_thw, + video_grid_thw=video_grid_thw, + image_index=image_index, + video_index=video_index, + spatial_merge_size=spatial_merge_size, + image_token_id=image_token_id, + video_token_id=video_token_id, + vision_start_token_id=vision_start_token_id, + ) + position_ids[ + ..., i, attention_mask[i] == 1 + ] = llm_positions.to(position_ids.device) + mrope_position_deltas.append( + llm_positions.max() + 1 - len(total_input_ids[i]), + ) + + mrope_position_deltas = torch.tensor( + mrope_position_deltas, device=total_input_ids.device, + ).unsqueeze(1) + return position_ids, mrope_position_deltas + + # ----------------------------------------------------------------- + # Text-only fallback + # ----------------------------------------------------------------- + if attention_mask is not None: + if attention_mask.dim() > 2: + attention_mask = attention_mask.any(dim=-1) + if attention_mask.dim() == 3: + attention_mask = attention_mask.squeeze(1) + attention_mask = attention_mask.to(dtype=torch.long) + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + position_ids = ( + position_ids.unsqueeze(0) + .expand(3, -1, -1) + .to(attention_mask.device) + ) + max_position_ids = ( + position_ids.max(0, keepdim=False)[0] + .max(-1, keepdim=True)[0] + ) + mrope_position_deltas = ( + max_position_ids + 1 - attention_mask.shape[-1] + ) + else: + position_ids = ( + torch.arange( + input_ids.shape[1], device=input_ids.device, + ) + .view(1, 1, -1) + .expand(3, input_ids.shape[0], -1) + ) + mrope_position_deltas = torch.zeros( + [input_ids.shape[0], 1], + device=input_ids.device, + dtype=input_ids.dtype, + ) + + return position_ids, mrope_position_deltas diff --git a/examples/multimodal_dev/models/qwen35_vl/specs.py b/examples/multimodal_dev/models/qwen35_vl/specs.py new file mode 100644 index 00000000000..22fb4e616eb --- /dev/null +++ b/examples/multimodal_dev/models/qwen35_vl/specs.py @@ -0,0 +1,131 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Layer spec helpers for Qwen3.5-VL vision encoder and language decoder. + +Provides ModuleSpec builders that define the transformer layer composition. +Both the standalone and MIMO training paths import from here. +""" + +from typing import Optional + +from examples.multimodal_dev.models.base import _NO_CP_GROUP +from megatron.core.models.gpt.experimental_attention_variant_module_specs import ( + get_transformer_block_with_experimental_attention_variant_spec, +) +from megatron.core.models.vision.vit_layer_specs import get_vit_layer_with_transformer_engine_spec +from megatron.core.transformer.attention import SelfAttention +from megatron.core.transformer.spec_utils import ModuleSpec +from megatron.core.transformer.transformer_block import TransformerBlockSubmodules +from megatron.core.transformer.transformer_config import TransformerConfig + + +def _apply_rope_fp32(t, freqs, config, cu_seqlens=None, mscale=1.0, cp_group=None): + """Apply rotary positional embedding in fp32, then cast back to original dtype. + + Mirrors ``Qwen3VLSelfAttention.apply_rotary_pos_emb_absolute`` in Megatron-Bridge + with ``apply_rotary_pos_emb_in_fp32=True``. + """ + from megatron.core import parallel_state + from megatron.core.models.common.embeddings.rope_utils import ( + _apply_rotary_pos_emb_bshd, + _apply_rotary_pos_emb_thd, + ) + + orig_dtype = t.dtype + t_fp32 = t.float() + + if cu_seqlens is None: + out = _apply_rotary_pos_emb_bshd( + t_fp32, + freqs, + rotary_interleaved=config.rotary_interleaved, + multi_latent_attention=getattr(config, 'multi_latent_attention', False), + mscale=mscale, + ) + else: + if cp_group is None: + cp_group = parallel_state.get_context_parallel_group() + out = _apply_rotary_pos_emb_thd( + t_fp32, + cu_seqlens, + freqs, + rotary_interleaved=config.rotary_interleaved, + multi_latent_attention=getattr(config, 'multi_latent_attention', False), + mscale=mscale, + cp_group=cp_group, + ) + return out.to(orig_dtype) + + +def _apply_rope_fp32_no_cp(t, freqs, config, cu_seqlens=None, mscale=1.0, cp_group=None): + """Same as ``_apply_rope_fp32`` but forces CP-size=1. + + The vision encoder uses THD packed sequences for variable-resolution + images. When the language model uses CP>1, the global CP group would + incorrectly split the vision seqlens. This wrapper substitutes a + trivial group so the vision RoPE sees the full packed sequence. + """ + return _apply_rope_fp32( + t, freqs, config, cu_seqlens, mscale, cp_group=_NO_CP_GROUP, + ) + + +class Qwen35VLVisionSelfAttention(SelfAttention): + """ViT self-attention with RoPE applied in fp32. + + Matches Bridge's ``Qwen3VLSelfAttention`` behaviour when + ``apply_rotary_pos_emb_in_fp32=True``: query and key are cast to float32 + before the rotary multiply and cast back to bf16 afterwards. The + monkey-patch approach avoids duplicating the 300-line ``SelfAttention.forward`` + while keeping the change local to this class. + """ + + def forward(self, *args, **kwargs): + import megatron.core.transformer.attention as _attn_mod + + _orig = _attn_mod.apply_rotary_pos_emb + _attn_mod.apply_rotary_pos_emb = _apply_rope_fp32_no_cp + try: + return super().forward(*args, **kwargs) + finally: + _attn_mod.apply_rotary_pos_emb = _orig + + +def get_qwen35_vl_language_spec( + config: TransformerConfig, + vp_stage: Optional[int] = None, + pp_rank: Optional[int] = None, +) -> TransformerBlockSubmodules: + """Transformer block spec for the Qwen3.5-VL language decoder. + + Uses the experimental attention variant infrastructure to build hybrid + GatedDeltaNet + full-attention layers with optional MoE interleaving. + + Args: + config: Language decoder TransformerConfig. + vp_stage: Virtual pipeline stage. + pp_rank: Pipeline parallel rank. + + Returns: + TransformerBlockSubmodules with per-layer specs. + """ + return get_transformer_block_with_experimental_attention_variant_spec( + config=config, + vp_stage=vp_stage, + pp_rank=pp_rank, + ) + + +def get_qwen35_vl_vision_spec() -> ModuleSpec: + """ModuleSpec for vision encoder transformer layers. + + Uses ``TEDotProductAttention`` which supports packed-sequence (THD) + attention via ``PackedSeqParams`` for variable-length images. + + ``Qwen35VLVisionSelfAttention`` replaces the default ``SelfAttention`` so + that RoPE is applied in fp32, matching Bridge's + ``apply_rotary_pos_emb_in_fp32=True`` behaviour. + """ + spec = get_vit_layer_with_transformer_engine_spec() + spec.submodules.self_attention.module = Qwen35VLVisionSelfAttention + return spec diff --git a/examples/multimodal_dev/models/qwen35_vl/vision_encoder.py b/examples/multimodal_dev/models/qwen35_vl/vision_encoder.py new file mode 100644 index 00000000000..d57d114374d --- /dev/null +++ b/examples/multimodal_dev/models/qwen35_vl/vision_encoder.py @@ -0,0 +1,593 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Megatron-native Qwen3.5-VL vision encoder. + +Architecture (matches HF ``Qwen3VLVisionModel`` exactly): + + PatchEmbed (Conv3d) + → learned position embedding (bilinear interpolation) + → 2D Vision RoPE + → TransformerBlock × N (with PackedSeqParams / THD attention) + → PatchMerger (per-token LN → spatial merge → MLP) + +Key design choices: + * ``Conv3d`` patch embedding is replicated across TP ranks (no MCore + equivalent for 3D convolutions). + * ``PatchMerger`` MLP uses ``ColumnParallelLinear`` / ``RowParallelLinear`` + for TP sharding. + * Inherits from ``VisionModule``. + * Expects pixel values in block-merge order (as produced by the HF + processor) so the merger's simple reshape is correct. +""" + +from typing import List, Optional + +import torch +import torch.nn.functional as F +from torch import Tensor + +from megatron.core.models.common.vision_module.vision_module import ( + VisionModule, +) +from megatron.core.packed_seq_params import PackedSeqParams +from megatron.core.tensor_parallel.layers import ( + ColumnParallelLinear, + RowParallelLinear, +) +from megatron.core.extensions.transformer_engine import TENorm +from megatron.core.transformer.module import MegatronModule +from megatron.core.transformer.spec_utils import ModuleSpec, build_module +from megatron.core.transformer.transformer_block import TransformerBlock +from megatron.core.transformer.transformer_config import TransformerConfig + +# ------------------------------------------------------------------- +# PatchEmbed — Conv3d (replicated, no TP sharding) +# ------------------------------------------------------------------- + +class Qwen35VLPatchEmbed(MegatronModule): + """3D convolution patch embedding matching HF ``Qwen3VLVisionPatchEmbed``. + + Uses ``nn.Conv3d`` with kernel = stride = ``[temporal_patch_size, + patch_size, patch_size]`` and ``bias=True``. The module is replicated + across TP ranks (no MCore equivalent for 3D conv). + + Args: + config: TransformerConfig (used by MegatronModule base). + in_channels: Number of input channels (3 for RGB). + hidden_size: Output embedding dimension. + patch_size: Spatial patch size. + temporal_patch_size: Temporal patch size. + """ + + def __init__( + self, + config: TransformerConfig, + in_channels: int = 3, + hidden_size: int = 1152, + patch_size: int = 16, + temporal_patch_size: int = 2, + ): + super().__init__(config=config) + self.patch_size = patch_size + self.temporal_patch_size = temporal_patch_size + self.in_channels = in_channels + self.hidden_size = hidden_size + + kernel = [temporal_patch_size, patch_size, patch_size] + self.proj = torch.nn.Conv3d( + in_channels, + hidden_size, + kernel_size=kernel, + stride=kernel, + bias=True, + ) + + def forward(self, pixel_values: Tensor) -> Tensor: + """Forward pass. + + Args: + pixel_values: ``[total_patches, C * T * pH * pW]`` + pre-extracted flat patches. + + Returns: + Patch embeddings ``[total_patches, hidden_size]``. + """ + target_dtype = self.proj.weight.dtype + pixel_values = pixel_values.view( + -1, + self.in_channels, + self.temporal_patch_size, + self.patch_size, + self.patch_size, + ) + return self.proj(pixel_values.to(dtype=target_dtype)).view( + -1, self.hidden_size + ) + + +# ------------------------------------------------------------------- +# VisionRotaryEmbedding — 1D frequency table +# ------------------------------------------------------------------- + +class Qwen35VLVisionRotaryEmbedding(MegatronModule): + """1D rotary position frequency table for the vision transformer. + + Generates RoPE frequencies for integer positions ``0 .. seqlen-1``. + The encoder maps 2D (row, col) positions to embeddings via table + lookup. Matches HF ``Qwen3VLVisionRotaryEmbedding``. + + Args: + dim: Frequency dimension (``head_dim // 2``). + theta: RoPE base frequency. + config: Optional TransformerConfig for MegatronModule base. + """ + + def __init__( + self, + dim: int, + theta: float = 10000.0, + config: Optional[TransformerConfig] = None, + ): + super().__init__(config=config) + self.dim = dim + self.theta = theta + inv_freq = 1.0 / ( + theta + ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim) + ) + self.register_buffer("inv_freq", inv_freq, persistent=False) + + def _get_inv_freq(self, device: torch.device) -> Tensor: + """Return ``inv_freq`` in float32 on *device*. + + Always recomputes in float32 regardless of the buffer's stored dtype. + This matches Bridge's lazy-init behaviour where ``inv_freq`` is + constructed fresh (in float32) on the first forward call, after any + ``model.bfloat16()`` cast has already occurred. + """ + return 1.0 / ( + self.theta + ** ( + torch.arange( + 0, self.dim, 2, + dtype=torch.float32, device=device, + ) + / self.dim + ) + ) + + def forward( + self, + seqlen: int, + device: Optional[torch.device] = None, + ) -> Tensor: + """Frequency lookup table for positions ``0 .. seqlen-1``. + + Args: + seqlen: Number of positions. + device: Runtime device (required for meta-init safety). + + Returns: + ``[seqlen, dim // 2]`` frequencies. + """ + if device is None: + if self.inv_freq.device.type != "meta": + device = self.inv_freq.device + else: + device = torch.device( + "cuda", torch.cuda.current_device() + ) + inv_freq = self._get_inv_freq(device) + seq = torch.arange(seqlen, device=device, dtype=inv_freq.dtype) + return torch.outer(seq, inv_freq) + + +# ------------------------------------------------------------------- +# PatchMerger — per-token LN, spatial merge, TP-sharded MLP +# ------------------------------------------------------------------- + +class Qwen35VLPatchMerger(MegatronModule): + """Spatial patch merger matching HF ``Qwen3VLVisionPatchMerger``. + + Per-token ``LayerNorm`` on ``hidden_size`` → reshape to merge + ``spatial_merge_size ** 2`` adjacent patches → two-layer MLP + (``ColumnParallelLinear`` → GELU → ``RowParallelLinear``). + + MLP dimensions: ``merge_dim → merge_dim → out_hidden_size`` + where ``merge_dim = hidden_size * spatial_merge_size ** 2``. + + Args: + config: TransformerConfig (provides TP settings, init_method). + hidden_size: Per-token hidden size from the ViT. + out_hidden_size: Output dimension (language model hidden_size). + spatial_merge_size: Merge factor per spatial dimension. + """ + + def __init__( + self, + config: TransformerConfig, + hidden_size: int = 1152, + out_hidden_size: int = 3584, + spatial_merge_size: int = 2, + ): + super().__init__(config=config) + self.spatial_merge_size = spatial_merge_size + self.merge_dim = hidden_size * (spatial_merge_size ** 2) + merge_dim = self.merge_dim + + self.patch_norm = TENorm(config=config, hidden_size=hidden_size, eps=1e-6) + self.linear_fc1 = build_module( + ColumnParallelLinear, + merge_dim, + merge_dim, + config=config, + init_method=config.init_method, + bias=True, + gather_output=False, + ) + self.linear_fc2 = build_module( + RowParallelLinear, + merge_dim, + out_hidden_size, + config=config, + init_method=config.output_layer_init_method, + bias=True, + input_is_parallel=True, + skip_bias_add=False, + ) + + def forward(self, hidden_states: Tensor) -> Tensor: + """Merge patches spatially. + + Args: + hidden_states: ``[total_patches, hidden_size]`` in block-merge + order from the ViT transformer blocks. + + Returns: + ``[total_merged_patches, out_hidden_size]``. + """ + hidden_states = self.patch_norm(hidden_states) + merged = hidden_states.view(-1, self.merge_dim) + merged, _ = self.linear_fc1(merged) + # Match official HuggingFace Qwen3VLVisionPatchMerger (default approximate='none'). + merged = torch.nn.functional.gelu(merged, approximate="none") + merged, _ = self.linear_fc2(merged) + return merged + + +# ------------------------------------------------------------------- +# Qwen35VLVisionEncoder — top-level encoder module +# ------------------------------------------------------------------- + +class Qwen35VLVisionEncoder(VisionModule): + """Megatron-native Qwen3.5-VL vision encoder. + + Processes image / video inputs through: + + 1. ``Qwen35VLPatchEmbed`` (Conv3d) + 2. Learned ``nn.Embedding`` position table with bilinear interpolation + 3. 2D Vision RoPE from ``(row, col)`` patch positions + 4. ``TransformerBlock`` × N with ``PackedSeqParams`` (THD attention) + 5. ``Qwen35VLPatchMerger`` + + Output dimension matches the language model ``hidden_size``. + + Args: + config: Vision ``TransformerConfig``. + transformer_layer_spec: ``ModuleSpec`` for ViT layers. + in_channels: Image channels (3 for RGB). + patch_size: Spatial patch size. + temporal_patch_size: Temporal patch size. + spatial_merge_size: Spatial merge factor. + out_hidden_size: Output dim (language decoder hidden_size). + max_num_positions: Size of the learned position table. + """ + + def __init__( + self, + config: TransformerConfig, + transformer_layer_spec: ModuleSpec = None, + in_channels: int = 3, + patch_size: int = 16, + temporal_patch_size: int = 2, + spatial_merge_size: int = 2, + out_hidden_size: int = 3584, + max_num_positions: int = 2304, + ): + super().__init__(config=config) + + self.hidden_size = config.hidden_size + self.spatial_merge_size = spatial_merge_size + + # --- Patch embedding (Conv3d) --- + self.patch_embed = Qwen35VLPatchEmbed( + config=config, + in_channels=in_channels, + hidden_size=config.hidden_size, + patch_size=patch_size, + temporal_patch_size=temporal_patch_size, + ) + + # --- Learned position embedding with bilinear interpolation --- + self.pos_embed = torch.nn.Embedding( + max_num_positions, config.hidden_size, + ) + self.num_grid_per_side = int(max_num_positions ** 0.5) + + # --- Vision rotary embeddings --- + head_dim = config.hidden_size // config.num_attention_heads + self.rot_pos_emb = Qwen35VLVisionRotaryEmbedding( + head_dim // 2, config=config, + ) + + # --- Transformer blocks --- + if transformer_layer_spec is None: + from examples.multimodal_dev.models.qwen35_vl.specs import ( + get_qwen35_vl_vision_spec, + ) + transformer_layer_spec = get_qwen35_vl_vision_spec() + + self.decoder = TransformerBlock( + config=config, + spec=transformer_layer_spec, + pre_process=True, + post_process=True, + post_layer_norm=False, + ) + + # --- Patch merger --- + self.merger = Qwen35VLPatchMerger( + config=config, + hidden_size=config.hidden_size, + out_hidden_size=out_hidden_size, + spatial_merge_size=spatial_merge_size, + ) + + # --------------------------------------------------------------- + # Learned position embedding with bilinear interpolation + # --------------------------------------------------------------- + + def _fast_pos_embed_interpolate( + self, grid_thw: Tensor, + ) -> Tensor: + """Bilinear interpolation of the learned 2D position table. + + Matches HF ``Qwen3VLVisionModel.fast_pos_embed_interpolate``. + + Args: + grid_thw: ``[num_images, 3]`` (T, H, W) in patch-grid units. + + Returns: + ``[total_patches, hidden_size]`` position embeddings in + block-merge order. + """ + grid_thw_list = grid_thw.tolist() + grid_ts = [int(row[0]) for row in grid_thw_list] + grid_hs = [int(row[1]) for row in grid_thw_list] + grid_ws = [int(row[2]) for row in grid_thw_list] + device = self.pos_embed.weight.device + n = self.num_grid_per_side + + idx_list: List[List[int]] = [[] for _ in range(4)] + weight_list: List[List[float]] = [[] for _ in range(4)] + + for t, h, w in grid_thw_list: + t, h, w = int(t), int(h), int(w) + h_idxs = torch.linspace(0, n - 1, h) + w_idxs = torch.linspace(0, n - 1, w) + + h_floor = h_idxs.int() + w_floor = w_idxs.int() + h_ceil = (h_floor + 1).clip(max=n - 1) + w_ceil = (w_floor + 1).clip(max=n - 1) + + dh = h_idxs - h_floor.float() + dw = w_idxs - w_floor.float() + + base_h = h_floor * n + base_h_ceil = h_ceil * n + + indices = [ + (base_h[None].T + w_floor[None]).flatten(), + (base_h[None].T + w_ceil[None]).flatten(), + (base_h_ceil[None].T + w_floor[None]).flatten(), + (base_h_ceil[None].T + w_ceil[None]).flatten(), + ] + weights = [ + ((1 - dh)[None].T * (1 - dw)[None]).flatten(), + ((1 - dh)[None].T * dw[None]).flatten(), + (dh[None].T * (1 - dw)[None]).flatten(), + (dh[None].T * dw[None]).flatten(), + ] + + for i in range(4): + idx_list[i].extend(indices[i].tolist()) + weight_list[i].extend(weights[i].tolist()) + + idx_tensor = torch.tensor( + idx_list, dtype=torch.long, device=device, + ) + weight_tensor = torch.tensor( + weight_list, + dtype=self.pos_embed.weight.dtype, + device=device, + ) + pos_embeds = ( + self.pos_embed(idx_tensor).to(device) + * weight_tensor[:, :, None] + ) + patch_pos_embeds = ( + pos_embeds[0] + pos_embeds[1] + + pos_embeds[2] + pos_embeds[3] + ) + + patch_pos_embeds = patch_pos_embeds.split( + [h * w for h, w in zip(grid_hs, grid_ws)] + ) + + merge = self.spatial_merge_size + result = [] + for pe, t, h, w in zip( + patch_pos_embeds, grid_ts, grid_hs, grid_ws, + ): + pe = pe.repeat(t, 1) + pe = ( + pe.view( + t, h // merge, merge, w // merge, merge, -1, + ) + .permute(0, 1, 3, 2, 4, 5) + .flatten(0, 4) + ) + result.append(pe) + + return torch.cat(result) + + # --------------------------------------------------------------- + # 2D Vision RoPE + # --------------------------------------------------------------- + + def _compute_rotary_pos_emb(self, grid_thw: Tensor) -> Tensor: + """Compute 2D Vision RoPE for all patches in block-merge order. + + Matches HF ``Qwen3VLVisionModel.rot_pos_emb``. + + Args: + grid_thw: ``[num_images, 3]`` (T, H, W) per image. + + Returns: + ``[total_patches, head_dim // 2]`` raw RoPE frequencies. + """ + merge = self.spatial_merge_size + grid_thw_list = grid_thw.tolist() + + max_hw = max(max(int(h), int(w)) for _, h, w in grid_thw_list) + freq_table = self.rot_pos_emb( + max_hw, device=grid_thw.device, + ) + device = freq_table.device + + total_tokens = sum( + int(t) * int(h) * int(w) for t, h, w in grid_thw_list + ) + pos_ids = torch.empty( + (total_tokens, 2), dtype=torch.long, device=device, + ) + + offset = 0 + for num_frames, height, width in grid_thw_list: + num_frames = int(num_frames) + height = int(height) + width = int(width) + merged_h = height // merge + merged_w = width // merge + + block_rows = torch.arange(merged_h, device=device) + block_cols = torch.arange(merged_w, device=device) + intra_row = torch.arange(merge, device=device) + intra_col = torch.arange(merge, device=device) + + row_idx = ( + block_rows[:, None, None, None] * merge + + intra_row[None, None, :, None] + ) + col_idx = ( + block_cols[None, :, None, None] * merge + + intra_col[None, None, None, :] + ) + + row_idx = row_idx.expand( + merged_h, merged_w, merge, merge, + ).reshape(-1) + col_idx = col_idx.expand( + merged_h, merged_w, merge, merge, + ).reshape(-1) + + coords = torch.stack((row_idx, col_idx), dim=-1) + if num_frames > 1: + coords = coords.repeat(num_frames, 1) + + n_tokens = coords.shape[0] + pos_ids[offset: offset + n_tokens] = coords + offset += n_tokens + + embeddings = freq_table[pos_ids] + embeddings = embeddings.flatten(1) + return embeddings + + # --------------------------------------------------------------- + # PackedSeqParams for variable-length attention + # --------------------------------------------------------------- + + @staticmethod + def _build_packed_seq_params(grid_thw: Tensor) -> PackedSeqParams: + """Build ``PackedSeqParams`` from grid dimensions. + + Each temporal frame of each image forms a separate sub-sequence + in the packed THD layout, matching HF's ``cu_seqlens`` computation. + + Args: + grid_thw: ``[num_images, 3]``. + + Returns: + ``PackedSeqParams`` for ``TransformerBlock``. + """ + cu_seqlens = torch.repeat_interleave( + grid_thw[:, 1] * grid_thw[:, 2], grid_thw[:, 0], + ).cumsum(dim=0, dtype=torch.int32) + cu_seqlens = F.pad(cu_seqlens, (1, 0), value=0) + max_seqlen = int( + (grid_thw[:, 1] * grid_thw[:, 2]).max().item() + ) + + return PackedSeqParams( + qkv_format="thd", + cu_seqlens_q=cu_seqlens, + cu_seqlens_kv=cu_seqlens, + max_seqlen_q=max_seqlen, + max_seqlen_kv=max_seqlen, + ) + + # --------------------------------------------------------------- + # Forward + # --------------------------------------------------------------- + + def forward( + self, + pixel_values: Tensor, + grid_thw: Tensor, + ) -> Tensor: + """Encode images / video frames. + + Args: + pixel_values: ``[total_patches, C * T * pH * pW]`` + pre-extracted flat patches in block-merge order. + grid_thw: ``[num_images, 3]`` (T, H, W) in patch-grid units. + + Returns: + ``[total_merged_patches, out_hidden_size]`` visual embeddings. + """ + # 1. Patch embedding (Conv3d) + hidden_states = self.patch_embed(pixel_values) + + # 2. Learned position embedding (bilinear interpolation) + pos_embeds = self._fast_pos_embed_interpolate(grid_thw) + hidden_states = hidden_states + pos_embeds + + # 3. 2D Vision RoPE + rot_freqs = self._compute_rotary_pos_emb(grid_thw) + emb = torch.cat((rot_freqs, rot_freqs), dim=-1) + rot_freqs_expanded = emb.unsqueeze(1).unsqueeze(1) + + # 4. Transformer blocks with PackedSeqParams + packed_seq_params = self._build_packed_seq_params(grid_thw) + hidden_states = hidden_states.unsqueeze(1) + hidden_states = self.decoder( + hidden_states=hidden_states, + attention_mask=None, + rotary_pos_emb=rot_freqs_expanded, + packed_seq_params=packed_seq_params, + ) + hidden_states = hidden_states.squeeze(1) + + # 5. Patch merger + return self.merger(hidden_states) diff --git a/examples/multimodal_dev/pretrain_multimodal.py b/examples/multimodal_dev/pretrain_multimodal.py new file mode 100644 index 00000000000..053fa00a5a2 --- /dev/null +++ b/examples/multimodal_dev/pretrain_multimodal.py @@ -0,0 +1,168 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Standalone entry point for multimodal_dev model training (FSDP + EP). + +This entry point is **model-agnostic**. All model-specific logic (layer +specs, model construction, FLOPs metadata, dataset generation) is +delegated to factory functions registered in +:data:`multimodal_dev.models.MODEL_REGISTRY`. + +Adding a new architecture only requires: + +1. Creating a new model package under ``multimodal_dev/models//`` + with the appropriate factory functions. +2. Registering an entry in ``MODEL_REGISTRY``. + +No changes to this file are necessary. + +Usage:: + + torchrun --nproc_per_node=8 multimodal_dev/pretrain_multimodal.py \\ + --model-arch qwen35_vl \\ + --dataset-provider mock \\ + ... (other megatron args) +""" + +import importlib +import os +import sys + +sys.path.insert( + 0, + os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")), +) + +from examples.multimodal_dev.arguments import add_multimodal_args +from examples.multimodal_dev.forward_step import forward_step +from megatron.core.enums import ModelType +from megatron.training import get_args, pretrain +from megatron.training.argument_utils import pretrain_cfg_container_from_args +from megatron.training.arguments import core_transformer_config_from_args, parse_and_validate_args + + +def model_provider( + pre_process: bool = True, + post_process: bool = True, + **kwargs, +): + """Build a multimodal model from ``--model-arch``. + + The language ``TransformerConfig`` is built from CLI args so that + parallelism settings, precision, and fusion flags are inherited. + Model-specific post-processing and construction are delegated to the + registry factory functions. + """ + args = get_args() + model_arch = getattr(args, "model_arch", "qwen35_vl") + + from examples.multimodal_dev.models import MODEL_REGISTRY + + if model_arch not in MODEL_REGISTRY: + raise ValueError( + f"Unknown model arch '{model_arch}'. " + f"Available: {list(MODEL_REGISTRY.keys())}" + ) + + registry = MODEL_REGISTRY[model_arch] + + # --- language config (generic + model-specific post-processing) --- + language_config = core_transformer_config_from_args(args) + post_language_config_fn = registry.get("post_language_config_fn") + if post_language_config_fn is not None: + post_language_config_fn(language_config, args) + + # --- vision config --- + vision_config = registry["vision_config_fn"]( + num_layers_override=getattr(args, "vision_num_layers", None), + variant=getattr(args, "model_variant", None), + ) + vision_config.bf16 = language_config.bf16 + vision_config.fp16 = language_config.fp16 + + if getattr(args, "recompute_vision", False): + vision_config.recompute_granularity = "full" + vision_config.recompute_method = "uniform" + vision_config.recompute_num_layers = 1 + + # --- vision FLOPs metadata --- + vision_flops_fn = registry.get("vision_flops_fn") + if vision_flops_fn is not None: + vision_flops_fn(args, language_config, vision_config) + + # --- build model (fully delegated to the arch factory) --- + model = registry["model_factory_fn"]( + args=args, + language_config=language_config, + vision_config=vision_config, + **kwargs, + ) + + return model + + +def _resolve_provider_fn(provider_fn): + """Resolve a provider that may be a dotted import path string.""" + if isinstance(provider_fn, str): + module_path, func_name = provider_fn.rsplit(".", 1) + provider_fn = getattr( + importlib.import_module(module_path), func_name, + ) + return provider_fn + + +def datasets_provider(train_val_test_num_samples): + """Dataset provider dispatcher. + + Routes to the dataset factory registered for the current + ``(--model-arch, --dataset-provider)`` combination. + """ + args = get_args() + model_arch = getattr(args, "model_arch", "qwen35_vl") + provider = getattr(args, "dataset_provider", "mock") + + from examples.multimodal_dev.models import MODEL_REGISTRY + + if model_arch not in MODEL_REGISTRY: + raise ValueError( + f"Unknown model arch '{model_arch}'. " + f"Available: {list(MODEL_REGISTRY.keys())}" + ) + + registry = MODEL_REGISTRY[model_arch] + available = registry.get("dataset_providers", {}) + + if provider not in available: + raise ValueError( + f"Unknown dataset provider '{provider}' for arch " + f"'{model_arch}'. Available: {list(available.keys())}" + ) + + provider_fn = _resolve_provider_fn(available[provider]) + return provider_fn(train_val_test_num_samples) + + +if __name__ == "__main__": + datasets_provider.is_distributed = True + + args = parse_and_validate_args( + extra_args_provider=add_multimodal_args, + args_defaults={}, + ) + # multimodal_dev's model_provider builds the full model on every rank and + # does not honor pre_process / post_process pipeline-stage flags. PP>1 + # would silently violate Megatron's pipeline-parallel contract. + if args.pipeline_model_parallel_size > 1: + raise ValueError( + "multimodal_dev does not support pipeline_model_parallel_size > 1 " + f"(got {args.pipeline_model_parallel_size}). The model provider " + "builds the full model on every rank; pipeline-stage splitting is " + "not wired through. Run with --pipeline-model-parallel-size 1." + ) + full_config = pretrain_cfg_container_from_args(args) + pretrain( + full_config, + datasets_provider, + model_provider, + ModelType.encoder_or_decoder, + forward_step, + ) diff --git a/examples/multimodal_dev/scripts/run_qwen35_vl.sh b/examples/multimodal_dev/scripts/run_qwen35_vl.sh new file mode 100755 index 00000000000..3a1ca55c826 --- /dev/null +++ b/examples/multimodal_dev/scripts/run_qwen35_vl.sh @@ -0,0 +1,526 @@ +#!/bin/bash + +# Launch script for Qwen3.5-VL training via multimodal_dev (FSDP + EP). +# +# Usage (from the Megatron-LM repo root): +# ./examples/multimodal_dev/scripts/run_qwen35_vl.sh +# +# Environment variables: +# MODEL_VARIANT: proxy (default), 0.8b, 2b, 4b, 9b, 27b, 35b_a3b, 122b_a10b, 397b_a17b, 35b_a3b_light +# CKPT_LOAD: path to a pre-converted checkpoint to load (enables --load + --finetune) +# CKPT_FORMAT: checkpoint format override (e.g. torch_dist); auto-detected when empty +# TP, EP, PP: parallelism sizes (PP must stay 1; multimodal_dev does not +# support pipeline parallelism) +# MBS, GBS: micro/global batch sizes +# NUM_LAYERS, NUM_EXPERTS: override for proxy testing +# FORCE_LOAD_BALANCING: set to 1 to enable --moe-router-force-load-balancing +# (perf / mock-data only; OFF for real finetuning) +# LAUNCHER: torchrun (default) or python +# PROFILE: set to 1 to enable Nsight Systems profiling (default: 0) +# PROFILE_STEP_START/PROFILE_STEP_END: profiled iteration window (default: 4-5) + +# example script: +# DRY_RUN=0 MODEL_VARIANT=proxy USE_PACKED_SEQUENCE=1 bash ./examples/multimodal_dev/scripts/run_qwen35_vl.sh + +set -euo pipefail + +export CUDA_DEVICE_MAX_CONNECTIONS=1 +export NCCL_IB_SL=1 +export NVTE_FUSED_ATTN=1 +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + +DRY_RUN=${DRY_RUN:-1} +GPUS_PER_NODE=${GPUS_PER_NODE:-8} +if [ -n "${SLURM_JOB_NUM_NODES:-}" ]; then + NUM_NODES="$SLURM_JOB_NUM_NODES" +else + NUM_NODES=${NNODES:-1} +fi +PROFILE=${PROFILE:-0} +PROFILE_STEP_START=${PROFILE_STEP_START:-4} +PROFILE_STEP_END=${PROFILE_STEP_END:-5} +PROFILE_RANKS=${PROFILE_RANKS:-0} +LAUNCHER=${LAUNCHER:-torchrun} + +MODEL_VARIANT=${MODEL_VARIANT:-proxy} +VISION_NUM_LAYERS=${VISION_NUM_LAYERS:-} + +# Batch sizes +MBS=${MBS:-2} +GBS=${GBS:-16} + +# Parallelism +TP=${TP:-1} +# EP defaults to 1; MoE variants override via the variant case block below. +EP=${EP:-1} +PP=${PP:-1} +CP=${CP:-1} +# Gate --moe-router-force-load-balancing behind an explicit opt-in. Useful for +# perf / mock-data benchmarking (it disables the auxiliary load-balancing loss +# coupling so router routes are perfectly uniform), but it must be off for any +# real fine-tuning / convergence run because it freezes data-dependent routing. +FORCE_LOAD_BALANCING=${FORCE_LOAD_BALANCING:-0} + +# Variant-aware architecture defaults. +# The model provider builds configs from the variant dict in +# multimodal_dev/models/qwen35_vl/configuration.py, but Megatron also +# uses these CLI args internally (PP splits, param counting). +case "$MODEL_VARIANT" in + 0.8b) + NUM_LAYERS=${NUM_LAYERS:-24} + NUM_EXPERTS=${NUM_EXPERTS:-0} + HIDDEN_SIZE=1024 + FFN_HIDDEN_SIZE=3584 + NUM_ATTN_HEADS=8 + NUM_QUERY_GROUPS=2 + LINEAR_NUM_VALUE_HEADS=16 + VISION_NUM_LAYERS=${VISION_NUM_LAYERS:-12} + ;; + 2b) + NUM_LAYERS=${NUM_LAYERS:-24} + NUM_EXPERTS=${NUM_EXPERTS:-0} + HIDDEN_SIZE=2048 + FFN_HIDDEN_SIZE=6144 + NUM_ATTN_HEADS=8 + NUM_QUERY_GROUPS=2 + LINEAR_NUM_VALUE_HEADS=16 + VISION_NUM_LAYERS=${VISION_NUM_LAYERS:-24} + ;; + 4b) + NUM_LAYERS=${NUM_LAYERS:-32} + NUM_EXPERTS=${NUM_EXPERTS:-0} + HIDDEN_SIZE=2560 + FFN_HIDDEN_SIZE=9216 + NUM_ATTN_HEADS=16 + NUM_QUERY_GROUPS=4 + LINEAR_NUM_VALUE_HEADS=32 + VISION_NUM_LAYERS=${VISION_NUM_LAYERS:-24} + ;; + proxy) + NUM_LAYERS=${NUM_LAYERS:-4} + NUM_EXPERTS=${NUM_EXPERTS:-16} + HIDDEN_SIZE=4096 + FFN_HIDDEN_SIZE=10240 + NUM_ATTN_HEADS=32 + NUM_QUERY_GROUPS=2 + LINEAR_NUM_VALUE_HEADS=64 + VISION_NUM_LAYERS=${VISION_NUM_LAYERS:-2} + ;; + 9b) + NUM_LAYERS=${NUM_LAYERS:-32} + NUM_EXPERTS=${NUM_EXPERTS:-0} + HIDDEN_SIZE=4096 + FFN_HIDDEN_SIZE=12288 + NUM_ATTN_HEADS=16 + NUM_QUERY_GROUPS=4 + LINEAR_NUM_VALUE_HEADS=32 + VISION_NUM_LAYERS=${VISION_NUM_LAYERS:-27} + ;; + 27b) + NUM_LAYERS=${NUM_LAYERS:-64} + NUM_EXPERTS=${NUM_EXPERTS:-0} + HIDDEN_SIZE=5120 + FFN_HIDDEN_SIZE=17408 + NUM_ATTN_HEADS=24 + NUM_QUERY_GROUPS=4 + LINEAR_NUM_VALUE_HEADS=48 + VISION_NUM_LAYERS=${VISION_NUM_LAYERS:-27} + ;; + 35b_a3b) + NUM_LAYERS=${NUM_LAYERS:-40} + NUM_EXPERTS=${NUM_EXPERTS:-256} + HIDDEN_SIZE=2048 + FFN_HIDDEN_SIZE=4096 + NUM_ATTN_HEADS=16 + NUM_QUERY_GROUPS=2 + LINEAR_NUM_VALUE_HEADS=32 + VISION_NUM_LAYERS=${VISION_NUM_LAYERS:-27} + ;; + 35b_a3b_light) + NUM_LAYERS=${NUM_LAYERS:-12} + NUM_EXPERTS=${NUM_EXPERTS:-128} + HIDDEN_SIZE=2048 + FFN_HIDDEN_SIZE=4096 + NUM_ATTN_HEADS=16 + NUM_QUERY_GROUPS=2 + LINEAR_NUM_VALUE_HEADS=32 + VISION_NUM_LAYERS=${VISION_NUM_LAYERS:-7} + ;; + 122b_a10b) + NUM_LAYERS=${NUM_LAYERS:-48} + NUM_EXPERTS=${NUM_EXPERTS:-256} + HIDDEN_SIZE=3072 + FFN_HIDDEN_SIZE=8192 + NUM_ATTN_HEADS=32 + NUM_QUERY_GROUPS=2 + LINEAR_NUM_VALUE_HEADS=64 + VISION_NUM_LAYERS=${VISION_NUM_LAYERS:-27} + ;; + 397b_a17b) + NUM_LAYERS=${NUM_LAYERS:-60} + NUM_EXPERTS=${NUM_EXPERTS:-512} + HIDDEN_SIZE=4096 + FFN_HIDDEN_SIZE=10240 + NUM_ATTN_HEADS=32 + NUM_QUERY_GROUPS=2 + LINEAR_NUM_VALUE_HEADS=64 + VISION_NUM_LAYERS=${VISION_NUM_LAYERS:-27} + ;; + *) + : "${NUM_LAYERS:?NUM_LAYERS must be set for MODEL_VARIANT=$MODEL_VARIANT}" + : "${NUM_EXPERTS:?NUM_EXPERTS must be set for MODEL_VARIANT=$MODEL_VARIANT}" + : "${HIDDEN_SIZE:?HIDDEN_SIZE must be set for MODEL_VARIANT=$MODEL_VARIANT}" + : "${FFN_HIDDEN_SIZE:?FFN_HIDDEN_SIZE must be set for MODEL_VARIANT=$MODEL_VARIANT}" + : "${NUM_ATTN_HEADS:?NUM_ATTN_HEADS must be set for MODEL_VARIANT=$MODEL_VARIANT}" + : "${NUM_QUERY_GROUPS:?NUM_QUERY_GROUPS must be set for MODEL_VARIANT=$MODEL_VARIANT}" + : "${LINEAR_NUM_VALUE_HEADS:?LINEAR_NUM_VALUE_HEADS must be set for MODEL_VARIANT=$MODEL_VARIANT}" + VISION_NUM_LAYERS=${VISION_NUM_LAYERS:-27} + ;; +esac + +# Fail fast on inconsistent expert-parallelism configuration. Dense variants +# (NUM_EXPERTS=0) do not emit any --num-experts / MoE args, so forwarding +# --expert-model-parallel-size > 1 would trip Megatron's arg validation. +if [ "${NUM_EXPERTS:-0}" -eq 0 ] && [ "$EP" -gt 1 ]; then + echo "ERROR: MODEL_VARIANT=$MODEL_VARIANT has NUM_EXPERTS=0 (dense) but EP=$EP." >&2 + echo " Set EP=1 for dense variants, or pick a MoE variant." >&2 + exit 1 +fi + +SEQ_LEN=${SEQ_LEN:-4096} + +WANDB_PROJECT=${WANDB_PROJECT:-'qwen35-vl-0524'} +EXP_NAME="qwen35vl_${MODEL_VARIANT}_tp${TP}_ep${EP}_pp${PP}_cp${CP}" + +RECOMPUTE_VISION=${RECOMPUTE_VISION:-0} +if [ "$RECOMPUTE_VISION" -eq 1 ]; then + EXP_NAME+="_recompute_encoder" +fi +RECOMPUTE=${RECOMPUTE:-0} +if [ "$RECOMPUTE" -eq 1 ]; then + EXP_NAME+="_recompute_decoder" +fi + +USE_PACKED_SEQUENCE=${USE_PACKED_SEQUENCE:-0} +if [ "$USE_PACKED_SEQUENCE" -eq 1 ]; then + EXP_NAME+="_thd" +fi + +MEGATRON_LM_PATH="${MEGATRON_LM_PATH:-$(cd "$(dirname "$0")/../../.." && pwd)}" +ROOT_DIR="${ROOT_DIR:-${MEGATRON_LM_PATH}/local/}" +CHECKPOINT_STORE_PATH="${ROOT_DIR}${EXP_NAME}" +mkdir -p "$CHECKPOINT_STORE_PATH" + +TENSORBOARD_LOGS_PATH="${TENSORBOARD_LOGS_PATH:-${MEGATRON_LM_PATH}/logs}" +mkdir -p "$TENSORBOARD_LOGS_PATH" + +DISTRIBUTED_ARGS=( + --nproc_per_node "$GPUS_PER_NODE" + --nnodes "$NUM_NODES" +) + +if [ "$NUM_NODES" -gt 1 ]; then + DISTRIBUTED_ARGS+=( + --master_addr "${MASTER_ADDR:-localhost}" + --master_port "${MASTER_PORT:-6000}" + ) +fi + +# --- Parallelism --- +MODEL_PARALLEL_ARGS=( + --tensor-model-parallel-size "$TP" + --pipeline-model-parallel-size "$PP" + --expert-model-parallel-size "$EP" + --context-parallel-size "$CP" + --cp-comm-type "a2a" + --expert-tensor-parallel-size 1 + --use-distributed-optimizer + --sequence-parallel +) + +# --- Training --- +TRAINING_ARGS=( + --micro-batch-size "$MBS" + --global-batch-size "$GBS" + --train-iters "${TRAIN_ITERS:-500}" + --adam-beta1 0.9 + --adam-beta2 0.95 + --lr 1.2e-4 + --min-lr 1.2e-5 + --lr-decay-style cosine + --lr-warmup-iters 100 + --lr-decay-iters 2000 + --weight-decay 0.1 + --clip-grad 1.0 + --bf16 + --use-mcore-models + --transformer-impl transformer_engine + --cross-entropy-loss-fusion + --cross-entropy-fusion-impl te + --enable-experimental + --manual-gc + --manual-gc-interval 50 + --mtp-num-layers 1 + --mtp-loss-scaling-factor 0.1 + --sft + --use-flash-attn + # --attention-backend flash + --calculate-per-token-loss +) + +PROFILE_ARGS=() +NSYS_CMD=() +if [ "$PROFILE" = "1" ]; then + PROFILE_ARGS=( + --profile + --profile-step-start "$PROFILE_STEP_START" + --profile-step-end "$PROFILE_STEP_END" + --profile-ranks "$PROFILE_RANKS" + ) + + NSYS_OUTPUT_DIR="${CHECKPOINT_STORE_PATH}/nsys" + mkdir -p "$NSYS_OUTPUT_DIR" + NSYS_CMD=( + nsys profile + --sample=none + --cpuctxsw=none + --trace=cuda,nvtx,cublas,cudnn + --force-overwrite=true + --capture-range=cudaProfilerApi + --capture-range-end=stop + -o "${NSYS_OUTPUT_DIR}/${EXP_NAME}_$(date +%Y%m%d_%H%M%S)" + ) +fi + +# --- Logging & Checkpointing --- +SAVE_INTERVAL=${SAVE_INTERVAL:-500} +EVAL_AND_LOGGING_ARGS=( + --log-interval 1 + --save-interval "$SAVE_INTERVAL" + --eval-interval 500 + --save "$CHECKPOINT_STORE_PATH" + --eval-iters 10 + --tensorboard-dir "$TENSORBOARD_LOGS_PATH" + --wandb-project "$WANDB_PROJECT" + --wandb-exp-name "$EXP_NAME" + --wandb-save-dir "$CHECKPOINT_STORE_PATH" + --log-throughput + --log-timers-to-tensorboard + --log-params-norm +) + +# --- Tokenizer --- +TOKENIZER_MODEL=${TOKENIZER_MODEL:-Qwen/Qwen3.5-397B-A17B} +TOKENIZER_ARGS=( + --tokenizer-type HuggingFaceTokenizer + --tokenizer-model "$TOKENIZER_MODEL" +) + +# --- Multimodal-specific --- +MULTIMODAL_ARGS=( + --model-arch qwen35_vl + --model-variant "$MODEL_VARIANT" + --dataset-provider cord_v2 + --hf-processor-path Qwen/Qwen3.5-397B-A17B + --use-vanilla-collate-fn + --image-token-id 248056 + --image-size 224 + --total-seq-length "$SEQ_LEN" + --image-seq-length 256 + --vision-num-layers "$VISION_NUM_LAYERS" +) + +if [ "$USE_PACKED_SEQUENCE" -eq 1 ]; then + MULTIMODAL_ARGS+=( --use-packed-sequence ) +fi + +# --- Qwen3.5 Decoder Architecture (variant-specific dims set above) --- +# These must match examples/multimodal_dev/models/qwen35_vl/configuration.py +GPT_MODEL_ARGS=( + --num-layers "$NUM_LAYERS" + --hidden-size "$HIDDEN_SIZE" + --ffn-hidden-size "$FFN_HIDDEN_SIZE" + --num-attention-heads "$NUM_ATTN_HEADS" + --group-query-attention + --num-query-groups "$NUM_QUERY_GROUPS" + --kv-channels 256 + --max-position-embeddings 262144 + --seq-length "$SEQ_LEN" + --normalization RMSNorm + --apply-layernorm-1p + --norm-epsilon 1e-06 + --swiglu + --disable-bias-linear + --position-embedding-type rope + --rotary-percent 0.25 + --rotary-base 10000000 + --rotary-seq-len-interpolation-factor 1 + --qk-layernorm + --attention-output-gate + --attention-dropout 0.0 + --hidden-dropout 0.0 + --experimental-attention-variant gated_delta_net + --linear-attention-freq 4 + --linear-conv-kernel-dim 4 + --linear-key-head-dim 128 + --linear-value-head-dim 128 + --linear-num-key-heads 16 + --linear-num-value-heads "$LINEAR_NUM_VALUE_HEADS" + --make-vocab-size-divisible-by 485 +) + +# --- Tied / untied embeddings --- +# 0.8B, 2B, 4B use tied embeddings; all other variants untie them. +case "$MODEL_VARIANT" in + 0.8b|2b|4b) ;; + *) GPT_MODEL_ARGS+=( --untie-embeddings-and-output-weights ) ;; +esac + +# --- MoE args (MoE variants only) --- +MOE_ARGS=() +case "$MODEL_VARIANT" in + proxy) + MOE_TOPK=2; MOE_FFN_HIDDEN=1024; MOE_SHARED_HIDDEN=1024 + ;; + 35b_a3b|35b_a3b_light) + MOE_TOPK=8; MOE_FFN_HIDDEN=512; MOE_SHARED_HIDDEN=512 + ;; + 122b_a10b) + MOE_TOPK=8; MOE_FFN_HIDDEN=1024; MOE_SHARED_HIDDEN=1024 + ;; + 397b_a17b) + MOE_TOPK=10; MOE_FFN_HIDDEN=1024; MOE_SHARED_HIDDEN=1024 + ;; + 0.8b|2b|4b|9b|27b) + ;; +esac +if [ "${NUM_EXPERTS:-0}" -gt 0 ]; then + MOE_ARGS=( + --num-experts "$NUM_EXPERTS" + --moe-ffn-hidden-size "$MOE_FFN_HIDDEN" + --moe-shared-expert-intermediate-size "$MOE_SHARED_HIDDEN" + --moe-shared-expert-gate + --moe-router-load-balancing-type aux_loss + --moe-router-topk "$MOE_TOPK" + --moe-grouped-gemm + --moe-aux-loss-coeff 1e-3 + --moe-token-dispatcher-type alltoall + --moe-router-dtype fp32 + --moe-permute-fusion + --moe-router-fusion + ) + # Perf / mock-data only: forces uniform router decisions; do NOT enable for + # real finetuning (it freezes data-dependent routing). + if [ "$FORCE_LOAD_BALANCING" -eq 1 ]; then + MOE_ARGS+=( --moe-router-force-load-balancing ) + fi +fi + +# --- Recompute --- +if [ "$RECOMPUTE" -eq 1 ]; then + RECOMPUTE_ARGS=( + --recompute-granularity full + --recompute-method uniform + --recompute-num-layers 1 + ) + # RECOMPUTE_ARGS=( + # --recompute-granularity selective + # --recompute-modules moe_act shared_experts layernorm moe + # ) +else + RECOMPUTE_ARGS=() +fi +if [ "$RECOMPUTE_VISION" -eq 1 ]; then + RECOMPUTE_ARGS+=( --recompute-vision ) +fi + +# --- Checkpoint loading --- +# CKPT_LOAD: path to checkpoint directory +# CKPT_FORMAT: override checkpoint format (default: auto-detect) +# CKPT_RESUME: set to 1 to resume training (keep iteration, optimizer, rng); +# default 0 = finetune mode (reset iteration, skip optim/rng) +CKPT_LOAD=${CKPT_LOAD:-} +CKPT_FORMAT=${CKPT_FORMAT:-} +CKPT_RESUME=${CKPT_RESUME:-0} +CKPT_OVERRIDE_SCHEDULER=${CKPT_OVERRIDE_SCHEDULER:-0} +CKPT_ARGS=() +if [ -n "$CKPT_LOAD" ]; then + CKPT_ARGS+=( --load "$CKPT_LOAD" ) + if [ "$CKPT_RESUME" -eq 0 ]; then + CKPT_ARGS+=( --finetune --no-load-optim --no-load-rng ) + fi + if [ -n "$CKPT_FORMAT" ]; then + CKPT_ARGS+=( --ckpt-format "$CKPT_FORMAT" ) + fi + if [ "$CKPT_OVERRIDE_SCHEDULER" -eq 1 ]; then + CKPT_ARGS+=( --override-opt-param-scheduler ) + fi +fi + +# --- FSDP --- +USE_FSDP=${USE_FSDP:-1} +if [ "$USE_FSDP" -eq 1 ]; then + FSDP_ARGS=( + --use-megatron-fsdp + --data-parallel-sharding-strategy optim_grads_params + --init-model-with-meta-device + --use-distributed-optimizer + --ckpt-format fsdp_dtensor + ) + export CUDA_DEVICE_MAX_CONNECTIONS=8 +else + FSDP_ARGS=() +fi + +echo "================================================================" +echo "Qwen3.5-VL Multimodal Training (multimodal_dev)" +echo " Variant: $MODEL_VARIANT" +echo " Vision layers: $VISION_NUM_LAYERS" +echo " GPUs per node: $GPUS_PER_NODE" +echo " Num nodes: $NUM_NODES" +echo " TP=$TP EP=$EP PP=$PP CP=$CP" +echo " MBS=$MBS GBS=$GBS" +echo " Launcher: $LAUNCHER" +echo " FSDP: $USE_FSDP" +echo " PROFILE: $PROFILE" +if [ -n "$CKPT_LOAD" ]; then + echo " CKPT_LOAD: $CKPT_LOAD" + echo " CKPT_FORMAT: ${CKPT_FORMAT:-auto}" + echo " CKPT_RESUME: $CKPT_RESUME" +fi +if [ "$PROFILE" = "1" ]; then + echo " Profile steps: ${PROFILE_STEP_START}-${PROFILE_STEP_END}" + echo " Profile ranks: $PROFILE_RANKS" +fi +echo "================================================================" + +if [ "$LAUNCHER" = "python" ]; then + LAUNCH_CMD=( python $MEGATRON_LM_PATH/examples/multimodal_dev/pretrain_multimodal.py ) +elif [ "$LAUNCHER" = "torchrun" ]; then + LAUNCH_CMD=( torchrun "${DISTRIBUTED_ARGS[@]}" $MEGATRON_LM_PATH/examples/multimodal_dev/pretrain_multimodal.py ) +else + echo "Unsupported LAUNCHER=$LAUNCHER (expected torchrun or python)" >&2 + exit 1 +fi + +cmd=( "${NSYS_CMD[@]}" "${LAUNCH_CMD[@]}" \ + "${TRAINING_ARGS[@]}" \ + "${PROFILE_ARGS[@]}" \ + "${MODEL_PARALLEL_ARGS[@]}" \ + "${EVAL_AND_LOGGING_ARGS[@]}" \ + "${TOKENIZER_ARGS[@]}" \ + "${MULTIMODAL_ARGS[@]}" \ + "${GPT_MODEL_ARGS[@]}" \ + "${MOE_ARGS[@]}" \ + "${RECOMPUTE_ARGS[@]}" \ + "${FSDP_ARGS[@]}" \ + "${CKPT_ARGS[@]}" ) + +echo "${cmd[@]}" + +if [ "$DRY_RUN" -eq 1 ]; then + echo "=== DRY RUN ===" + exit 0 +else + "${cmd[@]}" +fi diff --git a/examples/multimodal_dev/tests/__init__.py b/examples/multimodal_dev/tests/__init__.py new file mode 100644 index 00000000000..26496bfed70 --- /dev/null +++ b/examples/multimodal_dev/tests/__init__.py @@ -0,0 +1 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. diff --git a/examples/multimodal_dev/tests/_helpers.py b/examples/multimodal_dev/tests/_helpers.py new file mode 100644 index 00000000000..b0c69207f3a --- /dev/null +++ b/examples/multimodal_dev/tests/_helpers.py @@ -0,0 +1,19 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Test helpers shared across the multimodal_dev test suite.""" + + +def grad_norm(model): + """L2 norm of all populated parameter gradients on this rank.""" + total = 0.0 + for p in model.parameters(): + if p.grad is not None: + total += p.grad.data.float().norm(2).item() ** 2 + return total**0.5 + + +def mean_loss(per_token_loss, loss_mask): + """Mean per-token loss over valid (mask>0) positions on this rank.""" + flat = per_token_loss.float().view(-1) + mask = loss_mask.float().view(-1) + return (flat * mask).sum() / mask.sum().clamp(min=1) diff --git a/examples/multimodal_dev/tests/test_cp_correctness.py b/examples/multimodal_dev/tests/test_cp_correctness.py new file mode 100644 index 00000000000..fe156ed54af --- /dev/null +++ b/examples/multimodal_dev/tests/test_cp_correctness.py @@ -0,0 +1,313 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Distributed correctness test for Context Parallelism (CP) support. + +Verifies that CP>1 produces the same (or numerically close) loss as CP=1 +for the Qwen3.5-VL multimodal model by running forward passes with +deterministic data and comparing the per-rank reduced losses. + +Launch with torchrun (N must be >= 2*max_cp_size for zigzag splitting): + + # Test CP=2 on 2 GPUs: + torchrun --nproc_per_node=2 examples/multimodal_dev/tests/test_cp_correctness.py --cp-size 2 + + # Test CP=4 on 4 GPUs: + torchrun --nproc_per_node=4 examples/multimodal_dev/tests/test_cp_correctness.py --cp-size 4 + +The test: + 1. Builds a tiny proxy model (2 layers, no MoE, no vision encoder). + 2. Generates a deterministic batch (same seed on all ranks). + 3. Runs forward with CP=1 (each rank processes the full sequence independently). + 4. Re-initialises model-parallel groups with the target CP size. + 5. Runs forward with CP=target (sequence is split across ranks). + 6. Compares the all-reduced loss values. + +Exit code 0 = PASS, 1 = FAIL. +""" + +import argparse +import os +import sys + +import torch +import torch.distributed as dist + +# Ensure the repo root is on the path so that megatron and examples are importable. +_REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../..")) +if _REPO_ROOT not in sys.path: + sys.path.insert(0, _REPO_ROOT) + + +def _parse_args(): + parser = argparse.ArgumentParser(description="CP correctness test") + parser.add_argument( + "--cp-size", type=int, default=2, + help="Target context-parallel size to compare against CP=1 baseline", + ) + parser.add_argument( + "--seq-len", type=int, default=128, + help="Sequence length (must be divisible by 2*max(cp_size, tp_size*cp_size))", + ) + parser.add_argument( + "--atol", type=float, default=1e-4, + help="Absolute tolerance for loss comparison", + ) + parser.add_argument( + "--rtol", type=float, default=5e-2, + help="Relative tolerance for loss comparison (default 5%%)", + ) + parser.add_argument( + "--seed", type=int, default=42, + help="Random seed for reproducibility", + ) + # Megatron adds extra args; ignore them. + args, _ = parser.parse_known_args() + return args + + +def _init_distributed(): + """Initialise torch.distributed if not already done.""" + if not dist.is_initialized(): + dist.init_process_group(backend="nccl") + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + torch.cuda.set_device(local_rank) + return local_rank + + +def _init_megatron_parallel(tp_size=1, pp_size=1, cp_size=1, seed=42): + """(Re-)initialise Megatron model-parallel groups and RNG tracker.""" + from megatron.core import parallel_state as ps + from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed + ps.destroy_model_parallel() + ps.initialize_model_parallel( + tensor_model_parallel_size=tp_size, + pipeline_model_parallel_size=pp_size, + context_parallel_size=cp_size, + ) + model_parallel_cuda_manual_seed(seed) + + +def _make_deterministic_batch(seed, batch_size, seq_len, vocab_size, device): + """Create a deterministic batch identical on all ranks.""" + rng = torch.Generator(device="cpu") + rng.manual_seed(seed) + + input_ids = torch.randint( + 0, vocab_size, (batch_size, seq_len), generator=rng, + ).to(device) + labels = torch.randint( + 0, vocab_size, (batch_size, seq_len), generator=rng, + ).to(device) + loss_mask = torch.ones(batch_size, seq_len, device=device) + # Standard position_ids [B, S] + position_ids = torch.arange(seq_len, device=device).unsqueeze(0).expand(batch_size, -1) + + return { + "input_ids": input_ids, + "labels": labels, + "loss_mask": loss_mask, + "position_ids": position_ids, + } + + +def _build_tiny_model(cp_size, device): + """Build a minimal GPTModel for testing (no vision, no MoE).""" + from megatron.core.models.gpt import GPTModel + from megatron.core.models.gpt.gpt_layer_specs import get_gpt_layer_with_transformer_engine_spec + from megatron.core.transformer.spec_utils import ModuleSpec + from megatron.core.transformer.transformer_config import TransformerConfig + + hidden_size = 256 + num_heads = 4 + config = TransformerConfig( + num_layers=2, + hidden_size=hidden_size, + ffn_hidden_size=hidden_size * 4, + num_attention_heads=num_heads, + kv_channels=hidden_size // num_heads, + normalization="RMSNorm", + layernorm_epsilon=1e-6, + gated_linear_unit=True, + activation_func=torch.nn.functional.silu, + bf16=True, + context_parallel_size=cp_size, + add_bias_linear=False, + attention_dropout=0.0, + hidden_dropout=0.0, + sequence_parallel=False, + ) + + spec = get_gpt_layer_with_transformer_engine_spec() + + model = GPTModel( + config=config, + transformer_layer_spec=spec, + vocab_size=1024, + max_sequence_length=4096, + pre_process=True, + post_process=True, + parallel_output=False, + share_embeddings_and_output_weights=True, + position_embedding_type="rope", + rotary_percent=1.0, + rotary_base=10000, + ) + model = model.to(device=device, dtype=torch.bfloat16) + return model, config + + +def _forward_with_cp(model, batch, cp_size): + """Run forward pass, handling CP splitting of the batch. + + When cp_size > 1, splits the batch tensors using the same zigzag + logic as multimodal_dev/models/base.py. + """ + from examples.multimodal_dev.models.base import _cp_split_tensor + from megatron.core import parallel_state as ps + + input_ids = batch["input_ids"].clone() + labels = batch["labels"].clone() + loss_mask = batch["loss_mask"].clone() + position_ids = batch["position_ids"].clone() + + if cp_size > 1: + cp_rank = ps.get_context_parallel_rank() + input_ids = _cp_split_tensor(input_ids, seq_dim=1, cp_size=cp_size, cp_rank=cp_rank) + labels = _cp_split_tensor(labels, seq_dim=1, cp_size=cp_size, cp_rank=cp_rank) + loss_mask = _cp_split_tensor(loss_mask, seq_dim=1, cp_size=cp_size, cp_rank=cp_rank) + # position_ids are NOT split — the RoPE layer handles CP slicing internally. + + with torch.no_grad(): + output = model( + input_ids=input_ids, + position_ids=position_ids, + labels=labels, + attention_mask=None, + ) + + # output is the per-token loss [B, S/CP] + masked_loss = (output.float() * loss_mask.float()).sum() + num_tokens = loss_mask.sum() + + # All-reduce across CP ranks to get global loss + if cp_size > 1: + cp_group = ps.get_context_parallel_group() + dist.all_reduce(masked_loss, group=cp_group) + dist.all_reduce(num_tokens, group=cp_group) + + avg_loss = masked_loss / num_tokens.clamp(min=1) + return avg_loss.item() + + +def main(): + args = _parse_args() + local_rank = _init_distributed() + device = torch.device(f"cuda:{local_rank}") + world_size = dist.get_world_size() + rank = dist.get_rank() + + target_cp = args.cp_size + if world_size < target_cp: + if rank == 0: + print( + f"SKIP: world_size={world_size} < cp_size={target_cp}. " + f"Need at least {target_cp} GPUs.", + flush=True, + ) + dist.destroy_process_group() + sys.exit(0) + if world_size % target_cp != 0: + if rank == 0: + print( + f"SKIP: world_size={world_size} is not divisible by cp_size={target_cp}.", + flush=True, + ) + dist.destroy_process_group() + sys.exit(0) + + vocab_size = 1024 + + # Ensure seq_len is divisible by 2 * target_cp + seq_len = args.seq_len + align = 2 * target_cp + if seq_len % align != 0: + seq_len = ((seq_len + align - 1) // align) * align + if rank == 0: + print(f"Adjusted seq_len to {seq_len} for alignment with CP={target_cp}", flush=True) + + # --- Step 1: CP=1 baseline --- + if rank == 0: + print(f"=== CP=1 baseline (world_size={world_size}) ===", flush=True) + + _init_megatron_parallel(cp_size=1) + + # Set deterministic seed for model init + torch.manual_seed(args.seed) + torch.cuda.manual_seed_all(args.seed) + model_cp1, _ = _build_tiny_model(cp_size=1, device=device) + + batch = _make_deterministic_batch( + seed=args.seed + 1, batch_size=1, seq_len=seq_len, + vocab_size=vocab_size, device=device, + ) + + loss_cp1 = _forward_with_cp(model_cp1, batch, cp_size=1) + + if rank == 0: + print(f" CP=1 loss: {loss_cp1:.6f}", flush=True) + + # Save model state for reuse + state_dict = model_cp1.state_dict() + del model_cp1 + torch.cuda.empty_cache() + + # --- Step 2: CP=target --- + if rank == 0: + print(f"=== CP={target_cp} (world_size={world_size}) ===", flush=True) + + _init_megatron_parallel(cp_size=target_cp) + + torch.manual_seed(args.seed) + torch.cuda.manual_seed_all(args.seed) + model_cpN, _ = _build_tiny_model(cp_size=target_cp, device=device) + + # Load the same weights to ensure identical model + model_cpN.load_state_dict(state_dict, strict=True) + del state_dict + + loss_cpN = _forward_with_cp(model_cpN, batch, cp_size=target_cp) + + if rank == 0: + print(f" CP={target_cp} loss: {loss_cpN:.6f}", flush=True) + + del model_cpN + torch.cuda.empty_cache() + + # --- Step 3: Compare --- + if rank == 0: + diff = abs(loss_cpN - loss_cp1) + rel_diff = diff / max(abs(loss_cp1), 1e-10) + + print(f"\n=== Comparison ===", flush=True) + print(f" CP=1 loss: {loss_cp1:.6f}", flush=True) + print(f" CP={target_cp} loss: {loss_cpN:.6f}", flush=True) + print(f" Absolute diff: {diff:.6e}", flush=True) + print(f" Relative diff: {rel_diff:.6e}", flush=True) + print(f" Tolerance (atol): {args.atol:.6e}", flush=True) + print(f" Tolerance (rtol): {args.rtol:.6e}", flush=True) + + passed = diff <= args.atol + args.rtol * abs(loss_cp1) + if passed: + print(f"\nPASS: CP={target_cp} matches CP=1 baseline", flush=True) + else: + print(f"\nFAIL: CP={target_cp} loss differs from CP=1 beyond tolerance", flush=True) + + dist.barrier() + dist.destroy_process_group() + + if rank == 0 and not passed: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/examples/multimodal_dev/tests/test_cp_support.py b/examples/multimodal_dev/tests/test_cp_support.py new file mode 100644 index 00000000000..d46b5d8ef71 --- /dev/null +++ b/examples/multimodal_dev/tests/test_cp_support.py @@ -0,0 +1,347 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Unit tests for Context Parallelism (CP) support in multimodal_dev. + +Tests cover: + 1. _cp_split_tensor — zigzag split correctness, reconstruction, and edge cases + 2. _NoCPGroup — dummy process group behaviour + 3. _thd_cp_partition_index — TE-based per-sample THD CP partitioning + 4. Cross-validation against megatron.core.utils.get_batch_on_this_cp_rank + +Run with: pytest examples/multimodal_dev/tests/test_cp_support.py -v +""" + +import pytest +import torch + +from examples.multimodal_dev.models.base import _cp_split_tensor, _NoCPGroup + + +class TestCpSplitTensor: + """Tests for zigzag CP splitting.""" + + def test_basic_2d_cp2(self): + """[B, S] tensor with CP=2 splits and reconstructs correctly.""" + B, S = 2, 16 + t = torch.arange(B * S).reshape(B, S) + cp_size = 2 + + chunks = [] + for rank in range(cp_size): + chunks.append(_cp_split_tensor(t, seq_dim=1, cp_size=cp_size, cp_rank=rank)) + + # Each rank gets S / CP = 8 tokens + for c in chunks: + assert c.shape == (B, S // cp_size) + + # Reconstruct: rank 0 gets chunks [0, 3], rank 1 gets chunks [1, 2] + # Original split into 4 chunks of size 4: + # chunk0=[0..3], chunk1=[4..7], chunk2=[8..11], chunk3=[12..15] + # rank0 = [chunk0, chunk3] = [0..3, 12..15] + # rank1 = [chunk1, chunk2] = [4..7, 8..11] + assert torch.equal(chunks[0][0], torch.tensor([0, 1, 2, 3, 12, 13, 14, 15])) + assert torch.equal(chunks[1][0], torch.tensor([4, 5, 6, 7, 8, 9, 10, 11])) + + def test_3d_mrope_cp2(self): + """[3, B, S] MRoPE tensor with CP=2.""" + B, S = 1, 8 + cp_size = 2 + t = torch.arange(3 * B * S).reshape(3, B, S) + + chunk = _cp_split_tensor(t, seq_dim=2, cp_size=cp_size, cp_rank=0) + assert chunk.shape == (3, B, S // cp_size) + + # All 3 MRoPE components should be split consistently + for d in range(3): + original_row = t[d, 0] # [S] + # With S=8, CP=2: 4 chunks of size 2 + # rank0 gets chunks [0, 3] = positions [0,1, 6,7] + expected = torch.cat([original_row[0:2], original_row[6:8]]) + assert torch.equal(chunk[d, 0], expected) + + def test_sbh_decoder_input(self): + """[S, B, H] decoder input split along dim=0.""" + S, B, H = 16, 2, 4 + cp_size = 2 + t = torch.randn(S, B, H) + + chunk = _cp_split_tensor(t, seq_dim=0, cp_size=cp_size, cp_rank=0) + assert chunk.shape == (S // cp_size, B, H) + + def test_cp4(self): + """CP=4 zigzag pattern.""" + S = 32 + cp_size = 4 + t = torch.arange(S).unsqueeze(0) # [1, 32] + + all_chunks = [] + for rank in range(cp_size): + c = _cp_split_tensor(t, seq_dim=1, cp_size=cp_size, cp_rank=rank) + all_chunks.append(c) + assert c.shape == (1, S // cp_size) + + # All tokens should appear exactly once across ranks + combined = torch.cat(all_chunks, dim=1) + assert torch.equal(combined.sort(dim=1).values, t.sort(dim=1).values) + + def test_cp8(self): + """CP=8 zigzag pattern — all tokens appear exactly once.""" + S = 64 + cp_size = 8 + t = torch.arange(S).unsqueeze(0) # [1, 64] + + all_chunks = [] + for rank in range(cp_size): + c = _cp_split_tensor(t, seq_dim=1, cp_size=cp_size, cp_rank=rank) + all_chunks.append(c) + assert c.shape == (1, S // cp_size) + + combined = torch.cat(all_chunks, dim=1) + assert torch.equal(combined.sort(dim=1).values, t.sort(dim=1).values) + + def test_not_divisible_raises(self): + """Should raise when seq_len not divisible by 2*cp_size.""" + t = torch.randn(2, 10) # S=10, not divisible by 4 + with pytest.raises(AssertionError): + _cp_split_tensor(t, seq_dim=1, cp_size=2, cp_rank=0) + + def test_zigzag_symmetry(self): + """rank 0 and rank (cp_size-1) should get mirror chunks.""" + S = 16 + cp_size = 2 + t = torch.arange(S).unsqueeze(0) # [1, 16] + + c0 = _cp_split_tensor(t, seq_dim=1, cp_size=cp_size, cp_rank=0) + c1 = _cp_split_tensor(t, seq_dim=1, cp_size=cp_size, cp_rank=1) + + # rank0 gets chunks [0, 3], rank1 gets chunks [1, 2] + # chunk0=[0..3], chunk3=[12..15] -> rank0 gets [0..3, 12..15] + # chunk1=[4..7], chunk2=[8..11] -> rank1 gets [4..7, 8..11] + # rank0's first half is earliest, rank1's first half is next + assert c0[0, 0].item() < c1[0, 0].item() # rank0 starts earlier + + def test_matches_megatron_core(self): + """Cross-validate against megatron.core.utils.get_batch_on_this_cp_rank logic. + + We simulate the core function's logic (seq_dim=1, attention_mask seq_dim=2) + and compare. + """ + B, S = 2, 32 + cp_size = 4 + + input_ids = torch.arange(B * S).reshape(B, S) + labels = torch.arange(B * S).reshape(B, S) + 1000 + + for cp_rank in range(cp_size): + # Our implementation + our_ids = _cp_split_tensor(input_ids, seq_dim=1, cp_size=cp_size, cp_rank=cp_rank) + our_labels = _cp_split_tensor(labels, seq_dim=1, cp_size=cp_size, cp_rank=cp_rank) + + # Simulate megatron core logic inline + def core_split(val, seq_dim): + val = val.view( + *val.shape[0:seq_dim], + 2 * cp_size, + val.shape[seq_dim] // (2 * cp_size), + *val.shape[(seq_dim + 1):], + ) + index = torch.zeros(2, dtype=torch.int64, device=val.device) + index[0].fill_(cp_rank) + index[1].fill_(2 * cp_size - cp_rank - 1) + val = val.index_select(seq_dim, index) + val = val.view(*val.shape[0:seq_dim], -1, *val.shape[(seq_dim + 2):]) + return val + + ref_ids = core_split(input_ids.clone(), seq_dim=1) + ref_labels = core_split(labels.clone(), seq_dim=1) + + assert torch.equal(our_ids, ref_ids), f"input_ids mismatch at rank {cp_rank}" + assert torch.equal(our_labels, ref_labels), f"labels mismatch at rank {cp_rank}" + + def test_batch_dim_preserved(self): + """Batch dimension must be unchanged after split.""" + B, S = 4, 32 + cp_size = 4 + t = torch.randn(B, S) + + for rank in range(cp_size): + c = _cp_split_tensor(t, seq_dim=1, cp_size=cp_size, cp_rank=rank) + assert c.shape[0] == B + + +class TestNoCPGroup: + """Tests for the dummy CP group used by the vision encoder.""" + + def test_size_is_one(self): + g = _NoCPGroup() + assert g.size() == 1 + + def test_rank_is_zero(self): + g = _NoCPGroup() + assert g.rank() == 0 + + +try: + from transformer_engine.pytorch import cpp_extensions as _tex # noqa: F401 + + _HAS_TE = True +except Exception: + _HAS_TE = False + + +@pytest.mark.skipif(not _HAS_TE, reason="TransformerEngine not installed") +class TestThdCpPartition: + """Verify TE-based per-sample THD + CP partition matches THD semantics. + + Each packed sub-sample of length ``s_i`` (where ``s_i % (2*cp_size) == 0``) + is split into ``2*cp_size`` zigzag chunks per sample; rank ``r`` gets + chunks ``[r, 2*cp_size - r - 1]`` of every sample. The union across + ranks must cover every token position exactly once. + """ + + @staticmethod + def _make_padded_packed(seqlens, divisor): + """Concatenate per-sample dummy tokens after padding each sample to a + multiple of *divisor*. Returns ``(input_ids[1, T], cu_seqlens_padded)``. + """ + import math + padded = [math.ceil(s / divisor) * divisor for s in seqlens] + chunks = [] + next_id = 1 + for s, p in zip(seqlens, padded): + chunks.append(torch.arange(next_id, next_id + s, dtype=torch.int64)) + chunks.append(torch.zeros(p - s, dtype=torch.int64)) # padding + next_id += s + input_ids = torch.cat(chunks, dim=0).unsqueeze(0) # [1, T] + cu_seqlens_padded = torch.tensor( + [0] + list(torch.tensor(padded).cumsum(0).tolist()), + dtype=torch.int32, + ) + return input_ids, cu_seqlens_padded + + def _ensure_cuda(self, x): + return x.cuda() if torch.cuda.is_available() else x + + def test_partition_covers_all_positions_cp2(self): + from examples.multimodal_dev.models.base import _thd_cp_partition_index + + cp_size = 2 + seqlens = [5, 7, 3] # valid lengths + input_ids, cu_seqlens_padded = self._make_padded_packed( + seqlens, divisor=2 * cp_size, + ) + input_ids = self._ensure_cuda(input_ids) + cu_seqlens_padded = self._ensure_cuda(cu_seqlens_padded) + T = input_ids.shape[1] + + # Union of per-rank indices must be all positions exactly once. + seen = torch.zeros(T, dtype=torch.long, device=input_ids.device) + for cp_rank in range(cp_size): + idx = _thd_cp_partition_index( + cu_seqlens_padded, T, cp_size, cp_rank, + ) + assert idx.numel() == T // cp_size, ( + f"rank {cp_rank}: expected {T // cp_size} tokens, got {idx.numel()}" + ) + seen.scatter_add_( + 0, idx.long(), torch.ones_like(idx, dtype=seen.dtype), + ) + assert torch.all(seen == 1), ( + f"Position coverage broken: counts={seen.tolist()}" + ) + + def test_index_select_aligns_inputs_and_position_ids_cp2(self): + """input_ids, loss_mask, and (3, 1, T) position_ids index_select with + the same partition index produce shape-consistent per-rank tensors.""" + from examples.multimodal_dev.models.base import _thd_cp_partition_index + + cp_size = 2 + seqlens = [8, 4] + input_ids, cu_seqlens_padded = self._make_padded_packed( + seqlens, divisor=2 * cp_size, + ) + input_ids = self._ensure_cuda(input_ids) + cu_seqlens_padded = self._ensure_cuda(cu_seqlens_padded) + T = input_ids.shape[1] + labels = input_ids + 1000 + loss_mask = (input_ids != 0).float() + position_ids = ( + torch.arange(T, device=input_ids.device) + .unsqueeze(0).unsqueeze(0).expand(3, 1, T).contiguous() + ) + H = 4 + decoder_input = ( + torch.arange(T * H, dtype=torch.float32, device=input_ids.device) + .view(T, 1, H) + ) + + for cp_rank in range(cp_size): + idx = _thd_cp_partition_index( + cu_seqlens_padded, T, cp_size, cp_rank, + ) + ii = input_ids.index_select(1, idx) + ll = labels.index_select(1, idx) + lm = loss_mask.index_select(1, idx) + pi = position_ids.index_select(2, idx) + di = decoder_input.index_select(0, idx) + + assert ii.shape == (1, T // cp_size) + assert ll.shape == (1, T // cp_size) + assert lm.shape == (1, T // cp_size) + assert pi.shape == (3, 1, T // cp_size) + assert di.shape == (T // cp_size, 1, H) + # Sliced position_ids is just the partition index itself + # (since position_ids was arange(T) over all positions). + assert torch.equal(pi[0, 0], idx.to(pi.dtype)) + # All MRoPE rows agree. + assert torch.equal(pi[1, 0], pi[0, 0]) + assert torch.equal(pi[2, 0], pi[0, 0]) + + def test_partition_cp4_three_samples(self): + from examples.multimodal_dev.models.base import _thd_cp_partition_index + + cp_size = 4 + seqlens = [12, 4, 8] + input_ids, cu_seqlens_padded = self._make_padded_packed( + seqlens, divisor=2 * cp_size, + ) + input_ids = self._ensure_cuda(input_ids) + cu_seqlens_padded = self._ensure_cuda(cu_seqlens_padded) + T = input_ids.shape[1] + + seen = torch.zeros(T, dtype=torch.long, device=input_ids.device) + for cp_rank in range(cp_size): + idx = _thd_cp_partition_index( + cu_seqlens_padded, T, cp_size, cp_rank, + ) + assert idx.numel() == T // cp_size + seen.scatter_add_( + 0, idx.long(), torch.ones_like(idx, dtype=seen.dtype), + ) + assert torch.all(seen == 1) + + def test_loss_mask_zero_kept_per_rank(self): + """Pad-token positions (loss_mask=0) survive as 0 on whichever rank + they land — sanity check that we don't accidentally discard them.""" + from examples.multimodal_dev.models.base import _thd_cp_partition_index + + cp_size = 2 + seqlens = [5, 3] + input_ids, cu_seqlens_padded = self._make_padded_packed( + seqlens, divisor=2 * cp_size, + ) + input_ids = self._ensure_cuda(input_ids) + cu_seqlens_padded = self._ensure_cuda(cu_seqlens_padded) + T = input_ids.shape[1] + loss_mask = (input_ids != 0).float() + total_zeros = (loss_mask == 0).sum().item() + + zeros_seen = 0 + for cp_rank in range(cp_size): + idx = _thd_cp_partition_index( + cu_seqlens_padded, T, cp_size, cp_rank, + ) + zeros_seen += ( + loss_mask.index_select(1, idx) == 0 + ).sum().item() + assert zeros_seen == total_zeros diff --git a/examples/multimodal_dev/tests/test_cp_thd_correctness.py b/examples/multimodal_dev/tests/test_cp_thd_correctness.py new file mode 100644 index 00000000000..e815a948474 --- /dev/null +++ b/examples/multimodal_dev/tests/test_cp_thd_correctness.py @@ -0,0 +1,455 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# This is a stdout-reporting standalone script; `print` is intentional. +# pylint: disable=bad-builtin + +"""CP=1 vs CP=4 correctness test for THD and BSHD packing. + +Runs the production forward path (:class:`MultimodalModel`) twice in a +single ``torchrun`` invocation: + + Phase 1 — CP=1 baseline. All 4 ranks initialise with TP=1, CP=1 + (DP=4 implicit). Each rank computes the full sequence, + producing identical loss / grad_norm on every rank; rank 0's + value is the baseline. + + Phase 2 — CP=4. After ``destroy_model_parallel`` + ``initialize_model_parallel(CP=4)`` + the 4 ranks form a single CP group. The model's internal + ``_cp_split_for_forward`` slices inputs per rank; per-rank + loss / gradients are aggregated via AllReduce on the CP group. + +We compare CP=1 and CP=4 results for both BSHD and THD packing modes, +asserting that loss and grad_norm match within tolerance. + +Run with:: + + PYTHONPATH=. torchrun --nproc-per-node 4 \\ + examples/multimodal_dev/tests/test_cp_thd_correctness.py +""" + +import argparse +import os +import sys + +import torch + +_REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../..")) +if _REPO_ROOT not in sys.path: + sys.path.insert(0, _REPO_ROOT) + +from examples.multimodal_dev.forward_step import pack_or_pad_batch +from examples.multimodal_dev.models.base import ( + MultimodalModel, + _cp_split_tensor, + _thd_cp_partition_index, +) +from megatron.core.models.gpt.gpt_layer_specs import get_gpt_layer_with_transformer_engine_spec +from megatron.core.parallel_state import get_context_parallel_group, get_context_parallel_rank +from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed +from megatron.core.transformer.module import MegatronModule +from megatron.core.transformer.transformer_config import TransformerConfig +from tests.unit_tests.test_utilities import Utils + +# =================================================================== +# Stub vision encoder +# =================================================================== + + +class _StubVisionEncoder(MegatronModule): + """Vision encoder placeholder. The vision branch is skipped in + :meth:`MultimodalModel.forward` whenever ``pixel_values is None``, so + this module is never called — it only satisfies the constructor's + ``vision_encoder: MegatronModule`` requirement. + """ + + def __init__(self, config): + """Initialise the stub with the given TransformerConfig.""" + super().__init__(config=config) + + def forward(self, pixel_values, image_grid_thw): + """Never called when ``pixel_values=None``; raises if it ever is.""" + raise RuntimeError("vision branch should not run when pixel_values=None") + + +# =================================================================== +# Model builder +# =================================================================== + + +def _build_model(config, vocab_size, max_seq_len, image_token_id): + spec = get_gpt_layer_with_transformer_engine_spec() + vision = _StubVisionEncoder(config) + model = MultimodalModel( + language_config=config, + language_spec=spec, + vision_encoder=vision, + vocab_size=vocab_size, + max_sequence_length=max_seq_len, + image_token_id=image_token_id, + position_embedding_type="rope", + parallel_output=False, + ) + model.cuda() + return model + + +def _make_config( + num_layers, hidden_size, ffn_hidden_size, num_heads, num_kv_heads, context_parallel_size +): + return TransformerConfig( + num_layers=num_layers, + hidden_size=hidden_size, + ffn_hidden_size=ffn_hidden_size, + num_attention_heads=num_heads, + num_query_groups=num_kv_heads, + bf16=True, + params_dtype=torch.bfloat16, + pipeline_dtype=torch.bfloat16, + hidden_dropout=0.0, + attention_dropout=0.0, + tensor_model_parallel_size=1, + context_parallel_size=context_parallel_size, + sequence_parallel=False, + ) + + +# =================================================================== +# Loss / grad-norm aggregation +# =================================================================== + + +def _global_loss(output, rank_loss_mask, cp_size): + """Mean per-token loss over all CP shards (matches CP=1 mean exactly).""" + num = (output.float().view(-1) * rank_loss_mask.float().view(-1)).sum() + den = rank_loss_mask.float().view(-1).sum().clamp(min=1) + if cp_size > 1: + group = get_context_parallel_group() + torch.distributed.all_reduce(num, group=group) + torch.distributed.all_reduce(den, group=group) + return (num / den).item() + + +def _global_grad_norm(model, cp_size): + """Global L2 grad norm. For CP>1, AllReduce(SUM) gradients across CP + then divide by ``cp_size`` so each rank holds the CP-mean gradient + (matching CP=1's behaviour, where backward on the per-batch mean loss + yields exactly that gradient). + """ + if cp_size > 1: + group = get_context_parallel_group() + for p in model.parameters(): + if p.grad is not None: + torch.distributed.all_reduce(p.grad, group=group) + p.grad /= cp_size + + sq = 0.0 + for p in model.parameters(): + if p.grad is not None: + sq += p.grad.float().norm(2).item() ** 2 + return sq**0.5 + + +# =================================================================== +# Data — identical across all ranks (deterministic generator) +# =================================================================== + + +def _make_data(B, S, vocab_size, image_token_id, seed): + """Same input on every rank thanks to the seeded generator.""" + g = torch.Generator(device="cuda") + g.manual_seed(seed) + input_ids = torch.randint(0, vocab_size, (B, S), generator=g, device="cuda") + # Ensure no accidental image tokens (we never run the vision branch). + input_ids = torch.where(input_ids == image_token_id, (input_ids + 1) % vocab_size, input_ids) + labels = torch.randint(0, vocab_size, (B, S), generator=g, device="cuda") + loss_mask = torch.ones(B, S, device="cuda") + position_ids = torch.arange(S, device="cuda").unsqueeze(0).expand(B, -1).contiguous() + return input_ids, labels, loss_mask, position_ids + + +# =================================================================== +# One BSHD or THD forward+backward, returning (loss, grad_norm) +# =================================================================== + + +def _run_bshd(model, B, S, vocab_size, image_token_id, cp_size, seed): + input_ids, labels, loss_mask, position_ids = _make_data(B, S, vocab_size, image_token_id, seed) + + output = model( + input_ids=input_ids, + position_ids=position_ids, + attention_mask=None, + labels=labels, + loss_mask=loss_mask, + pixel_values=None, + image_grid_thw=None, + packed_seq_params=None, + ) + + # Slice loss_mask the same way forward_step does for BSHD + CP. + rank_loss_mask = loss_mask + if cp_size > 1: + rank_loss_mask = _cp_split_tensor( + rank_loss_mask, seq_dim=1, cp_size=cp_size, cp_rank=get_context_parallel_rank() + ) + + loss_val = _global_loss(output, rank_loss_mask, cp_size) + + # Backward on the LOCAL mean loss (each rank's contribution + # equal-weighted; SUM-then-divide across CP recovers CP=1's gradient). + local = ( + output.float().view(-1) * rank_loss_mask.float().view(-1) + ).sum() / rank_loss_mask.float().view(-1).sum().clamp(min=1) + model.zero_grad() + local.backward() + + gn = _global_grad_norm(model, cp_size) + return loss_val, gn + + +def _run_thd(model, B, S, vocab_size, image_token_id, cp_size, seed): + input_ids, labels, loss_mask, _ = _make_data(B, S, vocab_size, image_token_id, seed) + + # Build the per-sample dict list and pack to [1, T]. + samples = [] + for i in range(B): + samples.append( + { + "input_ids": input_ids[i].clone(), + "labels": labels[i].clone(), + "loss_mask": loss_mask[i].clone(), + # No vision; empty tensors satisfy pack_or_pad_batch. + "pixel_values": torch.zeros(0, 1, device="cuda"), + "image_grid_thw": torch.empty(0, 3, dtype=torch.long, device="cuda"), + } + ) + packed = pack_or_pad_batch(samples, use_packed_sequence=True, device="cuda") + psp = packed.pop("packed_seq_params") + + # THD position_ids: per-sample restart at 0. Each sample has length S + # (equal-length data), so this is arange(S) repeated B times. + thd_pos = ( + torch.cat([torch.arange(S, device="cuda") for _ in range(B)]).unsqueeze(0).contiguous() + ) + + output = model( + input_ids=packed["input_ids"], + position_ids=thd_pos, + attention_mask=None, + labels=packed["labels"], + loss_mask=packed["loss_mask"], + pixel_values=None, + image_grid_thw=None, + packed_seq_params=psp, + ) + + rank_loss_mask = packed["loss_mask"] + if cp_size > 1: + T = rank_loss_mask.shape[1] + idx = _thd_cp_partition_index( + psp.cu_seqlens_q_padded, T, cp_size, get_context_parallel_rank() + ) + rank_loss_mask = rank_loss_mask.index_select(1, idx) + + loss_val = _global_loss(output, rank_loss_mask, cp_size) + + local = ( + output.float().view(-1) * rank_loss_mask.float().view(-1) + ).sum() / rank_loss_mask.float().view(-1).sum().clamp(min=1) + model.zero_grad() + local.backward() + + gn = _global_grad_norm(model, cp_size) + return loss_val, gn + + +# =================================================================== +# State-dict roundtrip — keep weights identical across phases +# =================================================================== + + +def _cpu_state_dict(model): + """Snapshot of model.state_dict() detached to CPU (kept in memory). + + Some entries (TransformerEngine ``_extra_state``) are non-tensor or + ``None``; pass them through untouched. + """ + snap = {} + for k, v in model.state_dict().items(): + if isinstance(v, torch.Tensor): + snap[k] = v.detach().to("cpu").clone() + else: + snap[k] = v + return snap + + +def _restore_state_dict(model, snapshot): + """Load a saved snapshot back into a freshly built model.""" + payload = {k: (v.to("cuda") if isinstance(v, torch.Tensor) else v) for k, v in snapshot.items()} + model.load_state_dict(payload) + + +# =================================================================== +# Main +# =================================================================== + + +def _is_rank0(): + return not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0 + + +def _print_banner(title): + if _is_rank0(): + print(f"\n{'=' * 60}") + print(f" {title}") + print(f"{'=' * 60}") + + +def _print_compare(label, baseline, trial, atol, rtol): + """Print a CP=1 vs CP=4 comparison line and return whether it passed.""" + if not _is_rank0(): + return True + + abs_diff = abs(baseline - trial) + rel_diff = abs_diff / max(abs(baseline), 1e-8) + ok = abs_diff < atol or rel_diff < rtol + flag = "PASS" if ok else "FAIL" + print( + f" {label:<30s} CP=1: {baseline:.8f} CP=4: {trial:.8f}" + f" abs={abs_diff:.2e} rel={rel_diff:.2e} [{flag}]" + ) + return ok + + +def main(): + """Run CP=1 baseline + CP=4 trial and compare losses / grad_norms.""" + parser = argparse.ArgumentParser() + parser.add_argument("--batch-size", type=int, default=2) + # Must be divisible by 2*cp_size (=8 for CP=4 zigzag). + parser.add_argument("--seq-len", type=int, default=64) + parser.add_argument("--vocab-size", type=int, default=1024) + parser.add_argument("--hidden-size", type=int, default=256) + parser.add_argument("--num-layers", type=int, default=2) + parser.add_argument("--num-heads", type=int, default=4) + parser.add_argument("--num-kv-heads", type=int, default=2) + parser.add_argument("--ffn-hidden-size", type=int, default=512) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--atol-loss", type=float, default=1e-3) + parser.add_argument("--rtol-grad", type=float, default=5e-3) + parser.add_argument("--data-seed", type=int, default=123) + args = parser.parse_args() + + image_token_id = 0 # never appears in input (data filters this id out) + + # ---------------------------------------------------------------- + # Phase 1: CP=1 baseline + # ---------------------------------------------------------------- + _print_banner("Phase 1 — building CP=1 baseline (TP=1, CP=1, DP=4)") + Utils.initialize_model_parallel(tensor_model_parallel_size=1, context_parallel_size=1) + model_parallel_cuda_manual_seed(args.seed) + + config_cp1 = _make_config( + args.num_layers, + args.hidden_size, + args.ffn_hidden_size, + args.num_heads, + args.num_kv_heads, + context_parallel_size=1, + ) + torch.manual_seed(args.seed) + model_cp1 = _build_model(config_cp1, args.vocab_size, args.seq_len, image_token_id) + + bshd_loss_cp1, bshd_gn_cp1 = _run_bshd( + model_cp1, + args.batch_size, + args.seq_len, + args.vocab_size, + image_token_id, + cp_size=1, + seed=args.data_seed, + ) + thd_loss_cp1, thd_gn_cp1 = _run_thd( + model_cp1, + args.batch_size, + args.seq_len, + args.vocab_size, + image_token_id, + cp_size=1, + seed=args.data_seed, + ) + + # Snapshot weights *before* the optimizer would have touched them. + # (We've zeroed grads but never stepped; weights at this point are + # the just-initialised baseline.) + weights_snapshot = _cpu_state_dict(model_cp1) + del model_cp1 + torch.cuda.empty_cache() + + # ---------------------------------------------------------------- + # Phase 2: CP=4 + # ---------------------------------------------------------------- + _print_banner("Phase 2 — re-initialising for CP=4 (TP=1, CP=4)") + Utils.destroy_model_parallel() + Utils.initialize_model_parallel(tensor_model_parallel_size=1, context_parallel_size=4) + model_parallel_cuda_manual_seed(args.seed) + + config_cp4 = _make_config( + args.num_layers, + args.hidden_size, + args.ffn_hidden_size, + args.num_heads, + args.num_kv_heads, + context_parallel_size=4, + ) + torch.manual_seed(args.seed) + model_cp4 = _build_model(config_cp4, args.vocab_size, args.seq_len, image_token_id) + _restore_state_dict(model_cp4, weights_snapshot) + + bshd_loss_cp4, bshd_gn_cp4 = _run_bshd( + model_cp4, + args.batch_size, + args.seq_len, + args.vocab_size, + image_token_id, + cp_size=4, + seed=args.data_seed, + ) + thd_loss_cp4, thd_gn_cp4 = _run_thd( + model_cp4, + args.batch_size, + args.seq_len, + args.vocab_size, + image_token_id, + cp_size=4, + seed=args.data_seed, + ) + + # ---------------------------------------------------------------- + # Compare + # ---------------------------------------------------------------- + _print_banner("Results — CP=1 vs CP=4") + all_ok = True + all_ok &= _print_compare( + "BSHD loss", bshd_loss_cp1, bshd_loss_cp4, args.atol_loss, args.rtol_grad + ) + all_ok &= _print_compare( + "BSHD grad_norm", bshd_gn_cp1, bshd_gn_cp4, args.atol_loss, args.rtol_grad + ) + all_ok &= _print_compare( + "THD loss", thd_loss_cp1, thd_loss_cp4, args.atol_loss, args.rtol_grad + ) + all_ok &= _print_compare( + "THD grad_norm", thd_gn_cp1, thd_gn_cp4, args.atol_loss, args.rtol_grad + ) + + _print_banner("Summary") + if _is_rank0(): + print(f" {'ALL TESTS PASSED' if all_ok else 'SOME TESTS FAILED'}") + print(f"{'=' * 60}\n") + + Utils.destroy_model_parallel() + if not all_ok: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/examples/multimodal_dev/tests/test_mrope_parity.py b/examples/multimodal_dev/tests/test_mrope_parity.py new file mode 100644 index 00000000000..154284a4fc1 --- /dev/null +++ b/examples/multimodal_dev/tests/test_mrope_parity.py @@ -0,0 +1,663 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Parity tests for ``get_rope_index`` (MRoPE position-ID computation). + +Two properties are verified: + +1. **BSHD backwards compatibility** — the refactored ``get_rope_index`` + returns bit-identical ``(position_ids, mrope_position_deltas)`` to + the pre-refactor implementation on padded ``[B, S]`` batches. +2. **THD == BSHD on the valid region** — when the same variable-length + samples are fed through both layouts (BSHD with right-padding; THD + packed with ``cu_seqlens_q`` / ``cu_seqlens_q_padded``), positions at + every valid slot agree. + +The pre-refactor function is pinned inline as ``_old_get_rope_index`` +so this test stays self-contained. Run with:: + + python -m pytest examples/multimodal_dev/tests/test_mrope_parity.py -v + +or directly:: + + python examples/multimodal_dev/tests/test_mrope_parity.py +""" + +import math +import os +import sys +from itertools import accumulate + +import torch +import torch.nn.functional as F + +_REPO_ROOT = os.path.abspath( + os.path.join(os.path.dirname(__file__), "../../.."), +) +# Insert at position 0 unconditionally — other entries on sys.path +# (e.g. a sibling Megatron-LM checkout) have their own ``examples`` +# package that would otherwise shadow ours. +if _REPO_ROOT in sys.path: + sys.path.remove(_REPO_ROOT) +sys.path.insert(0, _REPO_ROOT) + +from megatron.core.packed_seq_params import PackedSeqParams + +from examples.multimodal_dev.models.qwen35_vl.mrope import get_rope_index + +# ----------------------------------------------------------------------------- +# Token-ID constants (match Qwen3.5-VL, but values are arbitrary for this test) +# ----------------------------------------------------------------------------- + +IMAGE_TOKEN_ID = 248056 +VIDEO_TOKEN_ID = 248057 +VISION_START_TOKEN_ID = 248053 +SPATIAL_MERGE_SIZE = 2 + + +# ----------------------------------------------------------------------------- +# Pinned reference implementation (pre-refactor BSHD path) +# ----------------------------------------------------------------------------- + +def _old_get_rope_index( + spatial_merge_size, + image_token_id, + video_token_id, + vision_start_token_id, + input_ids=None, + image_grid_thw=None, + video_grid_thw=None, + attention_mask=None, +): + """Pre-refactor BSHD implementation of ``get_rope_index``. + + Copied verbatim (modulo the broken cu_seqlens branch, which this + parity test does not exercise) so we can diff against the new + implementation on BSHD inputs. + """ + if video_grid_thw is not None: + video_grid_thw = torch.repeat_interleave( + video_grid_thw, video_grid_thw[:, 0], dim=0, + ) + video_grid_thw[:, 0] = 1 + + mrope_position_deltas = [] + + if input_ids is not None and ( + image_grid_thw is not None or video_grid_thw is not None + ): + total_input_ids = input_ids + if attention_mask is None: + attention_mask = torch.ones_like(total_input_ids) + + position_ids = torch.ones( + 3, + input_ids.shape[0], + input_ids.shape[1], + dtype=input_ids.dtype, + device=input_ids.device, + ) + image_index, video_index = 0, 0 + attention_mask = attention_mask.to(total_input_ids.device) + + for i, sample_input_ids in enumerate(total_input_ids): + sample_input_ids = sample_input_ids[attention_mask[i] == 1] + vision_start_indices = torch.argwhere( + sample_input_ids == vision_start_token_id, + ).squeeze(1) + vision_tokens = sample_input_ids[vision_start_indices + 1] + image_nums = (vision_tokens == image_token_id).sum() + video_nums = (vision_tokens == video_token_id).sum() + input_tokens = sample_input_ids.tolist() + llm_pos_ids_list = [] + st = 0 + remain_images, remain_videos = image_nums, video_nums + + for _ in range(image_nums + video_nums): + if image_token_id in input_tokens and remain_images > 0: + ed_image = input_tokens.index(image_token_id, st) + else: + ed_image = len(input_tokens) + 1 + if video_token_id in input_tokens and remain_videos > 0: + ed_video = input_tokens.index(video_token_id, st) + else: + ed_video = len(input_tokens) + 1 + + if ed_image < ed_video: + t, h, w = ( + image_grid_thw[image_index][0], + image_grid_thw[image_index][1], + image_grid_thw[image_index][2], + ) + image_index += 1 + remain_images -= 1 + ed = ed_image + else: + t, h, w = ( + video_grid_thw[video_index][0], + video_grid_thw[video_index][1], + video_grid_thw[video_index][2], + ) + video_index += 1 + remain_videos -= 1 + ed = ed_video + + llm_grid_t, llm_grid_h, llm_grid_w = ( + t.item(), + h.item() // spatial_merge_size, + w.item() // spatial_merge_size, + ) + text_len = ed - st + + st_idx = ( + llm_pos_ids_list[-1].max() + 1 + if llm_pos_ids_list + else 0 + ) + llm_pos_ids_list.append( + torch.arange(text_len).view(1, -1).expand(3, -1) + + st_idx + ) + + t_index = ( + torch.arange(llm_grid_t) + .view(-1, 1) + .expand(-1, llm_grid_h * llm_grid_w) + .flatten() + ) + h_index = ( + torch.arange(llm_grid_h) + .view(1, -1, 1) + .expand(llm_grid_t, -1, llm_grid_w) + .flatten() + ) + w_index = ( + torch.arange(llm_grid_w) + .view(1, 1, -1) + .expand(llm_grid_t, llm_grid_h, -1) + .flatten() + ) + llm_pos_ids_list.append( + torch.stack([t_index, h_index, w_index]) + + text_len + + st_idx + ) + st = ed + llm_grid_t * llm_grid_h * llm_grid_w + + if st < len(input_tokens): + st_idx = ( + llm_pos_ids_list[-1].max() + 1 + if llm_pos_ids_list + else 0 + ) + text_len = len(input_tokens) - st + llm_pos_ids_list.append( + torch.arange(text_len).view(1, -1).expand(3, -1) + + st_idx + ) + + llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1) + position_ids[ + ..., i, attention_mask[i] == 1 + ] = llm_positions.to(position_ids.device) + mrope_position_deltas.append( + llm_positions.max() + 1 - len(total_input_ids[i]), + ) + + mrope_position_deltas = torch.tensor( + mrope_position_deltas, device=total_input_ids.device, + ).unsqueeze(1) + return position_ids, mrope_position_deltas + + # Text-only fallback. + if attention_mask is not None: + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + position_ids = ( + position_ids.unsqueeze(0) + .expand(3, -1, -1) + .to(attention_mask.device) + ) + max_position_ids = ( + position_ids.max(0, keepdim=False)[0] + .max(-1, keepdim=True)[0] + ) + mrope_position_deltas = ( + max_position_ids + 1 - attention_mask.shape[-1] + ) + else: + position_ids = ( + torch.arange( + input_ids.shape[1], device=input_ids.device, + ) + .view(1, 1, -1) + .expand(3, input_ids.shape[0], -1) + ) + mrope_position_deltas = torch.zeros( + [input_ids.shape[0], 1], + device=input_ids.device, + dtype=input_ids.dtype, + ) + return position_ids, mrope_position_deltas + + +# ----------------------------------------------------------------------------- +# Synthetic-sample builder +# ----------------------------------------------------------------------------- + +def _build_sample( + prefix_text_len, + grids, + suffix_text_len, + text_token=100, +): + """Build one variable-length sample with ``len(grids)`` images. + + Layout per image: ``vision_start_id`` then + ``llm_grid_t * llm_grid_h * llm_grid_w`` ``image_token_id`` slots + (where ``llm_grid_* = grid_* // spatial_merge_size`` for h/w). + Grids use ``t=1``. + + Returns ``(input_ids [L], image_grid_thw [N, 3])``. + """ + tokens = [text_token] * prefix_text_len + grid_rows = [] + for t, h, w in grids: + n_image_tokens = ( + t * (h // SPATIAL_MERGE_SIZE) * (w // SPATIAL_MERGE_SIZE) + ) + tokens.append(VISION_START_TOKEN_ID) + tokens.extend([IMAGE_TOKEN_ID] * n_image_tokens) + grid_rows.append([t, h, w]) + tokens.extend([text_token + 1] * suffix_text_len) + input_ids = torch.tensor(tokens, dtype=torch.int64) + image_grid_thw = torch.tensor(grid_rows, dtype=torch.int64) + return input_ids, image_grid_thw + + +def _sample_bank(): + """A small bank of samples covering text-only, single-image, multi-image.""" + return [ + _build_sample( + prefix_text_len=5, + grids=[(1, 4, 4)], + suffix_text_len=7, + ), + _build_sample( + prefix_text_len=3, + grids=[(1, 2, 2), (1, 4, 6)], + suffix_text_len=4, + ), + _build_sample( + prefix_text_len=10, + grids=[], + suffix_text_len=0, + ), + _build_sample( + prefix_text_len=0, + grids=[(1, 6, 4)], + suffix_text_len=2, + ), + ] + + +# ----------------------------------------------------------------------------- +# Test 1: BSHD backwards compatibility +# ----------------------------------------------------------------------------- + +def test_bshd_matches_old_reference(): + """New ``get_rope_index`` equals the pinned reference on BSHD inputs.""" + samples = _sample_bank() + max_len = max(s.numel() for s, _ in samples) + + input_ids_rows = [] + mask_rows = [] + grid_rows = [] + for tokens, grids in samples: + L = tokens.numel() + padded = F.pad(tokens, (0, max_len - L), value=0) + mask = torch.zeros(max_len, dtype=torch.int64) + mask[:L] = 1 + input_ids_rows.append(padded) + mask_rows.append(mask) + if grids.numel() > 0: + grid_rows.append(grids) + + input_ids = torch.stack(input_ids_rows) # [B, S] + attention_mask = torch.stack(mask_rows) # [B, S] + image_grid_thw = ( + torch.cat(grid_rows, dim=0) if grid_rows else None + ) + + old_pos, old_delta = _old_get_rope_index( + spatial_merge_size=SPATIAL_MERGE_SIZE, + image_token_id=IMAGE_TOKEN_ID, + video_token_id=VIDEO_TOKEN_ID, + vision_start_token_id=VISION_START_TOKEN_ID, + input_ids=input_ids, + image_grid_thw=image_grid_thw, + attention_mask=attention_mask, + ) + new_pos, new_delta = get_rope_index( + spatial_merge_size=SPATIAL_MERGE_SIZE, + image_token_id=IMAGE_TOKEN_ID, + video_token_id=VIDEO_TOKEN_ID, + vision_start_token_id=VISION_START_TOKEN_ID, + input_ids=input_ids, + image_grid_thw=image_grid_thw, + attention_mask=attention_mask, + packed_seq_params=None, + ) + + assert torch.equal(old_pos, new_pos), ( + f"BSHD position_ids differ.\nold:\n{old_pos}\nnew:\n{new_pos}" + ) + assert torch.equal(old_delta, new_delta), ( + f"BSHD mrope_position_deltas differ.\n" + f"old: {old_delta}\nnew: {new_delta}" + ) + + +# ----------------------------------------------------------------------------- +# Test 2: THD positions match BSHD positions on the valid region +# ----------------------------------------------------------------------------- + +def _pack_samples(samples, divisible_by=1): + """Pack ``samples`` into a single ``[1, T]`` tensor the same way + ``pack_or_pad_batch`` does, and build ``PackedSeqParams``. + + Each per-sample tensor is right-padded to a multiple of + ``divisible_by`` before concatenation. ``cu_seqlens_q`` tracks + unpadded lengths; ``cu_seqlens_q_padded`` tracks the packed layout. + """ + padded_chunks = [] + seqlens = [] + seqlens_padded = [] + grid_rows = [] + for tokens, grids in samples: + L = tokens.numel() + target_L = math.ceil(L / divisible_by) * divisible_by + padded_chunks.append(F.pad(tokens, (0, target_L - L), value=0)) + seqlens.append(L) + seqlens_padded.append(target_L) + if grids.numel() > 0: + grid_rows.append(grids) + + packed = torch.cat(padded_chunks, dim=0).unsqueeze(0) # [1, T] + cu_seqlens = torch.tensor( + list(accumulate(seqlens, initial=0)), dtype=torch.int32, + ) + cu_seqlens_padded = torch.tensor( + list(accumulate(seqlens_padded, initial=0)), dtype=torch.int32, + ) + psp = PackedSeqParams( + cu_seqlens_q=cu_seqlens, + cu_seqlens_kv=cu_seqlens, + cu_seqlens_q_padded=cu_seqlens_padded, + cu_seqlens_kv_padded=cu_seqlens_padded, + max_seqlen_q=max(seqlens_padded), + max_seqlen_kv=max(seqlens_padded), + ) + image_grid_thw = ( + torch.cat(grid_rows, dim=0) if grid_rows else None + ) + return packed, psp, image_grid_thw, seqlens, seqlens_padded + + +def test_thd_matches_bshd_padded(): + """THD positions at every valid slot equal BSHD positions on the + equivalent right-padded batch. + """ + samples = _sample_bank() + + # BSHD side: right-pad to common max_len. + max_len = max(s.numel() for s, _ in samples) + input_ids_rows = [] + mask_rows = [] + grid_rows = [] + for tokens, grids in samples: + L = tokens.numel() + input_ids_rows.append(F.pad(tokens, (0, max_len - L), value=0)) + m = torch.zeros(max_len, dtype=torch.int64) + m[:L] = 1 + mask_rows.append(m) + if grids.numel() > 0: + grid_rows.append(grids) + bshd_input_ids = torch.stack(input_ids_rows) + bshd_mask = torch.stack(mask_rows) + bshd_grid = torch.cat(grid_rows, dim=0) if grid_rows else None + + bshd_pos, _ = get_rope_index( + spatial_merge_size=SPATIAL_MERGE_SIZE, + image_token_id=IMAGE_TOKEN_ID, + video_token_id=VIDEO_TOKEN_ID, + vision_start_token_id=VISION_START_TOKEN_ID, + input_ids=bshd_input_ids, + image_grid_thw=bshd_grid, + attention_mask=bshd_mask, + ) + # bshd_pos: [3, B, S_pad] + + # THD side: pack with a non-trivial divisor so the padded and + # unpadded cu_seqlens diverge — this exercises the distinction. + for divisible_by in (1, 4): + packed_input_ids, psp, thd_grid, seqlens, seqlens_padded = ( + _pack_samples(samples, divisible_by=divisible_by) + ) + thd_pos, _ = get_rope_index( + spatial_merge_size=SPATIAL_MERGE_SIZE, + image_token_id=IMAGE_TOKEN_ID, + video_token_id=VIDEO_TOKEN_ID, + vision_start_token_id=VISION_START_TOKEN_ID, + input_ids=packed_input_ids, + image_grid_thw=thd_grid, + packed_seq_params=psp, + ) + # thd_pos: [3, 1, T] + assert thd_pos.shape == ( + 3, 1, packed_input_ids.shape[1], + ), f"bad THD shape {thd_pos.shape}" + + seg_starts = list(accumulate(seqlens_padded, initial=0)) + for k, (valid_len, seg_start) in enumerate( + zip(seqlens, seg_starts) + ): + thd_slice = thd_pos[:, 0, seg_start:seg_start + valid_len] + bshd_slice = bshd_pos[:, k, :valid_len] + assert torch.equal(thd_slice, bshd_slice), ( + f"[divisible_by={divisible_by}] segment {k} " + f"(valid_len={valid_len}, seg_start={seg_start}) " + f"disagrees:\nTHD:\n{thd_slice}\nBSHD:\n{bshd_slice}" + ) + + +# ----------------------------------------------------------------------------- +# Test 3: THD with no images (text-only packed) +# ----------------------------------------------------------------------------- + +def test_thd_text_only_restarts_per_segment(): + """Text-only THD: each segment gets a fresh ``[0..valid_len-1]`` range.""" + samples = [ + _build_sample(prefix_text_len=6, grids=[], suffix_text_len=0), + _build_sample(prefix_text_len=11, grids=[], suffix_text_len=0), + _build_sample(prefix_text_len=3, grids=[], suffix_text_len=0), + ] + packed_input_ids, psp, _, seqlens, seqlens_padded = _pack_samples( + samples, divisible_by=4, + ) + thd_pos, _ = get_rope_index( + spatial_merge_size=SPATIAL_MERGE_SIZE, + image_token_id=IMAGE_TOKEN_ID, + video_token_id=VIDEO_TOKEN_ID, + vision_start_token_id=VISION_START_TOKEN_ID, + input_ids=packed_input_ids, + image_grid_thw=None, + packed_seq_params=psp, + ) + + seg_starts = list(accumulate(seqlens_padded, initial=0)) + for valid_len, seg_start in zip(seqlens, seg_starts): + expected = ( + torch.arange(valid_len, dtype=thd_pos.dtype) + .view(1, -1) + .expand(3, -1) + ) + got = thd_pos[:, 0, seg_start:seg_start + valid_len] + assert torch.equal(got, expected), ( + f"text-only segment mismatch at seg_start={seg_start}, " + f"valid_len={valid_len}:\n{got}\nexpected:\n{expected}" + ) + + +# ----------------------------------------------------------------------------- +# Test 4: Explicit two-sequence batch with vision, both in BSHD and THD +# ----------------------------------------------------------------------------- + +def _two_image_samples(): + """Two samples, each with one image — the smallest case that can + expose a bug where segment k > 0 positions leak state from segment + k - 1 (e.g. non-restarted ``st_idx`` or a stale ``image_index``). + """ + return [ + _build_sample( + prefix_text_len=5, + grids=[(1, 4, 4)], # 4 image tokens after spatial merge + suffix_text_len=3, + ), + _build_sample( + prefix_text_len=4, + grids=[(1, 4, 4)], + suffix_text_len=6, + ), + ] + + +def test_bshd_batch_size_2_with_vision(): + """BSHD with ``B == 2``: both rows' positions restart at 0 and match + the pinned reference. + """ + samples = _two_image_samples() + max_len = max(s.numel() for s, _ in samples) + + input_ids_rows, mask_rows, grid_rows = [], [], [] + for tokens, grids in samples: + L = tokens.numel() + input_ids_rows.append(F.pad(tokens, (0, max_len - L), value=0)) + m = torch.zeros(max_len, dtype=torch.int64) + m[:L] = 1 + mask_rows.append(m) + grid_rows.append(grids) + + input_ids = torch.stack(input_ids_rows) # [2, S] + attention_mask = torch.stack(mask_rows) + image_grid_thw = torch.cat(grid_rows, dim=0) + assert input_ids.shape[0] == 2 + + old_pos, old_delta = _old_get_rope_index( + spatial_merge_size=SPATIAL_MERGE_SIZE, + image_token_id=IMAGE_TOKEN_ID, + video_token_id=VIDEO_TOKEN_ID, + vision_start_token_id=VISION_START_TOKEN_ID, + input_ids=input_ids, + image_grid_thw=image_grid_thw, + attention_mask=attention_mask, + ) + new_pos, new_delta = get_rope_index( + spatial_merge_size=SPATIAL_MERGE_SIZE, + image_token_id=IMAGE_TOKEN_ID, + video_token_id=VIDEO_TOKEN_ID, + vision_start_token_id=VISION_START_TOKEN_ID, + input_ids=input_ids, + image_grid_thw=image_grid_thw, + attention_mask=attention_mask, + ) + assert torch.equal(old_pos, new_pos), ( + f"BSHD B=2 position mismatch vs reference.\n" + f"old:\n{old_pos}\nnew:\n{new_pos}" + ) + assert torch.equal(old_delta, new_delta) + + # Both rows must start at position 0. + for i in range(2): + valid_len = int(attention_mask[i].sum().item()) + assert torch.all(new_pos[:, i, 0] == 0), ( + f"row {i} does not start at 0: {new_pos[:, i, 0]}" + ) + # Sanity: positions within the valid region are strictly < valid_len + # would be wrong (MRoPE can skip positions), so just check max. + assert new_pos[:, i, :valid_len].max() < valid_len + + +def test_thd_batch_size_2_with_vision(): + """THD with 2 packed sequences: seg 1 positions restart at 0 and + equal BSHD row 1 on the valid region (bit-identical). + """ + samples = _two_image_samples() + + # BSHD reference. + max_len = max(s.numel() for s, _ in samples) + rows, masks, grids_bshd = [], [], [] + for tokens, grids in samples: + L = tokens.numel() + rows.append(F.pad(tokens, (0, max_len - L), value=0)) + m = torch.zeros(max_len, dtype=torch.int64) + m[:L] = 1 + masks.append(m) + grids_bshd.append(grids) + bshd_input_ids = torch.stack(rows) + bshd_mask = torch.stack(masks) + bshd_grid = torch.cat(grids_bshd, dim=0) + bshd_pos, _ = get_rope_index( + spatial_merge_size=SPATIAL_MERGE_SIZE, + image_token_id=IMAGE_TOKEN_ID, + video_token_id=VIDEO_TOKEN_ID, + vision_start_token_id=VISION_START_TOKEN_ID, + input_ids=bshd_input_ids, + image_grid_thw=bshd_grid, + attention_mask=bshd_mask, + ) + + # THD packed version with a non-trivial divisor so padded and + # unpadded cu_seqlens disagree. + packed_input_ids, psp, thd_grid, seqlens, seqlens_padded = ( + _pack_samples(samples, divisible_by=4) + ) + assert len(seqlens) == 2 + thd_pos, _ = get_rope_index( + spatial_merge_size=SPATIAL_MERGE_SIZE, + image_token_id=IMAGE_TOKEN_ID, + video_token_id=VIDEO_TOKEN_ID, + vision_start_token_id=VISION_START_TOKEN_ID, + input_ids=packed_input_ids, + image_grid_thw=thd_grid, + packed_seq_params=psp, + ) + + seg_starts = list(accumulate(seqlens_padded, initial=0)) + for k, (valid_len, seg_start) in enumerate( + zip(seqlens, seg_starts) + ): + thd_slice = thd_pos[:, 0, seg_start:seg_start + valid_len] + bshd_slice = bshd_pos[:, k, :valid_len] + assert torch.equal(thd_slice, bshd_slice), ( + f"seg {k} THD vs BSHD row {k} mismatch.\n" + f"THD:\n{thd_slice}\nBSHD:\n{bshd_slice}" + ) + # Critical: seg k must start at position 0 (bug 2 check). + assert torch.all(thd_slice[:, 0] == 0), ( + f"seg {k} does not start at 0 — positions leaked from " + f"previous segment: first col = {thd_slice[:, 0]}" + ) + + +if __name__ == "__main__": + test_bshd_matches_old_reference() + print("[ok] test_bshd_matches_old_reference") + test_thd_matches_bshd_padded() + print("[ok] test_thd_matches_bshd_padded") + test_thd_text_only_restarts_per_segment() + print("[ok] test_thd_text_only_restarts_per_segment") + test_bshd_batch_size_2_with_vision() + print("[ok] test_bshd_batch_size_2_with_vision") + test_thd_batch_size_2_with_vision() + print("[ok] test_thd_batch_size_2_with_vision") + print("All parity tests passed.") diff --git a/examples/multimodal_dev/tests/test_thd_correctness.py b/examples/multimodal_dev/tests/test_thd_correctness.py new file mode 100644 index 00000000000..1f92cc2e2f7 --- /dev/null +++ b/examples/multimodal_dev/tests/test_thd_correctness.py @@ -0,0 +1,380 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# This is a stdout-reporting standalone script; `print` is intentional. +# pylint: disable=bad-builtin + +"""BSHD vs THD correctness test for multimodal_dev packed sequence support. + +Validates that packing a [B, S] batch into [1, T] THD format produces +numerically equivalent loss values and gradient norms through a GPTModel. + +The test uses equal-length sequences (no padding) so that BSHD causal +attention and THD cu_seqlens-based causal attention are mathematically +identical. This makes any numerical deviation a real bug rather than an +expected consequence of different padding/masking strategies. + +Usage:: + + # Single GPU (flash attention): + torchrun --nproc_per_node=1 \\ + examples/multimodal_dev/tests/test_thd_correctness.py + + # Override model size: + torchrun --nproc_per_node=1 \\ + examples/multimodal_dev/tests/test_thd_correctness.py \\ + --num-layers 4 --hidden-size 512 --num-heads 8 --num-kv-heads 4 +""" + +import argparse +import os +import sys + +import torch + +_REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../..")) +if _REPO_ROOT not in sys.path: + sys.path.insert(0, _REPO_ROOT) + +from examples.multimodal_dev.forward_step import pack_or_pad_batch +from examples.multimodal_dev.tests._helpers import grad_norm, mean_loss +from megatron.core.models.gpt import GPTModel +from megatron.core.models.gpt.gpt_layer_specs import get_gpt_layer_with_transformer_engine_spec +from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed +from megatron.core.transformer.transformer_config import TransformerConfig +from tests.unit_tests.test_utilities import Utils + + +def _samples_from_bshd(input_ids, labels, loss_mask, seq_lengths=None): + """Slice ``[B, S]`` tensors into the per-sample dict list that + ``pack_or_pad_batch`` expects. ``seq_lengths`` lets variable-length + samples carry only their valid tokens (no attention_mask needed). + """ + B, S = input_ids.shape + if seq_lengths is None: + seq_lengths = [S] * B + samples = [] + for i, L in enumerate(seq_lengths): + samples.append( + { + "input_ids": input_ids[i, :L].clone(), + "labels": labels[i, :L].clone(), + "loss_mask": loss_mask[i, :L].clone(), + # pack_or_pad_batch requires these keys but the model call + # below ignores them; provide minimal dummies. + "pixel_values": torch.zeros(1, 1, device=input_ids.device), + "image_grid_thw": torch.tensor( + [[2, 1, 1]], dtype=torch.long, device=input_ids.device + ), + } + ) + return samples + + +def _thd_position_ids(seq_lengths, device): + """Build ``[1, T]`` THD position_ids: each segment restarts at 0.""" + return ( + torch.cat([torch.arange(L, device=device) for L in seq_lengths]).unsqueeze(0).contiguous() + ) + + +# =================================================================== +# Helpers +# =================================================================== + + +def _build_model(cfg, vocab_size, max_seq_len): + """Build a small GPTModel for testing.""" + spec = get_gpt_layer_with_transformer_engine_spec() + model = GPTModel( + config=cfg, + transformer_layer_spec=spec, + vocab_size=vocab_size, + max_sequence_length=max_seq_len, + pre_process=True, + post_process=True, + parallel_output=False, + position_embedding_type="rope", + ) + model.cuda() + return model + + +# =================================================================== +# Core test logic +# =================================================================== + + +def run_equal_length_test(model, batch_size, seq_len, vocab_size, seed, atol_loss, rtol_grad): + """Compare BSHD and THD with equal-length sequences (no padding). + + Returns a dict with test metrics for logging. + """ + B, S = batch_size, seq_len + + # Deterministic data generation. + torch.manual_seed(seed + 1) + input_ids = torch.randint(0, vocab_size, (B, S), device="cuda") + labels = torch.randint(0, vocab_size, (B, S), device="cuda") + loss_mask = torch.ones(B, S, device="cuda") + position_ids = torch.arange(S, device="cuda").unsqueeze(0).expand(B, -1).contiguous() + + # ---- BSHD forward / backward ---- + output_bshd = model( + input_ids=input_ids, + position_ids=position_ids, + attention_mask=None, + labels=labels, + loss_mask=loss_mask, + ) + bshd_loss = mean_loss(output_bshd, loss_mask) + bshd_loss.backward() + bshd_gn = grad_norm(model) + bshd_lv = bshd_loss.item() + bshd_per_token = output_bshd.detach().float().view(-1).clone() + + model.zero_grad() + + # ---- THD forward / backward ---- + samples = _samples_from_bshd(input_ids, labels, loss_mask) + packed = pack_or_pad_batch(samples, use_packed_sequence=True, device="cuda") + psp = packed.pop("packed_seq_params") + thd_position_ids = _thd_position_ids([S] * B, device="cuda") + + output_thd = model( + input_ids=packed["input_ids"], + position_ids=thd_position_ids, + attention_mask=None, + labels=packed["labels"], + loss_mask=packed["loss_mask"], + packed_seq_params=psp, + ) + thd_loss = mean_loss(output_thd, packed["loss_mask"]) + thd_loss.backward() + thd_gn = grad_norm(model) + thd_lv = thd_loss.item() + thd_per_token = output_thd.detach().float().view(-1).clone() + + model.zero_grad() + + # ---- Comparison ---- + loss_diff = abs(bshd_lv - thd_lv) + grad_diff = abs(bshd_gn - thd_gn) + grad_rel = grad_diff / max(bshd_gn, 1e-8) + token_max_diff = (bshd_per_token - thd_per_token).abs().max().item() + token_mean_diff = (bshd_per_token - thd_per_token).abs().mean().item() + + loss_ok = loss_diff < atol_loss + grad_ok = grad_rel < rtol_grad + + metrics = dict( + bshd_loss=bshd_lv, + thd_loss=thd_lv, + loss_diff=loss_diff, + bshd_grad_norm=bshd_gn, + thd_grad_norm=thd_gn, + grad_diff=grad_diff, + grad_rel=grad_rel, + token_max_diff=token_max_diff, + token_mean_diff=token_mean_diff, + loss_ok=loss_ok, + grad_ok=grad_ok, + ) + return metrics + + +def run_variable_length_smoke_test(model, vocab_size, seed): + """Smoke test: variable-length sequences packed to THD. + + Does NOT compare against BSHD (padding in BSHD changes attention + context). Validates that: + - Packing produces correct shapes + - Forward + backward complete without error + - Loss is finite + - Gradients are finite and non-zero + + Returns a dict with test metrics. + """ + seq_lengths = [128, 96, 112, 80] + S = max(seq_lengths) + B = len(seq_lengths) + + torch.manual_seed(seed + 2) + input_ids = torch.randint(0, vocab_size, (B, S), device="cuda") + labels = torch.randint(0, vocab_size, (B, S), device="cuda") + loss_mask = torch.ones(B, S, device="cuda") + + # Mask out padded positions in loss_mask for the input we hand to + # pack_or_pad_batch (variable-length samples carry only valid tokens). + for i, sl in enumerate(seq_lengths): + loss_mask[i, sl:] = 0.0 + + samples = _samples_from_bshd(input_ids, labels, loss_mask, seq_lengths=seq_lengths) + packed = pack_or_pad_batch(samples, use_packed_sequence=True, device="cuda") + psp = packed.pop("packed_seq_params") + thd_position_ids = _thd_position_ids(seq_lengths, device="cuda") + + T = sum(seq_lengths) + assert packed["input_ids"].shape == ( + 1, + T, + ), f"Expected [1, {T}], got {packed['input_ids'].shape}" + assert packed["labels"].shape == (1, T) + assert packed["loss_mask"].shape == (1, T) + assert psp.cu_seqlens_q.tolist() == [ + 0, + seq_lengths[0], + seq_lengths[0] + seq_lengths[1], + seq_lengths[0] + seq_lengths[1] + seq_lengths[2], + T, + ] + + output = model( + input_ids=packed["input_ids"], + position_ids=thd_position_ids, + attention_mask=None, + labels=packed["labels"], + loss_mask=packed["loss_mask"], + packed_seq_params=psp, + ) + loss = mean_loss(output, packed["loss_mask"]) + loss.backward() + gn = grad_norm(model) + loss_val = loss.item() + + model.zero_grad() + + loss_finite = torch.isfinite(torch.tensor(loss_val)).item() + grad_finite = torch.isfinite(torch.tensor(gn)).item() + grad_nonzero = gn > 0 + + return dict( + loss=loss_val, + grad_norm=gn, + total_tokens=T, + loss_finite=loss_finite, + grad_finite=grad_finite, + grad_nonzero=grad_nonzero, + passed=loss_finite and grad_finite and grad_nonzero, + ) + + +# =================================================================== +# Main +# =================================================================== + + +def _print_banner(title): + print(f"\n{'='*60}") + print(f" {title}") + print(f"{'='*60}") + + +def main(): + """CLI entry: run the equal-length parity test + variable-length smoke test.""" + parser = argparse.ArgumentParser(description="BSHD vs THD correctness test") + parser.add_argument("--batch-size", type=int, default=4) + parser.add_argument("--seq-len", type=int, default=128) + parser.add_argument("--vocab-size", type=int, default=1024) + parser.add_argument("--hidden-size", type=int, default=256) + parser.add_argument("--num-layers", type=int, default=2) + parser.add_argument("--num-heads", type=int, default=4) + parser.add_argument("--num-kv-heads", type=int, default=2) + parser.add_argument("--ffn-hidden-size", type=int, default=512) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument( + "--atol-loss", type=float, default=1e-5, help="Absolute tolerance for loss comparison" + ) + parser.add_argument( + "--rtol-grad", type=float, default=1e-3, help="Relative tolerance for grad norm comparison" + ) + args = parser.parse_args() + + Utils.initialize_model_parallel(tensor_model_parallel_size=1) + model_parallel_cuda_manual_seed(args.seed) + + config = TransformerConfig( + num_layers=args.num_layers, + hidden_size=args.hidden_size, + ffn_hidden_size=args.ffn_hidden_size, + num_attention_heads=args.num_heads, + num_query_groups=args.num_kv_heads, + bf16=True, + params_dtype=torch.bfloat16, + pipeline_dtype=torch.bfloat16, + hidden_dropout=0.0, + attention_dropout=0.0, + tensor_model_parallel_size=1, + sequence_parallel=False, + ) + + model = _build_model(config, args.vocab_size, args.seq_len) + + all_passed = True + + # ---------------------------------------------------------------- + # Test 1: equal-length correctness (BSHD vs THD) + # ---------------------------------------------------------------- + _print_banner("Test 1: Equal-length BSHD vs THD correctness") + m = run_equal_length_test( + model=model, + batch_size=args.batch_size, + seq_len=args.seq_len, + vocab_size=args.vocab_size, + seed=args.seed, + atol_loss=args.atol_loss, + rtol_grad=args.rtol_grad, + ) + print( + f" Config: B={args.batch_size}, S={args.seq_len}, " + f"H={args.hidden_size}, L={args.num_layers}, " + f"heads={args.num_heads}/{args.num_kv_heads}" + ) + print(f" BSHD loss: {m['bshd_loss']:.8f}") + print(f" THD loss: {m['thd_loss']:.8f}") + print(f" Loss abs diff: {m['loss_diff']:.2e}") + print(f" BSHD grad norm: {m['bshd_grad_norm']:.8f}") + print(f" THD grad norm: {m['thd_grad_norm']:.8f}") + print(f" Grad norm rel diff: {m['grad_rel']:.2e}") + print(f" Per-token max diff: {m['token_max_diff']:.2e}") + print(f" Per-token mean diff: {m['token_mean_diff']:.2e}") + print( + f" Loss match: {'PASS' if m['loss_ok'] else 'FAIL'} " f"(atol={args.atol_loss})" + ) + print( + f" Grad norm match: {'PASS' if m['grad_ok'] else 'FAIL'} " f"(rtol={args.rtol_grad})" + ) + if not (m["loss_ok"] and m["grad_ok"]): + all_passed = False + + # ---------------------------------------------------------------- + # Test 2: variable-length smoke test (THD only) + # ---------------------------------------------------------------- + _print_banner("Test 2: Variable-length THD smoke test") + v = run_variable_length_smoke_test(model, args.vocab_size, args.seed) + print(f" Seq lengths: [128, 96, 112, 80]") + print(f" Total packed tokens: {v['total_tokens']}") + print(f" Loss: {v['loss']:.8f}") + print(f" Grad norm: {v['grad_norm']:.8f}") + print(f" Loss finite: {'PASS' if v['loss_finite'] else 'FAIL'}") + print(f" Grad finite: {'PASS' if v['grad_finite'] else 'FAIL'}") + print(f" Grad nonzero: {'PASS' if v['grad_nonzero'] else 'FAIL'}") + if not v["passed"]: + all_passed = False + + # ---------------------------------------------------------------- + # Summary + # ---------------------------------------------------------------- + _print_banner("Summary") + if all_passed: + print(" ALL TESTS PASSED") + else: + print(" SOME TESTS FAILED") + print(f"{'='*60}\n") + + Utils.destroy_model_parallel() + + if not all_passed: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/examples/multimodal_dev/tests/test_thd_e2e.py b/examples/multimodal_dev/tests/test_thd_e2e.py new file mode 100644 index 00000000000..c8d9f2e1622 --- /dev/null +++ b/examples/multimodal_dev/tests/test_thd_e2e.py @@ -0,0 +1,314 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Tests for THD / padded batch construction in multimodal_dev. + +Exercises the production data path :func:`pack_or_pad_batch`, which +consumes a list of per-sample dicts produced by the dataset's +``__getitem__`` and produces either a packed THD batch (``[1, T]``) or a +padded BSHD batch (``[B, S]``). + +``pack_or_pad_batch`` ends with a TP-group broadcast, so these tests +require ``torch.distributed`` to be initialised. Run via:: + + torchrun --nproc-per-node 1 -m pytest -q \\ + examples/multimodal_dev/tests/test_thd_e2e.py +""" + +import os +import sys + +import pytest +import torch + +_REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../..")) +if _REPO_ROOT not in sys.path: + sys.path.insert(0, _REPO_ROOT) + +from examples.multimodal_dev.forward_step import _build_packed_seq_params, pack_or_pad_batch +from tests.unit_tests.test_utilities import Utils + + +@pytest.fixture(scope="module", autouse=True) +def _init_model_parallel(): + """Single-rank TP init so pack_or_pad_batch's TP broadcast is a no-op.""" + Utils.initialize_model_parallel(tensor_model_parallel_size=1) + yield + Utils.destroy_model_parallel() + + +def _make_sample( + seq_len: int, *, base: int = 0, num_patches: int = 4, pixel_dim: int = 8, device: str = "cuda" +): + """Per-sample dict in the shape produced by ``CordV2VLMDataset.__getitem__``. + + 1-D ``input_ids`` / ``labels`` / ``loss_mask`` over the sequence dim; + ``pixel_values`` is ``[num_patches, pixel_dim]``; ``image_grid_thw`` is + ``[1, 3]``. + """ + return { + "input_ids": torch.arange(seq_len, dtype=torch.long, device=device) + base, + "labels": (torch.arange(seq_len, dtype=torch.long, device=device) + base + 100), + "loss_mask": torch.ones(seq_len, dtype=torch.float, device=device), + "pixel_values": torch.full((num_patches, pixel_dim), float(base), device=device), + "image_grid_thw": torch.tensor([[2, 4, 4]], dtype=torch.long, device=device), + } + + +# =================================================================== +# _build_packed_seq_params — pure helper, exercised independently +# =================================================================== + + +class TestBuildPackedSeqParams: + """Tests for ``_build_packed_seq_params``.""" + + def test_basic(self): + """Mixed-length sample build sanity check.""" + params = _build_packed_seq_params(torch.tensor([5, 3, 7], dtype=torch.int32), device="cpu") + assert params.qkv_format == "thd" + assert params.cu_seqlens_q.tolist() == [0, 5, 8, 15] + assert params.cu_seqlens_kv.tolist() == [0, 5, 8, 15] + assert params.max_seqlen_q == 7 + assert params.max_seqlen_kv == 7 + assert params.total_tokens == 15 + assert params.cu_seqlens_q_padded.tolist() == [0, 5, 8, 15] + assert params.cu_seqlens_kv_padded.tolist() == [0, 5, 8, 15] + + def test_equal_lengths(self): + """Equal-length samples produce uniform cu_seqlens.""" + params = _build_packed_seq_params(torch.tensor([4, 4, 4], dtype=torch.int32), device="cpu") + assert params.cu_seqlens_q.tolist() == [0, 4, 8, 12] + assert params.max_seqlen_q == 4 + assert params.total_tokens == 12 + + def test_single_sample(self): + """Single-sample batch round-trips its own length.""" + params = _build_packed_seq_params(torch.tensor([10], dtype=torch.int32), device="cpu") + assert params.cu_seqlens_q.tolist() == [0, 10] + assert params.max_seqlen_q == 10 + assert params.total_tokens == 10 + + def test_dtype_is_int32(self): + """``cu_seqlens_q`` is cast to int32 regardless of input dtype.""" + params = _build_packed_seq_params(torch.tensor([3, 5], dtype=torch.int32), device="cpu") + assert params.cu_seqlens_q.dtype == torch.int32 + + def test_seq_idx_computed(self): + """``__post_init__`` computes ``seq_idx`` for Mamba compatibility.""" + params = _build_packed_seq_params(torch.tensor([3, 2], dtype=torch.int32), device="cpu") + assert params.seq_idx is not None + assert params.seq_idx.shape == (1, 5) + assert params.seq_idx[0].tolist() == [0, 0, 0, 1, 1] + + +# =================================================================== +# pack_or_pad_batch — packed (THD) mode +# =================================================================== + + +class TestPackOrPadBatchPacked: + """``pack_or_pad_batch(..., use_packed_sequence=True)`` produces ``[1, T]``.""" + + def test_equal_lengths(self): + """Two equal-length samples → packed ``[1, 2S]``.""" + S = 8 + batch = [_make_sample(S, base=0), _make_sample(S, base=1000)] + packed = pack_or_pad_batch(batch, use_packed_sequence=True, device="cuda") + + T = 2 * S + assert packed["input_ids"].shape == (1, T) + assert packed["labels"].shape == (1, T) + assert packed["loss_mask"].shape == (1, T) + psp = packed["packed_seq_params"] + assert psp.cu_seqlens_q.tolist() == [0, S, T] + assert psp.cu_seqlens_q_padded.tolist() == [0, S, T] + assert psp.max_seqlen_q == S + assert psp.total_tokens == T + + def test_variable_lengths(self): + """Variable-length samples concatenated end-to-end.""" + lens = [5, 8, 3] + batch = [_make_sample(L, base=i * 1000) for i, L in enumerate(lens)] + packed = pack_or_pad_batch(batch, use_packed_sequence=True, device="cuda") + + T = sum(lens) + assert packed["input_ids"].shape == (1, T) + psp = packed["packed_seq_params"] + assert psp.cu_seqlens_q.tolist() == [0, 5, 13, 16] + assert psp.cu_seqlens_q_padded.tolist() == [0, 5, 13, 16] + assert psp.max_seqlen_q == 8 + assert psp.total_tokens == T + + def test_token_order_preserved(self): + """Sample 0's tokens precede sample 1's tokens in the packed sequence.""" + s0 = _make_sample(3, base=10) + s1 = _make_sample(3, base=40) + packed = pack_or_pad_batch([s0, s1], use_packed_sequence=True, device="cuda") + assert packed["input_ids"][0].tolist() == [10, 11, 12, 40, 41, 42] + + def test_labels_loss_mask_content_preserved(self): + """labels and loss_mask carry through unchanged when divisible_by=1.""" + s = _make_sample(4, base=0) + packed = pack_or_pad_batch([s], use_packed_sequence=True, device="cuda") + assert packed["labels"][0].tolist() == [100, 101, 102, 103] + assert packed["loss_mask"][0].tolist() == [1.0, 1.0, 1.0, 1.0] + + def test_pixel_values_concatenated(self): + """``pixel_values`` are concatenated along the patch dim.""" + s0 = _make_sample(4, base=0, num_patches=4, pixel_dim=8) + s1 = _make_sample(4, base=10, num_patches=6, pixel_dim=8) + packed = pack_or_pad_batch([s0, s1], use_packed_sequence=True, device="cuda") + assert packed["pixel_values"].shape == (10, 8) + assert packed["pixel_values"][:4].eq(0.0).all().item() + assert packed["pixel_values"][4:].eq(10.0).all().item() + + def test_image_grid_thw_concatenated(self): + """``image_grid_thw`` rows are concatenated along the first dim.""" + s0 = _make_sample(4, base=0) + s1 = _make_sample(4, base=10) + packed = pack_or_pad_batch([s0, s1], use_packed_sequence=True, device="cuda") + assert packed["image_grid_thw"].shape == (2, 3) + + def test_single_sample_round_trip(self): + """A single sample packs to its own length.""" + s = _make_sample(7, base=0) + packed = pack_or_pad_batch([s], use_packed_sequence=True, device="cuda") + assert packed["input_ids"].shape == (1, 7) + psp = packed["packed_seq_params"] + assert psp.cu_seqlens_q.tolist() == [0, 7] + assert psp.max_seqlen_q == 7 + assert psp.total_tokens == 7 + + +# =================================================================== +# pack_or_pad_batch — padded (BSHD) mode +# =================================================================== + + +class TestPackOrPadBatchPadded: + """``pack_or_pad_batch(..., use_packed_sequence=False)`` produces ``[B, S]``.""" + + def test_equal_lengths(self): + """Equal-length samples → ``[B, S]`` without further padding.""" + S = 6 + batch = [_make_sample(S, base=0), _make_sample(S, base=10)] + padded = pack_or_pad_batch(batch, use_packed_sequence=False, seq_length=S, device="cuda") + assert padded["input_ids"].shape == (2, S) + assert padded["labels"].shape == (2, S) + assert padded["loss_mask"].shape == (2, S) + # Sample-0 content is preserved verbatim. + assert padded["input_ids"][0].tolist() == list(range(S)) + + def test_pads_short_sample_to_batch_max(self): + """Shorter sample is right-padded to match the batch max length.""" + long_sample = _make_sample(7, base=0) + short_sample = _make_sample(3, base=10) + padded = pack_or_pad_batch( + [long_sample, short_sample], use_packed_sequence=False, seq_length=7, device="cuda" + ) + assert padded["input_ids"].shape == (2, 7) + # Short sample: original [10, 11, 12] then pad zeros. + assert padded["input_ids"][1].tolist() == [10, 11, 12, 0, 0, 0, 0] + # labels pad with -100 (ignore index). + assert padded["labels"][1].tolist() == [110, 111, 112, -100, -100, -100, -100] + # loss_mask pads with 0. + assert padded["loss_mask"][1].tolist() == [1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0] + + def test_seq_length_required(self): + """``seq_length`` must be provided in padded mode.""" + s = _make_sample(4, base=0) + with pytest.raises(AssertionError, match="seq_length"): + pack_or_pad_batch([s], use_packed_sequence=False, seq_length=None, device="cuda") + + def test_pixel_values_concatenated(self): + """``pixel_values`` concat preserves both samples' patches.""" + s0 = _make_sample(4, base=0, num_patches=4, pixel_dim=8) + s1 = _make_sample(4, base=10, num_patches=6, pixel_dim=8) + padded = pack_or_pad_batch([s0, s1], use_packed_sequence=False, seq_length=4, device="cuda") + assert padded["pixel_values"].shape == (10, 8) + assert padded["pixel_values"][:4].eq(0.0).all().item() + assert padded["pixel_values"][4:].eq(10.0).all().item() + + +# =================================================================== +# pack_or_pad_batch — divisible_by = 4 alignment +# =================================================================== + + +class TestPackOrPadBatchDivisibleBy4: + """Per-sample sequence alignment when ``divisible_by = 4``. + + The function computes ``divisible_by`` from the parallel state. With + ``world_size=1`` we cannot stand up a real CP=2 group, so we patch + :func:`mpu.get_context_parallel_world_size` to return 2; the function + then takes the ``cp_size > 1`` branch and yields + ``divisible_by = cp_size * 2 = 4`` (no SP). + """ + + @pytest.fixture + def cp2(self, monkeypatch): + """Patch CP world size to 2 so ``divisible_by = 4``.""" + from examples.multimodal_dev import forward_step + + monkeypatch.setattr(forward_step.mpu, "get_context_parallel_world_size", lambda: 2) + + def test_packed_aligned_samples_no_padding(self, cp2): + """Samples already multiples of 4 → cu_seqlens == cu_seqlens_padded.""" + batch = [_make_sample(8, base=0), _make_sample(4, base=100)] + packed = pack_or_pad_batch(batch, use_packed_sequence=True, device="cuda") + + T = 12 + assert packed["input_ids"].shape == (1, T) + psp = packed["packed_seq_params"] + assert psp.cu_seqlens_q.tolist() == [0, 8, 12] + assert psp.cu_seqlens_q_padded.tolist() == [0, 8, 12] + assert psp.max_seqlen_q == 8 + assert psp.total_tokens == 12 + + def test_packed_misaligned_samples_padded_per_sample(self, cp2): + """Each sample padded up to the nearest multiple of 4.""" + # lens=[5, 8, 3] → padded=[8, 8, 4] → T_padded = 20. + batch = [_make_sample(5, base=0), _make_sample(8, base=100), _make_sample(3, base=200)] + packed = pack_or_pad_batch(batch, use_packed_sequence=True, device="cuda") + + T_padded = 20 + assert packed["input_ids"].shape == (1, T_padded) + psp = packed["packed_seq_params"] + # cu_seqlens reflects real per-sample lengths. + assert psp.cu_seqlens_q.tolist() == [0, 5, 13, 16] + # cu_seqlens_padded reflects per-sample alignment to 4. + assert psp.cu_seqlens_q_padded.tolist() == [0, 8, 16, 20] + # max_seqlen comes from the padded lengths. + assert psp.max_seqlen_q == 8 + assert psp.total_tokens == T_padded + + def test_packed_pad_values(self, cp2): + """Pad slots filled with input_ids=0, labels=-100, loss_mask=0.""" + # Single sample len=3 → target_len=4 → 1 pad slot at position 3. + batch = [_make_sample(3, base=10)] + packed = pack_or_pad_batch(batch, use_packed_sequence=True, device="cuda") + + assert packed["input_ids"].shape == (1, 4) + assert packed["input_ids"][0].tolist() == [10, 11, 12, 0] + assert packed["labels"][0].tolist() == [110, 111, 112, -100] + assert packed["loss_mask"][0].tolist() == [1.0, 1.0, 1.0, 0.0] + psp = packed["packed_seq_params"] + assert psp.cu_seqlens_q.tolist() == [0, 3] + assert psp.cu_seqlens_q_padded.tolist() == [0, 4] + assert psp.max_seqlen_q == 4 + assert psp.total_tokens == 4 + + def test_padded_target_rounded_up_to_multiple_of_4(self, cp2): + """Padded (BSHD) mode: ``target = ceil(min(max, seq_length) / 4) * 4``.""" + # lens=[5, 3], seq_length=10 → min(5, 10) = 5 → ceil(5/4)*4 = 8. + long_sample = _make_sample(5, base=0) + short_sample = _make_sample(3, base=10) + padded = pack_or_pad_batch( + [long_sample, short_sample], use_packed_sequence=False, seq_length=10, device="cuda" + ) + + assert padded["input_ids"].shape == (2, 8) + assert padded["input_ids"][0].tolist() == [0, 1, 2, 3, 4, 0, 0, 0] + assert padded["input_ids"][1].tolist() == [10, 11, 12, 0, 0, 0, 0, 0] + assert padded["labels"][1].tolist() == [110, 111, 112, -100, -100, -100, -100, -100] + assert padded["loss_mask"][1].tolist() == [1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0] diff --git a/examples/multimodal_dev/tests/test_vision_patch_merger_parity.py b/examples/multimodal_dev/tests/test_vision_patch_merger_parity.py new file mode 100644 index 00000000000..62c908ea91d --- /dev/null +++ b/examples/multimodal_dev/tests/test_vision_patch_merger_parity.py @@ -0,0 +1,187 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Numerical parity for ``Qwen35VLPatchMerger`` vs HuggingFace reference. + +The HuggingFace reference (``Qwen3VLVisionPatchMerger`` in +``transformers/models/qwen3_vl/modeling_qwen3_vl.py``, branch +``use_postshuffle_norm=False``) is reproduced inline so this test does not +require ``transformers`` to be installed. The HF module is verbatim:: + + self.norm = nn.LayerNorm(hidden_size, eps=1e-6) + self.linear_fc1 = nn.Linear(merge_dim, merge_dim) + self.act_fn = nn.GELU() # default approximate='none' + self.linear_fc2 = nn.Linear(merge_dim, out_hidden_size) + + x = self.norm(x) + x = x.view(-1, merge_dim) + x = self.linear_fc2(self.act_fn(self.linear_fc1(x))) + +With matching dims and weights copied across, the Megatron +implementation must agree: + + * fp32 forward: max-abs diff <= 1e-4 (TE LayerNorm vs nn.LayerNorm + have different fused reduction order; ~1e-5 absolute residual is + structural and not a real divergence) + * bf16 forward: max-abs diff <= 5e-2 (bf16 ceiling for two-layer MLP) + +Run with:: + + torchrun --nproc_per_node=1 \\ + examples/multimodal_dev/tests/test_vision_patch_merger_parity.py +""" + +import os +import sys + +import torch +import torch.distributed as dist +import torch.nn as nn + +_REPO_ROOT = os.path.abspath( + os.path.join(os.path.dirname(__file__), "../../.."), +) +if _REPO_ROOT in sys.path: + sys.path.remove(_REPO_ROOT) +sys.path.insert(0, _REPO_ROOT) + +from megatron.core import parallel_state as ps +from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed +from megatron.core.transformer.transformer_config import TransformerConfig + +from examples.multimodal_dev.models.qwen35_vl.vision_encoder import Qwen35VLPatchMerger + + +# Match Qwen3.5-VL 9B / 397B-A17B vision tower dims. +HIDDEN_SIZE = 1152 +OUT_HIDDEN_SIZE = 3584 +SPATIAL_MERGE_SIZE = 2 +NUM_PATCHES = 64 # must be divisible by spatial_merge_size ** 2 + +ATOL_FP32 = 1e-4 +RTOL_FP32 = 1e-3 +ATOL_BF16 = 5e-2 +RTOL_BF16 = 5e-2 + + +class HFPatchMergerReference(nn.Module): + """Inline HF ``Qwen3VLVisionPatchMerger`` (use_postshuffle_norm=False).""" + + def __init__(self, hidden_size: int, out_hidden_size: int, spatial_merge_size: int): + super().__init__() + self.merge_dim = hidden_size * (spatial_merge_size ** 2) + self.norm = nn.LayerNorm(hidden_size, eps=1e-6) + self.linear_fc1 = nn.Linear(self.merge_dim, self.merge_dim) + self.act_fn = nn.GELU() # approximate='none' by default + self.linear_fc2 = nn.Linear(self.merge_dim, out_hidden_size) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + x = self.norm(hidden_states) + x = x.view(-1, self.merge_dim) + x = self.linear_fc1(x) + x = self.act_fn(x) + x = self.linear_fc2(x) + return x + + +def _init_distributed() -> int: + if not dist.is_initialized(): + dist.init_process_group(backend="nccl") + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + torch.cuda.set_device(local_rank) + return local_rank + + +def _init_megatron_parallel() -> None: + ps.destroy_model_parallel() + ps.initialize_model_parallel(tensor_model_parallel_size=1) + model_parallel_cuda_manual_seed(42) + + +def _build_config(dtype: torch.dtype) -> TransformerConfig: + is_bf16 = dtype is torch.bfloat16 + return TransformerConfig( + num_layers=1, + hidden_size=HIDDEN_SIZE, + ffn_hidden_size=HIDDEN_SIZE, + num_attention_heads=8, + kv_channels=HIDDEN_SIZE // 8, + tensor_model_parallel_size=1, + pipeline_model_parallel_size=1, + sequence_parallel=False, + bf16=is_bf16, + params_dtype=dtype, + pipeline_dtype=dtype, + add_bias_linear=True, + gated_linear_unit=False, + normalization="LayerNorm", + layernorm_epsilon=1e-6, + attention_dropout=0.0, + hidden_dropout=0.0, + ) + + +def _copy_hf_to_megatron(hf: HFPatchMergerReference, mcore: Qwen35VLPatchMerger) -> None: + """TP=1: 1:1 parameter copy between HF nn.Module and the MCore module.""" + with torch.no_grad(): + mcore.patch_norm.weight.copy_(hf.norm.weight.to(mcore.patch_norm.weight.dtype)) + mcore.patch_norm.bias.copy_(hf.norm.bias.to(mcore.patch_norm.bias.dtype)) + mcore.linear_fc1.weight.copy_(hf.linear_fc1.weight.to(mcore.linear_fc1.weight.dtype)) + mcore.linear_fc1.bias.copy_(hf.linear_fc1.bias.to(mcore.linear_fc1.bias.dtype)) + mcore.linear_fc2.weight.copy_(hf.linear_fc2.weight.to(mcore.linear_fc2.weight.dtype)) + mcore.linear_fc2.bias.copy_(hf.linear_fc2.bias.to(mcore.linear_fc2.bias.dtype)) + + +def _run_one(dtype: torch.dtype, atol: float, rtol: float, device: torch.device, seed: int = 42) -> None: + torch.manual_seed(seed) + + hf_ref = HFPatchMergerReference( + hidden_size=HIDDEN_SIZE, + out_hidden_size=OUT_HIDDEN_SIZE, + spatial_merge_size=SPATIAL_MERGE_SIZE, + ).to(device=device, dtype=dtype).eval() + + config = _build_config(dtype) + mcore = Qwen35VLPatchMerger( + config=config, + hidden_size=HIDDEN_SIZE, + out_hidden_size=OUT_HIDDEN_SIZE, + spatial_merge_size=SPATIAL_MERGE_SIZE, + ).to(device=device, dtype=dtype).eval() + + _copy_hf_to_megatron(hf_ref, mcore) + + x = torch.randn(NUM_PATCHES, HIDDEN_SIZE, device=device, dtype=dtype) + + with torch.no_grad(): + y_hf = hf_ref(x) + y_mcore = mcore(x) + + assert y_hf.shape == y_mcore.shape, (y_hf.shape, y_mcore.shape) + diff = (y_hf - y_mcore).abs() + print( + f"[{dtype}] shape={tuple(y_hf.shape)} " + f"max_abs_diff={diff.max().item():.3e} " + f"mean_abs_diff={diff.mean().item():.3e} " + f"hf_norm={y_hf.float().norm().item():.4f} " + f"mcore_norm={y_mcore.float().norm().item():.4f}" + ) + torch.testing.assert_close(y_mcore, y_hf, atol=atol, rtol=rtol) + + +def main() -> None: + local_rank = _init_distributed() + _init_megatron_parallel() + device = torch.device(f"cuda:{local_rank}") + + _run_one(torch.float32, ATOL_FP32, RTOL_FP32, device) + _run_one(torch.bfloat16, ATOL_BF16, RTOL_BF16, device) + + if int(os.environ.get("RANK", 0)) == 0: + print( + "\nPASS: Qwen35VLPatchMerger logits match HF Qwen3VLVisionPatchMerger " + "in both fp32 and bf16." + ) + + +if __name__ == "__main__": + main() diff --git a/examples/post_training/modelopt/README.md b/examples/post_training/modelopt/README.md index a6fcaf8f5b7..c20476bde45 100644 --- a/examples/post_training/modelopt/README.md +++ b/examples/post_training/modelopt/README.md @@ -35,6 +35,7 @@ knowledge distillation, pruning, speculative decoding, and more. | `nvidia/NVIDIA-Nemotron-Nano-9B-v2` | ✅ | - | ✅ | ✅ | | `nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16` | ✅ | - | ✅ | ✅ | | `nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16` | ✅ | - | ✅ | ✅ | +| `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16` | ✅ | - | ✅ | ✅ | | `openai/gpt-oss-{20b, 120b}` | ✅ | **Online** | ✅ | ✅ | | `Qwen/Qwen3-{0.6B, 8B}` | ✅ | ✅ | ✅ | ✅ | | `Qwen/Qwen3-{30B-A3B, 235B-A22B}` | **WAR** | ✅ | ✅ | ✅ | @@ -165,45 +166,47 @@ Then only the draft model is called during training. AL is no longer reported du ### ⭐ Pruning -Checkout pruning getting started section and guidelines for configuring pruning parameters in the [ModelOpt pruning README](https://github.com/NVIDIA/Model-Optimizer/tree/main/examples/pruning). +Pruning is supported for GPT and Mamba models in Pipeline Parallel mode. The `prune.sh` script +prunes a model by passing `--prune-export-config ''` to `prune.py` via `MLM_EXTRA_ARGS`. +The JSON describes the target pruned architecture; calibration data is used to compute importance +scores that drive the dimension reduction. -Pruning is supported for GPT and Mamba models in Pipeline Parallel mode. Available pruning dimensions are: - -- `TARGET_FFN_HIDDEN_SIZE` -- `TARGET_HIDDEN_SIZE` -- `TARGET_NUM_ATTENTION_HEADS` -- `TARGET_NUM_QUERY_GROUPS` -- `TARGET_MAMBA_NUM_HEADS` -- `TARGET_MAMBA_HEAD_DIM` -- `TARGET_NUM_MOE_EXPERTS` -- `TARGET_MOE_FFN_HIDDEN_SIZE` -- `TARGET_MOE_SHARED_EXPERT_INTERMEDIATE_SIZE` -- `TARGET_NUM_LAYERS` -- `LAYERS_TO_DROP` (comma separated, 1-indexed list of layer numbers to directly drop) +Supported hyperparameters (any subset can appear as keys in `--prune-export-config`): +`hidden_size`, `ffn_hidden_size`, `num_attention_heads`, `num_query_groups`, `mamba_num_heads`, +`mamba_head_dim`, `num_moe_experts`, `moe_ffn_hidden_size`, `moe_shared_expert_intermediate_size`, +`num_layers`. Example for depth pruning Qwen3-8B from 36 to 24 layers: ```sh PP=1 \ -TARGET_NUM_LAYERS=24 \ +MLM_EXTRA_ARGS='--prune-export-config {"num_layers":24}' \ HF_MODEL_CKPT= \ MLM_MODEL_SAVE=Qwen3-8B-Pruned \ ./prune.sh Qwen/Qwen3-8B ``` +The default calibration dataset is `nemotron-post-training-dataset-v2` (gated, requires +`hf auth login`). Override it by adding `--calib-dataset ` +to `MLM_EXTRA_ARGS` (e.g. `cnn_dailymail` for an ungated alternative). + > [!TIP] > If number of layers in the model is not divisible by pipeline parallel size (PP), you can configure uneven -> PP by setting `MLM_EXTRA_ARGS="--decoder-first-pipeline-num-layers --decoder-last-pipeline-num-layers "` +> PP by adding `--decoder-first-pipeline-num-layers --decoder-last-pipeline-num-layers ` to `MLM_EXTRA_ARGS`. > [!TIP] -> You can reuse pruning scores for pruning same model again to different architectures by setting -> `PRUNE_ARGS="--pruning-scores-path "` +> You can reuse intermediate pruning scores when pruning the same model again to a different config +> by adding `--prune-intermediate-ckpt ` to `MLM_EXTRA_ARGS`. > [!NOTE] > When loading pruned M-LM checkpoint for subsequent steps, make sure overwrite the pruned parameters in the > default `conf/` by setting `MLM_EXTRA_ARGS`. E.g.: for loading above pruned Qwen3-8B checkpoint for mmlu, set: > `MLM_EXTRA_ARGS="--num-layers 24"` +For NAS-based automatic pruning (search across many candidate architectures and pick the best via +MMLU scoring), see the [Megatron-Bridge pruning example](https://github.com/NVIDIA/Model-Optimizer/tree/main/examples/megatron_bridge#pruning). +Checkout pruning getting started and general guidelines in the [ModelOpt pruning README](https://github.com/NVIDIA/Model-Optimizer/tree/main/examples/pruning). + ### ⭐ Inference and Training The saved Megatron-LM distributed checkpoint (output of above scripts) can be resumed for inference diff --git a/examples/post_training/modelopt/conf/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16.sh b/examples/post_training/modelopt/conf/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16.sh new file mode 100644 index 00000000000..6e9c99c42c5 --- /dev/null +++ b/examples/post_training/modelopt/conf/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +if [ -z "${HF_MODEL_CKPT:-}" ]; then + HF_MODEL_CKPT=nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16 +fi + +if [ -z "${TOKENIZER_MODEL:-}" ]; then + TOKENIZER_MODEL=${HF_MODEL_CKPT} +fi + +MODEL_ARGS=" \ + --trust-remote-code \ + --save-interval 100 \ + --micro-batch-size 1 \ + --enable-experimental \ + --use-fused-weighted-squared-relu \ + --cross-entropy-loss-fusion \ + --cross-entropy-fusion-impl native \ + --moe-permute-fusion \ + --moe-latent-size 2048 \ + --moe-router-score-function sigmoid \ + --moe-grouped-gemm \ + --num-experts 512 \ + --moe-router-topk 22 \ + --moe-shared-expert-intermediate-size 10240 \ + --moe-aux-loss-coeff 1e-4 \ + --moe-router-topk-scaling-factor 5.0 \ + --moe-router-enable-expert-bias \ + --moe-router-dtype fp32 \ + --moe-router-load-balancing-type seq_aux_loss \ + --moe-token-dispatcher-type alltoall \ + \ + --attention-backend flash \ + --disable-gloo-process-groups \ + --is-hybrid-model \ + --mamba-num-heads 256 \ + --mamba-head-dim 64 \ + --hybrid-override-pattern MEMEMEM*EMEMEM*EMEMEMEM*EMEMEMEM*EMEMEM*EMEMEMEM*EMEMEMEM*EMEMEM*EMEMEMEM*EMEMEMEM*EMEMEM*EMEMEMEM*EMEMEMEME \ + --use-mcore-models \ + --untie-embeddings-and-output-weights \ + --disable-bias-linear \ + --init-method-std 0.0099 \ + --position-embedding-type none \ + --squared-relu \ + --num-layers 108 \ + --hidden-size 8192 \ + --num-attention-heads 64 \ + --group-query-attention \ + --num-query-groups 2 \ + --ffn-hidden-size 5120 \ + --kv-channels 128 \ + --normalization RMSNorm \ + \ + --tokenizer-type HuggingFaceTokenizer \ + --bf16 \ + --seq-length 8192 \ + --max-position-embeddings 8192 \ + --export-model-type HybridModel \ + " diff --git a/examples/post_training/modelopt/convert_model.py b/examples/post_training/modelopt/convert_model.py index cc34e1e3e3c..0e449f6b576 100644 --- a/examples/post_training/modelopt/convert_model.py +++ b/examples/post_training/modelopt/convert_model.py @@ -17,6 +17,7 @@ from megatron.core import mpu from megatron.core.enums import ModelType from megatron.core.parallel_state import destroy_model_parallel +from megatron.core.utils import unwrap_model from megatron.post_training.arguments import add_modelopt_args from megatron.post_training.checkpointing import load_modelopt_checkpoint from megatron.post_training.model_builder import modelopt_gpt_hybrid_builder @@ -25,7 +26,7 @@ from megatron.training.arguments import parse_and_validate_args from megatron.training.checkpointing import save_checkpoint from megatron.training.initialize import initialize_megatron -from megatron.training.utils import print_rank_0, unwrap_model +from megatron.training.utils import print_rank_0 from model_provider import model_provider ALGO_TO_CONFIG = { diff --git a/examples/post_training/modelopt/export.py b/examples/post_training/modelopt/export.py index c080115d729..f905a2edeea 100755 --- a/examples/post_training/modelopt/export.py +++ b/examples/post_training/modelopt/export.py @@ -13,13 +13,13 @@ import modelopt.torch.export as mtex import torch +from megatron.core.utils import unwrap_model from megatron.post_training.arguments import add_modelopt_args from megatron.post_training.checkpointing import load_modelopt_checkpoint from megatron.post_training.model_builder import modelopt_gpt_hybrid_builder from megatron.training import get_args, get_model from megatron.training.arguments import parse_and_validate_args from megatron.training.initialize import initialize_megatron -from megatron.training.utils import unwrap_model from model_provider import model_provider warnings.filterwarnings('ignore') diff --git a/examples/post_training/modelopt/finetune.py b/examples/post_training/modelopt/finetune.py index adff7421fd3..7ca446dbcf4 100755 --- a/examples/post_training/modelopt/finetune.py +++ b/examples/post_training/modelopt/finetune.py @@ -2,11 +2,10 @@ """Supervised Finetuning GPT.""" import itertools -import json import os import sys from functools import partial -from typing import Any, Dict, Optional +from typing import Any, Dict sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../"))) @@ -18,16 +17,14 @@ from megatron.core import mpu, tensor_parallel from megatron.core.enums import ModelType from megatron.core.models.gpt import GPTModel +from megatron.core.parallel_state import get_context_parallel_group +from megatron.core.utils import get_batch_on_this_cp_rank from megatron.post_training.arguments import add_modelopt_args from megatron.post_training.loss_func import loss_func from megatron.post_training.model_builder import modelopt_gpt_hybrid_builder from megatron.post_training.non_loss_data_func import report_draft_acceptance_length from megatron.training import get_args, get_timers, pretrain -from megatron.training.utils import ( - get_batch_on_this_cp_rank, - get_ltor_masks_and_position_ids, - print_rank_0, -) +from megatron.training.utils import get_ltor_masks_and_position_ids, print_rank_0 from model_provider import model_provider REMOVE_THINK_CHAT_TEMPLATE = ( @@ -452,7 +449,9 @@ def get_batch(data_iterator): batch["hidden_states"] = feature_b["hidden_states"].transpose(0, 1)[: args.seq_length] # slice batch along sequence dimension for context parallelism - batch = get_batch_on_this_cp_rank(batch) + batch = get_batch_on_this_cp_rank( + batch, is_hybrid_cp=False, cp_group=get_context_parallel_group() + ) return batch @@ -507,13 +506,15 @@ def forward_step(data_iterator, model: GPTModel): if __name__ == "__main__": + from megatron.training.argument_utils import pretrain_cfg_container_from_args from megatron.training.arguments import parse_and_validate_args - parse_and_validate_args( + args = parse_and_validate_args( extra_args_provider=add_finetune_args, args_defaults={"tokenizer_type": "HuggingFaceTokenizer"}, ) pretrain( + pretrain_cfg_container_from_args(args), train_valid_test_sft_datasets_provider, partial(model_provider, modelopt_gpt_hybrid_builder), ModelType.encoder_or_decoder, diff --git a/examples/post_training/modelopt/generate.py b/examples/post_training/modelopt/generate.py index 75807cac7f7..7c6afa77847 100644 --- a/examples/post_training/modelopt/generate.py +++ b/examples/post_training/modelopt/generate.py @@ -11,16 +11,17 @@ import modelopt.torch.quantization as mtq import torch from datasets import load_dataset +from modelopt.torch.utils.plugins import megatron_generate from utils import get_hf_tokenizer +from megatron.core.utils import unwrap_model from megatron.post_training.arguments import add_modelopt_args from megatron.post_training.checkpointing import load_modelopt_checkpoint -from megatron.post_training.generate import simple_generate from megatron.post_training.model_builder import modelopt_gpt_hybrid_builder from megatron.post_training.utils import report_current_memory_info, to_empty_if_meta from megatron.training import get_args, get_model, initialize_megatron from megatron.training.arguments import parse_and_validate_args -from megatron.training.utils import print_rank_0, unwrap_model +from megatron.training.utils import print_rank_0 from model_provider import model_provider warnings.filterwarnings('once') @@ -167,7 +168,7 @@ def get_conversations(example): ) input_ids = encoding["input_ids"] with torch.no_grad(): - output_ids = simple_generate( + output_ids = megatron_generate( unwrapped_model, input_ids.cuda(), osl=args.osl, diff --git a/examples/post_training/modelopt/mmlu.py b/examples/post_training/modelopt/mmlu.py index 3ff4b51f957..e3d57063544 100644 --- a/examples/post_training/modelopt/mmlu.py +++ b/examples/post_training/modelopt/mmlu.py @@ -1,162 +1,87 @@ # Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. -"""Sample Generate GPT.""" +"""MMLU evaluation for Megatron-LM models. + +The plugin runs a single prefill pass per batch and selects the answer as argmax over +the choice token logits at the last prompt position (lm-evaluation-harness style), +instead of autoregressively generating tokens. +""" + +import argparse import functools -import logging import os import sys import warnings -import datasets -import torch.distributed as dist +import torch sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../"))) import modelopt.torch.quantization as mtq -import torch -from diskcache import Cache +from modelopt.torch.utils.plugins import megatron_mmlu from utils import get_hf_tokenizer +from megatron.core.utils import unwrap_model from megatron.post_training.arguments import add_modelopt_args from megatron.post_training.checkpointing import load_modelopt_checkpoint -from megatron.post_training.generate import simple_generate from megatron.post_training.model_builder import modelopt_gpt_hybrid_builder from megatron.post_training.utils import report_current_memory_info from megatron.training import get_args, get_model, initialize_megatron from megatron.training.arguments import parse_and_validate_args -from megatron.training.utils import print_rank_0, unwrap_model +from megatron.training.utils import print_rank_0 from model_provider import model_provider -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) # set to debug if you need more logging - -warnings.filterwarnings('ignore') +warnings.filterwarnings("ignore") def add_mmlu_args(parser): - """Add additional arguments for ModelOpt text generation PTQ.""" - group = parser.add_argument_group(title='ModelOpt text generation ptq') - group.add_argument("--disable-tqdm", action="store_true", help="Disable tqdm.") - group.add_argument("--fraction", type=float, default=1.0, help="Fraction of dataset to use.") - group.add_argument("--lower-bound", type=float, default=None) + """Add additional arguments for MMLU evaluation.""" + group = parser.add_argument_group(title="ModelOpt MMLU evaluation") + group.add_argument( + "--fraction", + type=float, + default=1.0, + help="Fraction of MMLU test set (per subject) to evaluate on.", + ) group.add_argument( - "--no-subject-prompt", - action="store_true", - help="Use empty prompt instead of subject-based prompt.", + "--few-shots", + type=int, + default=0, + help="Number of few-shot examples to prepend to each prompt.", ) group.add_argument( - "--mmlu-dataset", - type=str, - default="cais/mmlu", - help="The default dataset to use is cais/mmlu from the HG hub.", + "--mmlu-batch-size", + type=int, + default=1, + help="Batch size for the batched prefill evaluation.", ) - group.add_argument("--cache-dir", type=str, default=None) + group.add_argument( + "--lower-bound", + type=float, + default=None, + help="Optional accuracy threshold; the script asserts the average is above this value.", + ) + # Kept for backward compatibility with prior MLM_EXTRA_ARGS callers. Has no effect: + # `megatron_mmlu` already disables its progress bar on non-master ranks. + group.add_argument("--disable-tqdm", action="store_true", help=argparse.SUPPRESS) + group.add_argument("--mmlu-dataset", type=str, default="cais/mmlu", help=argparse.SUPPRESS) add_modelopt_args(parser) return parser -def get_all_subjects(): - """Return all MMLU subjects.""" - return [ - 'abstract_algebra', - 'anatomy', - 'astronomy', - 'business_ethics', - 'clinical_knowledge', - 'college_biology', - 'college_chemistry', - 'college_computer_science', - 'college_mathematics', - 'college_medicine', - 'college_physics', - 'computer_security', - 'conceptual_physics', - 'econometrics', - 'electrical_engineering', - 'elementary_mathematics', - 'formal_logic', - 'global_facts', - 'high_school_biology', - 'high_school_chemistry', - 'high_school_computer_science', - 'high_school_european_history', - 'high_school_geography', - 'high_school_government_and_politics', - 'high_school_macroeconomics', - 'high_school_mathematics', - 'high_school_microeconomics', - 'high_school_physics', - 'high_school_psychology', - 'high_school_statistics', - 'high_school_us_history', - 'high_school_world_history', - 'human_aging', - 'human_sexuality', - 'international_law', - 'jurisprudence', - 'logical_fallacies', - 'machine_learning', - 'management', - 'marketing', - 'medical_genetics', - 'miscellaneous', - 'moral_disputes', - 'moral_scenarios', - 'nutrition', - 'philosophy', - 'prehistory', - 'professional_accounting', - 'professional_law', - 'professional_medicine', - 'professional_psychology', - 'public_relations', - 'security_studies', - 'sociology', - 'us_foreign_policy', - 'virology', - 'world_religions', - ] - - -def format_example(example, include_answer: bool = True): - """Format an example into a multi-choices problem.""" - prompt = example["question"] - for choice, answer in zip(["A", "B", "C", "D"], example["choices"]): - prompt += "\n{}. {}".format(choice, answer) - if include_answer: - prompt += "\nAnswer: {}\n\n".format(["A", "B", "C", "D"][example["answer"]]) - else: - prompt += "\nAnswer:" - return prompt - - -def generate_prompt(test_example, dev_examples, few_shots=0, no_subject_prompt=False): - """Generating few-shot prompts.""" - if no_subject_prompt: - prompt = "" - else: - prompt = "The following are multiple choice questions (with answers) about {}.\n\n".format( - " ".join(test_example["subject"].split("_")) - ) - for i in range(few_shots): - prompt += format_example(dev_examples[i]) - prompt += format_example(test_example, include_answer=False) - return prompt - - if __name__ == "__main__": parse_and_validate_args( extra_args_provider=add_mmlu_args, args_defaults={ - 'tokenizer_type': 'HuggingFaceTokenizer', - 'no_load_rng': True, - 'no_load_optim': True, + "tokenizer_type": "HuggingFaceTokenizer", + "no_load_rng": True, + "no_load_optim": True, }, ) initialize_megatron() args = get_args() - cache = Cache(args.cache_dir) + # Meta device initialization for ParallelLinear only works if using cpu initialization. # Meta device initialization is used such that models can be materialized in low-precision # directly when ModelOpt real quant is used. Otherwise, the model is first initialized @@ -182,8 +107,6 @@ def generate_prompt(test_example, dev_examples, few_shots=0, no_subject_prompt=F unwrapped_model.to_empty(device="cuda") report_current_memory_info() - disable_tqdm = args.disable_tqdm or torch.distributed.get_rank() > 0 - tokenizer = get_hf_tokenizer() if args.load is not None: @@ -202,61 +125,16 @@ def generate_prompt(test_example, dev_examples, few_shots=0, no_subject_prompt=F ): mtq.fold_weight(unwrapped_model) - all_subjects = get_all_subjects() - - all_correct = {} - - for subject in all_subjects: - test_data = datasets.load_dataset(args.mmlu_dataset, subject, split="test") - dev_data = datasets.load_dataset(args.mmlu_dataset, subject, split="dev") - - correct = [] - for idx, test_example in enumerate(test_data): - if idx > args.fraction * len(test_data): - break - label = ["A", "B", "C", "D"][test_example["answer"]] - prompt = generate_prompt( - test_example, dev_data, few_shots=0, no_subject_prompt=args.no_subject_prompt - ) - cache_key = f"{args.load}_{subject}_{prompt}" # model name, subject, prompt - - if cache_key in cache: - predict = cache[cache_key] - if dist.get_rank() == 0: - logger.debug(f"Cache hit for {args.load}_{subject}") - else: - tokens = tokenizer(prompt, return_tensors="pt") - with torch.no_grad(): - generated_ids = simple_generate( - unwrapped_model, tokens.input_ids.cuda(), osl=2, disable_tqdm=disable_tqdm - ) - predict = tokenizer.batch_decode(generated_ids)[0].strip() - if torch.distributed.get_rank() == 0: - cache.add(cache_key, predict) - - correct += [True] if predict.startswith(label) else [False] - all_correct[subject] = correct - - if torch.distributed.get_rank() == 0: - print( - "{:48}| {:.3f} | {:5}/{:5}".format( - subject, sum(correct) / len(correct), sum(correct), len(correct) - ), - flush=True, - ) - - avg_correct = [] - - for subject, correct in all_correct.items(): - avg_correct += correct - - if torch.distributed.get_rank() == 0: - print( - "{:48}| {:.3f} | {:5}/{:5}".format( - "average", sum(avg_correct) / len(avg_correct), sum(avg_correct), len(avg_correct) - ), - flush=True, + with torch.no_grad(): + avg = megatron_mmlu( + unwrapped_model, + tokenizer, + few_shots=args.few_shots, + fraction=args.fraction, + batch_size=args.mmlu_batch_size, ) - if args.lower_bound is not None: - assert sum(avg_correct) / len(avg_correct) > args.lower_bound + if torch.distributed.get_rank() == 0 and args.lower_bound is not None: + assert ( + avg > args.lower_bound + ), f"MMLU accuracy {avg:.4f} below lower bound {args.lower_bound}" diff --git a/examples/post_training/modelopt/offline_feature_extract.py b/examples/post_training/modelopt/offline_feature_extract.py index 6ea44943f74..74b3da6d154 100644 --- a/examples/post_training/modelopt/offline_feature_extract.py +++ b/examples/post_training/modelopt/offline_feature_extract.py @@ -12,11 +12,12 @@ from examples.post_training.modelopt.finetune import SFTDataset from megatron.core import mpu +from megatron.core.utils import unwrap_model from megatron.post_training.arguments import add_modelopt_args from megatron.post_training.checkpointing import load_modelopt_checkpoint from megatron.post_training.model_builder import modelopt_gpt_hybrid_builder from megatron.training import get_args, get_model, get_tokenizer, initialize_megatron -from megatron.training.utils import print_rank_0, unwrap_model +from megatron.training.utils import print_rank_0 from model_provider import model_provider diff --git a/examples/post_training/modelopt/prune.py b/examples/post_training/modelopt/prune.py index 0e3e4e41dda..46b232b914f 100644 --- a/examples/post_training/modelopt/prune.py +++ b/examples/post_training/modelopt/prune.py @@ -6,13 +6,13 @@ """ import functools -import inspect +import gc +import json import os import sys import warnings import torch -from datasets import load_dataset from tqdm import tqdm sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../"))) @@ -20,21 +20,35 @@ import modelopt.torch.prune as mtp from modelopt.torch.export import import_mcore_gpt_from_hf from modelopt.torch.prune.plugins.mcore_minitron import SUPPORTED_HPARAMS +from modelopt.torch.utils import get_dataset_samples +from modelopt.torch.utils.dataset_utils import get_supported_datasets +from modelopt.torch.utils.plugins import megatron_generate, megatron_prefill + +# modelopt 0.45+ exposes a shared Megatron calibration forward loop. Fall back to an +# inline pack=True implementation on 0.44 so this script works on both releases. +try: + from modelopt.torch.utils.plugins.megatron_calibration import ( + get_megatron_calibration_forward_loop, + ) + + _HAS_SHARED_CALIB = True +except ImportError: + _HAS_SHARED_CALIB = False from utils import get_hf_tokenizer from megatron.core.parallel_state import ( get_pipeline_model_parallel_group, get_tensor_model_parallel_group, ) +from megatron.core.utils import unwrap_model from megatron.post_training.arguments import add_modelopt_args from megatron.post_training.checkpointing import load_modelopt_checkpoint -from megatron.post_training.generate import simple_generate from megatron.post_training.model_builder import modelopt_gpt_hybrid_builder from megatron.post_training.utils import report_current_memory_info from megatron.training import get_args, get_model, initialize_megatron from megatron.training.arguments import parse_and_validate_args from megatron.training.checkpointing import save_checkpoint -from megatron.training.utils import print_rank_0, unwrap_model +from megatron.training.utils import print_rank_0 from model_provider import model_provider warnings.filterwarnings("ignore") @@ -46,6 +60,22 @@ def add_prune_args(parser): group.add_argument( "--calib-size", type=int, default=1024, help="Samples to use for pruning calibration." ) + group.add_argument( + "--calib-dataset", + type=str, + default="nemotron-post-training-dataset-v2", + help=( + f"HF Dataset name or local .jsonl path for calibration " + f"(supported options: {', '.join(get_supported_datasets())}). " + "You can also pass any other dataset and see if auto-detection works." + ), + ) + group.add_argument( + "--calib-max-sequence-length", + type=int, + default=4096, + help="Maximum sequence length for calibration samples.", + ) group.add_argument( "--prompts", type=str, @@ -61,82 +91,61 @@ def add_prune_args(parser): group.add_argument( "--pretrained-model-path", type=str, default=None, help="HuggingFace pretrained model" ) - # Pruning parameters - group.add_argument( - "--target-ffn-hidden-size", type=int, help="Prune MLP FFN hidden size to this value" - ) - group.add_argument( - "--target-hidden-size", type=int, help="Prune hidden size (embedding dim) to this value" - ) - group.add_argument( - "--target-num-attention-heads", - type=int, - help="Prune number of attention heads to this value. Must be supplied with --target-num-query-groups", - ) - group.add_argument( - "--target-num-query-groups", - type=int, - help="Prune number of query groups to this value. Must be supplied with --target-num-attention-heads", - ) - group.add_argument( - "--target-mamba-num-heads", - type=int, - help="Prune number of Mamba attention heads to this value", - ) - group.add_argument( - "--target-mamba-head-dim", - type=int, - help="Prune dimension of Mamba attention heads to this value", - ) - group.add_argument( - "--target-num-moe-experts", type=int, help="Prune number of MoE experts to this value" - ) - group.add_argument( - "--target-moe-ffn-hidden-size", type=int, help="Prune MoE FFN hidden size to this value" - ) - group.add_argument( - "--target-moe-shared-expert-intermediate-size", - type=int, - help="Prune MoE shared expert intermediate size to this value", - ) group.add_argument( - "--target-num-layers", - type=int, - help="Prune number of transformer layers to this value based on " - "Block Influence metric (cosine similarity) as per https://arxiv.org/abs/2403.03853", + "--skip-generate", + action="store_true", + default=False, + help="Skip the post-pruning generate/validation step.", ) + # Pruning targets group.add_argument( - "--layers-to-drop", - type=int, - metavar="N", - nargs="*", - help="Drop specific model layers (1-indexed). Cannot be used with rest of the pruning options", + "--prune-export-config", + type=str, + required=True, + help=( + 'Target pruned config as a JSON object, e.g. \'{"hidden_size": 3584, ' + '"ffn_hidden_size": 9216}\'. ' + f"Supported hyperparameters: {sorted(SUPPORTED_HPARAMS)}." + ), ) group.add_argument( - "--pruning-scores-path", + "--prune-intermediate-ckpt", type=str, default=None, - help="Path to the cache and reuse pruning scores for pruning again to different params", + help=( + "Directory to cache and reuse per-rank intermediate pruning scores " + "for resuming / faster re-runs (e.g. pruning the same model to a different config)." + ), ) add_modelopt_args(parser) return parser def check_arguments(args): - """Checking user arguments.""" - if args.layers_to_drop: - if any(getattr(args, f"target_{k}", None) is not None for k in SUPPORTED_HPARAMS): - raise ValueError("--layers_to_drop cannot be used with other pruning parameters") - - -def get_calib_dataloader(calib_size=1024, max_sequence_length=512): - """Return a dataloader for calibration.""" - dataset = load_dataset("cnn_dailymail", name="3.0.0", split="train") - text_column = "article" + """Validate user-provided pruning arguments.""" + try: + args.prune_export_config = json.loads(args.prune_export_config) + except json.JSONDecodeError as exc: + raise ValueError( + f"Invalid JSON for --prune-export-config: {args.prune_export_config}" + ) from exc + if not isinstance(args.prune_export_config, dict): + raise ValueError("--prune-export-config must parse to a dictionary.") + unsupported = set(args.prune_export_config) - set(SUPPORTED_HPARAMS) + if unsupported: + raise ValueError( + f"Unsupported hyperparameters in --prune-export-config: {sorted(unsupported)}. " + f"Supported: {sorted(SUPPORTED_HPARAMS)}" + ) - calib_size = min(len(dataset), calib_size) - for i in range(calib_size): - yield dataset[i][text_column][:max_sequence_length] + # Default the intermediate-checkpoint location to /modelopt_pruning_scores + # so that re-running on the same --save target reuses cached per-rank scores + if args.prune_intermediate_ckpt is None and args.save is not None: + args.prune_intermediate_ckpt = os.path.join(args.save, "modelopt_pruning_scores") + print_rank_0( + "No directory provided to cache per-rank intermediate pruning scores. " + f"Setting to: {args.prune_intermediate_ckpt}" + ) def get_params(model): @@ -162,10 +171,14 @@ def get_params(model): check_arguments(args) tokenizer = get_hf_tokenizer() - model = get_model( - functools.partial(model_provider, modelopt_gpt_hybrid_builder), wrap_with_ddp=False - ) + # Pruning operates on per-expert linears (which only exist as separate modules under + # SequentialMLP, not the packed-tensor TEGroupedMLP). `disable_moe_grouped_gemm=True` + # forces the export spec to SequentialMLP so mtp.prune can act on individual experts. + # Other example scripts (quantize.py, generate.py, finetune.py) keep the default. + prune_builder = functools.partial(modelopt_gpt_hybrid_builder, disable_moe_grouped_gemm=True) + model = get_model(functools.partial(model_provider, prune_builder), wrap_with_ddp=False) unwrapped_model = unwrap_model(model)[0] + print_rank_0(f"Original Model: {unwrapped_model}") report_current_memory_info() @@ -176,9 +189,7 @@ def get_params(model): if args.pretrained_model_path is not None: import_dtype = torch.float16 if args.fp16 else torch.bfloat16 workspace_dir = os.environ.get("MLM_WORK_DIR", "/tmp") - import_kwargs = {"dtype": import_dtype} - if "trust_remote_code" in inspect.signature(import_mcore_gpt_from_hf).parameters: - import_kwargs.update({"trust_remote_code": args.trust_remote_code}) + import_kwargs = {"dtype": import_dtype, "trust_remote_code": args.trust_remote_code} import_mcore_gpt_from_hf( unwrapped_model, args.pretrained_model_path, workspace_dir, **import_kwargs ) @@ -192,50 +203,77 @@ def _custom_prompt_forward_loop_func(model): for idx, prompt in tqdm(enumerate(all_prompts), disable=torch.distributed.get_rank()): tokens = tokenizer(prompt, return_tensors="pt") - generated_ids = simple_generate(model, tokens.input_ids.cuda(), osl=32) + # enable_kv_cache=False to skip the static KV-cache pre-allocation; this is a + # sanity-check generation (32 tokens) and skipping the cache keeps memory headroom. + generated_ids = megatron_generate( + model, tokens.input_ids.cuda(), osl=32, enable_kv_cache=False + ) generated_texts = tokenizer.batch_decode(generated_ids) print_rank_0("{}".format(generated_texts)) if all_references[idx] is not None: assert all_references[idx] == generated_texts[0], all_references[idx] - def _hf_dataset_forword_loop_func(model): - dataloader = get_calib_dataloader(args.calib_size) - - for prompt in tqdm(dataloader, total=args.calib_size, disable=torch.distributed.get_rank()): - tokens = tokenizer(prompt, return_tensors="pt") - simple_generate(model, tokens.input_ids.cuda(), osl=1) - - if args.layers_to_drop: - mtp.mcore_minitron.drop_mcore_language_model_layers( - model, layers_to_drop=args.layers_to_drop + if _HAS_SHARED_CALIB: + forward_loop = get_megatron_calibration_forward_loop( + tokenizer, + dataset_name=args.calib_dataset, + num_samples=args.calib_size, + seq_length=args.calib_max_sequence_length, + batch_size=1, + # pack=True uses Megatron pretraining-style global-stream document packing + pack=True, ) else: - print_rank_0("Pruning model...") - export_config = { - k: getattr(args, f"target_{k}") - for k in SUPPORTED_HPARAMS - if getattr(args, f"target_{k}", None) is not None - } - config = {"forward_loop": _hf_dataset_forword_loop_func} - if args.pruning_scores_path is not None: - config["scores_path"] = args.pruning_scores_path - mtp.prune( - unwrapped_model, - mode="mcore_minitron", - constraints={"export_config": export_config}, - dummy_input=None, # Not used - config=config, - ) - # [WAR till modelopt 0.39]: Remove prune state to avoid converting again on restore which forces TP=1. - if mto.ModeloptStateManager.has_state_for_mode_type("prune", model=unwrapped_model): - mto.ModeloptStateManager.remove_state(unwrapped_model) + # modelopt 0.44 fallback: inline pack=True (concatenate raw samples into a single + # EOS-separated token stream, slice into fixed-length chunks). Equivalent in + # behavior to get_megatron_calibration_forward_loop at batch_size=1. + def forward_loop(model): + if not hasattr(tokenizer, "pad_token") or tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + seq_len = args.calib_max_sequence_length + samples = get_dataset_samples(args.calib_dataset, num_samples=args.calib_size * 2) + sep_id = tokenizer.eos_token_id + token_stream: list[int] = [] + for s in samples: + token_stream.extend(tokenizer.encode(s, add_special_tokens=False)) + token_stream.append(sep_id) + if len(token_stream) >= args.calib_size * seq_len: + break + n_chunks = min(args.calib_size, len(token_stream) // seq_len) + print_rank_0( + f"Calibration packing: {len(samples)} raw samples -> {len(token_stream)} tokens " + f"-> {n_chunks} chunks of {seq_len} tokens." + ) + for i in tqdm(range(n_chunks), disable=torch.distributed.get_rank()): + chunk = token_stream[i * seq_len : (i + 1) * seq_len] + input_ids = torch.tensor([chunk], dtype=torch.long, device="cuda") + megatron_prefill(model, input_ids, skip_return_logits=True) + + print_rank_0(f"Pruning model with export_config: {args.prune_export_config}") + config = {"forward_loop": forward_loop} + if args.prune_intermediate_ckpt is not None: + config["checkpoint"] = args.prune_intermediate_ckpt + mtp.prune( + unwrapped_model, + mode="mcore_minitron", + constraints={"export_config": args.prune_export_config}, + dummy_input=None, # Not used + config=config, + ) + # Remove unnecessary modelopt_state since ckpt is homogeneous + if mto.ModeloptStateManager.has_state_for_mode_type("prune", model=unwrapped_model): + mto.ModeloptStateManager.remove_state(unwrapped_model) print_rank_0(f"Pruned Model:\n {unwrapped_model}") - print_rank_0(f"Pruned Model Params: {get_params(unwrapped_model)/1e9:.2f}B") - - _custom_prompt_forward_loop_func(unwrapped_model) + print_rank_0(f"Pruned Model Params: {get_params(unwrapped_model) / 1e9:.2f}B") if args.save is not None: save_checkpoint(1, model, None, None, 0) + # Free pruning-side memory before the sanity-check generation (do this after saving in case it causes issues) + gc.collect() + torch.cuda.empty_cache() + if not args.skip_generate: + _custom_prompt_forward_loop_func(unwrapped_model) + print_rank_0("Done") diff --git a/examples/post_training/modelopt/prune.sh b/examples/post_training/modelopt/prune.sh index 33f3e615e96..cb5ef9014fd 100755 --- a/examples/post_training/modelopt/prune.sh +++ b/examples/post_training/modelopt/prune.sh @@ -15,42 +15,16 @@ MLM_DEFAULT_ARGS=" --distributed-timeout-minutes 30 \ --finetune --auto-detect-ckpt-format \ --no-gradient-accumulation-fusion \ - --export-te-mcore-model + --export-default-te-spec " -# Pruning target arguments - set these environment variables to enable pruning -# Example: export TARGET_HIDDEN_SIZE=3072 TARGET_FFN_HIDDEN_SIZE=9216 -# Example: export LAYERS_TO_DROP="1 5 10" - -# Define pruning argument mappings: "env_var:cli_arg" -# List of environment variables we want to check for pruning CLI args -PRUNE_ENV_VARS=( - TARGET_FFN_HIDDEN_SIZE - TARGET_HIDDEN_SIZE - TARGET_NUM_ATTENTION_HEADS - TARGET_NUM_QUERY_GROUPS - TARGET_MAMBA_NUM_HEADS - TARGET_MAMBA_HEAD_DIM - TARGET_NUM_MOE_EXPERTS - TARGET_MOE_FFN_HIDDEN_SIZE - TARGET_MOE_SHARED_EXPERT_INTERMEDIATE_SIZE - TARGET_NUM_LAYERS - LAYERS_TO_DROP -) - -# Build arguments from environment variables (TARGET_NUM_LAYERS -> --target-num-layers, etc.) -PRUNE_ARGS=${PRUNE_ARGS:-""} -for env_var in "${PRUNE_ENV_VARS[@]}"; do - if [ ! -z "${!env_var}" ]; then - # prepend --, convert to lowercase, replace _ with - - cli_arg="--$(echo "${env_var}" | tr '[:upper:]' '[:lower:]' | tr '_' '-')" - PRUNE_ARGS="${PRUNE_ARGS} ${cli_arg} ${!env_var}" - fi -done - -if [ -z "${PRUNE_ARGS}" ]; then - printf "${MLM_WARNING} No pruning arguments specified. Set TARGET_* or LAYERS_TO_DROP environment variables.\n" -fi +# Pruning configuration is supplied via MLM_EXTRA_ARGS. At minimum, pass +# --prune-export-config '' (required by prune.py). Example: +# MLM_EXTRA_ARGS='--prune-export-config {"hidden_size":3072,"ffn_hidden_size":9216}' ./prune.sh ... +# Optionally add --prune-intermediate-ckpt to cache scores for re-runs. +# Supported hparams: hidden_size, ffn_hidden_size, num_attention_heads, num_query_groups, +# mamba_num_heads, mamba_head_dim, num_moe_experts, moe_ffn_hidden_size, +# moe_shared_expert_intermediate_size, num_layers. if [ -z ${MLM_MODEL_SAVE} ]; then MLM_MODEL_SAVE=${MLM_WORK_DIR}/${MLM_MODEL_CFG}_pruned @@ -74,5 +48,4 @@ ${LAUNCH_SCRIPT} ${SCRIPT_DIR}/prune.py \ --tokenizer-model ${TOKENIZER_MODEL} \ --save ${MLM_MODEL_SAVE} \ --references "${MLM_REF_LABEL}" \ - ${PRUNE_ARGS} \ ${MLM_DEFAULT_ARGS} ${MLM_EXTRA_ARGS} diff --git a/examples/post_training/modelopt/quantize.py b/examples/post_training/modelopt/quantize.py index f9935d48a5d..d80fada68bd 100644 --- a/examples/post_training/modelopt/quantize.py +++ b/examples/post_training/modelopt/quantize.py @@ -3,6 +3,7 @@ """Script for quantizing a HuggingFace or Megatron-LM checkpoint using ModelOpt.""" import functools +import gc import inspect import json import os @@ -21,6 +22,19 @@ from modelopt.recipe import ModelOptPTQRecipe, load_recipe from modelopt.torch.export import import_mcore_gpt_from_hf from modelopt.torch.utils.dataset_utils import get_dataset_dataloader +from modelopt.torch.utils.plugins import megatron_generate, megatron_prefill + +# modelopt 0.45+ exposes a shared Megatron calibration forward loop. Fall back to the +# legacy local-JSONL + HF-dataset calibration path on 0.44 so this script works on both +# releases. +try: + from modelopt.torch.utils.plugins.megatron_calibration import ( + get_megatron_calibration_forward_loop, + ) + + _HAS_SHARED_CALIB = True +except ImportError: + _HAS_SHARED_CALIB = False try: import modelopt.torch.quantization.plugins.psx_formats as mtq_psx @@ -37,16 +51,16 @@ from utils import get_hf_tokenizer -from megatron.core.utils import get_batch_on_this_cp_rank +from megatron.core.parallel_state import get_context_parallel_group +from megatron.core.utils import get_batch_on_this_cp_rank, unwrap_model from megatron.post_training.arguments import add_modelopt_args from megatron.post_training.checkpointing import load_modelopt_checkpoint -from megatron.post_training.generate import simple_generate from megatron.post_training.model_builder import modelopt_gpt_hybrid_builder from megatron.post_training.utils import print_distributed_quant_summary, report_current_memory_info from megatron.training import get_args, get_model, initialize_megatron from megatron.training.arguments import parse_and_validate_args from megatron.training.checkpointing import save_checkpoint -from megatron.training.utils import print_rank_0, unwrap_model +from megatron.training.utils import print_rank_0 from model_provider import model_provider warnings.filterwarnings("ignore") @@ -77,18 +91,18 @@ def add_text_generate_ptq_args(parser): """Add additional arguments for ModelOpt text generation PTQ.""" group = parser.add_argument_group(title="ModelOpt text generation ptq") group.add_argument( - "--calib-size", type=int, default=512, help="Number of samples to use for ptq calibration." + "--calib-size", type=int, default=1024, help="Number of samples to use for ptq calibration." ) group.add_argument( "--calib-dataset-path-or-name", type=str, - default="cnn_dailymail", + default="nemotron-post-training-dataset-v2", help="Path to local calibration dataset file (.jsonl) or HuggingFace dataset name.", ) group.add_argument( "--calib-max-sequence-length", type=int, - default=512, + default=4096, help="Maximum sequence length for calibration.", ) group.add_argument( @@ -366,24 +380,44 @@ def _custom_prompt_forward_loop_func(model): for idx, prompt in tqdm(enumerate(all_prompts), disable=torch.distributed.get_rank()): tokens = tokenizer(prompt, return_tensors="pt") - generated_ids = simple_generate(model, tokens.input_ids.cuda(), osl=32) + # enable_kv_cache=False to avoid pre-allocating the static KV cache: this is a + # sanity-check generation (32 tokens), and the KV-cache allocation can OOM tight + # quantization runs on large MoE models. + generated_ids = megatron_generate( + model, tokens.input_ids.cuda(), osl=32, enable_kv_cache=False + ) generated_texts = tokenizer.batch_decode(generated_ids) print_rank_0("{}".format(generated_texts)) if all_references[idx] is not None: assert all_references[idx] == generated_texts[0], all_references[idx] - def _dataset_forward_loop_func(model): - dataloader = get_calib_dataloader( - dataset_path_or_name=args.calib_dataset_path_or_name, - tokenizer=tokenizer, - calib_size=args.calib_size, - max_sequence_length=args.calib_max_sequence_length, - use_random_offset=args.calib_use_random_offset, + if _HAS_SHARED_CALIB: + _dataset_forward_loop_func = get_megatron_calibration_forward_loop( + tokenizer, + dataset_name=args.calib_dataset_path_or_name, + num_samples=args.calib_size, + seq_length=args.calib_max_sequence_length, batch_size=args.calib_batch_size, + # pack=True uses Megatron pretraining-style global-stream document packing + # Leave to False for backward compatibility + pack=False, ) - for sample in tqdm(dataloader, disable=torch.distributed.get_rank()): - sample = get_batch_on_this_cp_rank(sample) - simple_generate(model, sample["input_ids"], osl=1, calibration_mode=True) + else: + + def _dataset_forward_loop_func(model): + dataloader = get_calib_dataloader( + dataset_path_or_name=args.calib_dataset_path_or_name, + tokenizer=tokenizer, + calib_size=args.calib_size, + max_sequence_length=args.calib_max_sequence_length, + use_random_offset=args.calib_use_random_offset, + batch_size=args.calib_batch_size, + ) + for sample in tqdm(dataloader, disable=torch.distributed.get_rank()): + sample = get_batch_on_this_cp_rank( + sample, is_hybrid_cp=False, cp_group=get_context_parallel_group() + ) + megatron_prefill(model, sample["input_ids"], skip_return_logits=True) unwrapped_model = unwrap_model(model)[0] @@ -413,12 +447,9 @@ def _dataset_forward_loop_func(model): if args.save is not None: save_checkpoint(1, model, None, None, 0, release=True) - # Free calibration/quantization memory before generate - import gc - + # Free calibration/quantization memory before generate (do this after saving in case it causes issues) gc.collect() torch.cuda.empty_cache() - # Do this after saving in case it causes issues if not args.skip_generate: _custom_prompt_forward_loop_func(unwrapped_model) diff --git a/examples/post_training/modelopt/validate.py b/examples/post_training/modelopt/validate.py index 3b3f855c393..f393f13c9e7 100644 --- a/examples/post_training/modelopt/validate.py +++ b/examples/post_training/modelopt/validate.py @@ -13,13 +13,14 @@ from modelopt.torch.speculative.plugins.megatron_eagle import MegatronARValidation from utils import get_hf_tokenizer +from megatron.core.utils import unwrap_model from megatron.post_training.arguments import add_modelopt_args from megatron.post_training.checkpointing import load_modelopt_checkpoint from megatron.post_training.model_builder import modelopt_gpt_hybrid_builder from megatron.post_training.utils import get_mtbench_chat_data from megatron.training import get_args, get_model, initialize_megatron from megatron.training.arguments import parse_and_validate_args -from megatron.training.utils import print_rank_0, unwrap_model +from megatron.training.utils import print_rank_0 from model_provider import model_provider warnings.filterwarnings('ignore') diff --git a/pretrain_t5.py b/examples/t5/pretrain_t5.py similarity index 100% rename from pretrain_t5.py rename to examples/t5/pretrain_t5.py diff --git a/experimental/lite/.pre-commit-config.yaml b/experimental/lite/.pre-commit-config.yaml new file mode 100644 index 00000000000..a1b8b2d897f --- /dev/null +++ b/experimental/lite/.pre-commit-config.yaml @@ -0,0 +1,16 @@ +repos: +- repo: local + hooks: + - id: isort + name: isort + entry: isort + language: python + additional_dependencies: ["isort==5.13.2"] + files: ^experimental/lite/.*\.py$ + - id: black + name: black + entry: black + language: python + additional_dependencies: ["black==24.4.2"] + files: ^experimental/lite/.*\.py$ + args: ["--skip-magic-trailing-comma", "--skip-string-normalization"] diff --git a/experimental/lite/README.md b/experimental/lite/README.md new file mode 100644 index 00000000000..e9b8f4d4082 --- /dev/null +++ b/experimental/lite/README.md @@ -0,0 +1,139 @@ +# Megatron Lite + +Megatron Lite is an experimental, agentic-native training runtime and native +model implementation layer for Megatron. It is designed for work that needs to +move quickly without giving up Megatron-Core performance: small composable +primitives, explicit model/runtime protocols, and validation recipes that make +changes easy to review and easy to reproduce. + +The source lives under `experimental/lite/megatron/lite`, and the public import +path is `megatron.lite`. + +Do not import `experimental.lite` from user code. Examples and public APIs should +refer to `megatron.lite`. + +## Scope + +This initial drop contains: + +- A lightweight runtime API in `megatron.lite.runtime`. +- Common training primitives in `megatron.lite.primitive`. +- Lite-only native model implementations for Qwen3 MoE and Qwen3.5 MoE. +- Hugging Face safetensors load/export helpers for the included models. +- Megatron-Core optimizer wrapping for the lite runtime. +- FSDP2 optimizer primitives for supported lite model protocols. +- Reference runtime backends for comparison runs: `mbridge` for the legacy + package and `bridge` for real Megatron-Bridge environments. +- A benchmark example that can dry-run or execute `mlite`, `mbridge`, and + `bridge` backends. + +This initial drop intentionally does not include: + +- Hybrid model implementations. +- Dense Qwen3 model support. The included Qwen3-family path is Qwen3 MoE only. + +## Why MLite + +- **Agentic-native development surface.** Runtime, model, and primitive code are + split into reviewable contracts so agents and humans can make targeted changes + without touching unrelated Megatron subsystems. +- **Native MLite models, not wrapper models.** `backend="mlite"` builds native + `megatron.lite` model code; reference backends are used only for comparison. +- **Megatron-Core distopt parity.** In deterministic correctness runs against the + `mbridge` reference backend on the Megatron-Core distributed optimizer path, + MLite matched loss and grad-norm exactly (`max_abs=0.0`) with no mismatches; + post-step weights and eval logits were checked by SHA256 fingerprints. +- **Speed-aligned with the Core path.** On an 8x H100 Qwen3.5 MoE benchmark using + `distopt`, MLite measured 309.433 ms/step and 105,896.935 tokens/s, compared + with 332.201 ms/step and 98,639.089 tokens/s for `mbridge`, and 334.936 + ms/step and 97,833.496 tokens/s for the real `bridge` path. + +The `mbridge` benchmark line is the validated Megatron-Core/distopt reference +used for this PR. The `bridge` line is a separate Megatron-Bridge environment +check and should not be confused with the Core/distopt parity claim. + +## Layout + +```text +experimental/lite/ + README.md + docs/ Design and usage notes + examples/ Optional integration and benchmark examples + skills/ Agent-agnostic maintenance skills + megatron/ + lite/ + runtime/ Runtime API, config, and backend registry + model/ Model registry and Qwen model implementations + primitive/ Parallel, checkpoint, optimizer, module, and op primitives +``` + +For local source-tree use: + +```bash +export PYTHONPATH=/path/to/Megatron-LM/experimental/lite:$PYTHONPATH +``` + +## Public API + +```python +from megatron.lite.runtime import MegatronLiteConfig, RuntimeConfig, create_runtime + +cfg = RuntimeConfig( + backend="mlite", + hf_path="/path/to/hf-model", + backend_cfg=MegatronLiteConfig(model_name="qwen3_moe", impl="lite"), +) +runtime = create_runtime(cfg) +handle = runtime.build_model() +``` + +`backend="mlite"` selects the Megatron Lite runtime backend. `impl="lite"` +selects the model implementation inside the registered model family. +`backend="mbridge"` selects the legacy `mbridge` reference backend used by the +validated benchmark example. `backend="bridge"` selects the Megatron-Bridge +runtime backend and requires an environment where `import megatron.bridge` works +when the model is built. + +Canonical model names currently registered by default: + +- `qwen3_moe`: Qwen3 MoE lite implementation. Use this name in new configs. +- `qwen3_5`: Qwen3.5 MoE lite implementation. + +Compatibility names: + +- `qwen3`: legacy alias for the Qwen3 MoE implementation only. It does not mean + dense Qwen3 support. HF `model_type` values `qwen3_moe` and `qwen2_moe` + currently resolve through this compatibility path. + +## Benchmark And Correctness Signoff + +The validated benchmark and correctness commands live in +[`examples/bench/README.md`](examples/bench/README.md). The signoff setup uses +Qwen3.5 MoE, `optimizer_backend=distopt`, deterministic mode for strict +correctness, and identical synthetic input streams for paired performance runs. + +Reproduce the strict MLite vs Megatron-Core/distopt comparison with: + +```bash +export MEGATRON_LITE_DETERMINISTIC=1 +export CUBLAS_WORKSPACE_CONFIG=:4096:8 + +HF_PATH=/models/Qwen3.5-35B-A3B \ +REFERENCE_BACKEND=mbridge \ +DRY_RUN=0 \ +bash experimental/lite/examples/bench/scripts/run_qwen35_correctness_pair.sh +``` + +## Docs + +- [Architecture](docs/architecture.md) +- [Runtime](docs/runtime.md) +- [Models](docs/models.md) +- [Porting Notes](docs/porting.md) +- [Skills](skills/README.md) +- [Bench Example](examples/bench/README.md) + +## Acknowledgements + +The Qwen3 MoE LoRA adapter support follows Mind-Lab's PEFT/Mint-compatible +adapter work. Thanks to Mind-Lab for the reference implementation and guidance. diff --git a/experimental/lite/docs/architecture.md b/experimental/lite/docs/architecture.md new file mode 100644 index 00000000000..a239b79997e --- /dev/null +++ b/experimental/lite/docs/architecture.md @@ -0,0 +1,46 @@ +# Architecture + +Megatron Lite source lives under `experimental/lite/megatron/lite` and is split +into three layers: + +- `runtime`: lifecycle and training-step orchestration. +- `model`: model registration plus model-specific build/load/export protocols. +- `primitive`: reusable lower-level pieces such as parallel state, tensor-parallel + layers, checkpoint conversion, MoE utilities, and optimizer wrapping. + +The runtime does not know Qwen implementation details. It imports a model +protocol from the model registry, builds the typed implementation config, then +delegates model construction to that protocol. + +## Import Boundary + +The source root for local use is `experimental/lite`; adding that directory to +`PYTHONPATH` exposes the package as `megatron.lite`. Internal imports also use +`megatron.lite` so user-facing code matches the final package path. + +## Runtime Boundary + +The runtime API owns: + +- Distributed initialization. +- Model protocol loading. +- Model checkpoint save/load dispatch. +- Forward/backward microbatch orchestration. +- Optimizer and learning-rate scheduler stepping. +- Optional model/optimizer offload hooks. + +The model protocol owns: + +- Architecture config creation. +- Model chunk construction. +- Model-specific recompute/offload wiring. +- Model-specific optimizer construction. +- HF checkpoint load/export mapping. + +## Current Deliberate Omissions + +This package currently includes only the lite model implementation path. It +intentionally excludes non-lite and hybrid model implementation packages. The +optional benchmark entrypoints under `examples/bench` are comparison tools and +are not imported by the package runtime. FSDP2 is included as an optimizer +primitive and can be selected by model protocols that support it. diff --git a/experimental/lite/docs/models.md b/experimental/lite/docs/models.md new file mode 100644 index 00000000000..98e17389a88 --- /dev/null +++ b/experimental/lite/docs/models.md @@ -0,0 +1,58 @@ +# Models + +Model code lives under `megatron.lite.model`. + +The model registry maps `(model_name, impl)` pairs to model protocol modules. +The runtime loads the protocol and expects the following required symbols: + +- `ImplConfig`: dataclass for model-specific knobs. +- `build_model_config(source, **overrides)`: returns a typed model config. +- `build_model(model_cfg, *, impl_cfg)`: returns a `ModelBundle`. + +Optional protocol symbols: + +- `load_hf_weights(chunk, hf_path, model_cfg, ps)` +- `export_hf_weights(chunks, model_cfg, ps, **kwargs)` +- `vocab_size(model_cfg)` + +## Included Models + +`qwen3_moe` is the canonical name for the Qwen3 MoE lite implementation: + +```text +megatron.lite.model.qwen3_moe.lite.protocol +``` + +`qwen3` is kept as a legacy compatibility alias for the same implementation. It +does not mean dense Qwen3 support. Hugging Face `model_type` values `qwen3_moe` +and `qwen2_moe` currently resolve through this compatibility path. + +`qwen3_5` maps to the Qwen3.5 MoE lite implementation: + +```text +megatron.lite.model.qwen3_5.lite.protocol +``` + +## Acknowledgements + +The Qwen3 MoE LoRA adapter support follows Mind-Lab's PEFT/Mint-compatible +adapter work. Thanks to Mind-Lab for the reference implementation and guidance. + +## Adding A Model + +Add a model package under `model/`, then register it in +`model/registry.py`: + +```python +register_model( + "my_model", + package="megatron.lite.model.my_model", + hf_model_types=["my_model"], + impls={ + "lite": "megatron.lite.model.my_model.lite.protocol", + }, +) +``` + +New models should keep heavyweight imports inside protocol functions when +possible so importing `megatron.lite` stays cheap. diff --git a/experimental/lite/docs/porting.md b/experimental/lite/docs/porting.md new file mode 100644 index 00000000000..2c762a9c3ba --- /dev/null +++ b/experimental/lite/docs/porting.md @@ -0,0 +1,43 @@ +# Porting Notes + +This tree is prepared as an experimental Megatron package. The package code +lives under `experimental/lite/megatron/lite`, so users can add +`experimental/lite` to `PYTHONPATH` and import `megatron.lite`. + +## Naming Rules + +- Use `Megatron Lite` for the component name in docs and comments. +- Use `megatron.lite` for public and internal imports. +- Do not introduce project-specific legacy branding. +- Use `mlite` for the runtime backend key. +- Use `lite` for model implementation names. + +## Included Surface + +Keep the PR focused on the lite model implementation path: + +- Runtime backend: `mlite`. +- Reference comparison backends: `mbridge` for the validated legacy + Megatron-Core/distopt path and `bridge` for real Megatron-Bridge environments. +- Models: Qwen3 MoE and Qwen3.5 MoE. Dense Qwen3 is not included. +- Model implementations: `lite` only. +- Optimizer primitives: Megatron-Core optimizer wrapping and FSDP2. +- Optional examples: benchmark and VERL launchers under `experimental/lite/examples`. + +Keep these out of the first PR unless the scope changes: + +- Hybrid model implementation packages. +- Megatron-Bridge model implementation packages. +- Undocumented experiment-specific entrypoints. + +## Package Integration + +No repository-level packaging changes are made in this experimental drop. The +current layout is importable from source with: + +```bash +export PYTHONPATH=/path/to/Megatron-LM/experimental/lite:$PYTHONPATH +``` + +A future integration step can decide whether to keep the experimental location +or move the tree into the final package location. diff --git a/experimental/lite/docs/runtime.md b/experimental/lite/docs/runtime.md new file mode 100644 index 00000000000..f74851f8fe2 --- /dev/null +++ b/experimental/lite/docs/runtime.md @@ -0,0 +1,83 @@ +# Runtime + +The public runtime entrypoint is `megatron.lite.runtime`. + +```python +from megatron.lite.runtime import MegatronLiteConfig, ParallelConfig, RuntimeConfig, create_runtime + +cfg = RuntimeConfig( + backend="mlite", + hf_path="/path/to/hf-model", + backend_cfg=MegatronLiteConfig( + model_name="qwen3_moe", + impl="lite", + parallel=ParallelConfig(tp=1, pp=1, cp=1, ep=1), + ), +) +runtime = create_runtime(cfg) +handle = runtime.build_model() +``` + +## API Tiers + +All runtime backends implement the pretraining tier: + +- `build_model` +- `save_checkpoint` +- `load_checkpoint` +- `train_mode` +- `eval_mode` +- `forward_backward` +- `zero_grad` +- `optimizer_step` +- `lr_scheduler_step` + +The lite runtime also implements `export_weights` and `to` when the underlying +model and optimizer support those operations. + +The `mbridge` runtime implements the same runtime contract through the legacy +`mbridge` package and Megatron-Core optimizer/checkpoint helpers. The benchmark +example currently uses this backend for validated reference runs. + +The `bridge` runtime is the real Megatron-Bridge path. It imports +`megatron.bridge` lazily from `build_model()`, so config construction and dry-run +examples can execute without Megatron-Bridge installed. + +## Config Types + +`RuntimeConfig` selects the backend and carries the Hugging Face model path. + +`MegatronLiteConfig` carries `mlite` backend settings: + +- `model_name`: `qwen3_moe` or `qwen3_5` for new configs. `qwen3` remains + accepted as a legacy alias for `qwen3_moe` only; dense Qwen3 is not included. +- `impl`: currently only `lite`. +- `parallel`: tensor, expert, pipeline, virtual pipeline, and context sizes. +- `optimizer`: Megatron-Core optimizer settings. +- `impl_cfg`: model-specific options consumed by each model protocol. + +`BridgeConfig` carries shared `mbridge` and `bridge` backend settings: + +- `model_name`: optional model identifier used for benchmark metadata. +- `parallel`: tensor, expert, pipeline, virtual pipeline, and context sizes. +- `optimizer`: Megatron-Core optimizer settings. +- `override_ddp_config`, `override_transformer_config`, and + `override_optimizer_config`: explicit reference-backend/Core override maps. +- `param_offload` and `optimizer_offload`: offload model/optimizer state between + train/eval contexts. + +## Backend Registry + +The built-in backend keys are `mlite`, `mbridge`, and `bridge`. Model +implementations for the native runtime remain selected through +`MegatronLiteConfig.impl`, which currently supports `impl="lite"`. + +Custom runtime backends can be registered with: + +```python +from megatron.lite.runtime import register_runtime + +register_runtime("my_backend", "my_package.my_runtime") +``` + +The target module must expose `create(hf_path, cfg)`. diff --git a/experimental/lite/examples/__init__.py b/experimental/lite/examples/__init__.py new file mode 100644 index 00000000000..742e27ee4bf --- /dev/null +++ b/experimental/lite/examples/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Megatron Lite examples package.""" diff --git a/experimental/lite/examples/bench/.gitignore b/experimental/lite/examples/bench/.gitignore new file mode 100644 index 00000000000..17aa483ab4e --- /dev/null +++ b/experimental/lite/examples/bench/.gitignore @@ -0,0 +1 @@ +outputs/ diff --git a/experimental/lite/examples/bench/README.md b/experimental/lite/examples/bench/README.md new file mode 100644 index 00000000000..a9a2ecf369e --- /dev/null +++ b/experimental/lite/examples/bench/README.md @@ -0,0 +1,205 @@ +# MLite Bench Example + +This example runs the same small pretrain-style benchmark through `backend=mlite` +and a reference backend. + +The validated reference backend in this PR is `backend=mbridge`, backed by the +legacy `mbridge` package. `backend=bridge` is reserved for the real +Megatron-Bridge package and requires an environment where `import +megatron.bridge` works. Dry-run mode does not import either reference package and +is safe for config validation. + +The `backend=mlite` path remains native MLite. For deterministic Qwen3.5 MoE +runs it mounts the Qwen3.5 vision module from the local Hugging Face model code +through `transformers`; it does not import `mbridge` or Megatron-Bridge to build +the MLite model. + +This benchmark separates two validation lines: + +- `mbridge`: the validated reference for MLite vs Megatron-Core distributed + optimizer (`distopt`) parity. +- `bridge`: a real Megatron-Bridge environment check when `import + megatron.bridge` works. + +Use the `mbridge` line for Core/distopt precision and speed claims. + +## Dry-Run + +```bash +export PYTHONPATH=/path/to/Megatron-LM/experimental/lite:$PYTHONPATH + +python experimental/lite/examples/bench/bench.py \ + --backend mlite \ + --hf-path /models/Qwen3.5-35B-A3B \ + --model-name qwen3_5 \ + --truncate-layers 2 \ + --disable-mtp \ + --dry-run + +python experimental/lite/examples/bench/bench.py \ + --backend mbridge \ + --hf-path /models/Qwen3.5-35B-A3B \ + --model-name qwen3_5 \ + --truncate-layers 2 \ + --disable-mtp \ + --dry-run +``` + +The dry-run output is JSON containing the resolved `RuntimeConfig` and session +settings. Use it to confirm the two backend runs differ only on the intended +axis. + +## Pair Script + +```bash +HF_PATH=/models/Qwen3.5-35B-A3B \ +REFERENCE_BACKEND=mbridge \ +DRY_RUN=1 \ +bash experimental/lite/examples/bench/scripts/run_qwen35_pair.sh +``` + +Set `DRY_RUN=0` to run the benchmark under `torchrun`. Results are written to +`experimental/lite/examples/bench/outputs/`. Set `REFERENCE_BACKEND=bridge` only +when Megatron-Bridge is installed and `import megatron.bridge` succeeds. + +## Validated Run + +The following paired run completed on 2026-06-07 with 8x NVIDIA H100 80GB GPUs: + +```bash +HF_PATH=/models/Qwen3.5-35B-A3B \ +OUTPUT_DIR=experimental/lite/examples/bench/outputs/qwen35_pair \ +REFERENCE_BACKEND=mbridge \ +DRY_RUN=0 \ +NPROC=8 \ +MASTER_PORT=31841 \ +MASTER_PORT_BRIDGE=31842 \ +STEPS=15 \ +WARMUP=5 \ +SEQ_LEN=1024 \ +NUM_MICROBATCHES=4 \ +TRUNCATE_LAYERS=8 \ +KEEP_EXPERTS=8 \ +SAME_DATA_ACROSS_DP=1 \ +bash experimental/lite/examples/bench/scripts/run_qwen35_pair.sh +``` + +Slurm job `12624917` completed with exit code `0:0`. The run used +`torch==2.10.0+cu129`; `transformer_engine`, `einops`, and `mbridge` were +available in the runtime environment. + +| Runtime | Impl | Optimizer backend | Measured steps | Avg step ms | Tokens/s | Tokens/s/GPU | Peak memory GB | TFLOPs/GPU | +| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | +| `mlite` | `lite` | `distopt` | 10 | 309.433 | 105896.935 | 13237.117 | 14.324 | 80.444 | +| `mbridge` | `bridge` | `distopt` | 10 | 332.201 | 98639.089 | 12329.886 | 17.987 | 74.931 | +| `bridge` | `bridge` | `distopt` | 10 | 334.936 | 97833.496 | 12229.187 | 16.403 | 74.319 | + +The two runs used the same synthetic input stream. Loss matched within +`atol=0.05, rtol=0.005` across 10 measured samples +(`max_abs_diff=0.000500`). This long benchmark is performance evidence; the +strict optimizer-metric evidence is the deterministic run below. + +## Deterministic mbridge Correctness + +Slurm job `12630675` completed a strict deterministic MLite vs `mbridge` run +with 1x GPU, `seed=42`, Qwen3.5 MoE, `seq_len=8`, `truncate_layers=1`, +`keep_experts=2`, `optimizer_backend=distopt`, and `steps=2`. + +The comparison passed with bitwise scalar parity and no mismatches: + +- `samples=2` +- `max_loss_abs=0.0` +- `max_grad_norm_abs=0.0` +- `mismatches=[]` +- step 0: `loss=13.027458190917969`, + `grad_norm=120.75512734973202`, post-step weight SHA256 + `1e3176a8cb18d68c5da9bfa5f31a507fa8d51a3a5ddc10fbcd821260a4c6c980` +- step 1: `loss=14.698704719543457`, + `grad_norm=96.15656991334498`, post-step weight SHA256 + `e6d034f7e05ee5ee6a42ceddda8970874c6742baf59cade38f53550abd7aec29` +- eval logits canonical bf16 SHA256 + `2f805802633927852c5ec87455b1afa3a68597e1e411b979f2678eaedfb1c710` + +## Real Run + +```bash +torchrun --nproc_per_node 1 experimental/lite/examples/bench/bench.py \ + --backend mlite \ + --hf-path /models/Qwen3.5-35B-A3B \ + --model-name qwen3_5 \ + --steps 5 \ + --warmup 1 \ + --seq-len 2048 \ + --num-microbatches 1 \ + --truncate-layers 2 \ + --disable-mtp \ + --output-json /tmp/qwen35_mlite_bench.json + +torchrun --nproc_per_node 1 experimental/lite/examples/bench/bench.py \ + --backend mbridge \ + --hf-path /models/Qwen3.5-35B-A3B \ + --model-name qwen3_5 \ + --steps 5 \ + --warmup 1 \ + --seq-len 2048 \ + --num-microbatches 1 \ + --truncate-layers 2 \ + --disable-mtp \ + --output-json /tmp/qwen35_mbridge_bench.json +``` + +Compare `loss`, `grad_norm`, `avg_step_ms`, `tok_per_s`, peak memory, and +`tflops_per_gpu` in the two JSON outputs. Benchmarks are performance evidence; +they are not a replacement for precision tests. + +## Deterministic Correctness + +Use `correctness.py` for strict deterministic parity. It emits exact scalar +fingerprints for `loss` and `grad_norm`, SHA256 fingerprints for logits and +post-step exported weights, and a strict comparison artifact. + +```bash +export MEGATRON_LITE_DETERMINISTIC=1 +export CUBLAS_WORKSPACE_CONFIG=:4096:8 + +torchrun --nproc_per_node 1 experimental/lite/examples/bench/correctness.py run \ + --backend mlite \ + --hf-path /models/Qwen3.5-35B-A3B \ + --model-name qwen3_5 \ + --steps 2 \ + --seq-len 128 \ + --num-microbatches 1 \ + --truncate-layers 2 \ + --disable-mtp \ + --same-data-across-dp \ + --output-json /tmp/qwen35_mlite_correctness.json + +torchrun --nproc_per_node 1 experimental/lite/examples/bench/correctness.py run \ + --backend mbridge \ + --hf-path /models/Qwen3.5-35B-A3B \ + --model-name qwen3_5 \ + --steps 2 \ + --seq-len 128 \ + --num-microbatches 1 \ + --truncate-layers 2 \ + --disable-mtp \ + --same-data-across-dp \ + --output-json /tmp/qwen35_mbridge_correctness.json + +python experimental/lite/examples/bench/correctness.py compare \ + /tmp/qwen35_mlite_correctness.json \ + /tmp/qwen35_mbridge_correctness.json \ + --fail-on-mismatch +``` + +For the PR signoff pair script: + +```bash +export MEGATRON_LITE_DETERMINISTIC=1 +export CUBLAS_WORKSPACE_CONFIG=:4096:8 + +HF_PATH=/models/Qwen3.5-35B-A3B \ +REFERENCE_BACKEND=mbridge \ +DRY_RUN=0 \ +bash experimental/lite/examples/bench/scripts/run_qwen35_correctness_pair.sh +``` diff --git a/experimental/lite/examples/bench/__init__.py b/experimental/lite/examples/bench/__init__.py new file mode 100644 index 00000000000..c910a4bfbf3 --- /dev/null +++ b/experimental/lite/examples/bench/__init__.py @@ -0,0 +1,7 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Benchmark example for Megatron Lite runtime backends.""" + +from .results import RunResult, StepTrace +from .session import PretrainSessionConfig, run_pretrain_session + +__all__ = ["PretrainSessionConfig", "RunResult", "StepTrace", "run_pretrain_session"] diff --git a/experimental/lite/examples/bench/bench.py b/experimental/lite/examples/bench/bench.py new file mode 100644 index 00000000000..709057388e0 --- /dev/null +++ b/experimental/lite/examples/bench/bench.py @@ -0,0 +1,444 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Benchmark MLite and reference runtime backends. + +Run from the Megatron-LM repo root after adding ``experimental/lite`` to +``PYTHONPATH``. ``--dry-run`` validates config construction without importing +reference backend packages or initializing distributed state. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from dataclasses import dataclass, fields, is_dataclass, replace +from pathlib import Path +from typing import Any + +_EXPERIMENTAL_LITE_ROOT = Path(__file__).resolve().parents[2] +_REPO_ROOT = Path(__file__).resolve().parents[4] +if str(_EXPERIMENTAL_LITE_ROOT) not in sys.path: + sys.path.insert(0, str(_EXPERIMENTAL_LITE_ROOT)) +if str(_REPO_ROOT) not in sys.path: + sys.path.insert(1, str(_REPO_ROOT)) + +from examples.bench.results import StepTrace +from examples.bench.session import PretrainSessionConfig, run_pretrain_session +from megatron.lite.runtime import RuntimeConfig, create_runtime +from megatron.lite.runtime.backends.bridge.config import BridgeConfig +from megatron.lite.runtime.backends.mlite.config import MegatronLiteConfig +from megatron.lite.runtime.contracts.config import OptimizerConfig, ParallelConfig + + +@dataclass +class BenchCliConfig: + backend: str = "mlite" + hf_path: str = "" + model_name: str = "qwen3_moe" + impl: str = "lite" + tp: int = 1 + etp: int | None = None + ep: int = 1 + pp: int = 1 + vpp: int = 1 + cp: int = 1 + steps: int = 2 + warmup: int = 0 + num_microbatches: int = 1 + seq_len: int = 2048 + seed: int = 42 + device: str = "cuda" + use_thd: bool = False + same_data_across_dp: bool = False + no_optimizer: bool = False + skip_load_hf_weights: bool = False + skip_optimizer_build: bool = False + keep_experts: int | None = None + truncate_layers: int | None = None + disable_mtp: bool = False + optimizer_lr: float = 1e-4 + optimizer_weight_decay: float = 0.1 + optimizer_clip_grad: float = 1.0 + override_ddp_json: str = "{}" + override_transformer_json: str = "{}" + override_optimizer_json: str = "{}" + impl_cfg_json: str = "{}" + dry_run: bool = False + output_json: str | None = None + + +def _json_mapping(raw: str, *, name: str) -> dict[str, Any]: + try: + value = json.loads(raw) + except json.JSONDecodeError as exc: + raise ValueError(f"{name} must be a JSON object: {exc}") from exc + if not isinstance(value, dict): + raise ValueError(f"{name} must be a JSON object.") + return value + + +def _parallel_config(cfg: BenchCliConfig) -> ParallelConfig: + return ParallelConfig(tp=cfg.tp, etp=cfg.etp, ep=cfg.ep, pp=cfg.pp, vpp=cfg.vpp, cp=cfg.cp) + + +def _optimizer_config(cfg: BenchCliConfig) -> OptimizerConfig: + return OptimizerConfig( + lr=cfg.optimizer_lr, + weight_decay=cfg.optimizer_weight_decay, + clip_grad=cfg.optimizer_clip_grad, + adam_beta1=0.9, + adam_beta2=0.95, + adam_eps=1e-8, + use_precision_aware_optimizer=True, + decoupled_weight_decay=True, + ) + + +def _set_field(config_obj: Any, name: str, value: Any) -> Any: + if is_dataclass(config_obj): + return replace(config_obj, **{name: value}) + setattr(config_obj, name, value) + return config_obj + + +def _get_model_config_root(config_obj: Any) -> Any: + return getattr(config_obj, "text_config", config_obj) + + +def _get_bridge_config_root(bridge: Any) -> Any: + hf_pretrained = getattr(bridge, "hf_pretrained", None) + if hf_pretrained is not None: + return _get_model_config_root(getattr(hf_pretrained, "config", hf_pretrained)) + hf_config = getattr(bridge, "hf_config", None) + if hf_config is not None: + return _get_model_config_root(hf_config) + raise ValueError("Bridge object does not expose hf_pretrained or hf_config.") + + +def _refresh_bridge_config(bridge: Any) -> None: + if hasattr(bridge, "_build_config"): + bridge.config = bridge._build_config() + + +def _disable_mtp(config_obj: Any) -> Any: + root = _get_model_config_root(config_obj) + for attr in ("mtp_num_hidden_layers", "num_nextn_predict_layers"): + if hasattr(root, attr): + root = _set_field(root, attr, 0) + if hasattr(root, "mtp_layer_types"): + root = _set_field(root, "mtp_layer_types", []) + if root is not config_obj and hasattr(config_obj, "text_config"): + return _set_field(config_obj, "text_config", root) + return root + + +def _make_mlite_model_config_hook(cfg: BenchCliConfig): + hooks = [] + if cfg.keep_experts is not None: + keep_experts = cfg.keep_experts + + def keep_experts_hook(model_cfg): + old_num = getattr(model_cfg, "num_experts", None) + old_topk = getattr(model_cfg, "num_experts_per_tok", None) + if old_num is None or old_topk is None: + raise ValueError("keep_experts requires model config with MoE expert metadata.") + if keep_experts <= 0 or keep_experts > old_num: + raise ValueError(f"keep_experts must be in [1, {old_num}], got {keep_experts}.") + return replace( + model_cfg, num_experts=keep_experts, num_experts_per_tok=min(old_topk, keep_experts) + ) + + hooks.append(keep_experts_hook) + + if cfg.truncate_layers is not None: + keep_layers = cfg.truncate_layers + + def truncate_layers_hook(model_cfg): + old_layers = getattr(model_cfg, "num_hidden_layers", None) + if old_layers is None: + raise ValueError("truncate_layers requires num_hidden_layers.") + if keep_layers <= 0 or keep_layers > old_layers: + raise ValueError( + f"truncate_layers must be in [1, {old_layers}], got {keep_layers}." + ) + updates = {"num_hidden_layers": keep_layers} + # layer_types is qwen-style metadata; deepseek_v4 / kimi_k2 configs don't carry it. + layer_types = getattr(model_cfg, "layer_types", None) + if layer_types is not None: + updates["layer_types"] = list(layer_types[:keep_layers]) + return replace(model_cfg, **updates) + + hooks.append(truncate_layers_hook) + + if cfg.disable_mtp: + hooks.append(_disable_mtp) + + if not hooks: + return None + + def composed(model_cfg): + for hook in hooks: + model_cfg = hook(model_cfg) + return model_cfg + + return composed + + +def _make_bridge_post_init_hook(cfg: BenchCliConfig): + hooks = [] + if cfg.keep_experts is not None: + keep_experts = cfg.keep_experts + + def keep_experts_hook(bridge) -> None: + hf_cfg = _get_bridge_config_root(bridge) + old_num = getattr(hf_cfg, "num_experts", None) + old_topk = getattr(hf_cfg, "num_experts_per_tok", None) + if old_num is None or old_topk is None: + raise ValueError("keep_experts requires HF config with MoE expert metadata.") + if keep_experts <= 0 or keep_experts > old_num: + raise ValueError(f"keep_experts must be in [1, {old_num}], got {keep_experts}.") + hf_cfg.num_experts = keep_experts + hf_cfg.num_experts_per_tok = min(old_topk, keep_experts) + _refresh_bridge_config(bridge) + + if hasattr(bridge, "_weight_to_mcore_format"): + original = bridge._weight_to_mcore_format + + def patched(name: str, hf_weights: list): + if "mlp.router.weight" in name and len(hf_weights) == 1: + hf_weights = [hf_weights[0][:keep_experts].contiguous()] + return original(name, hf_weights) + + bridge._weight_to_mcore_format = patched + + hooks.append(keep_experts_hook) + + if cfg.truncate_layers is not None: + keep_layers = cfg.truncate_layers + + def truncate_layers_hook(bridge) -> None: + hf_cfg = _get_bridge_config_root(bridge) + old_layers = getattr(hf_cfg, "num_hidden_layers", None) + if old_layers is None: + raise ValueError("truncate_layers requires HF config with num_hidden_layers.") + if keep_layers <= 0 or keep_layers > old_layers: + raise ValueError( + f"truncate_layers must be in [1, {old_layers}], got {keep_layers}." + ) + hf_cfg.num_hidden_layers = keep_layers + if hasattr(hf_cfg, "layer_types"): + hf_cfg.layer_types = list(hf_cfg.layer_types[:keep_layers]) + _refresh_bridge_config(bridge) + + hooks.append(truncate_layers_hook) + + if cfg.disable_mtp: + + def disable_mtp_hook(bridge) -> None: + hf_cfg = _get_bridge_config_root(bridge) + for attr in ("mtp_num_hidden_layers", "num_nextn_predict_layers"): + if hasattr(hf_cfg, attr): + setattr(hf_cfg, attr, 0) + if hasattr(hf_cfg, "mtp_layer_types"): + hf_cfg.mtp_layer_types = [] + _refresh_bridge_config(bridge) + + hooks.append(disable_mtp_hook) + + if not hooks: + return None + + def composed(bridge) -> None: + for hook in hooks: + hook(bridge) + + return composed + + +def build_runtime_config(cfg: BenchCliConfig) -> RuntimeConfig: + parallel = _parallel_config(cfg) + optimizer = _optimizer_config(cfg) + optimizer_overrides = _json_mapping(cfg.override_optimizer_json, name="override_optimizer_json") + + if cfg.backend == "mlite": + for key, value in optimizer_overrides.items(): + setattr(optimizer, key, value) + impl_cfg = _json_mapping(cfg.impl_cfg_json, name="impl_cfg_json") + impl_cfg.setdefault("use_thd", cfg.use_thd) + if cfg.model_name == "qwen3_5": + from megatron.lite.primitive.deterministic import deterministic_requested + + if deterministic_requested(): + impl_cfg.setdefault("mount_vision_model", True) + backend_cfg = MegatronLiteConfig( + model_name=cfg.model_name, + impl=cfg.impl, + hf_path=cfg.hf_path, + parallel=parallel, + optimizer=optimizer, + load_hf_weights=not cfg.skip_load_hf_weights, + impl_cfg=impl_cfg, + model_config_hook=_make_mlite_model_config_hook(cfg), + ) + elif cfg.backend in {"bridge", "mbridge"}: + impl_cfg = _json_mapping(cfg.impl_cfg_json, name="impl_cfg_json") + if impl_cfg: + raise ValueError(f"{cfg.backend} backend does not accept impl_cfg_json.") + backend_cfg = BridgeConfig( + model_name=cfg.model_name, + parallel=parallel, + optimizer=optimizer, + use_thd=cfg.use_thd, + load_hf_weights=not cfg.skip_load_hf_weights, + build_optimizer=not cfg.skip_optimizer_build, + override_ddp_config=_json_mapping(cfg.override_ddp_json, name="override_ddp_json"), + override_transformer_config=_json_mapping( + cfg.override_transformer_json, name="override_transformer_json" + ), + override_optimizer_config=optimizer_overrides, + bridge_post_init=_make_bridge_post_init_hook(cfg), + ) + else: + raise ValueError(f"backend must be 'mlite', 'bridge', or 'mbridge', got {cfg.backend!r}.") + + return RuntimeConfig(backend=cfg.backend, hf_path=cfg.hf_path, backend_cfg=backend_cfg) + + +def build_session_config(cfg: BenchCliConfig) -> PretrainSessionConfig: + return PretrainSessionConfig( + steps=cfg.steps, + warmup=cfg.warmup, + num_microbatches=cfg.num_microbatches, + seq_len=cfg.seq_len, + seed=cfg.seed, + device=cfg.device, + use_thd=cfg.use_thd, + same_data_across_dp=cfg.same_data_across_dp, + no_optimizer=cfg.no_optimizer, + ) + + +def _to_jsonable(value: Any) -> Any: + if is_dataclass(value): + return {field.name: _to_jsonable(getattr(value, field.name)) for field in fields(value)} + if isinstance(value, dict): + return {str(k): _to_jsonable(v) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [_to_jsonable(v) for v in value] + if callable(value): + return f"" + if isinstance(value, Path): + return str(value) + return value + + +def build_dry_run_plan(cfg: BenchCliConfig) -> dict[str, Any]: + return { + "dry_run": True, + "runtime": _to_jsonable(build_runtime_config(cfg)), + "session": _to_jsonable(build_session_config(cfg)), + "notes": [ + "Dry-run validates config construction only.", + "Run under torchrun for real benchmark execution.", + ], + } + + +def _distributed_rank() -> int: + for name in ("RANK", "SLURM_PROCID"): + raw = os.environ.get(name) + if raw is None: + continue + try: + return int(raw) + except ValueError: + continue + return 0 + + +def _step_reporter(trace: StepTrace) -> None: + parts = [ + "[MLITE_BENCH_STEP]", + f"step={trace.step}", + f"loss={trace.loss:.6f}", + f"grad_norm={trace.grad_norm:.6f}", + f"step_ms={trace.step_ms:.3f}", + f"peak_mem_gb={(trace.peak_mem_gb or 0.0):.3f}", + ] + if trace.tflops_per_gpu is not None: + parts.append(f"tflops_per_gpu={trace.tflops_per_gpu:.3f}") + print(" ".join(parts), flush=True) + + +def run(cfg: BenchCliConfig) -> dict[str, Any]: + if cfg.dry_run: + return build_dry_run_plan(cfg) + + rt_cfg = build_runtime_config(cfg) + rt = create_runtime(rt_cfg) + handle = rt.build_model() + result = run_pretrain_session( + rt, handle, build_session_config(cfg), step_reporter=_step_reporter + ) + return result.to_dict() + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--backend", choices=["mlite", "bridge", "mbridge"], default="mlite") + parser.add_argument("--hf-path", default="") + parser.add_argument("--model-name", default="qwen3_moe") + parser.add_argument("--impl", default="lite") + parser.add_argument("--tp", type=int, default=1) + parser.add_argument("--etp", type=int, default=None) + parser.add_argument("--ep", type=int, default=1) + parser.add_argument("--pp", type=int, default=1) + parser.add_argument("--vpp", type=int, default=1) + parser.add_argument("--cp", type=int, default=1) + parser.add_argument("--steps", type=int, default=2) + parser.add_argument("--warmup", type=int, default=0) + parser.add_argument("--num-microbatches", type=int, default=1) + parser.add_argument("--seq-len", type=int, default=2048) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--device", default="cuda") + parser.add_argument("--use-thd", action="store_true") + parser.add_argument("--same-data-across-dp", action="store_true") + parser.add_argument("--no-optimizer", action="store_true") + parser.add_argument("--skip-load-hf-weights", action="store_true") + parser.add_argument("--skip-optimizer-build", action="store_true") + parser.add_argument("--keep-experts", type=int, default=None) + parser.add_argument("--truncate-layers", type=int, default=None) + parser.add_argument("--disable-mtp", action="store_true") + parser.add_argument("--optimizer-lr", type=float, default=1e-4) + parser.add_argument("--optimizer-weight-decay", type=float, default=0.1) + parser.add_argument("--optimizer-clip-grad", type=float, default=1.0) + parser.add_argument("--override-ddp-json", default="{}") + parser.add_argument("--override-transformer-json", default="{}") + parser.add_argument("--override-optimizer-json", default="{}") + parser.add_argument("--impl-cfg-json", default="{}") + parser.add_argument("--dry-run", action="store_true") + parser.add_argument("--output-json", default=None) + return parser + + +def parse_args(argv: list[str] | None = None) -> BenchCliConfig: + ns = _parser().parse_args(argv) + return BenchCliConfig(**vars(ns)) + + +def main(argv: list[str] | None = None) -> dict[str, Any]: + cfg = parse_args(argv) + artifact = run(cfg) + if _distributed_rank() == 0: + text = json.dumps(artifact, indent=2, sort_keys=True) + print(text, flush=True) + if cfg.output_json is not None: + output_path = Path(cfg.output_json) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(text + "\n", encoding="utf-8") + return artifact + + +if __name__ == "__main__": + main() diff --git a/experimental/lite/examples/bench/correctness.py b/experimental/lite/examples/bench/correctness.py new file mode 100644 index 00000000000..7cb90d81d2f --- /dev/null +++ b/experimental/lite/examples/bench/correctness.py @@ -0,0 +1,493 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Deterministic correctness runner for MLite and reference backends.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import struct +import sys +from contextlib import contextmanager +from pathlib import Path +from typing import Any + +import torch + +_EXPERIMENTAL_LITE_ROOT = Path(__file__).resolve().parents[2] +_REPO_ROOT = Path(__file__).resolve().parents[4] +if str(_EXPERIMENTAL_LITE_ROOT) not in sys.path: + sys.path.insert(0, str(_EXPERIMENTAL_LITE_ROOT)) +if str(_REPO_ROOT) not in sys.path: + sys.path.insert(1, str(_REPO_ROOT)) + +from megatron.lite.primitive.deterministic import set_deterministic +from megatron.lite.runtime import create_runtime + +from examples.bench.bench import BenchCliConfig, build_runtime_config, build_session_config +from examples.bench.results import compare_correctness_artifacts, load_result_artifact +from examples.bench.session import _make_data_iter + + +def _distributed_rank() -> int: + for name in ("RANK", "SLURM_PROCID"): + raw = os.environ.get(name) + if raw is None: + continue + try: + return int(raw) + except ValueError: + continue + return 0 + + +def _sync(device: str) -> None: + if device.startswith("cuda") and torch.cuda.is_available(): + torch.cuda.synchronize() + + +def _scalar(value: float | int | torch.Tensor | None) -> dict[str, Any]: + if isinstance(value, torch.Tensor): + if value.numel() != 1: + raise ValueError("scalar fingerprint requires a scalar tensor.") + value = float(value.detach().cpu().float().item()) + value_f = float(0.0 if value is None else value) + return { + "value": value_f, + "float_hex": value_f.hex(), + "sha256_f64_be": hashlib.sha256(struct.pack(">d", value_f)).hexdigest(), + } + + +def _hash_tensor(tensor: torch.Tensor | None) -> dict[str, Any] | None: + if tensor is None: + return None + t = tensor.detach().contiguous().cpu() + raw = t.view(torch.uint8).numpy().tobytes() + as_bf16 = t.to(torch.bfloat16).contiguous() if t.is_floating_point() else None + summary = t.float() if t.is_floating_point() else None + result = { + "shape": list(t.shape), + "dtype": str(t.dtype), + "sha256": hashlib.sha256(raw).hexdigest(), + } + if as_bf16 is not None: + result["sha256_as_bf16"] = hashlib.sha256( + as_bf16.view(torch.uint8).numpy().tobytes() + ).hexdigest() + if summary is not None: + flat = summary.reshape(-1) + result["summary"] = { + "min": float(flat.min().item()) if flat.numel() else 0.0, + "max": float(flat.max().item()) if flat.numel() else 0.0, + "mean": float(flat.mean().item()) if flat.numel() else 0.0, + "first8": [float(x) for x in flat[:8].tolist()], + } + return result + + +def _first_tensor(value: Any) -> torch.Tensor | None: + if isinstance(value, torch.Tensor): + return value + if isinstance(value, dict): + for item in value.values(): + tensor = _first_tensor(item) + if tensor is not None: + return tensor + return None + if isinstance(value, (list, tuple)): + for item in value: + tensor = _first_tensor(item) + if tensor is not None: + return tensor + return None + + +def _record_activation_probe( + records: list[dict[str, Any]], + name: str, + output: Any, + *, + record_grad: bool = False, + tensor_hooks: list[Any] | None = None, +) -> None: + tensor = _first_tensor(output) + record = {"name": name, "found": True, "tensor": _hash_tensor(tensor)} + if record_grad: + record["grad"] = None + record["grad_found"] = isinstance(tensor, torch.Tensor) and tensor.requires_grad + if isinstance(tensor, torch.Tensor) and tensor.requires_grad: + + def _grad_hook(grad, _record=record): + _record["grad"] = _hash_tensor(grad) + + hook = tensor.register_hook(_grad_hook) + if tensor_hooks is not None: + tensor_hooks.append(hook) + records.append(record) + + +def _resolve_probe_module(modules: dict[str, Any], name: str) -> tuple[str, Any] | None: + module = modules.get(name) + if module is not None: + return name, module + matches = [ + (candidate_name, candidate) + for candidate_name, candidate in modules.items() + if candidate_name.endswith(f".{name}") + ] + if len(matches) == 1: + return matches[0] + return None + + +@contextmanager +def _activation_probe_context(handle, probe_names: list[str], *, record_grad: bool = False): + records: list[dict[str, Any]] = [] + hooks = [] + tensor_hooks = [] + patched_methods = [] + modules = dict(handle._model.named_modules()) + for name in probe_names: + probe_name = name + record_input = probe_name.endswith(":input") + lookup_name = probe_name[:-6] if record_input else probe_name + if "::" in lookup_name: + module_name, method_name = lookup_name.split("::", 1) + resolved = _resolve_probe_module(modules, module_name) + if resolved is None or not hasattr(resolved[1], method_name): + records.append({"name": name, "found": False}) + continue + resolved_name, module = resolved + original = getattr(module, method_name) + + def _wrapped( + *args, + _original=original, + _probe_name=name, + _resolved_name=resolved_name, + _module_type=type(module).__module__ + "." + type(module).__qualname__, + _record_input=record_input, + _method_name=method_name, + **kwargs, + ): + if _record_input: + _record_activation_probe( + records, + _probe_name, + args, + record_grad=record_grad, + tensor_hooks=tensor_hooks, + ) + records[-1]["resolved_name"] = f"{_resolved_name}::{_method_name}:input" + records[-1]["module_type"] = _module_type + return _original(*args, **kwargs) + output = _original(*args, **kwargs) + _record_activation_probe( + records, _probe_name, output, record_grad=record_grad, tensor_hooks=tensor_hooks + ) + records[-1]["resolved_name"] = f"{_resolved_name}::{_method_name}" + records[-1]["module_type"] = _module_type + return output + + setattr(module, method_name, _wrapped) + patched_methods.append((module, method_name, original)) + continue + + resolved = _resolve_probe_module(modules, lookup_name) + if resolved is None: + records.append({"name": name, "found": False}) + continue + resolved_name, module = resolved + + if record_input: + + def _pre_hook(_module, args, probe_name=name, resolved_probe_name=resolved_name): + _record_activation_probe( + records, probe_name, args, record_grad=record_grad, tensor_hooks=tensor_hooks + ) + records[-1]["resolved_name"] = f"{resolved_probe_name}:input" + records[-1]["module_type"] = ( + type(_module).__module__ + "." + type(_module).__qualname__ + ) + + hooks.append(module.register_forward_pre_hook(_pre_hook)) + continue + + def _hook(_module, _args, output, probe_name=name, resolved_probe_name=resolved_name): + _record_activation_probe( + records, probe_name, output, record_grad=record_grad, tensor_hooks=tensor_hooks + ) + records[-1]["resolved_name"] = resolved_probe_name + records[-1]["module_type"] = type(_module).__module__ + "." + type(_module).__qualname__ + + hooks.append(module.register_forward_hook(_hook)) + try: + yield records + finally: + for hook in tensor_hooks: + hook.remove() + for hook in hooks: + hook.remove() + for module, method_name, original in patched_methods: + setattr(module, method_name, original) + + +def _update_hash_with_tensor(h: Any, name: str, tensor: torch.Tensor) -> None: + if hasattr(tensor, "to_local"): + tensor = tensor.to_local() + t = tensor.detach().contiguous().cpu() + h.update(name.encode("utf-8")) + h.update(b"\0") + h.update(str(t.dtype).encode("ascii")) + h.update(b"\0") + h.update(json.dumps(list(t.shape), separators=(",", ":")).encode("ascii")) + h.update(b"\0") + h.update(t.view(torch.uint8).numpy().tobytes()) + h.update(b"\0") + + +def _model_chunks(handle) -> list[Any]: + chunks = handle._extras.get("model_chunks") + if chunks is None: + chunks = handle._extras.get("model_list") + if chunks is None: + chunks = [handle._model] + return list(chunks) + + +def _grad_fingerprint(handle) -> dict[str, Any]: + h = hashlib.sha256() + count = 0 + details = [] + include_details = os.environ.get("MLITE_CORRECTNESS_GRAD_DETAILS") == "1" + for chunk_idx, chunk in enumerate(_model_chunks(handle)): + for name, param in sorted(chunk.named_parameters(), key=lambda item: item[0]): + grad = param.grad + if grad is None: + grad = getattr(param, "main_grad", None) + if grad is None: + continue + fingerprint_name = f"{chunk_idx}:{name}" + _update_hash_with_tensor(h, fingerprint_name, grad) + if include_details: + detail = _hash_tensor(grad) + assert detail is not None + detail["name"] = fingerprint_name + details.append(detail) + count += 1 + result = {"sha256": h.hexdigest(), "tensor_count": count} + if include_details: + result["details"] = details + return result + + +def _weight_fingerprint(rt, handle) -> dict[str, Any]: + h = hashlib.sha256() + count = 0 + details = [] + include_details = os.environ.get("MLITE_CORRECTNESS_WEIGHT_DETAILS") == "1" + for name, tensor in sorted(rt.export_weights(handle), key=lambda item: item[0]): + _update_hash_with_tensor(h, str(name), tensor) + if include_details: + detail = _hash_tensor(tensor) + assert detail is not None + detail["name"] = str(name) + details.append(detail) + count += 1 + result = {"sha256": h.hexdigest(), "tensor_count": count} + if include_details: + result["details"] = details + return result + + +def _forward_logits(rt, handle, batch: Any) -> torch.Tensor | None: + result = rt.forward_backward( + handle, + iter([batch]), + loss_fn=None, + num_microbatches=1, + forward_only=True, + ) + output = result.model_output + return output.vocab_parallel_logits if output.vocab_parallel_logits is not None else (-output.log_probs if output.log_probs is not None else None) + + +def run_backend( + cfg: BenchCliConfig, + *, + hash_weights: bool = True, + activation_probe_names: list[str] | None = None, +) -> dict[str, Any]: + os.environ["MEGATRON_LITE_DETERMINISTIC"] = "1" + set_deterministic(cfg.seed) + + rt_cfg = build_runtime_config(cfg) + rt = create_runtime(rt_cfg) + handle = rt.build_model() + session_cfg = build_session_config(cfg) + + eval_iter = _make_data_iter(handle, session_cfg) + eval_batch = next(eval_iter) + activation_probe_names = list(activation_probe_names or []) + with _activation_probe_context(handle, activation_probe_names) as activation_probes: + with rt.eval_mode(handle): + eval_logits = _hash_tensor(_forward_logits(rt, handle, eval_batch)) + + data_iter = _make_data_iter(handle, session_cfg) + steps: list[dict[str, Any]] = [] + with rt.train_mode(handle): + for step in range(session_cfg.steps): + with _activation_probe_context( + handle, activation_probe_names, record_grad=True + ) as train_activation_probes: + rt.zero_grad(handle) + _sync(session_cfg.device) + result = rt.forward_backward( + handle, data_iter, loss_fn=None, num_microbatches=session_cfg.num_microbatches + ) + _sync(session_cfg.device) + output = result.model_output + logits = _hash_tensor(output.vocab_parallel_logits if output.vocab_parallel_logits is not None else (-output.log_probs if output.log_probs is not None else None)) + grads = _grad_fingerprint(handle) + + if session_cfg.no_optimizer: + update_successful, grad_norm, num_zeros = True, 0.0, 0 + else: + update_successful, grad_norm, num_zeros = rt.optimizer_step(handle) + rt.lr_scheduler_step(handle) + _sync(session_cfg.device) + loss = result.metrics.get("loss") + if loss is None: + loss = result.model_output.loss + + steps.append( + { + "step": step, + "loss": _scalar(0.0 if loss is None else loss), + "logits": logits, + "grad_fingerprint": grads, + "grad_norm": _scalar(grad_norm), + "update_successful": bool(update_successful), + "num_zeros": None if num_zeros is None else int(num_zeros), + "post_step_weights": _weight_fingerprint(rt, handle) if hash_weights else None, + "train_activation_probes": train_activation_probes, + } + ) + + return { + "kind": "mlite_bench_correctness", + "backend": cfg.backend, + "model_name": cfg.model_name, + "seed": cfg.seed, + "seq_len": cfg.seq_len, + "num_microbatches": cfg.num_microbatches, + "steps": steps, + "eval_logits": eval_logits, + "activation_probes": activation_probes, + "metadata": { + "deterministic": True, + "hash_weights": hash_weights, + "same_data_across_dp": cfg.same_data_across_dp, + "use_thd": cfg.use_thd, + }, + } + + +def _add_run_args(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--backend", choices=["mlite", "bridge", "mbridge"], required=True) + parser.add_argument("--hf-path", required=True) + parser.add_argument("--model-name", default="qwen3_5") + parser.add_argument("--impl", default="lite") + parser.add_argument("--tp", type=int, default=1) + parser.add_argument("--etp", type=int, default=None) + parser.add_argument("--ep", type=int, default=1) + parser.add_argument("--pp", type=int, default=1) + parser.add_argument("--vpp", type=int, default=1) + parser.add_argument("--cp", type=int, default=1) + parser.add_argument("--steps", type=int, default=2) + parser.add_argument("--warmup", type=int, default=0) + parser.add_argument("--num-microbatches", type=int, default=1) + parser.add_argument("--seq-len", type=int, default=128) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--device", default="cuda") + parser.add_argument("--use-thd", action="store_true") + parser.add_argument("--same-data-across-dp", action="store_true") + parser.add_argument("--no-optimizer", action="store_true") + parser.add_argument("--skip-load-hf-weights", action="store_true") + parser.add_argument("--skip-optimizer-build", action="store_true") + parser.add_argument("--keep-experts", type=int, default=None) + parser.add_argument("--truncate-layers", type=int, default=None) + parser.add_argument("--disable-mtp", action="store_true") + parser.add_argument("--optimizer-lr", type=float, default=1e-4) + parser.add_argument("--optimizer-weight-decay", type=float, default=0.1) + parser.add_argument("--optimizer-clip-grad", type=float, default=1.0) + parser.add_argument("--override-ddp-json", default="{}") + parser.add_argument("--override-transformer-json", default="{}") + parser.add_argument("--override-optimizer-json", default="{}") + parser.add_argument("--impl-cfg-json", default="{}") + parser.add_argument("--output-json", required=True) + parser.add_argument("--skip-weight-hash", action="store_true") + parser.add_argument("--activation-probes-json", default="{}") + + +def _activation_probe_names(raw: str, backend: str) -> list[str]: + value = json.loads(raw) + if isinstance(value, list): + return [str(item) for item in value] + if isinstance(value, dict): + selected = value.get(backend, []) + if not isinstance(selected, list): + raise ValueError(f"activation probe list for {backend!r} must be a JSON list.") + return [str(item) for item in selected] + raise ValueError("activation_probes_json must be a JSON list or backend-to-list mapping.") + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + run_p = sub.add_parser("run", help="run one backend and write a correctness artifact") + _add_run_args(run_p) + + cmp_p = sub.add_parser("compare", help="strictly compare two correctness artifacts") + cmp_p.add_argument("baseline") + cmp_p.add_argument("candidate") + cmp_p.add_argument("--output-json", default=None) + cmp_p.add_argument("--fail-on-mismatch", action="store_true") + return parser + + +def main(argv: list[str] | None = None) -> dict[str, Any]: + ns = _parser().parse_args(argv) + if ns.command == "compare": + result = compare_correctness_artifacts( + load_result_artifact(ns.baseline), load_result_artifact(ns.candidate) + ) + text = json.dumps(result, indent=2, sort_keys=True) + print(text, flush=True) + if ns.output_json: + Path(ns.output_json).write_text(text + "\n", encoding="utf-8") + if ns.fail_on_mismatch and not result["passed"]: + raise SystemExit(1) + return result + + cfg = BenchCliConfig( + **{k: v for k, v in vars(ns).items() if k in BenchCliConfig.__dataclass_fields__} + ) + artifact = run_backend( + cfg, + hash_weights=not ns.skip_weight_hash, + activation_probe_names=_activation_probe_names(ns.activation_probes_json, cfg.backend), + ) + if _distributed_rank() == 0: + text = json.dumps(artifact, indent=2, sort_keys=True) + print(text, flush=True) + output_path = Path(ns.output_json) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(text + "\n", encoding="utf-8") + return artifact + + +if __name__ == "__main__": + main() diff --git a/experimental/lite/examples/bench/results.py b/experimental/lite/examples/bench/results.py new file mode 100644 index 00000000000..6a015c10458 --- /dev/null +++ b/experimental/lite/examples/bench/results.py @@ -0,0 +1,239 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Benchmark result dataclasses.""" + +from __future__ import annotations + +import json +import math +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + + +@dataclass(slots=True) +class StepTrace: + step: int + loss: float + grad_norm: float + step_ms: float + peak_mem_gb: float | None = None + tflops_per_gpu: float | None = None + + def to_dict(self) -> dict[str, Any]: + result: dict[str, Any] = { + "step": self.step, + "loss": self.loss, + "grad_norm": self.grad_norm, + "step_ms": self.step_ms, + } + if self.peak_mem_gb is not None: + result["peak_mem_gb"] = self.peak_mem_gb + if self.tflops_per_gpu is not None: + result["tflops_per_gpu"] = self.tflops_per_gpu + return result + + +@dataclass(slots=True) +class RunResult: + backend: str + model_name: str + impl: str + optimizer_backend: str + tp: int + etp: int | None + ep: int + pp: int + vpp: int + cp: int + seq_len: int + num_microbatches: int + step_traces: list[StepTrace] = field(default_factory=list) + avg_step_ms: float = 0.0 + peak_mem_gb: float = 0.0 + tok_per_s: float = 0.0 + tok_per_s_per_gpu: float = 0.0 + tflops_per_gpu: float | None = None + metadata: dict[str, Any] = field(default_factory=dict) + + def summary_dict(self) -> dict[str, Any]: + return { + "backend": self.backend, + "model_name": self.model_name, + "impl": self.impl, + "optimizer_backend": self.optimizer_backend, + "avg_step_ms": self.avg_step_ms, + "tok_per_s": self.tok_per_s, + "tok_per_s_per_gpu": self.tok_per_s_per_gpu, + "peak_mem_gb": self.peak_mem_gb, + "tflops_per_gpu": self.tflops_per_gpu, + "steps_measured": len(self.step_traces), + } + + def to_dict(self) -> dict[str, Any]: + return { + "summary": self.summary_dict(), + "result": { + "backend": self.backend, + "model_name": self.model_name, + "impl": self.impl, + "optimizer_backend": self.optimizer_backend, + "tp": self.tp, + "etp": self.etp, + "ep": self.ep, + "pp": self.pp, + "vpp": self.vpp, + "cp": self.cp, + "seq_len": self.seq_len, + "num_microbatches": self.num_microbatches, + "step_traces": [trace.to_dict() for trace in self.step_traces], + "avg_step_ms": self.avg_step_ms, + "peak_mem_gb": self.peak_mem_gb, + "tok_per_s": self.tok_per_s, + "tok_per_s_per_gpu": self.tok_per_s_per_gpu, + "tflops_per_gpu": self.tflops_per_gpu, + "metadata": dict(self.metadata), + }, + } + + +def load_result_artifact(path: str | Path) -> dict[str, Any]: + """Load a benchmark JSON artifact from ``bench.py --output-json``.""" + with Path(path).open(encoding="utf-8") as f: + value = json.load(f) + if not isinstance(value, dict): + raise ValueError(f"Benchmark artifact must be a JSON object: {path}") + return value + + +def result_summary(artifact: dict[str, Any]) -> dict[str, Any]: + """Return the summary block from a benchmark artifact.""" + summary = artifact.get("summary") + if isinstance(summary, dict): + return dict(summary) + result = artifact.get("result") + if not isinstance(result, dict): + raise ValueError("Benchmark artifact must contain `summary` or `result`.") + return { + "backend": result.get("backend"), + "model_name": result.get("model_name"), + "impl": result.get("impl"), + "optimizer_backend": result.get("optimizer_backend"), + "avg_step_ms": result.get("avg_step_ms"), + "tok_per_s": result.get("tok_per_s"), + "tok_per_s_per_gpu": result.get("tok_per_s_per_gpu"), + "peak_mem_gb": result.get("peak_mem_gb"), + "tflops_per_gpu": result.get("tflops_per_gpu"), + "steps_measured": len(result.get("step_traces", [])), + } + + +def compare_step_traces( + baseline: dict[str, Any], candidate: dict[str, Any], *, atol: float = 1e-4, rtol: float = 1e-4 +) -> dict[str, Any]: + """Compare loss and grad-norm traces from two benchmark artifacts.""" + base_steps = baseline.get("result", {}).get("step_traces", []) + cand_steps = candidate.get("result", {}).get("step_traces", []) + sample_count = min(len(base_steps), len(cand_steps)) + max_loss_abs = 0.0 + max_grad_norm_abs = 0.0 + for idx in range(sample_count): + base = base_steps[idx] + cand = cand_steps[idx] + max_loss_abs = max(max_loss_abs, abs(float(base["loss"]) - float(cand["loss"]))) + max_grad_norm_abs = max( + max_grad_norm_abs, abs(float(base["grad_norm"]) - float(cand["grad_norm"])) + ) + + lengths_match = sample_count == len(base_steps) == len(cand_steps) + loss_ref_max = max([abs(float(step["loss"])) for step in base_steps[:sample_count]] + [0.0]) + grad_norm_ref_max = max( + [abs(float(step["grad_norm"])) for step in base_steps[:sample_count]] + [0.0] + ) + loss_passed = lengths_match and max_loss_abs <= atol + rtol * loss_ref_max + grad_norm_passed = lengths_match and max_grad_norm_abs <= atol + rtol * grad_norm_ref_max + + return { + "samples": sample_count, + "atol": atol, + "rtol": rtol, + "passed": loss_passed and grad_norm_passed, + "loss_passed": loss_passed, + "grad_norm_passed": grad_norm_passed, + "max_loss_abs": max_loss_abs, + "max_grad_norm_abs": max_grad_norm_abs, + } + + +def compare_correctness_artifacts( + baseline: dict[str, Any], candidate: dict[str, Any] +) -> dict[str, Any]: + """Strict bitwise comparison for deterministic correctness artifacts.""" + base_steps = baseline.get("steps", []) + cand_steps = candidate.get("steps", []) + sample_count = min(len(base_steps), len(cand_steps)) + lengths_match = sample_count == len(base_steps) == len(cand_steps) + + max_loss_abs = 0.0 + max_grad_norm_abs = 0.0 + mismatches: list[dict[str, Any]] = [] + + def _tensor_fingerprint_matches(base: Any, cand: Any) -> bool: + if base == cand: + return True + if not isinstance(base, dict) or not isinstance(cand, dict): + return False + if base.get("shape") != cand.get("shape"): + return False + base_bf16 = base.get("sha256_as_bf16") + cand_bf16 = cand.get("sha256_as_bf16") + return bool(base_bf16 and base_bf16 == cand_bf16) + + base_eval = baseline.get("eval_logits") + cand_eval = candidate.get("eval_logits") + if base_eval is not None or cand_eval is not None: + if not _tensor_fingerprint_matches(base_eval, cand_eval): + mismatches.append({"field": "eval_logits"}) + + for idx in range(sample_count): + base = base_steps[idx] + cand = cand_steps[idx] + loss_abs = abs(float(base["loss"]["value"]) - float(cand["loss"]["value"])) + grad_abs = abs(float(base["grad_norm"]["value"]) - float(cand["grad_norm"]["value"])) + if math.isfinite(loss_abs): + max_loss_abs = max(max_loss_abs, loss_abs) + if math.isfinite(grad_abs): + max_grad_norm_abs = max(max_grad_norm_abs, grad_abs) + + for field in ("loss", "grad_norm", "post_step_weights", "update_successful", "num_zeros"): + if base.get(field) != cand.get(field): + mismatches.append({"step": idx, "field": field}) + if not _tensor_fingerprint_matches(base.get("logits"), cand.get("logits")): + mismatches.append({"step": idx, "field": "logits"}) + + if not lengths_match: + mismatches.append( + { + "field": "steps", + "baseline_count": len(base_steps), + "candidate_count": len(cand_steps), + } + ) + + return { + "samples": sample_count, + "passed": lengths_match and not mismatches, + "max_loss_abs": max_loss_abs, + "max_grad_norm_abs": max_grad_norm_abs, + "tensor_fingerprint_rule": "raw_sha256_or_bf16_canonical_sha256", + "mismatches": mismatches, + } + + +__all__ = [ + "RunResult", + "StepTrace", + "compare_correctness_artifacts", + "compare_step_traces", + "load_result_artifact", + "result_summary", +] diff --git a/experimental/lite/examples/bench/scripts/run_qwen35_correctness_pair.sh b/experimental/lite/examples/bench/scripts/run_qwen35_correctness_pair.sh new file mode 100755 index 00000000000..5fa5eff9c90 --- /dev/null +++ b/experimental/lite/examples/bench/scripts/run_qwen35_correctness_pair.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +set -euo pipefail + +HF_PATH=${HF_PATH:?set HF_PATH to a HuggingFace Qwen3.5 model directory} +REPO_ROOT=${REPO_ROOT:-$(pwd)} +PYTHON_BIN=${PYTHON_BIN:-python} +NPROC=${NPROC:-1} +OUTPUT_DIR=${OUTPUT_DIR:-"${REPO_ROOT}/experimental/lite/examples/bench/outputs/correctness"} +REFERENCE_BACKEND=${REFERENCE_BACKEND:-mbridge} + +export PYTHONPATH="${REPO_ROOT}/experimental/lite:${REPO_ROOT}${PYTHONPATH:+:${PYTHONPATH}}" +export MEGATRON_LITE_DETERMINISTIC=1 +export CUBLAS_WORKSPACE_CONFIG=${CUBLAS_WORKSPACE_CONFIG:-:4096:8} + +mkdir -p "${OUTPUT_DIR}" + +COMMON_ARGS=( + --hf-path "${HF_PATH}" + --model-name qwen3_5 + --tp "${TP:-1}" + --etp "${ETP:-1}" + --ep "${EP:-1}" + --pp "${PP:-1}" + --cp "${CP:-1}" + --steps "${STEPS:-2}" + --num-microbatches "${NUM_MICROBATCHES:-1}" + --seq-len "${SEQ_LEN:-128}" + --seed "${SEED:-42}" + --truncate-layers "${TRUNCATE_LAYERS:-2}" + --disable-mtp + --same-data-across-dp +) + +if [[ -n "${KEEP_EXPERTS:-}" ]]; then + COMMON_ARGS+=(--keep-experts "${KEEP_EXPERTS}") +fi +if [[ "${SKIP_LOAD_HF_WEIGHTS:-0}" == "1" ]]; then + COMMON_ARGS+=(--skip-load-hf-weights) +fi +if [[ "${SKIP_WEIGHT_HASH:-0}" == "1" ]]; then + COMMON_ARGS+=(--skip-weight-hash) +fi +if [[ -n "${ACTIVATION_PROBES_JSON:-}" ]]; then + COMMON_ARGS+=(--activation-probes-json "${ACTIVATION_PROBES_JSON}") +fi + +MLITE_TORCHRUN_ARGS=(--nproc_per_node "${NPROC}") +BRIDGE_TORCHRUN_ARGS=(--nproc_per_node "${NPROC}") +if [[ -n "${MASTER_PORT:-}" ]]; then + MLITE_TORCHRUN_ARGS+=(--master_port "${MASTER_PORT}") +fi +if [[ -n "${MASTER_PORT_BRIDGE:-}" ]]; then + BRIDGE_TORCHRUN_ARGS+=(--master_port "${MASTER_PORT_BRIDGE}") +fi + +torchrun "${MLITE_TORCHRUN_ARGS[@]}" \ + "${REPO_ROOT}/experimental/lite/examples/bench/correctness.py" run \ + --backend mlite "${COMMON_ARGS[@]}" \ + --output-json "${OUTPUT_DIR}/qwen35_mlite_correctness.json" \ + 2>&1 | tee "${OUTPUT_DIR}/qwen35_mlite_correctness.log" + +torchrun "${BRIDGE_TORCHRUN_ARGS[@]}" \ + "${REPO_ROOT}/experimental/lite/examples/bench/correctness.py" run \ + --backend "${REFERENCE_BACKEND}" "${COMMON_ARGS[@]}" \ + --output-json "${OUTPUT_DIR}/qwen35_${REFERENCE_BACKEND}_correctness.json" \ + 2>&1 | tee "${OUTPUT_DIR}/qwen35_${REFERENCE_BACKEND}_correctness.log" + +"${PYTHON_BIN}" "${REPO_ROOT}/experimental/lite/examples/bench/correctness.py" compare \ + "${OUTPUT_DIR}/qwen35_mlite_correctness.json" \ + "${OUTPUT_DIR}/qwen35_${REFERENCE_BACKEND}_correctness.json" \ + --output-json "${OUTPUT_DIR}/qwen35_correctness_compare.json" \ + --fail-on-mismatch \ + 2>&1 | tee "${OUTPUT_DIR}/qwen35_correctness_compare.log" diff --git a/experimental/lite/examples/bench/scripts/run_qwen35_pair.sh b/experimental/lite/examples/bench/scripts/run_qwen35_pair.sh new file mode 100755 index 00000000000..2c2b7e8eb49 --- /dev/null +++ b/experimental/lite/examples/bench/scripts/run_qwen35_pair.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +set -euo pipefail + +HF_PATH=${HF_PATH:?set HF_PATH to a HuggingFace Qwen3.5 model directory} +REPO_ROOT=${REPO_ROOT:-$(pwd)} +PYTHON_BIN=${PYTHON_BIN:-python} +NPROC=${NPROC:-1} +DRY_RUN=${DRY_RUN:-1} +OUTPUT_DIR=${OUTPUT_DIR:-"${REPO_ROOT}/experimental/lite/examples/bench/outputs"} +REFERENCE_BACKEND=${REFERENCE_BACKEND:-mbridge} + +export PYTHONPATH="${REPO_ROOT}/experimental/lite:${REPO_ROOT}${PYTHONPATH:+:${PYTHONPATH}}" + +COMMON_ARGS=( + --hf-path "${HF_PATH}" + --model-name qwen3_5 + --tp "${TP:-1}" + --etp "${ETP:-1}" + --ep "${EP:-1}" + --pp "${PP:-1}" + --cp "${CP:-1}" + --steps "${STEPS:-2}" + --warmup "${WARMUP:-0}" + --num-microbatches "${NUM_MICROBATCHES:-1}" + --seq-len "${SEQ_LEN:-2048}" + --truncate-layers "${TRUNCATE_LAYERS:-2}" + --disable-mtp +) + +if [[ -n "${KEEP_EXPERTS:-}" ]]; then + COMMON_ARGS+=(--keep-experts "${KEEP_EXPERTS}") +fi +if [[ "${SAME_DATA_ACROSS_DP:-0}" == "1" ]]; then + COMMON_ARGS+=(--same-data-across-dp) +fi +if [[ "${SKIP_LOAD_HF_WEIGHTS:-0}" == "1" ]]; then + COMMON_ARGS+=(--skip-load-hf-weights) +fi +if [[ "${SKIP_OPTIMIZER_BUILD:-0}" == "1" ]]; then + COMMON_ARGS+=(--skip-optimizer-build --no-optimizer) +fi + +if [[ "${DRY_RUN}" == "1" ]]; then + "${PYTHON_BIN}" "${REPO_ROOT}/experimental/lite/examples/bench/bench.py" \ + --backend mlite "${COMMON_ARGS[@]}" --dry-run + "${PYTHON_BIN}" "${REPO_ROOT}/experimental/lite/examples/bench/bench.py" \ + --backend "${REFERENCE_BACKEND}" "${COMMON_ARGS[@]}" --dry-run +else + mkdir -p "${OUTPUT_DIR}" + MLITE_TORCHRUN_ARGS=(--nproc_per_node "${NPROC}") + BRIDGE_TORCHRUN_ARGS=(--nproc_per_node "${NPROC}") + if [[ -n "${MASTER_PORT:-}" ]]; then + MLITE_TORCHRUN_ARGS+=(--master_port "${MASTER_PORT}") + fi + if [[ -n "${MASTER_PORT_BRIDGE:-}" ]]; then + BRIDGE_TORCHRUN_ARGS+=(--master_port "${MASTER_PORT_BRIDGE}") + fi + torchrun "${MLITE_TORCHRUN_ARGS[@]}" \ + "${REPO_ROOT}/experimental/lite/examples/bench/bench.py" \ + --backend mlite "${COMMON_ARGS[@]}" \ + --output-json "${OUTPUT_DIR}/qwen35_mlite.json" \ + 2>&1 | tee "${OUTPUT_DIR}/qwen35_mlite.log" + torchrun "${BRIDGE_TORCHRUN_ARGS[@]}" \ + "${REPO_ROOT}/experimental/lite/examples/bench/bench.py" \ + --backend "${REFERENCE_BACKEND}" "${COMMON_ARGS[@]}" \ + --output-json "${OUTPUT_DIR}/qwen35_${REFERENCE_BACKEND}.json" \ + 2>&1 | tee "${OUTPUT_DIR}/qwen35_${REFERENCE_BACKEND}.log" +fi diff --git a/experimental/lite/examples/bench/session.py b/experimental/lite/examples/bench/session.py new file mode 100644 index 00000000000..5f471e24148 --- /dev/null +++ b/experimental/lite/examples/bench/session.py @@ -0,0 +1,262 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Small pretrain benchmark session composed from runtime atoms.""" + +from __future__ import annotations + +import importlib +import time +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any + +import torch +from megatron.lite.runtime.backends import Runtime +from megatron.lite.runtime.contracts.handle import ModelHandle + +from .results import RunResult, StepTrace + + +@dataclass +class PretrainSessionConfig: + steps: int = 2 + warmup: int = 0 + num_microbatches: int = 1 + seq_len: int = 2048 + seed: int = 42 + device: str = "cuda" + use_thd: bool = False + same_data_across_dp: bool = False + no_optimizer: bool = False + + +def _is_cuda_device(device: str) -> bool: + return device.startswith("cuda") and torch.cuda.is_available() + + +def _sync(device: str) -> None: + if _is_cuda_device(device): + torch.cuda.synchronize() + + +def _reset_peak_memory(device: str) -> None: + if _is_cuda_device(device): + torch.cuda.reset_peak_memory_stats() + + +def _peak_memory_gb(device: str) -> float: + if _is_cuda_device(device): + return torch.cuda.max_memory_allocated() / 1e9 + return 0.0 + + +def _world_size() -> int: + if torch.distributed.is_available() and torch.distributed.is_initialized(): + return torch.distributed.get_world_size() + return 1 + + +def _resolve_vocab_size(handle: ModelHandle) -> int: + proto = handle._extras.get("protocol") + model_cfg = handle._extras.get("model_cfg") + if proto is not None and model_cfg is not None and hasattr(proto, "vocab_size"): + return int(proto.vocab_size(model_cfg)) + if model_cfg is not None and hasattr(model_cfg, "vocab_size"): + return int(model_cfg.vocab_size) + return 151936 + + +def _infinite_packed_batches( + vocab_size: int, seq_len: int, *, device: str, seed: int +): + """Yield raw, model-agnostic :class:`PackedBatch` objects for the bench. + + The bench is the single source of truth for one unpadded packed batch (1-D + ``input_ids``/``labels`` plus true per-sequence ``seq_lens``). Padding, CP + layout and THD metadata (``packed_seq_params``) are derived by whichever + runtime/model consumes the batch at the forward boundary, never baked into + bench data — that is what keeps the mlite-vs-bridge comparison fair. + """ + from megatron.lite.runtime.contracts.data import PackedBatch + + g = torch.Generator(device=device).manual_seed(seed) + seq_lens = torch.tensor([seq_len], dtype=torch.int64, device=device) + while True: + yield PackedBatch( + input_ids=torch.randint(0, vocab_size, (seq_len,), device=device, generator=g), + labels=torch.randint(0, vocab_size, (seq_len,), device=device, generator=g), + seq_lens=seq_lens.clone(), + ) + + +def _make_data_iter(handle: ModelHandle, cfg: PretrainSessionConfig): + data_seed = cfg.seed if cfg.same_data_across_dp else cfg.seed + handle.dp_rank + vocab_size = _resolve_vocab_size(handle) + return _infinite_packed_batches(vocab_size, cfg.seq_len, device=cfg.device, seed=data_seed) + + +def _calc_tflops_per_gpu( + *, + num_floating_point_operations: int | None, + activated_params: int | None, + tokens_per_step: int, + step_s: float, + world_size: int, +) -> float | None: + if step_s <= 0: + return None + if num_floating_point_operations: + return num_floating_point_operations / (step_s * world_size * 1e12) + if activated_params: + return 6 * activated_params * tokens_per_step / (step_s * world_size * 1e12) + return None + + +def _resolve_model_stats(config: Any, proto: Any) -> Any | None: + model_name = getattr(config, "model_name", None) + if model_name and model_name != "auto": + stats_module = f"megatron.lite.model.{model_name}.stats" + try: + return importlib.import_module(stats_module) + except ModuleNotFoundError as exc: + if exc.name is not None and not stats_module.startswith(exc.name): + raise + return proto + + +def _resolve_step_flops( + handle: ModelHandle, cfg: PretrainSessionConfig +) -> tuple[int | None, int | None]: + config = handle.config + proto = handle._extras.get("protocol") + model_stats = _resolve_model_stats(config, proto) + model_cfg = handle._extras.get("model_cfg") + if model_stats is None or model_cfg is None: + return None, None + + step_flops = None + if hasattr(model_stats, "num_floating_point_operations"): + parallel_cfg = getattr(config, "parallel", None) + tp_size = getattr(parallel_cfg, "tp", 1) + step_flops = model_stats.num_floating_point_operations( + model_cfg, + seq_len=cfg.seq_len, + global_batch_size=cfg.num_microbatches * handle.dp_size, + tp_size=tp_size, + ) + + activated_params = None + if step_flops is None and hasattr(model_stats, "activated_params"): + activated_params = model_stats.activated_params(model_cfg) + + return step_flops, activated_params + + +def run_pretrain_session( + rt: Runtime, + handle: ModelHandle, + cfg: PretrainSessionConfig, + *, + data_iter: Any = None, + step_reporter: Callable[[StepTrace], None] | None = None, +) -> RunResult: + """Run a fixed-shape benchmark loop through the public runtime API.""" + if cfg.steps < 1: + raise ValueError("steps must be >= 1") + if cfg.warmup < 0 or cfg.warmup >= cfg.steps: + raise ValueError("warmup must satisfy 0 <= warmup < steps") + if cfg.num_microbatches < 1: + raise ValueError("num_microbatches must be >= 1") + + if data_iter is None: + data_iter = _make_data_iter(handle, cfg) + + world_size = _world_size() + tokens_per_step = cfg.num_microbatches * cfg.seq_len * world_size + step_flops, activated_params = _resolve_step_flops(handle, cfg) + + step_traces: list[StepTrace] = [] + timings: list[float] = [] + + _reset_peak_memory(cfg.device) + with rt.train_mode(handle): + for step in range(cfg.steps): + if step == cfg.warmup: + _reset_peak_memory(cfg.device) + + rt.zero_grad(handle) + _sync(cfg.device) + t0 = time.perf_counter() + result = rt.forward_backward( + handle, data_iter, loss_fn=None, num_microbatches=cfg.num_microbatches + ) + if cfg.no_optimizer: + grad_norm = 0.0 + else: + _, grad_norm, _ = rt.optimizer_step(handle) + rt.lr_scheduler_step(handle) + _sync(cfg.device) + + elapsed_ms = (time.perf_counter() - t0) * 1000 + tflops_per_gpu = _calc_tflops_per_gpu( + num_floating_point_operations=step_flops, + activated_params=activated_params, + tokens_per_step=tokens_per_step, + step_s=elapsed_ms / 1000, + world_size=world_size, + ) + trace = StepTrace( + step=step, + loss=float(result.metrics.get("loss", result.model_output.loss) or 0.0), + grad_norm=float(grad_norm), + step_ms=elapsed_ms, + peak_mem_gb=_peak_memory_gb(cfg.device), + tflops_per_gpu=tflops_per_gpu, + ) + if step_reporter is not None: + step_reporter(trace) + if step >= cfg.warmup: + timings.append(elapsed_ms) + trace.step = step - cfg.warmup + step_traces.append(trace) + + avg_step_ms = sum(timings) / len(timings) if timings else 0.0 + avg_step_s = avg_step_ms / 1000 + tok_per_s = tokens_per_step / avg_step_s if avg_step_s > 0 else 0.0 + avg_tflops = _calc_tflops_per_gpu( + num_floating_point_operations=step_flops, + activated_params=activated_params, + tokens_per_step=tokens_per_step, + step_s=avg_step_s, + world_size=world_size, + ) + + config = handle.config + parallel = config.parallel + backend = "bridge" if type(config).__name__ == "BridgeConfig" else "mlite" + return RunResult( + backend=backend, + model_name=getattr(config, "model_name", "unknown"), + impl=getattr(config, "impl", "bridge"), + optimizer_backend=handle._extras.get( + "optimizer_backend", + getattr(handle._optimizer, "name", "none") if handle._optimizer is not None else "none", + ), + tp=parallel.tp, + etp=parallel.etp, + ep=parallel.ep, + pp=parallel.pp, + vpp=parallel.vpp, + cp=parallel.cp, + seq_len=cfg.seq_len, + num_microbatches=cfg.num_microbatches, + step_traces=step_traces, + avg_step_ms=avg_step_ms, + peak_mem_gb=_peak_memory_gb(cfg.device), + tok_per_s=tok_per_s, + tok_per_s_per_gpu=tok_per_s / world_size, + tflops_per_gpu=avg_tflops, + metadata={"warmup": cfg.warmup, "device": cfg.device, "use_thd": cfg.use_thd}, + ) + + +__all__ = ["PretrainSessionConfig", "run_pretrain_session"] diff --git a/experimental/lite/examples/miles/README.md b/experimental/lite/examples/miles/README.md new file mode 100644 index 00000000000..d14f26331e9 --- /dev/null +++ b/experimental/lite/examples/miles/README.md @@ -0,0 +1,66 @@ +# Miles Megatron Lite Example + +This example runs radixark/miles with Megatron Lite as the training actor by +applying the miles runtime patch from +`experimental/lite/examples/miles/miles_mlite/backend_patch.py`. + +The patch intentionally keeps `--train-backend megatron`: miles imports +`MegatronTrainRayActor` from its Megatron backend module inside the actor +allocation path, so replacing that source symbol before allocation routes the +existing Megatron slot to `MLiteTrainRayActor` without changing miles code or CLI +choices. + +## Layout + +- `miles_mlite/launch.py`: registers MLite CLI flags, optionally imports the + patch, parses miles arguments, and calls the normal miles train loop. +- `scripts/run_qwen3moe_sft.sh`: short SFT launcher for Qwen3-30B-A3B style + MoE checkpoints. +- `scripts/run_qwen3moe_grpo.sh`: GRPO launcher using miles rollout and MLite + policy-loss training. +- `REQUIRED_MILES.txt`: source commit pinned for the miles API shape. + +## SFT + +Run inside a Slurm GPU allocation/container with miles, Megatron-LM, and +Megatron Lite importable: + +```bash +export MILES_ROOT=/path/to/miles +export MEGATRON_ROOT=/path/to/full/Megatron-LM # if Megatron-Core is not installed +export MODEL_PATH=/path/to/Qwen3-30B-A3B +export TRAIN_DATA=/path/to/messages.parquet +export CONTAINER_IMAGE=/path/to/miles.sqsh +bash experimental/lite/examples/miles/scripts/run_qwen3moe_sft.sh +``` + +Useful knobs: + +- `TP_SIZE`, `PP_SIZE`, `CP_SIZE`, `EP_SIZE`, `ETP_SIZE` +- `GLOBAL_BATCH_SIZE`, `ROLLOUT_BATCH_SIZE`, `MAX_TOKENS_PER_GPU` +- `OPTIMIZER_OFFLOAD=1`, `PARAM_OFFLOAD=1` +- `MLITE_MODEL_NAME=qwen3_moe`, `MLITE_OPTIMIZER_BACKEND=dist_opt` +- `TRAIN_BACKEND=mlite` uses the patch; `TRAIN_BACKEND=megatron` leaves the + native miles Megatron actor untouched for A/B runs. +- `DRY_RUN=1` prints the resolved Ray job command without launching + +## GRPO + +```bash +export MILES_ROOT=/path/to/miles +export MEGATRON_ROOT=/path/to/full/Megatron-LM # if Megatron-Core is not installed +export MODEL_PATH=/path/to/Qwen3-30B-A3B +export PROMPT_DATA=/path/to/prompts.jsonl +export CONTAINER_IMAGE=/path/to/miles.sqsh +bash experimental/lite/examples/miles/scripts/run_qwen3moe_grpo.sh +``` + +The GRPO launcher selects `--loss-type policy_loss`, +`--advantage-estimator grpo`, `--use-rollout-logprobs`, and +`--megatron-to-hf-mode raw`. MLite exports HF-format weights directly, so the +rollout resync path bypasses mbridge conversion and sends raw HF tensors through +miles update-weight APIs. + +GPU validation should be run through Slurm. A successful smoke must use +`DRY_RUN=0`, finish with `sacct` exit code `0:0`, and show real training loss +progression in the Slurm log. diff --git a/experimental/lite/examples/miles/REQUIRED_MILES.txt b/experimental/lite/examples/miles/REQUIRED_MILES.txt new file mode 100644 index 00000000000..ab123dc0fad --- /dev/null +++ b/experimental/lite/examples/miles/REQUIRED_MILES.txt @@ -0,0 +1,11 @@ +radixark/miles source reference for this example. + +Repository: https://github.com/radixark/miles +Branch: main +Commit: 55c8d8b22f82a5afa0db69b758e5df281c6f8f2d + +The runtime patch relies on miles.ray.actor_group importing +miles.backends.megatron_utils.actor.MegatronTrainRayActor inside actor +allocation, and on miles.ray.train_actor.TrainRayActor exposing init, train, +save_model, update_weights, sleep, wake_up, connect_actor_critic, and +_get_parallel_config. diff --git a/experimental/lite/examples/miles/miles_mlite/__init__.py b/experimental/lite/examples/miles/miles_mlite/__init__.py new file mode 100644 index 00000000000..3dbd29a860c --- /dev/null +++ b/experimental/lite/examples/miles/miles_mlite/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Miles launcher helpers for the Megatron Lite backend patch.""" diff --git a/experimental/lite/examples/miles/miles_mlite/arguments.py b/experimental/lite/examples/miles/miles_mlite/arguments.py new file mode 100644 index 00000000000..048ab9de593 --- /dev/null +++ b/experimental/lite/examples/miles/miles_mlite/arguments.py @@ -0,0 +1,84 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Megatron Lite flags for miles launchers.""" + +from __future__ import annotations + +_OPTIMIZER_BACKEND_TO_IMPL = { + "dist_opt": "dist_opt", + "fsdp2": "fsdp2", +} + + +def optimizer_backend_to_impl(backend: str) -> str: + if backend not in _OPTIMIZER_BACKEND_TO_IMPL: + raise ValueError( + f"Unsupported --mlite-optimizer-backend {backend!r}; " + f"expected one of {sorted(_OPTIMIZER_BACKEND_TO_IMPL)}." + ) + return _OPTIMIZER_BACKEND_TO_IMPL[backend] + + +def add_mlite_arguments(parser): + """Register Megatron Lite flags on a miles parser.""" + group = parser.add_argument_group(title="megatron-lite") + group.add_argument( + "--mlite-backend-patch", + action="store_true", + default=False, + help="Patch the miles Megatron actor slot to use Megatron Lite.", + ) + group.add_argument( + "--mlite-model-name", + type=str, + default="auto", + help="Megatron Lite model registry name; 'auto' infers it from the HF config.", + ) + group.add_argument( + "--mlite-impl", + type=str, + default="lite", + help="Megatron Lite model implementation variant.", + ) + group.add_argument( + "--mlite-optimizer-backend", + type=str, + default="dist_opt", + choices=sorted(_OPTIMIZER_BACKEND_TO_IMPL), + help="Optimizer backend: dist_opt or fsdp2.", + ) + group.add_argument( + "--mlite-attention-backend", + type=str, + default=None, + help="Override attention backend. Defaults to --attention-backend, then 'flash'.", + ) + group.add_argument( + "--mlite-optimizer-offload", + action="store_true", + default=False, + help="Offload optimizer state to CPU via offload_fraction=1.0.", + ) + group.add_argument( + "--mlite-param-offload", + action="store_true", + default=False, + help="Offload model parameters to CPU during training/rollout alternation.", + ) + group.add_argument( + "--mlite-export-dtype", + type=str, + default=None, + choices=["bf16", "bfloat16", "fp16", "float16", "fp32", "float32", "float"], + help="Optional dtype cast when exporting HF-format rollout weights.", + ) + return parser + + +def validate_mlite_args(args) -> None: + if not getattr(args, "hf_checkpoint", None): + raise ValueError("--hf-checkpoint is required for the Megatron Lite backend patch.") + args.variable_seq_lengths = True + if not hasattr(args, "calculate_per_token_loss"): + args.calculate_per_token_loss = False + if not hasattr(args, "use_rollout_logprobs"): + args.use_rollout_logprobs = False diff --git a/experimental/lite/examples/miles/miles_mlite/backend_patch.py b/experimental/lite/examples/miles/miles_mlite/backend_patch.py new file mode 100644 index 00000000000..bfa1afc578c --- /dev/null +++ b/experimental/lite/examples/miles/miles_mlite/backend_patch.py @@ -0,0 +1,504 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Monkeypatch the miles Megatron train actor to use Megatron Lite. + +miles chooses its train actor by a function-local import of +``miles.backends.megatron_utils.actor.MegatronTrainRayActor``. Replacing that +source symbol before actor-group construction lets the example use the existing +``--train-backend megatron`` slot without changing miles code or CLI choices. +""" + +from __future__ import annotations + +import importlib +import logging +import os +import shutil +import sys +from argparse import Namespace +from pathlib import Path +from types import ModuleType, SimpleNamespace +from typing import Any + +import torch +from miles.ray.train_actor import TrainRayActor as _MilesTrainRayActor + +from .arguments import optimizer_backend_to_impl, validate_mlite_args +from .data import build_runtime_microbatches +from .loss import make_runtime_loss_fn +from .weight_update import RawHFWeightUpdater + +logger = logging.getLogger(__name__) + + +def _env_flag(name: str, default: bool = False) -> bool: + raw = os.environ.get(name) + if raw is None: + return default + return raw.lower() in {"1", "true", "yes", "on"} + + +def _rank() -> int: + if torch.distributed.is_available() and torch.distributed.is_initialized(): + return torch.distributed.get_rank() + return 0 + + +def _barrier() -> None: + if torch.distributed.is_available() and torch.distributed.is_initialized(): + torch.distributed.barrier() + + +def _iter_local_parameters(handle): + model_chunks = handle._extras.get("model_chunks", [handle._model]) + for chunk_id, chunk in enumerate(model_chunks): + for name, param in chunk.named_parameters(): + yield f"{chunk_id}:{name}", param + + +def _install_set_input_tensor_proxy(handle) -> None: + for chunk in handle._extras.get("model_chunks", [handle._model]): + if hasattr(chunk, "set_input_tensor"): + continue + module = getattr(chunk, "module", None) + setter = getattr(module, "set_input_tensor", None) + if not callable(setter): + continue + try: + setattr(chunk, "set_input_tensor", setter) + except Exception: + object.__setattr__(chunk, "set_input_tensor", setter) + + +def _capture_checkpoint_probe(handle): + fallback = None + for key, param in _iter_local_parameters(handle): + if fallback is None: + fallback = (key, param) + if "norm" in key and param.numel() <= 1_000_000: + return key, param.detach().float().cpu().clone() + if fallback is None: + return None, None + key, param = fallback + return key, param.detach().float().cpu().clone() + + +def _find_local_parameter(handle, key: str): + for candidate_key, param in _iter_local_parameters(handle): + if candidate_key == key: + return param + return None + + +def _clear_checkpoint_contents(path: str) -> None: + root = Path(path) + root.mkdir(parents=True, exist_ok=True) + for child in root.iterdir(): + if child.name == "ray_done": + continue + if child.is_dir(): + shutil.rmtree(child) + else: + child.unlink() + + +def _looks_like_mlite_checkpoint(path: str) -> bool: + root = Path(path) + if root.name.startswith("step_"): + return (root / "metadata.json").exists() or (root / "common.pt").exists() + if not root.is_dir(): + return False + for child in root.iterdir(): + if child.name.startswith("step_") and child.is_dir(): + return True + return (root / "metadata.json").exists() or (root / "common.pt").exists() + + +def _group(rank: int, size: int, group): + return SimpleNamespace(rank=rank, size=size, group=group, gloo_group=None) + + +def _install_miles_parallel_state(ps) -> None: + try: + parallel_mod = importlib.import_module("miles.backends.training_utils.parallel") + except ImportError: + return + state = parallel_mod.ParallelState( + intra_dp=_group(ps.dp_rank, ps.dp_size, ps.dp_group), + intra_dp_cp=_group(getattr(ps, "dp_cp_rank", ps.dp_rank), getattr(ps, "dp_cp_size", ps.dp_size), getattr(ps, "dp_cp_group", ps.dp_group)), + cp=_group(ps.cp_rank, ps.cp_size, ps.cp_group), + tp=_group(ps.tp_rank, ps.tp_size, ps.tp_group), + pp=_group(ps.pp_rank, ps.pp_size, ps.pp_group), + ep=_group(getattr(ps, "ep_rank", 0), getattr(ps, "ep_size", 1), getattr(ps, "ep_group", None)), + etp=_group(getattr(ps, "etp_rank", 0), getattr(ps, "etp_size", 1), getattr(ps, "etp_group", None)), + cp_comm_type=getattr(ps, "cp_comm_type", None), + is_pp_last_stage=ps.pp_rank == ps.pp_size - 1, + vpp_size=1, + microbatch_group_size_per_vp_stage=1, + ) + parallel_mod.set_parallel_state(state) + + +class _MLiteTrainRayActorMixin: + def _build_mlite_config(self, args: Namespace): + from megatron.lite.runtime.backends.mlite.config import MegatronLiteConfig + from megatron.lite.runtime.contracts.config import OptimizerConfig, ParallelConfig + + validate_mlite_args(args) + parallel = ParallelConfig( + tp=args.tensor_model_parallel_size, + etp=getattr(args, "expert_tensor_parallel_size", None), + ep=getattr(args, "expert_model_parallel_size", 1), + pp=args.pipeline_model_parallel_size, + vpp=getattr(args, "virtual_pipeline_model_parallel_size", None) or 1, + cp=args.context_parallel_size, + ) + optimizer = OptimizerConfig( + optimizer=args.optimizer, + lr=args.lr, + min_lr=args.min_lr if args.min_lr is not None else 0.0, + clip_grad=args.clip_grad, + weight_decay=args.weight_decay, + lr_decay_style=args.lr_decay_style, + adam_beta1=getattr(args, "adam_beta1", None), + adam_beta2=getattr(args, "adam_beta2", None), + adam_eps=getattr(args, "adam_eps", None), + ) + if getattr(args, "mlite_optimizer_offload", False): + optimizer.offload_fraction = 1.0 + optimizer.use_precision_aware_optimizer = True + optimizer.decoupled_weight_decay = True + + attention_backend = args.mlite_attention_backend + if attention_backend is None: + raw = getattr(args, "attention_backend", None) + attention_backend = getattr(raw, "name", raw) + attention_backend = attention_backend or "flash" + + return MegatronLiteConfig( + model_name=args.mlite_model_name, + impl=args.mlite_impl, + hf_path=args.hf_checkpoint, + parallel=parallel, + optimizer=optimizer, + attention_backend_override=attention_backend, + load_hf_weights=True, + impl_cfg={ + "use_thd": True, + "optimizer": optimizer_backend_to_impl(args.mlite_optimizer_backend), + }, + ) + + def init( + self, + args: Namespace, + role: str, + with_ref: bool = False, + with_opd_teacher: bool = False, + ) -> int | None: + super().init(args, role, with_ref, with_opd_teacher=with_opd_teacher) + if role != "actor": + raise NotImplementedError("Megatron Lite miles backend supports actor training only.") + if with_ref or with_opd_teacher: + raise NotImplementedError("Reference/teacher model swapping is not implemented for the MLite patch.") + + if args.debug_rollout_only: + self.args = args + return 0 + + from megatron.lite.runtime import RuntimeConfig, create_runtime + + self._cfg = self._build_mlite_config(args) + self.runtime = create_runtime( + RuntimeConfig(backend="mlite", hf_path=args.hf_checkpoint, backend_cfg=self._cfg) + ) + self.handle = self.runtime.build_model() + _install_set_input_tensor_proxy(self.handle) + + ps = self.handle._parallel_state + _install_miles_parallel_state(ps) + self.train_parallel_config = { + "dp_size": ps.dp_size, + "cp_size": ps.cp_size, + "vpp_size": self._cfg.parallel.vpp or 1, + "microbatch_group_size_per_vp_stage": 1, + } + self.weight_updater = RawHFWeightUpdater(args, self.runtime, self.handle) + + start_rollout_id = 0 + if getattr(args, "load", None): + if _looks_like_mlite_checkpoint(args.load): + loaded = self.runtime.load_checkpoint( + self.handle, + args.load, + load_optimizer=_env_flag("MLITE_LOAD_OPTIMIZER_CHECKPOINT", False), + load_rng=_env_flag("MLITE_LOAD_RNG_CHECKPOINT", False), + ) + if _env_flag("MLITE_RESET_ROLLOUT_AFTER_LOAD", False): + start_rollout_id = 0 + else: + start_rollout_id = int(loaded) + 1 + if _rank() == 0: + logger.info( + "MLITE_MILES_GRPO_INITIAL_LOAD_DONE path=%s loaded_step=%s start_rollout_id=%s", + args.load, + loaded, + start_rollout_id, + ) + else: + if _rank() == 0: + logger.info( + "MLITE_MILES_GRPO_INITIAL_LOAD_SKIPPED path=%s reason=not_mlite_checkpoint", + args.load, + ) + + if getattr(args, "offload_train", False) or getattr(args, "mlite_param_offload", False): + self.sleep() + return start_rollout_id + + def _process_rollout_data(self, rollout_data_ref): + data_mod = importlib.import_module("miles.utils.data") + ps = self.handle._parallel_state + rollout_data = data_mod.process_rollout_data(self.args, rollout_data_ref, ps.dp_rank, ps.dp_size) + rollout_data["tokens"] = [torch.as_tensor(t, dtype=torch.long) for t in rollout_data["tokens"]] + rollout_data["loss_masks"] = [torch.as_tensor(t, dtype=torch.float32) for t in rollout_data["loss_masks"]] + device = torch.device("cuda", torch.cuda.current_device()) + for key in ("rollout_log_probs", "log_probs", "ref_log_probs", "advantages", "returns"): + if key in rollout_data and rollout_data[key] is not None: + rollout_data[key] = [ + torch.as_tensor(t, dtype=torch.float32, device=device).reshape(-1) + for t in rollout_data[key] + ] + return rollout_data + + def _compute_advantages_and_returns(self, rollout_data) -> None: + loss_mod = importlib.import_module("miles.backends.training_utils.loss") + loss_mod.compute_advantages_and_returns(self.args, rollout_data) + + def _build_microbatches(self, rollout_data, *, calculate_entropy: bool = False): + return build_runtime_microbatches( + rollout_data, + micro_batch_size=getattr(self.args, "micro_batch_size", 1) or 1, + use_dynamic_batch_size=getattr(self.args, "use_dynamic_batch_size", False), + max_tokens_per_gpu=getattr(self.args, "max_tokens_per_gpu", 0) or 0, + calculate_entropy=calculate_entropy, + temperature=float(getattr(self.args, "rollout_temperature", 1.0) or 1.0), + ) + + def _compute_log_probs(self, rollout_data) -> list[torch.Tensor] | None: + microbatches = self._build_microbatches( + rollout_data, + calculate_entropy=bool(getattr(self.args, "use_rollout_entropy", False)), + ) + if not microbatches: + return [] + store: list[dict[str, list[torch.Tensor]]] = [] + with self.runtime.eval_mode(self.handle): + self.runtime.forward_backward( + self.handle, + (mb.as_runtime_item() for mb in microbatches), + loss_fn=make_runtime_loss_fn( + self.args, + self.handle, + forward_store=store, + loss_context_iter=(mb.loss_context() for mb in microbatches), + ), + num_microbatches=len(microbatches), + forward_only=True, + ) + if not store and not self.runtime.is_mp_src_rank_with_outputs(self.handle): + return None + return [item for micro in store for item in micro["log_probs"]] + + def train(self, rollout_id: int, rollout_data_ref) -> None: + self._last_rollout_id = rollout_id + if getattr(self.args, "offload_train", False) or getattr(self.args, "mlite_param_offload", False): + self.wake_up() + + rollout_data = self._process_rollout_data(rollout_data_ref) + if self.args.debug_rollout_only: + return None + + loss_type = getattr(self.args, "loss_type", "sft_loss") + if loss_type == "policy_loss" and getattr(self.args, "compute_advantages_and_returns", True): + if not getattr(self.args, "use_rollout_logprobs", False) or getattr(self.args, "get_mismatch_metrics", False): + log_probs = self._compute_log_probs(rollout_data) + if log_probs is not None: + rollout_data["log_probs"] = log_probs + elif "rollout_log_probs" not in rollout_data: + log_probs = self._compute_log_probs(rollout_data) + if log_probs is not None: + rollout_data["log_probs"] = log_probs + elif _rank() == 0: + logger.info( + "MLITE_MILES_GRPO_USING_ROLLOUT_LOGPROBS rollout=%s count=%s", + rollout_id, + len(rollout_data["rollout_log_probs"]), + ) + self._compute_advantages_and_returns(rollout_data) + + microbatches = self._build_microbatches(rollout_data) + if not microbatches: + logger.warning("rollout %s: empty data shard", rollout_id) + return None + + with self.runtime.train_mode(self.handle): + self.runtime.zero_grad(self.handle) + result = self.runtime.forward_backward( + self.handle, + (mb.as_runtime_item() for mb in microbatches), + loss_fn=make_runtime_loss_fn( + self.args, + self.handle, + loss_context_iter=(mb.loss_context() for mb in microbatches), + ), + num_microbatches=len(microbatches), + forward_only=False, + ) + _, grad_norm, _ = self.runtime.optimizer_step(self.handle) + lr = self.runtime.lr_scheduler_step(self.handle) + + logger.info( + "rollout %s | train/loss %s | grad_norm %.4f | lr %s | num_microbatches %s", + rollout_id, + result.metrics.get("loss"), + float(grad_norm), + lr, + len(microbatches), + ) + return None + + def save_model(self, rollout_id: int, force_sync: bool = False) -> None: + if self.args.debug_rollout_only: + return + save_dir = getattr(self.args, "save", None) + if not save_dir: + return + save_optimizer = _env_flag("MLITE_SAVE_OPTIMIZER_CHECKPOINT", False) + save_rng = _env_flag("MLITE_SAVE_RNG_CHECKPOINT", False) + verify_load = _env_flag("MLITE_VERIFY_CHECKPOINT_LOAD", True) + delete_after_load = _env_flag("MLITE_DELETE_CHECKPOINT_AFTER_LOAD", True) + + probe_key, before = _capture_checkpoint_probe(self.handle) + self.runtime.save_checkpoint( + self.handle, + save_dir, + step=rollout_id, + save_optimizer=save_optimizer, + save_rng=save_rng, + ) + + max_abs = None + loaded = None + if verify_load: + loaded = self.runtime.load_checkpoint( + self.handle, + save_dir, + load_optimizer=save_optimizer, + load_rng=save_rng, + ) + if probe_key is not None and before is not None: + after_param = _find_local_parameter(self.handle, probe_key) + if after_param is None: + raise RuntimeError(f"checkpoint load probe parameter missing after load: {probe_key}") + after = after_param.detach().float().cpu() + max_abs = float(torch.max(torch.abs(after - before)).item()) + if max_abs != 0.0: + raise RuntimeError( + f"checkpoint load probe mismatch for {probe_key}: max_abs={max_abs}" + ) + + if _rank() == 0: + logger.info( + "MLITE_MILES_GRPO_SAVE_LOAD_DONE rollout=%s path=%s loaded_step=%s " + "probe=%s max_abs=%s save_optimizer=%s save_rng=%s", + rollout_id, + save_dir, + loaded, + probe_key, + max_abs, + save_optimizer, + save_rng, + ) + + _barrier() + if delete_after_load and _rank() == 0: + _clear_checkpoint_contents(save_dir) + logger.info("MLITE_MILES_GRPO_CHECKPOINT_CLEANED path=%s", save_dir) + _barrier() + + def update_weights(self, info=None) -> None: + if self.args.debug_train_only or self.args.debug_rollout_only: + return + if info is None: + raise ValueError("update_weights requires rollout engine info from the miles rollout manager.") + if getattr(self.args, "offload_train", False) or getattr(self.args, "mlite_param_offload", False): + self.wake_up() + + rollout_engines = info.rollout_engines + rollout_engine_lock = info.rollout_engine_lock + has_new_engines = info.has_new_engines + engine_gpu_counts = getattr(info, "engine_gpu_counts", None) + engine_gpu_offsets = getattr(info, "engine_gpu_offsets", None) + del info + + if has_new_engines: + self.weight_updater.connect_rollout_engines( + rollout_engines, + rollout_engine_lock, + engine_gpu_counts=engine_gpu_counts, + engine_gpu_offsets=engine_gpu_offsets, + ) + import ray + + if torch.distributed.get_rank() == 0: + ray.get(self.rollout_manager.clear_updatable_has_new_engines.remote()) + + if getattr(self.args, "debug_skip_weight_update", False): + logger.warning("Skipping MLite actor-to-rollout weight update because --debug-skip-weight-update is set.") + return + self.weight_updater.update_weights() + + def sleep(self, *args, **kwargs) -> None: + if not (getattr(self.args, "offload_train", False) or getattr(self.args, "mlite_param_offload", False)): + return + self.runtime.to(self.handle, "cpu") + + def wake_up(self, *args, **kwargs) -> None: + if not (getattr(self.args, "offload_train", False) or getattr(self.args, "mlite_param_offload", False)): + return + self.runtime.to(self.handle, "cuda") + + def connect_actor_critic(self, critic_group=None, **kwargs): + raise NotImplementedError("Megatron Lite miles backend does not support critic training yet.") + + def _get_parallel_config(self): + return self.train_parallel_config + + +class MLiteTrainRayActor(_MLiteTrainRayActorMixin, _MilesTrainRayActor): + """Megatron Lite TrainRayActor patched into miles.""" + + +def _load_or_synthesize_actor_module(import_error: ImportError) -> ModuleType: + module_name = "miles.backends.megatron_utils.actor" + actor_mod = ModuleType(module_name) + actor_mod.__package__ = "miles.backends.megatron_utils" + actor_mod.__doc__ = "Synthetic MLite actor patch module for miles." + sys.modules[module_name] = actor_mod + + parent_mod = importlib.import_module(actor_mod.__package__) + setattr(parent_mod, "actor", actor_mod) + logger.warning("Using synthetic %s because the original module failed to import: %s", module_name, import_error) + return actor_mod + + +def patch_miles_backend() -> type: + """Patch installed miles and return the patched actor class.""" + importlib.import_module("miles") + try: + actor_mod = importlib.import_module("miles.backends.megatron_utils.actor") + except ImportError as exc: + actor_mod = _load_or_synthesize_actor_module(exc) + actor_mod.MegatronTrainRayActor = MLiteTrainRayActor + logger.info("Patched miles MegatronTrainRayActor with Megatron Lite.") + return MLiteTrainRayActor diff --git a/experimental/lite/examples/miles/miles_mlite/config/qwen3moe.yaml b/experimental/lite/examples/miles/miles_mlite/config/qwen3moe.yaml new file mode 100644 index 00000000000..a97f3759193 --- /dev/null +++ b/experimental/lite/examples/miles/miles_mlite/config/qwen3moe.yaml @@ -0,0 +1,12 @@ +# Reference settings consumed by the example shell launchers. +model_name: qwen3_moe +mlite_impl: lite +train_backend: megatron +megatron_to_hf_mode: raw +optimizer_backend: dist_opt +default_parallel: + tp: 2 + pp: 1 + cp: 1 + ep: 8 + etp: 1 diff --git a/experimental/lite/examples/miles/miles_mlite/data.py b/experimental/lite/examples/miles/miles_mlite/data.py new file mode 100644 index 00000000000..fd01809ea8b --- /dev/null +++ b/experimental/lite/examples/miles/miles_mlite/data.py @@ -0,0 +1,180 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Rollout-data conversion for the miles MLite actor.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import torch +import torch.nn.functional as F +from megatron.lite.runtime.contracts.data import PackedBatch +from megatron.lite.runtime.contracts.loss import LossContext + + +@dataclass(slots=True) +class RuntimeMicroBatch: + runtime_batch: PackedBatch + source_batch: dict[str, Any] + + def loss_context(self) -> LossContext: + return LossContext( + temperature=float(self.source_batch.get("temperature", 1.0)), + calculate_entropy=bool(self.source_batch.get("calculate_entropy", False)), + return_log_probs=True, + source_batch=self.source_batch, + ) + + def as_runtime_item(self): + return self.runtime_batch + + +def _as_tensor_list(values, *, dtype, device) -> list[torch.Tensor]: + return [torch.as_tensor(v, dtype=dtype, device=device).reshape(-1) for v in values] + + +def _optional_tensor_list(data: dict[str, Any], key: str, *, dtype, device) -> list[torch.Tensor] | None: + if key not in data or data[key] is None: + return None + return _as_tensor_list(data[key], dtype=dtype, device=device) + + +def _group_microbatches( + total_lengths: list[int], + *, + micro_batch_size: int, + use_dynamic_batch_size: bool, + max_tokens_per_gpu: int, +) -> list[list[int]]: + num_samples = len(total_lengths) + if num_samples == 0: + return [] + if not use_dynamic_batch_size: + mbs = max(int(micro_batch_size), 1) + return [list(range(i, min(i + mbs, num_samples))) for i in range(0, num_samples, mbs)] + + budget = max(int(max_tokens_per_gpu), 1) + groups: list[list[int]] = [] + current: list[int] = [] + current_tokens = 0 + for idx, length in enumerate(total_lengths): + length = int(length) + if current and current_tokens + length > budget: + groups.append(current) + current = [] + current_tokens = 0 + current.append(idx) + current_tokens += length + if current: + groups.append(current) + return groups + + +def response_mask_to_full(loss_mask: torch.Tensor, total_length: int, response_length: int) -> torch.Tensor: + prompt_length = total_length - response_length + if loss_mask.numel() != response_length: + raise ValueError( + f"response loss mask length {loss_mask.numel()} does not match response_length={response_length}." + ) + return F.pad(loss_mask.float(), (prompt_length, 0), value=0.0) + + +def response_mask_to_aligned_full( + loss_mask: torch.Tensor, total_length: int, response_length: int +) -> torch.Tensor: + prompt_length = total_length - response_length + left_pad = max(prompt_length - 1, 0) + right_pad = total_length - response_length - left_pad + return F.pad(loss_mask.float(), (left_pad, right_pad), value=0.0) + + +def _select(values: list[Any] | None, group: list[int]) -> list[Any] | None: + if values is None: + return None + return [values[i] for i in group] + + +def _put_if_present(batch: dict[str, Any], key: str, values: list[Any] | None, group: list[int]) -> None: + selected = _select(values, group) + if selected is not None: + batch[key] = selected + + +def build_runtime_microbatches( + rollout_data: dict[str, Any], + *, + micro_batch_size: int, + use_dynamic_batch_size: bool, + max_tokens_per_gpu: int, + calculate_entropy: bool = False, + temperature: float = 1.0, +) -> list[RuntimeMicroBatch]: + """Build model-agnostic PackedBatch items plus loss-side source metadata.""" + device = torch.device("cuda", torch.cuda.current_device()) + tokens = _as_tensor_list(rollout_data["tokens"], dtype=torch.long, device=device) + response_loss_masks = _as_tensor_list(rollout_data["loss_masks"], dtype=torch.float32, device=device) + rollout_log_probs = _optional_tensor_list(rollout_data, "rollout_log_probs", dtype=torch.float32, device=device) + old_log_probs = _optional_tensor_list(rollout_data, "log_probs", dtype=torch.float32, device=device) + ref_log_probs = _optional_tensor_list(rollout_data, "ref_log_probs", dtype=torch.float32, device=device) + advantages = _optional_tensor_list(rollout_data, "advantages", dtype=torch.float32, device=device) + returns = _optional_tensor_list(rollout_data, "returns", dtype=torch.float32, device=device) + + total_lengths = [int(x) for x in rollout_data["total_lengths"]] + response_lengths = [int(x) for x in rollout_data["response_lengths"]] + rewards = rollout_data.get("rewards") + + groups = _group_microbatches( + total_lengths, + micro_batch_size=micro_batch_size, + use_dynamic_batch_size=use_dynamic_batch_size, + max_tokens_per_gpu=max_tokens_per_gpu, + ) + + microbatches: list[RuntimeMicroBatch] = [] + for group in groups: + group_tokens = [tokens[i] for i in group] + group_total_lengths = [total_lengths[i] for i in group] + group_response_lengths = [response_lengths[i] for i in group] + group_response_masks = [response_loss_masks[i] for i in group] + full_masks = [ + response_mask_to_full(mask, total, response) + for mask, total, response in zip( + group_response_masks, group_total_lengths, group_response_lengths, strict=True + ) + ] + aligned_masks = [ + response_mask_to_aligned_full(mask, total, response) + for mask, total, response in zip( + group_response_masks, group_total_lengths, group_response_lengths, strict=True + ) + ] + + flat_tokens = torch.cat(group_tokens, dim=0).contiguous() + runtime_batch = PackedBatch( + input_ids=flat_tokens, + labels=flat_tokens, + loss_mask=torch.cat(full_masks, dim=0).contiguous(), + seq_lens=torch.tensor(group_total_lengths, dtype=torch.int64, device=device), + ) + + source_batch: dict[str, Any] = { + "unconcat_tokens": group_tokens, + "total_lengths": group_total_lengths, + "response_lengths": group_response_lengths, + "loss_masks": group_response_masks, + "full_loss_masks": full_masks, + "aligned_loss_masks": aligned_masks, + "temperature": temperature, + "calculate_entropy": calculate_entropy, + } + _put_if_present(source_batch, "rollout_log_probs", rollout_log_probs, group) + _put_if_present(source_batch, "log_probs", old_log_probs, group) + _put_if_present(source_batch, "ref_log_probs", ref_log_probs, group) + _put_if_present(source_batch, "advantages", advantages, group) + _put_if_present(source_batch, "returns", returns, group) + if rewards is not None: + source_batch["rewards"] = [rewards[i] for i in group] + + microbatches.append(RuntimeMicroBatch(runtime_batch, source_batch)) + + return microbatches diff --git a/experimental/lite/examples/miles/miles_mlite/launch.py b/experimental/lite/examples/miles/miles_mlite/launch.py new file mode 100644 index 00000000000..84ea2247680 --- /dev/null +++ b/experimental/lite/examples/miles/miles_mlite/launch.py @@ -0,0 +1,53 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Launch miles with the Megatron Lite backend patch.""" + +from __future__ import annotations + +import asyncio + +from miles.utils import arguments as miles_arguments +from miles.utils.tracking_utils import finish_tracking +from miles_mlite.arguments import add_mlite_arguments, validate_mlite_args + + +def _patch_epsilon_alias_for_miles_validate() -> None: + original_validate = miles_arguments.hf_validate_args + if getattr(original_validate, "_mlite_epsilon_alias", False): + return + + def validate_with_epsilon_alias(args, hf_config): + if hasattr(args, "norm_epsilon") and not hasattr(args, "layernorm_epsilon"): + args.layernorm_epsilon = args.norm_epsilon + if hasattr(args, "layernorm_epsilon") and not hasattr(args, "norm_epsilon"): + args.norm_epsilon = args.layernorm_epsilon + return original_validate(args, hf_config) + + validate_with_epsilon_alias._mlite_epsilon_alias = True + miles_arguments.hf_validate_args = validate_with_epsilon_alias + + +def _select_train_loop(args): + if getattr(args, "colocate", False): + from train import train + else: + from train_async import train + return train + + +def main() -> None: + _patch_epsilon_alias_for_miles_validate() + args = miles_arguments.parse_args(add_custom_arguments=add_mlite_arguments) + if getattr(args, "mlite_backend_patch", False): + from miles_mlite.backend_patch import patch_miles_backend + + patch_miles_backend() + validate_mlite_args(args) + train = _select_train_loop(args) + try: + asyncio.run(train(args)) + finally: + finish_tracking() + + +if __name__ == "__main__": + main() diff --git a/experimental/lite/examples/miles/miles_mlite/loss.py b/experimental/lite/examples/miles/miles_mlite/loss.py new file mode 100644 index 00000000000..bcb48a96896 --- /dev/null +++ b/experimental/lite/examples/miles/miles_mlite/loss.py @@ -0,0 +1,168 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""External loss functions for the miles MLite actor.""" + +from __future__ import annotations + +from collections.abc import Callable, Iterator +from typing import Any + +import torch +from megatron.lite.runtime.contracts.loss import LossContext + + +def _nested_to_list(output: torch.Tensor, seq_lens: torch.Tensor) -> list[torch.Tensor]: + if getattr(output, "is_nested", False): + return [x.reshape(-1) for x in output.unbind()] + tensor = output + if tensor.dim() == 2 and tensor.size(0) == 1: + tensor = tensor.squeeze(0) + pieces = [] + offset = 0 + for length_t in seq_lens: + length = int(length_t.item()) + pieces.append(tensor.narrow(0, offset, length).reshape(-1)) + offset += length + return pieces + + +def _unpack_output(handle, runtime_batch, output: torch.Tensor) -> list[torch.Tensor]: + proto = handle._extras.get("protocol") + unpack = getattr(proto, "unpack_forward_output", None) + if unpack is None: + raise ValueError("Model protocol must expose unpack_forward_output for miles losses.") + model = handle._model[0] if isinstance(handle._model, list | tuple) else handle._model + return _nested_to_list(unpack(model, runtime_batch, output), runtime_batch.seq_lens) + + +def extract_response_log_probs(raw_output: dict[str, torch.Tensor], runtime_batch, source_batch, handle): + log_probs = raw_output.get("log_probs") + if log_probs is None: + raise ValueError("Megatron Lite model output must contain token log_probs.") + full_log_probs = _unpack_output(handle, runtime_batch, log_probs) + response_log_probs = [] + for values, mask in zip(full_log_probs, source_batch["aligned_loss_masks"], strict=True): + active = mask.to(device=values.device).bool() + if values.numel() != active.numel(): + raise ValueError( + f"log_probs length {values.numel()} does not match aligned mask length {active.numel()}." + ) + response_log_probs.append(values[active]) + return response_log_probs + + +def _masked_sample_mean(values: torch.Tensor, mask: torch.Tensor) -> torch.Tensor: + mask = mask.to(device=values.device, dtype=values.dtype) + return (values * mask).sum() / mask.sum().clamp_min(1.0) + + +def _mean_scalars(values: list[torch.Tensor], device) -> torch.Tensor: + if not values: + return torch.zeros((), device=device, dtype=torch.float32) + return torch.stack([v.float() for v in values]).mean() + + +def _policy_loss( + args, + current: list[torch.Tensor], + source_batch: dict[str, Any], +) -> tuple[torch.Tensor, dict[str, torch.Tensor]]: + old = source_batch.get("rollout_log_probs") if getattr(args, "use_rollout_logprobs", False) else None + if old is None: + old = source_batch.get("log_probs") + if old is None: + raise ValueError("policy_loss requires actor log_probs or rollout_log_probs.") + advantages = source_batch.get("advantages") + if advantages is None: + raise ValueError("policy_loss requires advantages. Did GRPO preprocessing run?") + + eps_clip = float(getattr(args, "eps_clip", 0.2)) + eps_clip_high = float(getattr(args, "eps_clip_high", eps_clip)) + losses = [] + clipfracs = [] + kls = [] + for cur, old_lp, adv, mask in zip(current, old, advantages, source_batch["loss_masks"], strict=True): + cur = cur.float() + old_lp = old_lp.to(device=cur.device, dtype=cur.dtype) + adv = adv.to(device=cur.device, dtype=cur.dtype) + if cur.numel() != old_lp.numel() or cur.numel() != adv.numel(): + raise ValueError( + "policy_loss tensor length mismatch: " + f"current={cur.numel()} old={old_lp.numel()} advantages={adv.numel()}." + ) + active = mask.to(device=cur.device, dtype=cur.dtype) + ppo_kl = old_lp - cur + ratio = (-ppo_kl).exp() + pg_losses1 = -ratio * adv + pg_losses2 = -ratio.clamp(1.0 - eps_clip, 1.0 + eps_clip_high) * adv + pg_losses = torch.maximum(pg_losses1, pg_losses2) + losses.append(_masked_sample_mean(pg_losses, active)) + clipfracs.append(_masked_sample_mean((pg_losses2 > pg_losses1).float(), active)) + kls.append(_masked_sample_mean(ppo_kl, active)) + + loss = _mean_scalars(losses, current[0].device) + entropy_loss = torch.zeros_like(loss) + if getattr(args, "entropy_coef", 0.0): + # MLite model protocols can return entropy, but qwen3_moe keeps this optional. + entropy_loss = torch.zeros_like(loss) + loss = loss - float(args.entropy_coef) * entropy_loss + + metrics = { + "loss": loss.detach(), + "pg_loss": loss.detach(), + "pg_clipfrac": _mean_scalars(clipfracs, current[0].device).detach(), + "ppo_kl": _mean_scalars(kls, current[0].device).detach(), + "entropy_loss": entropy_loss.detach(), + } + return loss, metrics + + +def _sft_loss(current: list[torch.Tensor], source_batch: dict[str, Any]) -> tuple[torch.Tensor, dict[str, torch.Tensor]]: + losses = [] + for log_probs, mask in zip(current, source_batch["loss_masks"], strict=True): + mask = mask.to(device=log_probs.device, dtype=log_probs.dtype) + if log_probs.numel() != mask.numel(): + raise ValueError( + f"SFT log_probs length {log_probs.numel()} does not match response mask length {mask.numel()}." + ) + losses.append(-_masked_sample_mean(log_probs.float(), mask)) + loss = _mean_scalars(losses, current[0].device) + return loss, {"loss": loss.detach()} + + +def make_runtime_loss_fn( + args, + handle, + *, + forward_store: list[dict[str, list[torch.Tensor]]] | None = None, + loss_context_iter: Iterator[LossContext] | None = None, +) -> Callable: + """Build a Megatron Lite runtime loss_fn for SFT, forward-only logprob, or policy loss.""" + + def _next_loss_context() -> LossContext | None: + if loss_context_iter is None: + return None + try: + return next(loss_context_iter) + except StopIteration as exc: + raise RuntimeError("MLite miles loss context iterator ended before loss_fn calls.") from exc + + def _loss_fn(raw_output: dict[str, torch.Tensor], runtime_batch, loss_context=None): + if loss_context is None: + loss_context = _next_loss_context() + if loss_context is None or loss_context.source_batch is None: + raise ValueError("MLite miles loss_fn requires a LossContext with source_batch.") + source_batch = loss_context.source_batch + current = extract_response_log_probs(raw_output, runtime_batch, source_batch, handle) + if forward_store is not None: + forward_store.append({"log_probs": [x.detach() for x in current]}) + zero = torch.zeros((), device=current[0].device, dtype=torch.float32) + return zero, {"loss": zero.detach()} + + loss_type = getattr(args, "loss_type", "sft_loss") + if loss_type == "sft_loss": + return _sft_loss(current, source_batch) + if loss_type == "policy_loss": + return _policy_loss(args, current, source_batch) + raise NotImplementedError(f"Megatron Lite miles actor does not support loss_type={loss_type!r}.") + + return _loss_fn diff --git a/experimental/lite/examples/miles/miles_mlite/reward_log.py b/experimental/lite/examples/miles/miles_mlite/reward_log.py new file mode 100644 index 00000000000..92490a6e65f --- /dev/null +++ b/experimental/lite/examples/miles/miles_mlite/reward_log.py @@ -0,0 +1,38 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Rollout-side reward marker logging for the miles MLite example. + +miles' built-in rollout logger (``miles.ray.rollout.metrics.log_rollout_data``) +emits response-length / perf / zero-std stats but no reward mean. This thin hook, +wired through ``--custom-rollout-log-function-path``, runs in the rollout manager +(which holds every sample, so no parallel gather is needed) and logs a reward / +pass-rate marker. Reward scoring itself stays with miles (``--rm-type``); this only +reuses miles' ``compute_pass_rate`` to surface the signal. It is reward-rule +agnostic. Returning False lets miles' built-in logger run as well. +""" + +from __future__ import annotations + +import logging + +from miles.utils.metric_utils import compute_pass_rate + +logger = logging.getLogger(__name__) + + +def log_rollout_data(rollout_id, args, samples, rollout_extra_metrics, rollout_time) -> bool: + rewards = [float(sample.get_reward_value(args)) for sample in samples] + if not rewards: + return False + passrate = compute_pass_rate( + rewards, + group_size=args.n_samples_per_prompt, + num_groups=args.rollout_batch_size, + ) + metrics = { + "reward/score/mean": sum(rewards) / len(rewards), + "reward/score/nonzero": sum(1 for reward in rewards if reward != 0.0), + "reward/score/count": len(rewards), + } + metrics.update({f"reward/passrate/{key}": value for key, value in passrate.items()}) + logger.info("reward/score/passrate rollout %s: %s", rollout_id, metrics) + return False diff --git a/experimental/lite/examples/miles/miles_mlite/weight_update.py b/experimental/lite/examples/miles/miles_mlite/weight_update.py new file mode 100644 index 00000000000..f76672af448 --- /dev/null +++ b/experimental/lite/examples/miles/miles_mlite/weight_update.py @@ -0,0 +1,294 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Raw HF weight update bridge for miles rollout engines.""" + +from __future__ import annotations + +import logging +import importlib +from dataclasses import replace +from collections.abc import Sequence + +import ray +import torch +import torch.distributed as dist + +logger = logging.getLogger(__name__) + + +class RawHFWeightUpdater: + """Send MLite-exported HF-format weights through miles update APIs.""" + + def __init__(self, args, runtime, handle) -> None: + self.args = args + self.runtime = runtime + self.handle = handle + self.weight_version = 0 + self._ipc_gather_group = None + self._ipc_gather_src = None + self._ipc_engine = None + self._model_update_groups = None + self._distributed_group_name = None + self.rollout_engines = [] + self.distributed_rollout_engines = [] + self.use_distribute = False + + @property + def _ps(self): + return self.handle._parallel_state + + @property + def _is_distributed_src_rank(self) -> bool: + ps = self._ps + return ( + int(getattr(ps, "dp_rank", 0) or 0) == 0 + and int(getattr(ps, "cp_rank", 0) or 0) == 0 + and int(getattr(ps, "tp_rank", 0) or 0) == 0 + ) + + def connect_rollout_engines( + self, + rollout_engines: Sequence, + rollout_engine_lock, + *, + engine_gpu_counts: Sequence[int] | None = None, + engine_gpu_offsets: Sequence[int] | None = None, + ) -> None: + broadcast = importlib.import_module( + "miles.backends.megatron_utils.update_weight.update_weight_from_distributed.broadcast" + ) + connect_rollout_engines_from_distributed = broadcast.connect_rollout_engines_from_distributed + disconnect_rollout_engines_from_distributed = broadcast.disconnect_rollout_engines_from_distributed + + self.rollout_engines = list(rollout_engines) + self.rollout_engine_lock = rollout_engine_lock + + if engine_gpu_counts is None: + engine_gpu_counts = [self.args.rollout_num_gpus_per_engine] * len(rollout_engines) + if engine_gpu_offsets is None: + engine_gpu_offsets = [] + offset = 0 + for count in engine_gpu_counts: + engine_gpu_offsets.append(offset) + offset += count + + if not getattr(self.args, "colocate", False): + colocate_engine_nums = 0 + else: + total_actor_gpus = self.args.actor_num_nodes * self.args.actor_num_gpus_per_node + colocate_engine_nums = 0 + for gpu_offset, gpu_count in zip(engine_gpu_offsets, engine_gpu_counts, strict=True): + if gpu_offset + gpu_count > total_actor_gpus: + break + colocate_engine_nums += 1 + + self.use_distribute = len(rollout_engines) > colocate_engine_nums + self.distributed_rollout_engines = list(rollout_engines[colocate_engine_nums:]) + self.rollout_engines = list(rollout_engines[:colocate_engine_nums]) + + if self.use_distribute and self._is_distributed_src_rank: + self._distributed_group_name = f"mlite-pp_{getattr(self._ps, 'pp_rank', 0)}" + if self._model_update_groups is not None: + disconnect_rollout_engines_from_distributed( + self.args, + self._distributed_group_name, + self._model_update_groups, + self.distributed_rollout_engines, + ) + self._model_update_groups = connect_rollout_engines_from_distributed( + self.args, + self._distributed_group_name, + self.distributed_rollout_engines, + engine_gpu_counts=engine_gpu_counts[colocate_engine_nums:], + ) + + self._connect_colocated_groups(engine_gpu_counts[:colocate_engine_nums], engine_gpu_offsets[:colocate_engine_nums]) + + @property + def _active_rollout_engines(self): + return [*self.rollout_engines, *self.distributed_rollout_engines] + + def _connect_colocated_groups(self, engine_gpu_counts, engine_gpu_offsets) -> None: + rank = dist.get_rank() + self._ipc_gather_group = None + self._ipc_gather_src = None + self._ipc_engine = None + + for engine_idx, (offset, count) in enumerate(zip(engine_gpu_offsets, engine_gpu_counts, strict=True)): + group_ranks = list(range(offset, offset + count)) + new_group = dist.new_group(ranks=group_ranks, backend="gloo") + if rank in group_ranks: + self._ipc_gather_group = new_group + self._ipc_gather_src = offset + self._ipc_engine = self.rollout_engines[engine_idx] + + def update_weights(self) -> None: + common = importlib.import_module("miles.backends.megatron_utils.update_weight.common") + broadcast = importlib.import_module( + "miles.backends.megatron_utils.update_weight.update_weight_from_distributed.broadcast" + ) + distributed_utils = importlib.import_module("miles.utils.distributed_utils") + sglang_utils = importlib.import_module("miles.backends.megatron_utils.sglang") + _check_weight_sync_results = common._check_weight_sync_results + post_process_weights = common.post_process_weights + update_weights_from_distributed = broadcast.update_weights_from_distributed + get_gloo_group = distributed_utils.get_gloo_group + monkey_patch_torch_reductions = sglang_utils.monkey_patch_torch_reductions + + self.weight_version += 1 + monkey_patch_torch_reductions() + rank = dist.get_rank() + if rank == 0: + mode = self.args.pause_generation_mode + ray.get([engine.pause_generation.remote(mode=mode) for engine in self._active_rollout_engines]) + ray.get([engine.flush_cache.remote() for engine in self._active_rollout_engines]) + dist.barrier(group=get_gloo_group()) + + refs = [] + live_tensors = [] + for chunk in self._export_weight_chunks(): + colocated_refs, long_lived = _send_to_colocated_engine_direct( + hf_named_tensors=chunk, + ipc_engine=self._ipc_engine, + ipc_gather_src=self._ipc_gather_src, + ipc_gather_group=self._ipc_gather_group, + weight_version=self.weight_version, + ) + refs.extend(colocated_refs) + if long_lived is not None: + live_tensors.append(long_lived) + if self.use_distribute and self._is_distributed_src_rank: + refs.extend( + update_weights_from_distributed( + self._distributed_group_name, + self._model_update_groups, + self.weight_version, + self.distributed_rollout_engines, + chunk, + ) + ) + if refs: + _check_weight_sync_results(ray.get(refs), is_lora=False) + del live_tensors + + dist.barrier(group=get_gloo_group()) + if rank == 0: + post_process_weights( + rollout_engines=self._active_rollout_engines, + restore_weights_before_load=False, + post_process_quantization=True, + ) + ray.get([engine.continue_generation.remote() for engine in self._active_rollout_engines]) + dist.barrier(group=get_gloo_group()) + + def _export_weight_chunks(self): + chunk = [] + chunk_bytes = 0 + limit = int(getattr(self.args, "update_weight_buffer_size", 2**30)) + export_kwargs = {} + if getattr(self.args, "mlite_export_dtype", None): + export_kwargs["export_dtype"] = self.args.mlite_export_dtype + for name, tensor in self._iter_local_hf_weights(**export_kwargs): + if not self._needs_weight_payload: + del tensor + continue + tensor = tensor.detach() + if not tensor.is_cuda: + tensor = tensor.to(device=torch.cuda.current_device(), non_blocking=True) + item_bytes = tensor.numel() * tensor.element_size() + if chunk and chunk_bytes + item_bytes > limit: + torch.cuda.synchronize() + yield chunk + chunk = [] + chunk_bytes = 0 + chunk.append((name, tensor)) + chunk_bytes += item_bytes + if chunk: + torch.cuda.synchronize() + yield chunk + + @property + def _needs_weight_payload(self) -> bool: + return self._is_distributed_src_rank or self._ipc_gather_group is not None + + def _iter_local_hf_weights(self, **export_kwargs): + """Export this PP stage only; each PP source broadcasts its own group.""" + + ps = self._ps + if int(getattr(ps, "pp_size", 1) or 1) <= 1: + yield from self.runtime.export_weights(self.handle, **export_kwargs) + return + + proto = self.handle._extras.get("protocol") + model_cfg = self.handle._extras.get("model_cfg") + model_chunks = self.handle._extras.get("model_chunks", [self.handle._model]) + if proto and hasattr(proto, "export_hf_weights"): + local_pp_ps = replace( + ps, + pp_group=None, + pp_cpu_group=None, + pp_global_ranks=None, + pp_size=1, + pp_rank=0, + pp_is_first=True, + pp_is_last=True, + pp_next_rank=-1, + pp_prev_rank=-1, + ) + yield from proto.export_hf_weights(model_chunks, model_cfg, local_pp_ps, **export_kwargs) + return + + for chunk in model_chunks: + yield from chunk.named_parameters() + + +def _send_to_colocated_engine_direct( + hf_named_tensors, + *, + ipc_engine, + ipc_gather_src, + ipc_gather_group, + weight_version=None, +): + if ipc_gather_group is None: + return [], None + + sglang_utils = importlib.import_module("miles.backends.megatron_utils.sglang") + model_runner = importlib.import_module("sglang.srt.model_executor.model_runner") + MultiprocessingSerializer = sglang_utils.MultiprocessingSerializer + LocalSerializedTensor = model_runner.LocalSerializedTensor + + is_gather_src = dist.get_rank() == ipc_gather_src + serialized_tensors = [ + (name, MultiprocessingSerializer.serialize(tensor)) for name, tensor in hf_named_tensors + ] + serialized_named_tensors = [None] * dist.get_world_size(ipc_gather_group) if is_gather_src else None + dist.gather_object( + serialized_tensors, + object_gather_list=serialized_named_tensors, + dst=ipc_gather_src, + group=ipc_gather_group, + ) + + refs = [] + if is_gather_src: + named_tensors = [] + for tensor_group in zip(*serialized_named_tensors, strict=True): + names = {name for name, _ in tensor_group} + if len(names) != 1: + raise RuntimeError(f"Mismatched TP tensor names during weight sync: {sorted(names)}") + name = tensor_group[0][0] + named_tensors.append( + (name, LocalSerializedTensor(values=[rank_part[1] for rank_part in tensor_group])) + ) + payload = [ + MultiprocessingSerializer.serialize(named_tensors, output_str=True) + for _ in range(len(serialized_named_tensors)) + ] + refs.append( + ipc_engine.update_weights_from_tensor.remote( + serialized_named_tensors=payload, + weight_version=str(weight_version), + ) + ) + + return refs, [tensor for _, tensor in hf_named_tensors] diff --git a/experimental/lite/examples/miles/scripts/run_qwen3moe_grpo.sh b/experimental/lite/examples/miles/scripts/run_qwen3moe_grpo.sh new file mode 100755 index 00000000000..e4746410d20 --- /dev/null +++ b/experimental/lite/examples/miles/scripts/run_qwen3moe_grpo.sh @@ -0,0 +1,422 @@ +#!/usr/bin/env bash +#SBATCH --job-name=mlite_miles_grpo +#SBATCH --partition=batch +#SBATCH --account=coreai_devtech_all +#SBATCH --nodes=2 +#SBATCH --ntasks-per-node=1 +#SBATCH --gres=gpu:8 +#SBATCH --cpus-per-task=64 +#SBATCH --mem=1000G +#SBATCH --time=04:00:00 +#SBATCH --output=/lustre/fs1/portfolios/coreai/projects/coreai_devtech_all/users/bayan/code/env/mlite_miles_grpo-%j.log + +# Qwen3 MoE GRPO using miles' qwen3-next-80B-A3B 8-GPU recipe, with only +# the training backend swapped to Megatron Lite. +set -euo pipefail + +if [[ "${VERBOSE:-0}" == "1" ]]; then set -x; fi + +resolve_script_path() { + if [[ -n "${MLITE_MILES_SCRIPT_PATH:-}" ]]; then + readlink -f "${MLITE_MILES_SCRIPT_PATH}" + return + fi + if [[ -n "${SLURM_JOB_ID:-}" ]] && command -v scontrol >/dev/null 2>&1; then + local command_path + command_path="$( + scontrol show job "${SLURM_JOB_ID}" | tr ' ' '\n' | sed -n 's/^Command=//p' | head -1 + )" + if [[ -n "${command_path}" && -r "${command_path}" ]]; then + readlink -f "${command_path}" + return + fi + fi + readlink -f "${BASH_SOURCE[0]}" +} + +SCRIPT_PATH="$(resolve_script_path)" +CONTAINER_IMAGE="${CONTAINER_IMAGE:-/lustre/fs1/portfolios/coreai/projects/coreai_devtech_all/users/bayan/code/env/miles.sqsh}" +CONTAINER_MOUNTS="${CONTAINER_MOUNTS:-/lustre:/lustre}" +DRY_RUN="${DRY_RUN:-0}" +RAY_PORT="${RAY_PORT:-6379}" +RAY_DASHBOARD_PORT="${RAY_DASHBOARD_PORT:-8265}" +RAY_EXPECTED_NODES="${RAY_EXPECTED_NODES:-2}" +SYSTEM_PATH="${MLITE_SYSTEM_PATH:-/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin}" +SYSTEM_CC="${MLITE_CC:-/usr/bin/gcc}" +SYSTEM_CXX="${MLITE_CXX:-/usr/bin/g++}" +SYSTEM_CPATH="${MLITE_CPATH:-/usr/include:/usr/include/x86_64-linux-gnu}" + +MLITE_SAVE_OPTIMIZER_CHECKPOINT="${MLITE_SAVE_OPTIMIZER_CHECKPOINT:-0}" +MLITE_SAVE_RNG_CHECKPOINT="${MLITE_SAVE_RNG_CHECKPOINT:-0}" +MLITE_LOAD_OPTIMIZER_CHECKPOINT="${MLITE_LOAD_OPTIMIZER_CHECKPOINT:-0}" +MLITE_LOAD_RNG_CHECKPOINT="${MLITE_LOAD_RNG_CHECKPOINT:-0}" +MLITE_VERIFY_CHECKPOINT_LOAD="${MLITE_VERIFY_CHECKPOINT_LOAD:-1}" +MLITE_DELETE_CHECKPOINT_AFTER_LOAD="${MLITE_DELETE_CHECKPOINT_AFTER_LOAD:-1}" +MLITE_RESET_ROLLOUT_AFTER_LOAD="${MLITE_RESET_ROLLOUT_AFTER_LOAD:-1}" + +if [[ "${IN_MILES_CONTAINER:-0}" != "1" ]]; then + if [[ ! -r "${CONTAINER_IMAGE}" ]]; then + echo "Container image not readable: ${CONTAINER_IMAGE}" >&2 + exit 2 + fi + if [[ -z "${MASTER_ADDR:-}" && -n "${SLURM_JOB_NODELIST:-}" ]] && command -v scontrol >/dev/null 2>&1; then + MASTER_ADDR="$(scontrol show hostnames "${SLURM_JOB_NODELIST}" | head -n1)" + fi + MASTER_ADDR="${MASTER_ADDR:-127.0.0.1}" + + SRUN_CMD=( + srun + --nodes=2 + --ntasks=2 + --ntasks-per-node=1 + --container-image="${CONTAINER_IMAGE}" + --container-mounts="${CONTAINER_MOUNTS}" + --container-workdir=/ + env + -u CONDA_PREFIX + -u CONDA_DEFAULT_ENV + -u CONDA_EXE + -u CONDA_PYTHON_EXE + -u CONDA_SHLVL + -u _CE_CONDA + -u _CE_M + PATH="${SYSTEM_PATH}" + CC="${SYSTEM_CC}" + CXX="${SYSTEM_CXX}" + CPATH="${SYSTEM_CPATH}" + IN_MILES_CONTAINER=1 + MLITE_MILES_SCRIPT_PATH="${SCRIPT_PATH}" + MASTER_ADDR="${MASTER_ADDR}" + RAY_PORT="${RAY_PORT}" + RAY_DASHBOARD_PORT="${RAY_DASHBOARD_PORT}" + RAY_EXPECTED_NODES="${RAY_EXPECTED_NODES}" + MLITE_SAVE_OPTIMIZER_CHECKPOINT="${MLITE_SAVE_OPTIMIZER_CHECKPOINT}" + MLITE_SAVE_RNG_CHECKPOINT="${MLITE_SAVE_RNG_CHECKPOINT}" + MLITE_LOAD_OPTIMIZER_CHECKPOINT="${MLITE_LOAD_OPTIMIZER_CHECKPOINT}" + MLITE_LOAD_RNG_CHECKPOINT="${MLITE_LOAD_RNG_CHECKPOINT}" + MLITE_VERIFY_CHECKPOINT_LOAD="${MLITE_VERIFY_CHECKPOINT_LOAD}" + MLITE_DELETE_CHECKPOINT_AFTER_LOAD="${MLITE_DELETE_CHECKPOINT_AFTER_LOAD}" + MLITE_RESET_ROLLOUT_AFTER_LOAD="${MLITE_RESET_ROLLOUT_AFTER_LOAD}" + bash "${SCRIPT_PATH}" + ) + + if [[ "${DRY_RUN}" == "1" ]]; then + printf 'outer: ' + printf '%q ' "${SRUN_CMD[@]}" + printf '\n' + IN_MILES_CONTAINER=1 DRY_RUN=1 bash "${SCRIPT_PATH}" + exit 0 + fi + + if [[ -z "${SLURM_JOB_ID:-}" ]]; then + echo "Submit this script with sbatch, or run DRY_RUN=1 bash ${SCRIPT_PATH} for a local command check." >&2 + exit 2 + fi + + exec "${SRUN_CMD[@]}" +fi + +set -ex + +export PATH="${SYSTEM_PATH}" +export CC="${SYSTEM_CC}" +export CXX="${SYSTEM_CXX}" +export CPATH="${SYSTEM_CPATH}" +unset CONDA_PREFIX CONDA_DEFAULT_ENV CONDA_EXE CONDA_PYTHON_EXE CONDA_SHLVL _CE_CONDA _CE_M + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd -L)" +EXAMPLE_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd -L)" +LITE_ROOT="$(cd "${EXAMPLE_ROOT}/../.." && pwd -L)" +REPO_ROOT="$(cd "${LITE_ROOT}/../.." && pwd -L)" + +MILES_ROOT="${MILES_ROOT:-/lustre/fs1/portfolios/coreai/projects/coreai_devtech_all/users/bayan/code/miles}" +MEGATRON_ROOT="${MEGATRON_ROOT:-/lustre/fs1/portfolios/coreai/projects/coreai_devtech_all/users/bayan/code/megatron_lite/Megatron-LM}" +MODEL_PATH="${MODEL_PATH:-/lustre/fs1/portfolios/coreai/projects/coreai_devtech_all/users/shunyad/models/Qwen/Qwen3-30B-A3B}" +RUN_ROOT="${RUN_ROOT:-/lustre/fs1/portfolios/coreai/projects/coreai_devtech_all/users/bayan/code/env/mlite_miles_runs}" +DAPO_MATH_DATA="${DAPO_MATH_DATA:-/lustre/fs1/portfolios/coreai/projects/coreai_devtech_all/users/achartier/dapo-math-17k/dapo-math-17k.jsonl}" +LOAD_DIR="${LOAD_DIR:-${RUN_ROOT}/qwen3moe_sft_mlite/13021984}" +SAVE_DIR="${SAVE_DIR:-${RUN_ROOT}/qwen3moe_grpo_mlite/${SLURM_JOB_ID:-dryrun}}" +REF_LOAD="${REF_LOAD:-${MODEL_PATH}}" + +if [[ ! -d "${MODEL_PATH}" ]]; then + echo "MODEL_PATH does not exist: ${MODEL_PATH}" >&2 + exit 2 +fi +if [[ ! -s "${DAPO_MATH_DATA}" ]]; then + echo "DAPO_MATH_DATA does not exist: ${DAPO_MATH_DATA}" >&2 + exit 2 +fi +if [[ ! -d "${LOAD_DIR}" ]]; then + echo "LOAD_DIR does not exist: ${LOAD_DIR}" >&2 + exit 2 +fi +mkdir -p "${SAVE_DIR}" + +if [[ "${DRY_RUN}" == "1" ]]; then + NVLINK_COUNT=0 + HAS_NVLINK=0 +else + NVLINK_COUNT=$(nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l) + if [[ "${NVLINK_COUNT}" -gt 0 ]]; then + HAS_NVLINK=1 + else + HAS_NVLINK=0 + fi +fi +echo "HAS_NVLINK: ${HAS_NVLINK} (detected ${NVLINK_COUNT} NVLink references)" + +source "${MILES_ROOT}/scripts/models/qwen3-30B-A3B.sh" + +export PYTHONBUFFERED=16 +export PYTHONUNBUFFERED=1 +export CUDA_DEVICE_MAX_CONNECTIONS="${CUDA_DEVICE_MAX_CONNECTIONS:-1}" +export MASTER_ADDR="${MASTER_ADDR:-127.0.0.1}" +export RAY_PORT RAY_DASHBOARD_PORT RAY_EXPECTED_NODES +export no_proxy="127.0.0.1,${MASTER_ADDR}" +export MLITE_SAVE_OPTIMIZER_CHECKPOINT MLITE_SAVE_RNG_CHECKPOINT +export MLITE_LOAD_OPTIMIZER_CHECKPOINT MLITE_LOAD_RNG_CHECKPOINT +export MLITE_VERIFY_CHECKPOINT_LOAD MLITE_DELETE_CHECKPOINT_AFTER_LOAD +export MLITE_RESET_ROLLOUT_AFTER_LOAD + +add_pythonpath() { [[ -n "${1:-}" ]] && export PYTHONPATH="${1}:${PYTHONPATH:-}"; } +add_pythonpath "${EXAMPLE_ROOT}" +add_pythonpath "${LITE_ROOT}/examples" +add_pythonpath "${LITE_ROOT}" +add_pythonpath "${REPO_ROOT}" +add_pythonpath "${MEGATRON_ROOT}" +add_pythonpath "${MILES_ROOT}" + +CKPT_ARGS=( + --hf-checkpoint "${MODEL_PATH}" + --ref-load "${REF_LOAD}" + --load "${LOAD_DIR}" + --save "${SAVE_DIR}" + --save-interval 20 +) + +ROLLOUT_ARGS=( + --prompt-data "${DAPO_MATH_DATA}" + --input-key prompt + --label-key label + --apply-chat-template + --rollout-shuffle + --rm-type deepscaler + --num-rollout "${NUM_ROLLOUT:-5}" + --rollout-batch-size 16 + --n-samples-per-prompt 4 + --rollout-max-response-len 8192 + --rollout-temperature 0.8 + --global-batch-size 64 + --balance-data + --custom-rollout-log-function-path miles_mlite.reward_log.log_rollout_data +) + +PERF_ARGS=( + --tensor-model-parallel-size 2 + --sequence-parallel + --pipeline-model-parallel-size 1 + --context-parallel-size 1 + --expert-model-parallel-size 8 + --expert-tensor-parallel-size 1 + --recompute-granularity full + --recompute-method uniform + --recompute-num-layers 1 + --use-dynamic-batch-size + --max-tokens-per-gpu 2048 +) + +GRPO_ARGS=( + --advantage-estimator gspo + --use-rollout-logprobs + --kl-loss-coef 0.00 + --kl-loss-type low_var_kl + --kl-coef 0.00 + --entropy-coef 0.00 + --eps-clip 4e-4 +) + +OPTIMIZER_ARGS=( + --optimizer adam + --lr 1e-6 + --lr-decay-style constant + --weight-decay 0.1 + --adam-beta1 0.9 + --adam-beta2 0.98 +) + +BACKEND_ARGS=( + --train-backend megatron + --model-name qwen3_moe + --megatron-to-hf-mode raw + --mlite-backend-patch + --mlite-model-name qwen3_moe + --mlite-impl lite + --mlite-optimizer-backend dist_opt + --mlite-optimizer-offload +) + +SGLANG_ARGS=( + --rollout-num-gpus-per-engine 2 + --rollout-num-gpus 8 + --sglang-mem-fraction-static 0.8 + --sglang-ep-size 1 + --sglang-cuda-graph-bs 1 2 4 8 $(seq 16 8 128) + --update-weight-buffer-size $((1024 * 1024 * 1024 * 4)) +) + +MISC_ARGS=( + --attention-dropout 0.0 + --hidden-dropout 0.0 + --accumulate-allreduce-grads-in-fp32 + --attention-softmax-in-fp32 + --attention-backend flash + --moe-token-dispatcher-type alltoall + --log-passrate +) + +JOB_ARGS=( + python3 -m miles_mlite.launch + --actor-num-nodes 1 + --actor-num-gpus-per-node 8 + --num-gpus-per-node 8 + "${MODEL_ARGS[@]}" + "${CKPT_ARGS[@]}" + "${ROLLOUT_ARGS[@]}" + "${OPTIMIZER_ARGS[@]}" + "${GRPO_ARGS[@]}" + "${PERF_ARGS[@]}" + "${SGLANG_ARGS[@]}" + "${MISC_ARGS[@]}" + "${BACKEND_ARGS[@]}" +) + +RUNTIME_ENV_JSON="$( +python3 - < --disable-usage-stats --dashboard-host=0.0.0.0 --dashboard-port=%q\n' \ + "${MASTER_ADDR}:${RAY_PORT}" "${RAY_DASHBOARD_PORT}" + printf 'inner ray job: ray job submit --address=%q --runtime-env-json=%q -- ' \ + "${RAY_ADDRESS:-http://127.0.0.1:${RAY_DASHBOARD_PORT}}" "${RUNTIME_ENV_JSON}" + printf '%q ' "${JOB_ARGS[@]}" + printf '\n' + exit 0 +fi + +RAY_NODE_RANK="${SLURM_PROCID:-0}" +RAY_DONE_FILE="${RUN_ROOT}/.mlite_miles_grpo_${SLURM_JOB_ID:-manual}.done" +NODE_ADDR="$(hostname -s)" + +ray stop --force || true +pkill -9 sglang || true +pkill -9 ray || true +sleep 3 +ray stop --force || true + +cleanup() { + if [[ "${RAY_NODE_RANK}" == "0" ]]; then + touch "${RAY_DONE_FILE}" || true + fi + ray stop --force || true +} +trap cleanup EXIT + +if [[ "${RAY_NODE_RANK}" != "0" ]]; then + for attempt in $(seq 1 120); do + if ray start \ + --address="${MASTER_ADDR}:${RAY_PORT}" \ + --num-gpus 8 \ + --node-ip-address "${NODE_ADDR}" \ + --disable-usage-stats \ + --dashboard-host=0.0.0.0 \ + --dashboard-port="${RAY_DASHBOARD_PORT}"; then + break + fi + if [[ "${attempt}" == "120" ]]; then + echo "Failed to join Ray head at ${MASTER_ADDR}:${RAY_PORT}" >&2 + exit 1 + fi + sleep 2 + done + while [[ ! -f "${RAY_DONE_FILE}" ]]; do + sleep 10 + done + exit 0 +fi + +rm -f "${RAY_DONE_FILE}" + +ray start \ + --head \ + --node-ip-address "${MASTER_ADDR}" \ + --port="${RAY_PORT}" \ + --num-gpus 8 \ + --disable-usage-stats \ + --dashboard-host=0.0.0.0 \ + --dashboard-port="${RAY_DASHBOARD_PORT}" + +python3 - <<'PY' +import os +import time + +import ray + +expected = int(os.environ.get("RAY_EXPECTED_NODES", "1")) +address = f"{os.environ['MASTER_ADDR']}:{os.environ['RAY_PORT']}" +ray.init(address=address) +deadline = time.time() + 300 +while True: + alive = [node for node in ray.nodes() if node.get("Alive")] + print(f"RAY_CLUSTER_NODES {len(alive)}/{expected}", flush=True) + if len(alive) >= expected: + break + if time.time() > deadline: + raise SystemExit(f"Timed out waiting for Ray nodes: {len(alive)}/{expected}") + time.sleep(2) +ray.shutdown() +PY + +set +e +ray job submit \ + --address "${RAY_ADDRESS:-http://127.0.0.1:${RAY_DASHBOARD_PORT}}" \ + --runtime-env-json "${RUNTIME_ENV_JSON}" \ + -- \ + "${JOB_ARGS[@]}" +rc=$? +set -e +echo "MILES_MLITE_GRPO_DONE rc=${rc}" +exit "${rc}" diff --git a/experimental/lite/examples/miles/scripts/run_qwen3moe_sft.sh b/experimental/lite/examples/miles/scripts/run_qwen3moe_sft.sh new file mode 100755 index 00000000000..f9f07b48b89 --- /dev/null +++ b/experimental/lite/examples/miles/scripts/run_qwen3moe_sft.sh @@ -0,0 +1,258 @@ +#!/usr/bin/env bash +#SBATCH --job-name=mlite_miles_sft +#SBATCH --partition=batch +#SBATCH --account=coreai_devtech_all +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --gres=gpu:8 +#SBATCH --cpus-per-task=64 +#SBATCH --mem=2010G +#SBATCH --time=00:45:00 +#SBATCH --output=/lustre/fs1/portfolios/coreai/projects/coreai_devtech_all/users/bayan/code/env/mlite_miles_sft-%j.log + +# Qwen3 MoE SFT with miles using either the Megatron Lite patch or native Megatron. +set -euo pipefail + +if [[ "${VERBOSE:-0}" == "1" ]]; then set -x; fi + +resolve_script_path() { + if [[ -n "${MLITE_MILES_SCRIPT_PATH:-}" ]]; then + readlink -f "${MLITE_MILES_SCRIPT_PATH}" + return + fi + if [[ -n "${SLURM_JOB_ID:-}" ]] && command -v scontrol >/dev/null 2>&1; then + local command_path + command_path="$( + scontrol show job "${SLURM_JOB_ID}" | tr ' ' '\n' | sed -n 's/^Command=//p' | head -1 + )" + if [[ -n "${command_path}" && -r "${command_path}" ]]; then + readlink -f "${command_path}" + return + fi + fi + readlink -f "${BASH_SOURCE[0]}" +} + +SCRIPT_PATH="$(resolve_script_path)" +CONTAINER_IMAGE="${CONTAINER_IMAGE:-/lustre/fs1/portfolios/coreai/projects/coreai_devtech_all/users/bayan/code/env/miles.sqsh}" +CONTAINER_MOUNTS="${CONTAINER_MOUNTS:-/lustre:/lustre}" +DRY_RUN="${DRY_RUN:-0}" +unset PYTORCH_CUDA_ALLOC_CONF PYTORCH_ALLOC_CONF + +if [[ "${IN_MILES_CONTAINER:-0}" != "1" ]]; then + if [[ ! -r "${CONTAINER_IMAGE}" ]]; then + echo "Container image not readable: ${CONTAINER_IMAGE}" >&2 + exit 2 + fi + + SRUN_CMD=( + srun + --container-image="${CONTAINER_IMAGE}" + --container-mounts="${CONTAINER_MOUNTS}" + --container-workdir=/ + env IN_MILES_CONTAINER=1 MLITE_MILES_SCRIPT_PATH="${SCRIPT_PATH}" + bash "${SCRIPT_PATH}" + ) + + if [[ "${DRY_RUN}" == "1" ]]; then + printf 'outer: ' + printf '%q ' "${SRUN_CMD[@]}" + printf '\n' + IN_MILES_CONTAINER=1 DRY_RUN=1 bash "${SCRIPT_PATH}" + exit 0 + fi + + if [[ -z "${SLURM_JOB_ID:-}" ]]; then + echo "Submit this script with sbatch, or run DRY_RUN=1 bash ${SCRIPT_PATH} for a local command check." >&2 + exit 2 + fi + + exec "${SRUN_CMD[@]}" +fi + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -L)" +EXAMPLE_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd -L)" +LITE_ROOT="$(cd "${EXAMPLE_ROOT}/../.." && pwd -L)" +REPO_ROOT="$(cd "${LITE_ROOT}/../.." && pwd -L)" + +add_pythonpath() { [[ -n "${1:-}" ]] && export PYTHONPATH="${1}:${PYTHONPATH:-}"; } + +MILES_ROOT="${MILES_ROOT:-/lustre/fs1/portfolios/coreai/projects/coreai_devtech_all/users/bayan/code/miles}" +MEGATRON_ROOT="${MEGATRON_ROOT:-/lustre/fs1/portfolios/coreai/projects/coreai_devtech_all/users/bayan/code/megatron_lite/Megatron-LM}" +MODEL_PATH="${MODEL_PATH:-/lustre/fs1/portfolios/coreai/projects/coreai_devtech_all/users/shunyad/models/Qwen/Qwen3-30B-A3B}" +TRAIN_DATA="${TRAIN_DATA:?Set TRAIN_DATA to a messages parquet file for miles SFT.}" + +add_pythonpath "${EXAMPLE_ROOT}" +add_pythonpath "${LITE_ROOT}/examples" +add_pythonpath "${LITE_ROOT}" +add_pythonpath "${REPO_ROOT}" +add_pythonpath "${MEGATRON_ROOT}" +add_pythonpath "${MILES_ROOT}" + +MODEL_SCRIPT="${MODEL_SCRIPT:-${MILES_ROOT}/scripts/models/qwen3-30B-A3B.sh}" +NUM_GPUS="${NUM_GPUS:-8}" +TRAIN_BACKEND="${TRAIN_BACKEND:-mlite}" +case "${TRAIN_BACKEND}" in + mlite|megatron) ;; + *) echo "TRAIN_BACKEND must be 'mlite' or 'megatron'." >&2; exit 2 ;; +esac +RUN_ROOT="${RUN_ROOT:-/lustre/fs1/portfolios/coreai/projects/coreai_devtech_all/users/bayan/code/env/mlite_miles_runs}" +SAVE_DIR="${SAVE_DIR:-${RUN_ROOT}/qwen3moe_sft_${TRAIN_BACKEND}/${SLURM_JOB_ID:-dryrun}}" +LOG_DIR="${LOG_DIR:-${RUN_ROOT}/logs}" + +TP_SIZE="${TP_SIZE:-2}" +PP_SIZE="${PP_SIZE:-1}" +CP_SIZE="${CP_SIZE:-1}" +EP_SIZE="${EP_SIZE:-8}" +ETP_SIZE="${ETP_SIZE:-1}" +MLITE_MODEL_NAME="${MLITE_MODEL_NAME:-qwen3_moe}" +MLITE_OPTIMIZER_BACKEND="${MLITE_OPTIMIZER_BACKEND:-dist_opt}" +OPTIMIZER_OFFLOAD="${OPTIMIZER_OFFLOAD:-1}" +PARAM_OFFLOAD="${PARAM_OFFLOAD:-0}" +if [[ -z "${MEGATRON_TO_HF_MODE:-}" ]]; then + if [[ "${TRAIN_BACKEND}" == "mlite" ]]; then + MEGATRON_TO_HF_MODE="raw" + else + MEGATRON_TO_HF_MODE="bridge" + fi +fi + +LR="${LR:-1e-5}" +ROLLOUT_BATCH_SIZE="${ROLLOUT_BATCH_SIZE:-128}" +GLOBAL_BATCH_SIZE="${GLOBAL_BATCH_SIZE:-128}" +MAX_TOKENS_PER_GPU="${MAX_TOKENS_PER_GPU:-9216}" +NUM_EPOCH="${NUM_EPOCH:-1}" +NUM_ROLLOUT="${NUM_ROLLOUT:-3}" +SAVE_INTERVAL="${SAVE_INTERVAL:-100000}" + +mkdir -p "${SAVE_DIR}" "${LOG_DIR}" +source "${MODEL_SCRIPT}" + +export CUDA_DEVICE_MAX_CONNECTIONS="${CUDA_DEVICE_MAX_CONNECTIONS:-1}" +export PYTHONUNBUFFERED=1 +export MASTER_ADDR="${MASTER_ADDR:-127.0.0.1}" +export no_proxy="${no_proxy:-127.0.0.1,${MASTER_ADDR}}" + +BACKEND_ARGS=( + --train-backend megatron + --model-name qwen3_moe + --megatron-to-hf-mode "${MEGATRON_TO_HF_MODE}" +) +if [[ "${TRAIN_BACKEND}" == "mlite" ]]; then + BACKEND_ARGS+=( + --mlite-backend-patch + --mlite-model-name "${MLITE_MODEL_NAME}" + --mlite-impl lite + --mlite-optimizer-backend "${MLITE_OPTIMIZER_BACKEND}" + ) + if [[ "${OPTIMIZER_OFFLOAD}" == "1" || "${OPTIMIZER_OFFLOAD}" == "true" || "${OPTIMIZER_OFFLOAD}" == "True" ]]; then + BACKEND_ARGS+=(--mlite-optimizer-offload) + fi + if [[ "${PARAM_OFFLOAD}" == "1" || "${PARAM_OFFLOAD}" == "true" || "${PARAM_OFFLOAD}" == "True" ]]; then + BACKEND_ARGS+=(--mlite-param-offload) + fi +fi + +CKPT_ARGS=( + --hf-checkpoint "${MODEL_PATH}" + --save "${SAVE_DIR}" + --save-interval "${SAVE_INTERVAL}" +) + +SFT_ARGS=( + --rollout-function-path miles.rollout.sft_rollout.generate_rollout + --prompt-data "${TRAIN_DATA}" + --input-key messages + --rollout-shuffle + --num-epoch "${NUM_EPOCH}" + --num-rollout "${NUM_ROLLOUT}" + --rollout-batch-size "${ROLLOUT_BATCH_SIZE}" + --global-batch-size "${GLOBAL_BATCH_SIZE}" + --loss-type sft_loss + --calculate-per-token-loss + --disable-compute-advantages-and-returns + --debug-train-only +) + +PERF_ARGS=( + --tensor-model-parallel-size "${TP_SIZE}" + --sequence-parallel + --pipeline-model-parallel-size "${PP_SIZE}" + --context-parallel-size "${CP_SIZE}" + --expert-model-parallel-size "${EP_SIZE}" + --expert-tensor-parallel-size "${ETP_SIZE}" + --use-dynamic-batch-size + --max-tokens-per-gpu "${MAX_TOKENS_PER_GPU}" +) + +OPTIMIZER_ARGS=( + --optimizer adam + --lr "${LR}" + --lr-decay-style constant + --weight-decay 0.1 + --adam-beta1 0.9 + --adam-beta2 0.95 + --clip-grad 1.0 +) +if [[ "${TRAIN_BACKEND}" == "megatron" ]] && [[ "${OPTIMIZER_OFFLOAD}" == "1" || "${OPTIMIZER_OFFLOAD}" == "true" || "${OPTIMIZER_OFFLOAD}" == "True" ]]; then + OPTIMIZER_ARGS+=( + --optimizer-cpu-offload + --overlap-cpu-optimizer-d2h-h2d + --use-precision-aware-optimizer + ) +fi + +MISC_ARGS=( + --attention-dropout 0.0 + --hidden-dropout 0.0 + --attention-backend flash + --log-throughput +) + +JOB_ARGS=( + python3 -m miles_mlite.launch + --actor-num-nodes 1 + --actor-num-gpus-per-node "${NUM_GPUS}" + --num-gpus-per-node "${NUM_GPUS}" + "${MODEL_ARGS[@]}" + "${BACKEND_ARGS[@]}" + "${CKPT_ARGS[@]}" + "${SFT_ARGS[@]}" + "${OPTIMIZER_ARGS[@]}" + "${PERF_ARGS[@]}" + "${MISC_ARGS[@]}" +) + +RUNTIME_ENV_JSON="$( +python3 - <&2 + exit 1 +fi + +case "${MLITE_OPTIMIZER_BACKEND}" in + dist_opt) + MLITE_IMPL_OPTIMIZER="dist_opt" + ;; + fsdp2) + MLITE_IMPL_OPTIMIZER="fsdp2" + ;; + *) + echo "Unsupported MLITE_OPTIMIZER_BACKEND=${MLITE_OPTIMIZER_BACKEND}. Expected dist_opt or fsdp2." >&2 + exit 1 + ;; +esac + +if [[ "${INFER_BACKEND}" == "vllm" ]]; then + export VLLM_USE_V1="${VLLM_USE_V1:-1}" + export VLLM_ALLREDUCE_USE_SYMM_MEM="${VLLM_ALLREDUCE_USE_SYMM_MEM:-0}" +fi + +MLITE_VPP_SIZE="${ACTOR_VPP}" +if [[ "${MLITE_VPP_SIZE}" == "null" ]]; then + MLITE_VPP_SIZE=1 +fi + +RUN_NAME="${RUN_NAME:-qwen35_gsm8k_grpo_mlite_${INFER_BACKEND}_tp${ACTOR_TP}_pp${ACTOR_PP}_cp${ACTOR_CP}_ep${ACTOR_EP}}" +CKPT_DIR="${CKPT_DIR:-${OUTPUT_ROOT}/checkpoints/${RUN_NAME}}" +LOG_FILE="${LOG_FILE:-${OUTPUT_ROOT}/${RUN_NAME}.log}" +JSONL_FILE="${JSONL_FILE:-${OUTPUT_ROOT}/${RUN_NAME}.jsonl}" +CMD_FILE="${CMD_FILE:-${OUTPUT_ROOT}/${RUN_NAME}.cmd.sh}" + +mkdir -p "${OUTPUT_ROOT}" "${CKPT_DIR}" "$(dirname "${LOG_FILE}")" "$(dirname "${JSONL_FILE}")" "$(dirname "${CMD_FILE}")" +export VERL_FILE_LOGGER_PATH="${JSONL_FILE}" + +CACHE_ROOT="${VERL_MLITE_CACHE_ROOT:-${TMPDIR:-/tmp}/verl_mlite}" +mkdir -p "${CACHE_ROOT}/pycache_${USER:-user}" "${CACHE_ROOT}/torchinductor_${USER:-user}" "${CACHE_ROOT}/triton_${USER:-user}" +export PYTHONPYCACHEPREFIX="${PYTHONPYCACHEPREFIX:-${CACHE_ROOT}/pycache_${USER:-user}}" +export TORCHINDUCTOR_CACHE_DIR="${TORCHINDUCTOR_CACHE_DIR:-${CACHE_ROOT}/torchinductor_${USER:-user}}" +export TRITON_CACHE_DIR="${TRITON_CACHE_DIR:-${CACHE_ROOT}/triton_${USER:-user}}" + +ALGORITHM=( + "algorithm.adv_estimator=grpo" + "algorithm.use_kl_in_reward=${USE_KL_IN_REWARD:-False}" + "algorithm.kl_ctrl.kl_coef=${KL_COEF:-0.0}" + "algorithm.rollout_correction.bypass_mode=True" + "algorithm.norm_adv_by_std_in_grpo=False" +) + +DATA=( + "data.train_files=${TRAIN_FILES}" + "data.val_files=${VAL_FILES}" + "data.train_batch_size=${TRAIN_BATCH_SIZE}" + "data.prompt_key=prompt" + "data.return_raw_chat=True" + "data.max_prompt_length=${MAX_PROMPT_LENGTH}" + "data.max_response_length=${MAX_RESPONSE_LENGTH}" + "data.filter_overlong_prompts=True" + "data.truncation=error" +) + +MODEL=( + "actor_rollout_ref.model.path=${MODEL_PATH}" + "actor_rollout_ref.model.trust_remote_code=True" + "actor_rollout_ref.model.use_fused_kernels=False" +) + +ACTOR=( + "actor@actor_rollout_ref.actor=mlite_actor" + "actor_rollout_ref.actor.optim.lr=${ACTOR_LR}" + "actor_rollout_ref.actor.optim.weight_decay=${WEIGHT_DECAY}" + "actor_rollout_ref.actor.optim.betas=${BETAS}" + "actor_rollout_ref.actor.optim.clip_grad=${CLIP_GRAD}" + "actor_rollout_ref.actor.optim.lr_warmup_steps=${LR_WARMUP_STEPS}" + "actor_rollout_ref.actor.optim.lr_warmup_init=0" + "actor_rollout_ref.actor.optim.lr_decay_style=${LR_DECAY_STYLE}" + "actor_rollout_ref.actor.ppo_mini_batch_size=${PPO_MINI_BATCH_SIZE}" + "actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=${ACTOR_PPO_MICRO_BATCH_SIZE_PER_GPU}" + "actor_rollout_ref.actor.use_dynamic_bsz=${USE_DYNAMIC_BSZ}" + "actor_rollout_ref.actor.ppo_max_token_len_per_gpu=${PPO_MAX_TOKEN_LEN_PER_GPU}" + "actor_rollout_ref.actor.use_kl_loss=${USE_KL_LOSS:-False}" + "actor_rollout_ref.actor.kl_loss_coef=${KL_LOSS_COEF:-0.0}" + "actor_rollout_ref.actor.entropy_coeff=${ENTROPY_COEFF}" + "actor_rollout_ref.actor.policy_loss.loss_mode=${POLICY_LOSS_MODE}" + "actor_rollout_ref.actor.loss_agg_mode=${LOSS_AGG_MODE}" + "actor_rollout_ref.actor.engine.dtype=${DTYPE}" + "actor_rollout_ref.actor.engine.model_name=${MLITE_MODEL_NAME}" + "actor_rollout_ref.actor.engine.impl=${MLITE_IMPL}" + "actor_rollout_ref.actor.engine.tp=${ACTOR_TP}" + "actor_rollout_ref.actor.engine.pp=${ACTOR_PP}" + "actor_rollout_ref.actor.engine.vpp=${MLITE_VPP_SIZE}" + "actor_rollout_ref.actor.engine.cp=${ACTOR_CP}" + "actor_rollout_ref.actor.engine.ep=${ACTOR_EP}" + "actor_rollout_ref.actor.engine.etp=${ACTOR_ETP}" + "actor_rollout_ref.actor.engine.param_offload=${PARAM_OFFLOAD}" + "actor_rollout_ref.actor.engine.optimizer_offload=${OPTIMIZER_OFFLOAD}" + "actor_rollout_ref.actor.engine.grad_offload=${GRAD_OFFLOAD}" + "actor_rollout_ref.actor.engine.attention_backend_override=${ATTENTION_BACKEND}" + "actor_rollout_ref.actor.engine.impl_cfg.use_thd=True" + "+actor_rollout_ref.actor.engine.impl_cfg.optimizer=${MLITE_IMPL_OPTIMIZER}" +) + +if [[ "${OPTIMIZER_OFFLOAD}" == "True" || "${OPTIMIZER_OFFLOAD}" == "true" || "${OPTIMIZER_OFFLOAD}" == "1" ]]; then + ACTOR+=( + "+actor_rollout_ref.actor.optim.override_optimizer_config.offload_fraction=${OPTIMIZER_STATE_OFFLOAD_FRACTION}" + "+actor_rollout_ref.actor.optim.override_optimizer_config.use_precision_aware_optimizer=${USE_PRECISION_AWARE_OPTIMIZER}" + "+actor_rollout_ref.actor.optim.override_optimizer_config.decoupled_weight_decay=${DECOUPLED_WEIGHT_DECAY}" + ) +fi + +# Reference policy: only built when KL loss/reward is enabled. Route it through +# the mlite engine (forward-only) so it runs on the same mesh as the actor; +# otherwise the ref worker falls back to the FSDP engine config and mlite's +# get_data_parallel_size() raises AttributeError on the missing `tp` field. Left +# off the default path so KL-free GRPO never composes the ref override. +if [[ "${USE_KL_LOSS:-False}" =~ ^(True|true|1)$ || "${USE_KL_IN_REWARD:-False}" =~ ^(True|true|1)$ ]]; then + ACTOR+=("ref@actor_rollout_ref.ref=mlite_ref") +fi + +ROLLOUT=( + "actor_rollout_ref.rollout.name=${INFER_BACKEND}" + "actor_rollout_ref.rollout.mode=${ROLLOUT_MODE}" + "actor_rollout_ref.rollout.tensor_model_parallel_size=${ROLLOUT_TP}" + "actor_rollout_ref.rollout.gpu_memory_utilization=${ROLLOUT_GPU_MEMORY_UTILIZATION}" + "actor_rollout_ref.rollout.n=${ROLLOUT_N}" + "actor_rollout_ref.rollout.calculate_log_probs=True" + "actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${USE_DYNAMIC_BSZ}" + "actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=${ROLLOUT_LOG_PROB_MAX_TOKEN_LEN_PER_GPU}" + "actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=${ROLLOUT_LOG_PROB_MICRO_BATCH_SIZE_PER_GPU}" + "actor_rollout_ref.rollout.prompt_length=${MAX_PROMPT_LENGTH}" + "actor_rollout_ref.rollout.response_length=${MAX_RESPONSE_LENGTH}" + "actor_rollout_ref.rollout.max_model_len=${ROLLOUT_MAX_MODEL_LEN}" + "actor_rollout_ref.rollout.max_num_seqs=${ROLLOUT_MAX_NUM_SEQS}" + "actor_rollout_ref.rollout.max_num_batched_tokens=${ROLLOUT_MAX_NUM_BATCHED_TOKENS}" + "actor_rollout_ref.rollout.temperature=${ROLLOUT_TEMPERATURE}" + "actor_rollout_ref.rollout.top_p=${ROLLOUT_TOP_P}" + "actor_rollout_ref.rollout.top_k=${ROLLOUT_TOP_K}" + "actor_rollout_ref.rollout.val_kwargs.temperature=${VAL_TEMPERATURE}" + "actor_rollout_ref.rollout.val_kwargs.top_p=${VAL_TOP_P}" + "actor_rollout_ref.rollout.val_kwargs.do_sample=${VAL_DO_SAMPLE}" + "actor_rollout_ref.rollout.val_kwargs.n=${VAL_N}" + "actor_rollout_ref.rollout.free_cache_engine=True" +) + +if [[ "${INFER_BACKEND}" == "vllm" ]]; then + ROLLOUT+=( + "+actor_rollout_ref.rollout.engine_kwargs.vllm.limit_mm_per_prompt.image=${ROLLOUT_LIMIT_IMAGES}" + "+actor_rollout_ref.rollout.engine_kwargs.vllm.limit_mm_per_prompt.video=${ROLLOUT_LIMIT_VIDEOS}" + ) +fi + +TRAINER=( + "critic.enable=False" + "trainer.balance_batch=True" + "trainer.logger=${LOGGER}" + "trainer.project_name=${PROJECT_NAME}" + "trainer.experiment_name=${RUN_NAME}" + "trainer.n_gpus_per_node=${NGPUS_PER_NODE}" + "trainer.nnodes=${NNODES}" + "trainer.save_freq=${SAVE_FREQ}" + "trainer.test_freq=${TEST_FREQ}" + "trainer.total_epochs=${TOTAL_EPOCHS}" + "trainer.total_training_steps=${TOTAL_TRAINING_STEPS}" + "trainer.resume_mode=${RESUME_MODE}" + "trainer.resume_from_path=${RESUME_FROM_PATH}" + "trainer.default_local_dir=${CKPT_DIR}" + "trainer.val_before_train=False" + "trainer.log_val_generations=${LOG_VAL_GENERATIONS}" +) + +COMMAND=( + python3 + -m + verl.trainer.main_ppo + "hydra.searchpath=[pkg://verl_mlite.config]" + "${ALGORITHM[@]}" + "${DATA[@]}" + "${MODEL[@]}" + "${ACTOR[@]}" + "${ROLLOUT[@]}" + "${TRAINER[@]}" + "${EXTRA_ARGS[@]}" +) + +printf '%q ' "${COMMAND[@]}" > "${CMD_FILE}" +printf '\n' >> "${CMD_FILE}" + +if [[ "${DRY_RUN}" == "1" ]]; then + printf '%q ' "${COMMAND[@]}" + printf '\n' + exit 0 +fi + +echo "[mlite] output_root=${OUTPUT_ROOT}" +echo "[mlite] log=${LOG_FILE}" +echo "[mlite] jsonl=${JSONL_FILE}" +echo "[mlite] cmd=${CMD_FILE}" +echo "[mlite] optimizer_backend=${MLITE_OPTIMIZER_BACKEND} impl_cfg.optimizer=${MLITE_IMPL_OPTIMIZER}" + +set +e +"${COMMAND[@]}" 2>&1 | tee "${LOG_FILE}" +cmd_rc="${PIPESTATUS[0]}" +set -e +exit "${cmd_rc}" diff --git a/experimental/lite/examples/verl/scripts/run_qwen3moe_gsm8k_sft.sh b/experimental/lite/examples/verl/scripts/run_qwen3moe_gsm8k_sft.sh new file mode 100755 index 00000000000..9e59d530e01 --- /dev/null +++ b/experimental/lite/examples/verl/scripts/run_qwen3moe_gsm8k_sft.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ "${VERBOSE:-0}" == "1" ]]; then + set -x +fi + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -L)" + +DATASET_DIR="${DATASET_DIR:-${HOME}/data/gsm8k_sft}" +export MODEL_PATH="${MODEL_PATH:-Qwen/Qwen3.5-35B-A3B}" +export TRAIN_FILES="${TRAIN_FILES:-${DATASET_DIR}/train.parquet}" +export VAL_FILES="${VAL_FILES:-${DATASET_DIR}/test.parquet}" +export OUTPUT_ROOT="${OUTPUT_ROOT:-${SCRIPT_DIR}/../outputs/qwen35_gsm8k_sft}" +export PROJECT_NAME="${PROJECT_NAME:-verl-mlite-qwen35-gsm8k-sft}" +export RUN_NAME="${RUN_NAME:-qwen35_gsm8k_sft_mlite}" + +export TOTAL_STEPS="${TOTAL_STEPS:-100}" +export TOTAL_EPOCHS="${TOTAL_EPOCHS:-1}" +export TRAIN_BATCH_SIZE="${TRAIN_BATCH_SIZE:-64}" +export MICRO_BATCH_SIZE="${MICRO_BATCH_SIZE:-1}" +export MAX_TOKENS_PER_GPU="${MAX_TOKENS_PER_GPU:-8192}" +export MAX_LENGTH="${MAX_LENGTH:-2048}" + +exec bash "${SCRIPT_DIR}/run_qwen3moe_sft.sh" "$@" diff --git a/experimental/lite/examples/verl/scripts/run_qwen3moe_sft.sh b/experimental/lite/examples/verl/scripts/run_qwen3moe_sft.sh new file mode 100755 index 00000000000..8ef6d15179e --- /dev/null +++ b/experimental/lite/examples/verl/scripts/run_qwen3moe_sft.sh @@ -0,0 +1,249 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ "${VERBOSE:-0}" == "1" ]]; then + set -x +fi + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -L)" +EXAMPLE_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd -L)" +LITE_ROOT="$(cd "${EXAMPLE_ROOT}/../.." && pwd -L)" +REPO_ROOT="$(cd "${LITE_ROOT}/../.." && pwd -L)" + +add_pythonpath() { + local path="${1:-}" + if [[ -n "${path}" ]]; then + export PYTHONPATH="${path}:${PYTHONPATH:-}" + fi +} + +add_pythonpath "${EXAMPLE_ROOT}" +add_pythonpath "${LITE_ROOT}" +add_pythonpath "${REPO_ROOT}" +add_pythonpath "${VERL_ROOT:-}" +add_pythonpath "${MEGATRON_ROOT:-}" + +export CUDA_DEVICE_MAX_CONNECTIONS="${CUDA_DEVICE_MAX_CONNECTIONS:-1}" +if [[ -n "${CUDA_VISIBLE_DEVICES:-}" ]]; then + unset ROCR_VISIBLE_DEVICES + unset HIP_VISIBLE_DEVICES +fi + +: "${MODEL_PATH:?set MODEL_PATH to a Hugging Face checkpoint directory or model id}" +: "${TRAIN_FILES:?set TRAIN_FILES to a messages parquet path or comma-separated parquet paths}" + +BACKEND="${BACKEND:-mlite}" +VAL_FILES="${VAL_FILES:-}" +OUTPUT_ROOT="${OUTPUT_ROOT:-${EXAMPLE_ROOT}/outputs/qwen3moe_sft}" +PROJECT_NAME="${PROJECT_NAME:-verl-mlite-qwen3moe-sft}" + +NUM_GPUS="${NUM_GPUS:-${NPROC_PER_NODE:-8}}" +NPROC_PER_NODE="${NPROC_PER_NODE:-${NUM_GPUS}}" +NNODES="${NNODES:-1}" +NODE_RANK="${NODE_RANK:-0}" +MASTER_ADDR="${MASTER_ADDR:-127.0.0.1}" +MASTER_PORT="${MASTER_PORT:-29500}" + +TOTAL_STEPS="${TOTAL_STEPS:-100}" +TOTAL_EPOCHS="${TOTAL_EPOCHS:-1}" +SAVE_FREQ="${SAVE_FREQ:-${TOTAL_STEPS}}" +TEST_FREQ="${TEST_FREQ:--1}" +RESUME_MODE="${RESUME_MODE:-disable}" +RESUME_FROM_PATH="${RESUME_FROM_PATH:-null}" +CHECKPOINT_SAVE_CONTENTS="${CHECKPOINT_SAVE_CONTENTS:-[model,optimizer,extra]}" +LOAD_HF_WEIGHTS="${LOAD_HF_WEIGHTS:-True}" +TRAIN_BATCH_SIZE="${TRAIN_BATCH_SIZE:-64}" +MICRO_BATCH_SIZE="${MICRO_BATCH_SIZE:-1}" +MAX_TOKENS_PER_GPU="${MAX_TOKENS_PER_GPU:-8192}" +MAX_LENGTH="${MAX_LENGTH:-${MAX_TOKENS_PER_GPU}}" +PAD_MODE="${PAD_MODE:-no_padding}" +USE_DYNAMIC_BSZ="${USE_DYNAMIC_BSZ:-True}" +IGNORE_INPUT_IDS_MISMATCH="${IGNORE_INPUT_IDS_MISMATCH:-True}" +TRUST_REMOTE_CODE="${TRUST_REMOTE_CODE:-True}" +MESSAGES_KEY="${MESSAGES_KEY:-messages}" +NUM_WORKERS="${NUM_WORKERS:-0}" +SEED="${SEED:-1}" + +TP_SIZE="${TP_SIZE:-2}" +PP_SIZE="${PP_SIZE:-1}" +VPP_SIZE="${VPP_SIZE:-null}" +CP_SIZE="${CP_SIZE:-1}" +EP_SIZE="${EP_SIZE:-8}" +ETP_SIZE="${ETP_SIZE:-1}" +DTYPE="${DTYPE:-bfloat16}" +MLITE_MODEL_NAME="${MLITE_MODEL_NAME:-auto}" +MLITE_IMPL="${MLITE_IMPL:-lite}" +ATTENTION_BACKEND="${ATTENTION_BACKEND:-flash}" +# Optimizer backend: +# - dist_opt (default): Megatron-Core DDP + distributed optimizer. +# - fsdp2: Megatron Lite FSDP2 wrapper + optimizer. +MLITE_OPTIMIZER_BACKEND="${MLITE_OPTIMIZER_BACKEND:-dist_opt}" + +LR="${LR:-1e-5}" +MIN_LR="${MIN_LR:-${LR}}" +WEIGHT_DECAY="${WEIGHT_DECAY:-0.1}" +BETAS="${BETAS:-[0.9,0.95]}" +CLIP_GRAD="${CLIP_GRAD:-1.0}" +LR_WARMUP_STEPS="${LR_WARMUP_STEPS:-0}" +LR_DECAY_STYLE="${LR_DECAY_STYLE:-constant}" + +PARAM_OFFLOAD="${PARAM_OFFLOAD:-False}" +OPTIMIZER_OFFLOAD="${OPTIMIZER_OFFLOAD:-True}" +GRAD_OFFLOAD="${GRAD_OFFLOAD:-False}" +OPTIMIZER_STATE_OFFLOAD_FRACTION="${OPTIMIZER_STATE_OFFLOAD_FRACTION:-1.0}" +USE_PRECISION_AWARE_OPTIMIZER="${USE_PRECISION_AWARE_OPTIMIZER:-True}" +DECOUPLED_WEIGHT_DECAY="${DECOUPLED_WEIGHT_DECAY:-True}" +DRY_RUN="${DRY_RUN:-0}" +EXTRA_ARGS=("$@") + +if [[ "${BACKEND}" != "mlite" ]]; then + echo "Unsupported BACKEND=${BACKEND}. This example is for BACKEND=mlite." >&2 + exit 1 +fi + +if [[ "${PAD_MODE}" != "no_padding" ]]; then + echo "Megatron Lite VERL example currently supports PAD_MODE=no_padding only." >&2 + exit 1 +fi + +case "${MLITE_OPTIMIZER_BACKEND}" in + dist_opt) + MLITE_IMPL_OPTIMIZER="dist_opt" + ;; + fsdp2) + MLITE_IMPL_OPTIMIZER="fsdp2" + ;; + *) + echo "Unsupported MLITE_OPTIMIZER_BACKEND=${MLITE_OPTIMIZER_BACKEND}. Expected dist_opt or fsdp2." >&2 + exit 1 + ;; +esac + +MLITE_VPP_SIZE="${VPP_SIZE}" +if [[ "${MLITE_VPP_SIZE}" == "null" ]]; then + MLITE_VPP_SIZE=1 +fi + +RUN_NAME="${RUN_NAME:-qwen3moe_sft_mlite_tp${TP_SIZE}_pp${PP_SIZE}_cp${CP_SIZE}_ep${EP_SIZE}_etp${ETP_SIZE}}" +CKPT_DIR="${CKPT_DIR:-${OUTPUT_ROOT}/checkpoints/${RUN_NAME}}" +LOG_FILE="${LOG_FILE:-${OUTPUT_ROOT}/${RUN_NAME}.log}" +JSONL_FILE="${JSONL_FILE:-${OUTPUT_ROOT}/${RUN_NAME}.jsonl}" +CMD_FILE="${CMD_FILE:-${OUTPUT_ROOT}/${RUN_NAME}.cmd.sh}" + +mkdir -p "${OUTPUT_ROOT}" "${CKPT_DIR}" "$(dirname "${LOG_FILE}")" "$(dirname "${JSONL_FILE}")" "$(dirname "${CMD_FILE}")" +export VERL_FILE_LOGGER_PATH="${JSONL_FILE}" + +CACHE_ROOT="${VERL_MLITE_CACHE_ROOT:-${TMPDIR:-/tmp}/verl_mlite}" +mkdir -p "${CACHE_ROOT}/pycache_${USER:-user}" "${CACHE_ROOT}/torchinductor_${USER:-user}" "${CACHE_ROOT}/triton_${USER:-user}" +export PYTHONPYCACHEPREFIX="${PYTHONPYCACHEPREFIX:-${CACHE_ROOT}/pycache_${USER:-user}}" +export TORCHINDUCTOR_CACHE_DIR="${TORCHINDUCTOR_CACHE_DIR:-${CACHE_ROOT}/torchinductor_${USER:-user}}" +export TRITON_CACHE_DIR="${TRITON_CACHE_DIR:-${CACHE_ROOT}/triton_${USER:-user}}" + +COMMON_ARGS=( + "data.train_files=${TRAIN_FILES}" + "data.train_batch_size=${TRAIN_BATCH_SIZE}" + "data.micro_batch_size_per_gpu=${MICRO_BATCH_SIZE}" + "data.use_dynamic_bsz=${USE_DYNAMIC_BSZ}" + "data.max_token_len_per_gpu=${MAX_TOKENS_PER_GPU}" + "data.max_length=${MAX_LENGTH}" + "data.pad_mode=${PAD_MODE}" + "data.truncation=error" + "data.messages_key=${MESSAGES_KEY}" + "data.ignore_input_ids_mismatch=${IGNORE_INPUT_IDS_MISMATCH}" + "data.num_workers=${NUM_WORKERS}" + "model=hf_model" + "model.path=${MODEL_PATH}" + "model.trust_remote_code=${TRUST_REMOTE_CODE}" + "optim=megatron" + "optim.lr=${LR}" + "optim.min_lr=${MIN_LR}" + "optim.weight_decay=${WEIGHT_DECAY}" + "optim.betas=${BETAS}" + "optim.clip_grad=${CLIP_GRAD}" + "optim.lr_warmup_steps=${LR_WARMUP_STEPS}" + "optim.lr_warmup_init=0" + "optim.lr_decay_style=${LR_DECAY_STYLE}" + "trainer.logger=[console,file]" + "trainer.project_name=${PROJECT_NAME}" + "trainer.experiment_name=${RUN_NAME}" + "trainer.default_local_dir=${CKPT_DIR}" + "trainer.total_epochs=${TOTAL_EPOCHS}" + "trainer.total_training_steps=${TOTAL_STEPS}" + "trainer.save_freq=${SAVE_FREQ}" + "trainer.test_freq=${TEST_FREQ}" + "trainer.seed=${SEED}" + "trainer.resume_mode=${RESUME_MODE}" + "trainer.resume_from_path=${RESUME_FROM_PATH}" + "trainer.nnodes=${NNODES}" + "trainer.n_gpus_per_node=${NPROC_PER_NODE}" + "checkpoint.save_contents=${CHECKPOINT_SAVE_CONTENTS}" +) + +if [[ -n "${VAL_FILES}" ]]; then + COMMON_ARGS+=("data.val_files=${VAL_FILES}") +fi + +BACKEND_ARGS=( + "hydra.searchpath=[pkg://verl_mlite.config]" + "engine=mlite" + "engine.dtype=${DTYPE}" + "engine.model_name=${MLITE_MODEL_NAME}" + "engine.impl=${MLITE_IMPL}" + "engine.tp=${TP_SIZE}" + "engine.pp=${PP_SIZE}" + "engine.vpp=${MLITE_VPP_SIZE}" + "engine.cp=${CP_SIZE}" + "engine.ep=${EP_SIZE}" + "engine.etp=${ETP_SIZE}" + "engine.param_offload=${PARAM_OFFLOAD}" + "engine.optimizer_offload=${OPTIMIZER_OFFLOAD}" + "engine.grad_offload=${GRAD_OFFLOAD}" + "engine.attention_backend_override=${ATTENTION_BACKEND}" + "engine.load_hf_weights=${LOAD_HF_WEIGHTS}" + "engine.impl_cfg.use_thd=True" + "+engine.impl_cfg.optimizer=${MLITE_IMPL_OPTIMIZER}" +) + +if [[ "${OPTIMIZER_OFFLOAD}" == "True" || "${OPTIMIZER_OFFLOAD}" == "true" || "${OPTIMIZER_OFFLOAD}" == "1" ]]; then + BACKEND_ARGS+=( + "+optim.override_optimizer_config.offload_fraction=${OPTIMIZER_STATE_OFFLOAD_FRACTION}" + "+optim.override_optimizer_config.use_precision_aware_optimizer=${USE_PRECISION_AWARE_OPTIMIZER}" + "+optim.override_optimizer_config.decoupled_weight_decay=${DECOUPLED_WEIGHT_DECAY}" + ) +fi + +COMMAND=( + torchrun + --nnodes="${NNODES}" + --node_rank="${NODE_RANK}" + --master_addr="${MASTER_ADDR}" + --master_port="${MASTER_PORT}" + --nproc_per_node="${NPROC_PER_NODE}" + -m + verl_mlite.launch + verl.trainer.sft_trainer + "${COMMON_ARGS[@]}" + "${BACKEND_ARGS[@]}" + "${EXTRA_ARGS[@]}" +) + +printf '%q ' "${COMMAND[@]}" > "${CMD_FILE}" +printf '\n' >> "${CMD_FILE}" + +if [[ "${DRY_RUN}" == "1" ]]; then + printf '%q ' "${COMMAND[@]}" + printf '\n' + exit 0 +fi + +echo "[${BACKEND}] output_root=${OUTPUT_ROOT}" +echo "[${BACKEND}] log=${LOG_FILE}" +echo "[${BACKEND}] jsonl=${JSONL_FILE}" +echo "[${BACKEND}] cmd=${CMD_FILE}" +echo "[${BACKEND}] optimizer_backend=${MLITE_OPTIMIZER_BACKEND} impl_cfg.optimizer=${MLITE_IMPL_OPTIMIZER}" + +set +e +"${COMMAND[@]}" 2>&1 | tee "${LOG_FILE}" +cmd_rc="${PIPESTATUS[0]}" +set -e +exit "${cmd_rc}" diff --git a/experimental/lite/examples/verl/sitecustomize.py b/experimental/lite/examples/verl/sitecustomize.py new file mode 100644 index 00000000000..dfe77bf5951 --- /dev/null +++ b/experimental/lite/examples/verl/sitecustomize.py @@ -0,0 +1,6 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Process-wide compatibility hooks for the VERL MLite examples.""" + +from verl_mlite.compat import apply_runtime_patches + +apply_runtime_patches() diff --git a/experimental/lite/examples/verl/verl_mlite/__init__.py b/experimental/lite/examples/verl/verl_mlite/__init__.py new file mode 100644 index 00000000000..25500fa4d12 --- /dev/null +++ b/experimental/lite/examples/verl/verl_mlite/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""VERL integration package for Megatron Lite.""" diff --git a/experimental/lite/examples/verl/verl_mlite/compat.py b/experimental/lite/examples/verl/verl_mlite/compat.py new file mode 100644 index 00000000000..f49d6f50949 --- /dev/null +++ b/experimental/lite/examples/verl/verl_mlite/compat.py @@ -0,0 +1,96 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Small compatibility patches for dependency-version gaps in examples.""" + +from __future__ import annotations + +import importlib.util +import sys +from collections.abc import Iterable +from functools import wraps +from pathlib import Path +from typing import Any + + +def _patch_transformers_rope_ignore_keys() -> None: + try: + import transformers.modeling_rope_utils as rope_utils + except Exception: + return + + for cls in vars(rope_utils).values(): + if not isinstance(cls, type): + continue + if getattr(cls, "_verl_mlite_rope_ignore_keys_patch", False): + continue + descriptor = vars(cls).get("_check_received_keys") + if descriptor is None: + continue + + is_staticmethod = isinstance(descriptor, staticmethod) + is_classmethod = isinstance(descriptor, classmethod) + original = descriptor.__func__ if is_staticmethod or is_classmethod else descriptor + + def build_wrapper(check_received_keys: Any) -> Any: + @wraps(check_received_keys) + def patched(*args: Any, **kwargs: Any) -> Any: + ignore_keys = kwargs.get("ignore_keys") + if isinstance(ignore_keys, list): + kwargs["ignore_keys"] = set(ignore_keys) + elif ignore_keys is not None and not isinstance(ignore_keys, set): + if isinstance(ignore_keys, Iterable) and not isinstance( + ignore_keys, (str, bytes) + ): + kwargs["ignore_keys"] = set(ignore_keys) + return check_received_keys(*args, **kwargs) + + return patched + + patched = build_wrapper(original) + if is_staticmethod: + cls._check_received_keys = staticmethod(patched) + elif is_classmethod: + cls._check_received_keys = classmethod(patched) + else: + cls._check_received_keys = patched + cls._verl_mlite_rope_ignore_keys_patch = True + + +def apply_runtime_patches() -> None: + _patch_transformers_rope_ignore_keys() + + +def _load_verl_file(relative_path: str, module_name: str): + spec = importlib.util.find_spec("verl") + if spec is None or spec.submodule_search_locations is None: + raise ModuleNotFoundError("No module named 'verl'") + + path = Path(next(iter(spec.submodule_search_locations))) / relative_path + file_spec = importlib.util.spec_from_file_location(module_name, path) + if file_spec is None or file_spec.loader is None: + raise ImportError(f"Unable to load VERL module from {path}") + + module = importlib.util.module_from_spec(file_spec) + sys.modules[module_name] = module + file_spec.loader.exec_module(module) + return module + + +def load_verl_engine_api(): + # Prefer the canonical package import so the MLite engine registers into the + # SAME EngineRegistry that verl's trainers resolve against. Loading base.py as + # a standalone module (below) creates a *duplicate* registry, which silently + # drops the mlite backend ("Unknown backend: mlite"). The file-load path is + # only a fallback for environments where verl isn't importable as a package. + try: + from verl.workers.engine.base import BaseEngine, BaseEngineCtx, EngineRegistry + from verl.workers.engine.utils import postprocess_batch_func, prepare_micro_batches + except (ModuleNotFoundError, ImportError): + base = _load_verl_file("workers/engine/base.py", "_verl_mlite_verl_engine_base") + utils = _load_verl_file("workers/engine/utils.py", "_verl_mlite_verl_engine_utils") + BaseEngine = base.BaseEngine + BaseEngineCtx = base.BaseEngineCtx + EngineRegistry = base.EngineRegistry + postprocess_batch_func = utils.postprocess_batch_func + prepare_micro_batches = utils.prepare_micro_batches + + return BaseEngine, BaseEngineCtx, EngineRegistry, postprocess_batch_func, prepare_micro_batches diff --git a/experimental/lite/examples/verl/verl_mlite/config/__init__.py b/experimental/lite/examples/verl/verl_mlite/config/__init__.py new file mode 100644 index 00000000000..15d962701b4 --- /dev/null +++ b/experimental/lite/examples/verl/verl_mlite/config/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Hydra config package for Verl MLite.""" diff --git a/experimental/lite/examples/verl/verl_mlite/config/actor/__init__.py b/experimental/lite/examples/verl/verl_mlite/config/actor/__init__.py new file mode 100644 index 00000000000..b7d8532bf20 --- /dev/null +++ b/experimental/lite/examples/verl/verl_mlite/config/actor/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Hydra actor config group for Verl MLite.""" diff --git a/experimental/lite/examples/verl/verl_mlite/config/actor/mlite_actor.yaml b/experimental/lite/examples/verl/verl_mlite/config/actor/mlite_actor.yaml new file mode 100644 index 00000000000..7efeab8431b --- /dev/null +++ b/experimental/lite/examples/verl/verl_mlite/config/actor/mlite_actor.yaml @@ -0,0 +1,10 @@ +# Megatron Lite actor config for VERL's new engine worker path. +defaults: + - /optim@optim: megatron + - /engine@engine: mlite + - actor + - _self_ + +_target_: verl.workers.config.ActorConfig + +strategy: mlite diff --git a/experimental/lite/examples/verl/verl_mlite/config/engine/__init__.py b/experimental/lite/examples/verl/verl_mlite/config/engine/__init__.py new file mode 100644 index 00000000000..fe007d793a8 --- /dev/null +++ b/experimental/lite/examples/verl/verl_mlite/config/engine/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Engine config group for Verl MLite.""" diff --git a/experimental/lite/examples/verl/verl_mlite/config/engine/mlite.yaml b/experimental/lite/examples/verl/verl_mlite/config/engine/mlite.yaml new file mode 100644 index 00000000000..dbdd132feb2 --- /dev/null +++ b/experimental/lite/examples/verl/verl_mlite/config/engine/mlite.yaml @@ -0,0 +1,26 @@ +_target_: verl_mlite.engine.config.MegatronLiteEngineConfig + +strategy: mlite +custom_backend_module: verl_mlite.engine.mlite_engine +param_offload: false +optimizer_offload: false +grad_offload: false +forward_only: false +dtype: bfloat16 +export_dtype: bfloat16 +load_hf_weights: true + +model_name: auto +impl: lite + +tp: 1 +etp: null +ep: 1 +pp: 1 +vpp: 1 +cp: 1 + +attention_backend_override: flash +router_aux_loss_coef: null +impl_cfg: + use_thd: true diff --git a/experimental/lite/examples/verl/verl_mlite/config/ref/__init__.py b/experimental/lite/examples/verl/verl_mlite/config/ref/__init__.py new file mode 100644 index 00000000000..b5dff7b5663 --- /dev/null +++ b/experimental/lite/examples/verl/verl_mlite/config/ref/__init__.py @@ -0,0 +1 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. diff --git a/experimental/lite/examples/verl/verl_mlite/config/ref/mlite_ref.yaml b/experimental/lite/examples/verl/verl_mlite/config/ref/mlite_ref.yaml new file mode 100644 index 00000000000..48e832e6d3f --- /dev/null +++ b/experimental/lite/examples/verl/verl_mlite/config/ref/mlite_ref.yaml @@ -0,0 +1,41 @@ +# Megatron Lite reference-model config for VERL's new engine worker path. +# +# The reference policy is built by ActorRolloutRefWorker from +# `actor_rollout_ref.ref.engine`, independently of the actor. Without an mlite +# ref config the ref worker falls back to the default FSDP engine config, whose +# object has no `tp`/`cp`/`pp`, so mlite's get_data_parallel_size() raises +# `AttributeError: 'FSDPEngineConfig' object has no attribute 'tp'` as soon as a +# reference policy is requested (use_kl_loss / use_kl_in_reward). This config +# routes the ref worker through the mlite engine, forward-only, mirroring the +# actor's parallel layout so the reference log-probs are computed on the same +# mesh as the actor. +# +# Wire it from the launcher with: ref@actor_rollout_ref.ref=mlite_ref +defaults: + - ref + - /optim@optim: megatron + - /engine@engine: mlite + - _self_ + +_target_: verl.workers.config.ActorConfig + +strategy: mlite + +engine: + # ref model is forward-only (no optimizer / backward) + forward_only: True + # mirror the actor's parallel layout so ref runs on the same mesh + model_name: ${oc.select:actor_rollout_ref.actor.engine.model_name,auto} + impl: ${oc.select:actor_rollout_ref.actor.engine.impl,lite} + dtype: ${oc.select:actor_rollout_ref.actor.engine.dtype,bfloat16} + tp: ${oc.select:actor_rollout_ref.actor.engine.tp,1} + pp: ${oc.select:actor_rollout_ref.actor.engine.pp,1} + vpp: ${oc.select:actor_rollout_ref.actor.engine.vpp,1} + cp: ${oc.select:actor_rollout_ref.actor.engine.cp,1} + ep: ${oc.select:actor_rollout_ref.actor.engine.ep,1} + etp: ${oc.select:actor_rollout_ref.actor.engine.etp,null} + attention_backend_override: ${oc.select:actor_rollout_ref.actor.engine.attention_backend_override,flash} + # ref only runs forward, so default to offloading params to free GPU memory + param_offload: ${oc.select:actor_rollout_ref.ref.param_offload,True} + impl_cfg: + use_thd: ${oc.select:actor_rollout_ref.actor.engine.impl_cfg.use_thd,true} diff --git a/experimental/lite/examples/verl/verl_mlite/engine/__init__.py b/experimental/lite/examples/verl/verl_mlite/engine/__init__.py new file mode 100644 index 00000000000..7662660591b --- /dev/null +++ b/experimental/lite/examples/verl/verl_mlite/engine/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Engine entrypoints for Verl MLite.""" + +from verl_mlite.engine.mlite_engine import MegatronLiteEngine + +__all__ = ["MegatronLiteEngine"] diff --git a/experimental/lite/examples/verl/verl_mlite/engine/config.py b/experimental/lite/examples/verl/verl_mlite/engine/config.py new file mode 100644 index 00000000000..4ee184b752b --- /dev/null +++ b/experimental/lite/examples/verl/verl_mlite/engine/config.py @@ -0,0 +1,43 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Config objects for the Verl MLite Megatron Lite engine.""" + +from __future__ import annotations + +import importlib +from dataclasses import dataclass, field +from typing import Any + +from verl.workers.config.engine import EngineConfig + + +@dataclass +class MegatronLiteEngineConfig(EngineConfig): + """Minimal VERL-facing config for the external Megatron Lite engine.""" + + strategy: str = "mlite" + custom_backend_module: str | None = "verl_mlite.engine.mlite_engine" + model_name: str = "auto" + impl: str = "lite" + + tp: int = 1 + etp: int | None = None + ep: int = 1 + pp: int = 1 + vpp: int = 1 + cp: int = 1 + + attention_backend_override: str | None = "flash" + router_aux_loss_coef: float | None = None + cross_entropy_fusion: bool | None = None + export_dtype: str | None = "bfloat16" + load_hf_weights: bool = True + impl_cfg: dict[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + super().__post_init__() + if self.strategy != "mlite": + raise ValueError( + f"MegatronLiteEngineConfig expects strategy='mlite', got {self.strategy!r}" + ) + if self.custom_backend_module: + importlib.import_module(self.custom_backend_module) diff --git a/experimental/lite/examples/verl/verl_mlite/engine/mlite_engine.py b/experimental/lite/examples/verl/verl_mlite/engine/mlite_engine.py new file mode 100644 index 00000000000..f1fe54dd4e6 --- /dev/null +++ b/experimental/lite/examples/verl/verl_mlite/engine/mlite_engine.py @@ -0,0 +1,869 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""External VERL engine backed by Megatron Lite runtime primitives.""" + +from __future__ import annotations + +import math +import os +from enum import Enum +from typing import Any + +import torch +import torch.distributed as dist +from megatron.lite.model import resolve_model_type_from_hf +from megatron.lite.primitive.ckpt import load_training_checkpoint, save_training_checkpoint +from megatron.lite.primitive.protocols import default_expert_classifier, default_placement_fn +from megatron.lite.runtime import create_runtime +from megatron.lite.runtime.backends.mlite.config import MegatronLiteConfig +from megatron.lite.runtime.contracts import LossContext, PackedBatch +from megatron.lite.runtime.contracts.config import OptimizerConfig as MegatronLiteOptimizerConfig +from megatron.lite.runtime.contracts.config import ParallelConfig, RuntimeConfig +from tensordict import TensorDict + +from verl.trainer.config import CheckpointConfig +from verl.utils import tensordict_utils as tu +from verl.utils.device import get_device_id, get_device_name +from verl.utils.memory_utils import aggressive_empty_cache +from verl.workers.config import HFModelConfig, OptimizerConfig +from verl_mlite.compat import load_verl_engine_api + +try: + # Recent VERL wraps per-step metric values in a Metric aggregator that + # reduce_metrics() knows how to fold; older VERL expects list-of-scalars. + from verl.utils.metric import Metric as _VerlMetric +except Exception: # pragma: no cover - older VERL without Metric + _VerlMetric = None + +from .config import MegatronLiteEngineConfig + +BaseEngine, BaseEngineCtx, EngineRegistry, postprocess_batch_func, prepare_micro_batches = ( + load_verl_engine_api() +) + +try: + from verl.utils.dataset.dataset_utils import DatasetPadMode +except ImportError: + + class DatasetPadMode(Enum): + NO_PADDING = "no_padding" + + +_LR_SCHEDULER_STATE = "lr_scheduler.pt" + + +def _isolate_compile_cache_per_rank() -> None: + """Avoid torchinductor/triton cache races between local torchrun ranks.""" + rank = os.environ.get("LOCAL_RANK") or os.environ.get("RANK") + if rank is None: + return + for var in ("TORCHINDUCTOR_CACHE_DIR", "TRITON_CACHE_DIR"): + base = os.environ.get(var) + if not base: + continue + base_var = f"VERL_MLITE_BASE_{var}" + root = os.environ.setdefault(base_var, base) + rank_dir = os.path.join(root, f"rank_{rank}") + os.makedirs(rank_dir, exist_ok=True) + os.environ[var] = rank_dir + + +def _is_no_padding_pad_mode(pad_mode: Any) -> bool: + return ( + pad_mode == DatasetPadMode.NO_PADDING + or getattr(pad_mode, "name", None) == "NO_PADDING" + or getattr(pad_mode, "value", None) == "no_padding" + or str(pad_mode) in {"no_padding", "DatasetPadMode.NO_PADDING"} + ) + + +class _MegatronLiteLRScheduler: + def __init__( + self, + optimizer, + *, + init_lr: float, + max_lr: float, + min_lr: float, + lr_warmup_steps: int, + lr_decay_steps: int, + lr_decay_style: str, + start_wd: float, + end_wd: float, + wd_incr_steps: int, + wd_incr_style: str, + wsd_decay_steps: int | None, + lr_wsd_decay_style: str, + ): + self.optimizer = optimizer + self.init_lr = init_lr + self.max_lr = max_lr + self.min_lr = min_lr + self.lr_warmup_steps = max(lr_warmup_steps, 0) + self.lr_decay_steps = max(lr_decay_steps, self.lr_warmup_steps + 1) + self.lr_decay_style = lr_decay_style.lower() + self.start_wd = start_wd + self.end_wd = end_wd + self.wd_incr_steps = max(wd_incr_steps, 1) + self.wd_incr_style = wd_incr_style.lower() + self.wsd_decay_steps = wsd_decay_steps + self.lr_wsd_decay_style = lr_wsd_decay_style.lower() + self.num_steps = 0 + self._apply() + + def state_dict(self) -> dict[str, Any]: + return {"num_steps": self.num_steps} + + def load_state_dict(self, state: dict[str, Any]) -> None: + self.num_steps = int(state.get("num_steps", state.get("step", 0))) + self._apply() + + def step(self, increment: int = 1) -> None: + self.num_steps += increment + self._apply() + + def get_last_lr(self) -> list[float]: + return [group["lr"] for group in self.optimizer.param_groups] + + def _apply(self) -> None: + lr = self._get_lr() + wd = self._get_wd() + for param_group in self.optimizer.param_groups: + param_group["lr"] = lr + if param_group.get("weight_decay", None) is not None: + param_group["weight_decay"] = wd + + def _get_lr(self) -> float: + if self.lr_warmup_steps > 0 and self.num_steps <= self.lr_warmup_steps: + ratio = self.num_steps / self.lr_warmup_steps + return self.init_lr + (self.max_lr - self.init_lr) * ratio + + if self.lr_decay_style == "constant": + return self.max_lr + + if self.lr_decay_style == "inverse-square-root": + warmup = max(self.lr_warmup_steps, 1) + step = max(self.num_steps, 1) + return max(self.min_lr, self.max_lr * math.sqrt(warmup) / math.sqrt(step)) + + if self.lr_decay_style == "wsd": + return self._get_wsd_lr() + + decay_span = max(self.lr_decay_steps - self.lr_warmup_steps, 1) + ratio = min(max((self.num_steps - self.lr_warmup_steps) / decay_span, 0.0), 1.0) + return self._decay(self.max_lr, self.min_lr, ratio, self.lr_decay_style) + + def _get_wsd_lr(self) -> float: + decay_steps = self.wsd_decay_steps or 0 + decay_start = max(self.lr_decay_steps - decay_steps, self.lr_warmup_steps) + if decay_steps <= 0 or self.num_steps <= decay_start: + return self.max_lr + ratio = min((self.num_steps - decay_start) / max(decay_steps, 1), 1.0) + return self._decay(self.max_lr, self.min_lr, ratio, self.lr_wsd_decay_style) + + def _get_wd(self) -> float: + if self.wd_incr_style == "constant": + return self.end_wd + ratio = min(max(self.num_steps / self.wd_incr_steps, 0.0), 1.0) + return self._decay(self.start_wd, self.end_wd, ratio, self.wd_incr_style) + + @staticmethod + def _decay(start: float, end: float, ratio: float, style: str) -> float: + if style == "linear": + return start + (end - start) * ratio + if style == "cosine": + coeff = 0.5 * (math.cos(math.pi * ratio) + 1.0) + return end + (start - end) * coeff + if style == "exponential": + if start == 0.0: + return 0.0 + if end == 0.0: + return start * (1.0 - ratio) + return start * ((end / start) ** ratio) + if style == "constant": + return start + raise ValueError(f"Unsupported scheduler decay style: {style!r}") + + +def _build_lr_scheduler(optimizer, opt: MegatronLiteOptimizerConfig): + """Build a Megatron-style LR scheduler for Megatron Lite's optimizer.""" + total_steps = opt.total_training_steps + if total_steps <= 0: + return None + + warmup_steps = opt.lr_warmup_steps if opt.lr_warmup_steps is not None else -1 + if warmup_steps <= 0 and opt.lr_warmup_steps_ratio > 0: + warmup_steps = int(opt.lr_warmup_steps_ratio * total_steps) + warmup_steps = max(warmup_steps, 0) + + decay_steps = opt.lr_decay_steps if opt.lr_decay_steps is not None else total_steps + min_lr = opt.min_lr if opt.min_lr is not None else 0.0 + for param_group in optimizer.param_groups: + if param_group.get("min_lr") is None: + param_group["min_lr"] = min_lr + + return _MegatronLiteLRScheduler( + optimizer, + init_lr=opt.lr_warmup_init, + max_lr=opt.lr, + min_lr=min_lr, + lr_warmup_steps=warmup_steps, + lr_decay_steps=decay_steps, + lr_decay_style=opt.lr_decay_style, + start_wd=opt.weight_decay, + end_wd=opt.weight_decay, + wd_incr_steps=total_steps, + wd_incr_style=opt.weight_decay_incr_style, + wsd_decay_steps=opt.lr_wsd_decay_steps, + lr_wsd_decay_style=opt.lr_wsd_decay_style, + ) + + +class _MegatronLiteModeCtx(BaseEngineCtx): + """Wrap Megatron Lite runtime contexts with VERL's offload behavior.""" + + def __init__(self, engine: MegatronLiteEngine, mode: str, **kwargs): + super().__init__(engine=engine, mode=mode, **kwargs) + self._runtime_ctx = None + + def __enter__(self): + super().__enter__() + assert self.engine.runtime is not None and self.engine.handle is not None + if self.mode == "train": + self._runtime_ctx = self.engine.runtime.train_mode(self.engine.handle) + else: + self._runtime_ctx = self.engine.runtime.eval_mode(self.engine.handle) + self._runtime_ctx.__enter__() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + assert self._runtime_ctx is not None + self._runtime_ctx.__exit__(exc_type, exc_val, exc_tb) + super().__exit__(exc_type, exc_val, exc_tb) + return False + + +@EngineRegistry.register(model_type="language_model", backend="mlite", device="cuda") +class MegatronLiteEngine(BaseEngine): + """VERL BaseEngine implementation that delegates model lifecycle to Megatron Lite.""" + + def __init__( + self, + model_config: HFModelConfig, + engine_config: MegatronLiteEngineConfig, + optimizer_config: OptimizerConfig, + checkpoint_config: CheckpointConfig, + ): + super().__init__() + _isolate_compile_cache_per_rank() + self.model_config = model_config + self.engine_config = engine_config + self.optimizer_config = optimizer_config + self.checkpoint_config = checkpoint_config + + self.mode = None + self.device_name = get_device_name() + self.runtime = None + self.handle = None + self.module = None + self._mlite_config = None + self._rank = dist.get_rank() if dist.is_initialized() else 0 + + @property + def is_param_offload_enabled(self) -> bool: + return self.engine_config.param_offload + + @property + def is_optimizer_offload_enabled(self) -> bool: + return self.engine_config.optimizer_offload + + def initialize(self): + if self.engine_config.full_determinism: + from verl.workers.engine.utils import enable_full_determinism + + enable_full_determinism(seed=self.engine_config.seed) + + self._mlite_config = self._build_mlite_config() + self.runtime = create_runtime( + RuntimeConfig( + backend="mlite", + hf_path=self.model_config.local_path, + backend_cfg=self._mlite_config, + ) + ) + self.handle = self.runtime.build_model() + self.module = self._extract_primary_module() + + if self.handle._optimizer is not None and self.handle._lr_scheduler is None: + self.handle._lr_scheduler = _build_lr_scheduler( + self.handle._optimizer, self._mlite_config.optimizer + ) + + self.to( + device="cpu", + model=self.is_param_offload_enabled, + optimizer=self.is_optimizer_offload_enabled, + grad=self.is_param_offload_enabled, + ) + + def train_mode(self, **kwargs): + self._require_initialized() + return _MegatronLiteModeCtx(self, mode="train", **kwargs) + + def eval_mode(self, **kwargs): + self._require_initialized() + return _MegatronLiteModeCtx(self, mode="eval", **kwargs) + + def optimizer_zero_grad(self): + self._require_initialized() + self.runtime.zero_grad(self.handle) + + def optimizer_step(self): + self._require_initialized() + _, grad_norm, _ = self.runtime.optimizer_step(self.handle) + return grad_norm + + def lr_scheduler_step(self): + self._require_initialized() + if self.handle._lr_scheduler is not None: + self.handle._lr_scheduler.step(1) + return self.handle._optimizer.param_groups[0]["lr"] + return 0.0 + + def forward_backward_batch( + self, data: TensorDict, loss_function, forward_only: bool = False + ) -> dict[str, Any]: + self._require_initialized() + pad_mode = tu.get_non_tensor_data( + data=data, key="pad_mode", default=DatasetPadMode.NO_PADDING + ) + if not _is_no_padding_pad_mode(pad_mode): + raise NotImplementedError( + "MegatronLiteEngine only supports pad_mode=no_padding for now." + ) + + tu.assign_non_tensor(data, sp_size=self.engine_config.cp) + + token_mask = data["loss_mask"] if "loss_mask" in data.keys() else data["response_mask"] + batch_num_tokens = token_mask.sum().to(get_device_id()) + torch.distributed.all_reduce( + batch_num_tokens, + op=torch.distributed.ReduceOp.SUM, + group=self.get_data_parallel_group(), + ) + tu.assign_non_tensor(data, batch_num_tokens=batch_num_tokens.item()) + tu.assign_non_tensor(data, dp_size=self.get_data_parallel_size()) + + micro_batches, indices = prepare_micro_batches( + data=data, dp_group=self.get_data_parallel_group(), same_micro_num_in_dp=True + ) + + # Megatron drives every forward through the runtime's forward_backward + # callback; the engine never calls the module directly. + return self._forward_backward_batch_with_runtime( + data=data, + micro_batches=micro_batches, + indices=indices, + loss_function=loss_function, + forward_only=forward_only, + ) + + def get_per_tensor_param(self, **kwargs): + self._require_initialized() + if self.is_param_offload_enabled: + self.to("cuda", model=True, optimizer=False, grad=False) + if not getattr(self, "_initial_sync_cache_cleared", False): + aggressive_empty_cache(force_sync=True) + self._initial_sync_cache_cleared = True + export_kwargs = { + key: kwargs[key] + for key in ("limit", "include_mtp_only", "include_local_prefixes") + if key in kwargs + } + if self.engine_config.model_name == "qwen3_5": + export_kwargs["target"] = "vllm" + if self.engine_config.export_dtype: + export_kwargs["export_dtype"] = self.engine_config.export_dtype + return self.runtime.export_weights(self.handle, **export_kwargs), None + + def get_data_parallel_size(self): + if self.handle is None: + world_size = dist.get_world_size() if dist.is_initialized() else 1 + return world_size // ( + self.engine_config.tp * self.engine_config.cp * self.engine_config.pp + ) + return self.handle.dp_size + + def get_data_parallel_rank(self): + if self.handle is None: + rank = dist.get_rank() if dist.is_initialized() else 0 + dense_dp = self.get_data_parallel_size() + return (rank // (self.engine_config.tp * self.engine_config.cp)) % dense_dp + return self.handle.dp_rank + + def get_data_parallel_group(self): + if self.handle is None: + if ( + self.engine_config.tp == 1 + and self.engine_config.cp == 1 + and self.engine_config.pp == 1 + and dist.is_initialized() + ): + return dist.group.WORLD + return None + return self.handle.dp_group + + def to(self, device: str, model: bool = True, optimizer: bool = True, grad: bool = True): + self._require_initialized() + if model or not (optimizer or grad): + super().to(device=device, model=model, optimizer=optimizer, grad=grad) + self.runtime.to(self.handle, device, model=model, optimizer=optimizer, grad=grad) + + def save_checkpoint( + self, + local_path: str, + hdfs_path: str | None = None, + global_step: int = 0, + max_ckpt_to_keep: int | None = None, + **kwargs, + ) -> None: + del hdfs_path, max_ckpt_to_keep, kwargs + self._require_initialized() + + save_contents = self.checkpoint_config.get("save_contents", None) + save_model = save_contents is None or "model" in save_contents + save_optimizer = save_contents is None or "optimizer" in save_contents + if not save_model and not save_optimizer: + if self._rank == 0: + print( + f"Skipping Megatron Lite checkpoint save at step {global_step}: save_contents={save_contents}" + ) + if dist.is_initialized(): + dist.barrier() + return + + os.makedirs(local_path, exist_ok=True) + placement_fn, expert_classifier = self._checkpoint_hooks() + reload_params_for_save = self.is_param_offload_enabled + if reload_params_for_save: + self.to(device="cuda", model=True, optimizer=False, grad=False) + torch.cuda.synchronize() + try: + save_training_checkpoint( + self.module, + self.handle._optimizer, + global_step, + local_path, + self.handle._config.parallel, + self.handle._parallel_state, + get_placements=placement_fn, + is_expert=expert_classifier, + save_model=save_model, + save_optimizer=save_optimizer, + ) + if self.handle._lr_scheduler is not None and self._rank == 0: + torch.save( + self.handle._lr_scheduler.state_dict(), + os.path.join(local_path, _LR_SCHEDULER_STATE), + ) + if dist.is_initialized(): + dist.barrier() + finally: + if reload_params_for_save: + self.to(device="cpu", model=True, optimizer=False, grad=False) + + def load_checkpoint( + self, + local_path: str, + hdfs_path: str | None = None, + del_local_after_load: bool = True, + **kwargs, + ) -> None: + del hdfs_path, del_local_after_load, kwargs + self._require_initialized() + + placement_fn, expert_classifier = self._checkpoint_hooks() + reload_params_for_load = self.is_param_offload_enabled + if reload_params_for_load: + self.to(device="cuda", model=True, optimizer=False, grad=False) + torch.cuda.synchronize() + try: + load_training_checkpoint( + self.module, + self.handle._optimizer, + local_path, + self.handle._config.parallel, + self.handle._parallel_state, + get_placements=placement_fn, + is_expert=expert_classifier, + load_model=True, + load_optimizer=True, + ) + scheduler_path = os.path.join(local_path, _LR_SCHEDULER_STATE) + if self.handle._lr_scheduler is not None and os.path.exists(scheduler_path): + state = torch.load(scheduler_path, map_location="cpu", weights_only=False) + self.handle._lr_scheduler.load_state_dict(state) + if dist.is_initialized(): + dist.barrier() + finally: + if reload_params_for_load: + self.to(device="cpu", model=True, optimizer=False, grad=False) + + def is_mp_src_rank_with_outputs(self): + if self.handle is None: + rank = dist.get_rank() if dist.is_initialized() else 0 + dense_dp = self.get_data_parallel_size() + tp_rank = rank % self.engine_config.tp + cp_rank = (rank // self.engine_config.tp) % self.engine_config.cp + pp_rank = rank // (self.engine_config.tp * self.engine_config.cp * dense_dp) + return tp_rank == 0 and cp_rank == 0 and pp_rank == self.engine_config.pp - 1 + return self.runtime.is_mp_src_rank_with_outputs(self.handle) + + def _require_initialized(self) -> None: + if self.runtime is None or self.handle is None: + raise RuntimeError("MegatronLiteEngine is not initialized yet.") + + def _build_mlite_config(self) -> MegatronLiteConfig: + return MegatronLiteConfig( + model_name=self._resolve_model_name(), + impl=self.engine_config.impl, + hf_path=self.model_config.local_path, + parallel=ParallelConfig( + tp=self.engine_config.tp, + etp=self.engine_config.etp or 1, + ep=self.engine_config.ep, + pp=self.engine_config.pp, + vpp=self.engine_config.vpp, + cp=self.engine_config.cp, + ), + optimizer=self._build_mlite_optimizer_config(), + attention_backend_override=self.engine_config.attention_backend_override, + router_aux_loss_coef=self.engine_config.router_aux_loss_coef, + load_hf_weights=self.engine_config.load_hf_weights, + impl_cfg=self._build_impl_cfg(), + ) + + def _resolve_model_name(self) -> str: + if self.engine_config.model_name != "auto": + return self.engine_config.model_name + return resolve_model_type_from_hf(self.model_config.hf_config) + + def _build_impl_cfg(self) -> dict[str, Any]: + impl_cfg = dict(self.engine_config.impl_cfg) + if impl_cfg.get("use_thd", True) is not True: + raise ValueError( + "MegatronLiteEngine supports only THD/no-padding SFT; set engine.impl_cfg.use_thd=True." + ) + impl_cfg["use_thd"] = True + cross_entropy_fusion = getattr(self.engine_config, "cross_entropy_fusion", None) + if cross_entropy_fusion is None: + cross_entropy_fusion = getattr(self.engine_config, "use_fused_kernels", False) + impl_cfg.setdefault("cross_entropy_fusion", bool(cross_entropy_fusion)) + mtp_cfg = getattr(self.model_config, "mtp", None) + if mtp_cfg is not None: + mtp_enable = bool(getattr(mtp_cfg, "enable", False)) + mtp_enable_train = mtp_enable and bool(getattr(mtp_cfg, "enable_train", False)) + impl_cfg["mtp_enable"] = mtp_enable + impl_cfg["mtp_enable_train"] = mtp_enable_train + impl_cfg["mtp_detach_encoder"] = bool(getattr(mtp_cfg, "detach_encoder", False)) + impl_cfg["mtp_loss_scaling_factor"] = float( + getattr(mtp_cfg, "mtp_loss_scaling_factor", 0.1) + ) + if self.engine_config.full_determinism: + impl_cfg.setdefault("deterministic", True) + if self.engine_config.forward_only: + impl_cfg["optimizer"] = None + return impl_cfg + + def _build_mlite_optimizer_config(self) -> MegatronLiteOptimizerConfig: + optimizer_name = self._normalize_optimizer_name(self.optimizer_config) + betas = tuple(getattr(self.optimizer_config, "betas", (0.9, 0.999))) + override = getattr(self.optimizer_config, "override_optimizer_config", {}) or {} + offload_fraction = override.get( + "offload_fraction", override.get("optimizer_offload_fraction") + ) + if offload_fraction is None and override.get("optimizer_cpu_offload"): + offload_fraction = 1.0 + if offload_fraction is None and self.is_optimizer_offload_enabled: + offload_fraction = 1.0 + + min_lr = getattr(self.optimizer_config, "min_lr", None) + min_lr_ratio = getattr(self.optimizer_config, "min_lr_ratio", None) + if min_lr is None: + min_lr = 0.0 if min_lr_ratio is None else self.optimizer_config.lr * min_lr_ratio + + lr_decay_style = getattr(self.optimizer_config, "lr_decay_style", None) + if lr_decay_style is None: + lr_decay_style = getattr(self.optimizer_config, "lr_scheduler_type", "constant") + + return MegatronLiteOptimizerConfig( + optimizer=optimizer_name, + lr=self.optimizer_config.lr, + min_lr=min_lr, + clip_grad=self.optimizer_config.clip_grad, + weight_decay=self.optimizer_config.weight_decay, + lr_warmup_steps_ratio=self.optimizer_config.lr_warmup_steps_ratio, + total_training_steps=self.optimizer_config.total_training_steps, + lr_warmup_steps=self.optimizer_config.lr_warmup_steps, + lr_warmup_init=getattr(self.optimizer_config, "lr_warmup_init", 0.0), + lr_decay_steps=getattr(self.optimizer_config, "lr_decay_steps", None), + lr_decay_style=lr_decay_style, + weight_decay_incr_style=getattr( + self.optimizer_config, "weight_decay_incr_style", "constant" + ), + lr_wsd_decay_style=getattr(self.optimizer_config, "lr_wsd_decay_style", "exponential"), + lr_wsd_decay_steps=getattr(self.optimizer_config, "lr_wsd_decay_steps", None), + use_checkpoint_opt_param_scheduler=getattr( + self.optimizer_config, "use_checkpoint_opt_param_scheduler", False + ), + adam_beta1=betas[0], + adam_beta2=betas[1], + adam_eps=override.get("adam_eps", override.get("eps")), + offload_fraction=offload_fraction, + use_precision_aware_optimizer=override.get("use_precision_aware_optimizer"), + decoupled_weight_decay=override.get("decoupled_weight_decay"), + ) + + @staticmethod + def _normalize_optimizer_name(config: OptimizerConfig) -> str: + optimizer_name = getattr(config, "optimizer", "adam") + lower = str(optimizer_name).lower() + if "adam" in lower: + return "adam" + raise ValueError( + f"MegatronLiteEngine only supports Adam-style optimizers today, got {optimizer_name!r}" + ) + + def _extract_primary_module(self): + model = self.handle._model + if isinstance(model, list | tuple): + if not model: + raise RuntimeError("Megatron Lite runtime returned an empty model chunk list.") + if len(model) > 1: + return torch.nn.ModuleList(model) + return model[0] + return model + + def _forward_backward_batch_with_runtime( + self, + *, + data: TensorDict, + micro_batches: list[TensorDict], + indices, + loss_function, + forward_only: bool, + ) -> dict[str, Any]: + runtime_batches = [] + num_micro_batches = len(micro_batches) + batch_num_tokens = tu.get_non_tensor_data(data=data, key="batch_num_tokens", default=None) + if batch_num_tokens is None: + raise ValueError( + "MegatronLiteEngine PP/CP SFT requires batch_num_tokens for VERL-compatible loss scaling." + ) + if batch_num_tokens <= 0: + raise ValueError(f"batch_num_tokens must be positive, got {batch_num_tokens}.") + loss_scale = self.get_data_parallel_size() * num_micro_batches / float(batch_num_tokens) + for micro_idx, micro_batch in enumerate(micro_batches): + tu.assign_non_tensor(micro_batch, micro_batch_idx=micro_idx) + micro_batch = micro_batch.to(get_device_id()) + runtime_batches.append( + ( + self._make_runtime_batch(micro_batch), + self._make_runtime_loss_context(micro_batch, loss_scale=loss_scale), + ) + ) + + runtime_loss_fn = None + reduced_outputs = [] if (loss_function is not None or forward_only) and self.is_mp_src_rank_with_outputs() else None + if loss_function is not None or forward_only: + runtime_loss_fn = self._make_runtime_loss_fn(loss_function, num_micro_batches, reduced_outputs) + + result = self.runtime.forward_backward( + self.handle, + iter(runtime_batches), + loss_fn=runtime_loss_fn, + num_microbatches=num_micro_batches, + forward_only=forward_only, + ) + if reduced_outputs is not None: + return postprocess_batch_func(output_lst=reduced_outputs, indices=indices, data=data) + metrics = dict(result.metrics) + loss = result.model_output.loss + losses = [] if loss is None else torch.as_tensor(loss).detach().flatten().cpu().tolist() + return { + "model_output": {}, + "loss": losses, + # Pass Metric aggregators through unchanged (reduce_metrics folds them); + # list-wrap plain scalars as the legacy contract expects. + "metrics": { + key: value + if isinstance(value, list) + or (_VerlMetric is not None and isinstance(value, _VerlMetric)) + else [value] + for key, value in metrics.items() + }, + } + + def _make_runtime_batch(self, micro_batch: TensorDict) -> PackedBatch: + """Flatten a jagged no-padding batch to a model-agnostic ``PackedBatch``. + + No CP split, no padding, no ``PackedSeqParams`` here: each model's + protocol owns its pack/unpack pair (zigzag vs contiguous). ``labels`` are + the unrolled tokens; the protocol rolls them while packing. + """ + input_ids = micro_batch["input_ids"] + if not getattr(input_ids, "is_nested", False): + raise NotImplementedError( + "MegatronLiteEngine supports only nested no-padding THD batches." + ) + loss_mask = self._loss_mask_for_packing(micro_batch, input_ids) + return PackedBatch( + input_ids=input_ids.values().contiguous(), + labels=input_ids.values().contiguous(), + loss_mask=None if loss_mask is None else loss_mask.values().contiguous().float(), + seq_lens=input_ids.offsets().diff().to(dtype=torch.int64), + ) + + def _make_runtime_loss_context( + self, + micro_batch: TensorDict, + *, + loss_scale: float, + ) -> LossContext: + return LossContext( + temperature=float(self._scalar_temperature(micro_batch)), + calculate_entropy=bool( + tu.get_non_tensor_data(data=micro_batch, key="calculate_entropy", default=False) + ), + return_log_probs=True, + loss_scale=loss_scale, + source_batch=micro_batch, + ) + + @staticmethod + def _loss_mask_for_packing( + micro_batch: TensorDict, input_ids: torch.Tensor + ) -> torch.Tensor | None: + if "loss_mask" not in micro_batch.keys(): + return None + + loss_mask = micro_batch["loss_mask"] + if getattr(loss_mask, "is_nested", False): + return loss_mask + + rows = [] + for seq_ids, row_mask in zip(input_ids.unbind(0), loss_mask, strict=True): + seq_len = seq_ids.numel() + response_tokens = int(row_mask.sum().item()) + if response_tokens > seq_len: + raise ValueError( + f"response loss mask has {response_tokens} tokens but packed input sequence has {seq_len} tokens" + ) + full_mask = torch.zeros(seq_len, dtype=row_mask.dtype, device=row_mask.device) + if response_tokens: + full_mask[-response_tokens:] = row_mask[:response_tokens] + rows.append(full_mask) + return torch.nested.as_nested_tensor(rows, layout=torch.jagged) + + def _build_verl_model_output( + self, + *, + raw_output: dict[str, torch.Tensor], + runtime_batch: PackedBatch, + ) -> dict[str, torch.Tensor]: + log_probs = raw_output.get("log_probs") + if log_probs is None: + raise ValueError("Megatron Lite THD model output must contain token log_probs.") + proto = self.handle._extras.get("protocol") + unpack = getattr(proto, "unpack_forward_output", None) + if unpack is None: + raise ValueError( + "Model protocol must expose unpack_forward_output to reverse THD outputs." + ) + output = {"log_probs": unpack(self.module, runtime_batch, log_probs)} + entropy = raw_output.get("entropy") + if entropy is not None: + output["entropy"] = unpack(self.module, runtime_batch, entropy) + return output + + def _make_runtime_loss_fn(self, loss_function, num_microbatches: int, output_lst=None): + def _loss_fn( + raw_output: dict[str, torch.Tensor], + runtime_batch: PackedBatch, + loss_context: LossContext, + ): + micro_batch = loss_context.source_batch + model_output = self._build_verl_model_output( + raw_output=raw_output, runtime_batch=runtime_batch + ) + raw_output["_verl_model_output"] = model_output + if loss_function is not None: + loss, metrics = loss_function( + model_output=model_output, + data=micro_batch, + dp_group=self.get_data_parallel_group(), + ) + else: + loss = torch.zeros((), device=get_device_id(), dtype=torch.float32) + metrics = {} + + if raw_output.get("mtp_loss") is not None: + metrics = dict(metrics) + mtp_loss = self._reduce_mtp_metric(raw_output["mtp_loss"]) + metrics["mtp_losses/mtp_1_loss"] = ( + float(mtp_loss.item()) if mtp_loss.numel() == 1 else mtp_loss.cpu().tolist() + ) + + raw_output["_verl_metrics"] = metrics + if output_lst is not None: + output_lst.append( + { + "model_output": model_output, + "loss": float(loss.detach().item()), + "metrics": metrics, + } + ) + return (loss * num_microbatches if loss_function is not None else loss), metrics + + return _loss_fn + + def _mtp_enable_train(self) -> bool: + mtp_cfg = getattr(self.model_config, "mtp", None) + return bool( + mtp_cfg is not None + and getattr(mtp_cfg, "enable", False) + and getattr(mtp_cfg, "enable_train", False) + ) + + def _reduce_mtp_metric(self, mtp_loss: torch.Tensor) -> torch.Tensor: + mtp_loss = mtp_loss.detach().float().clone() + dp_group = self.get_data_parallel_group() + if dist.is_initialized() and dp_group is not None: + dist.all_reduce(mtp_loss, op=dist.ReduceOp.AVG, group=dp_group) + return mtp_loss + + @staticmethod + def _scalar_temperature(micro_batch: TensorDict) -> float: + if "temperature" not in micro_batch.keys(): + return 1.0 + temperature = micro_batch["temperature"] + if not isinstance(temperature, torch.Tensor): + return float(temperature) + values = ( + temperature.values() + if getattr(temperature, "is_nested", False) + else temperature.reshape(-1) + ) + if values.numel() == 0: + return 1.0 + first = values[0].detach() + if not torch.all(values.detach() == first).item(): + raise NotImplementedError( + "MegatronLiteEngine currently supports scalar temperature only." + ) + return float(first.float().item()) + + def _checkpoint_hooks(self): + proto = self.handle._extras.get("protocol") + placement_fn = getattr(proto, "PLACEMENT_FN", default_placement_fn) + expert_classifier = getattr(proto, "EXPERT_CLASSIFIER", default_expert_classifier) + return placement_fn, expert_classifier diff --git a/experimental/lite/examples/verl/verl_mlite/launch.py b/experimental/lite/examples/verl/verl_mlite/launch.py new file mode 100644 index 00000000000..c862affb172 --- /dev/null +++ b/experimental/lite/examples/verl/verl_mlite/launch.py @@ -0,0 +1,26 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Launch a VERL module after applying MLite compatibility patches.""" + +from __future__ import annotations + +import runpy +import sys + +from verl_mlite.compat import apply_runtime_patches + + +def main() -> None: + if len(sys.argv) < 2: + raise SystemExit("Usage: python -m verl_mlite.launch [args...]") + module = sys.argv[1] + sys.argv = [module, *sys.argv[2:]] + apply_runtime_patches() + # Import the engine so its EngineRegistry.register decorator runs before the + # verl trainer resolves the "mlite" backend. + import verl_mlite.engine # noqa: F401 + + runpy.run_module(module, run_name="__main__", alter_sys=True) + + +if __name__ == "__main__": + main() diff --git a/experimental/lite/megatron/lite/__init__.py b/experimental/lite/megatron/lite/__init__.py new file mode 100644 index 00000000000..7478deab37b --- /dev/null +++ b/experimental/lite/megatron/lite/__init__.py @@ -0,0 +1,36 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Top-level Megatron Lite package exports.""" + +from __future__ import annotations + +import importlib +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from megatron.lite.runtime.backends.bridge.config import BridgeConfig + from megatron.lite.runtime.backends.mlite.config import DebugConfig, MegatronLiteConfig + from megatron.lite.runtime.contracts import OptimizerConfig, ParallelConfig, RuntimeConfig + +__all__ = [ + "BridgeConfig", + "DebugConfig", + "MegatronLiteConfig", + "OptimizerConfig", + "ParallelConfig", + "RuntimeConfig", +] + + +def __getattr__(name: str): + _lazy = { + "BridgeConfig": "megatron.lite.runtime.backends.bridge.config", + "DebugConfig": "megatron.lite.runtime.backends.mlite.config", + "MegatronLiteConfig": "megatron.lite.runtime.backends.mlite.config", + "OptimizerConfig": "megatron.lite.runtime.contracts", + "ParallelConfig": "megatron.lite.runtime.contracts", + "RuntimeConfig": "megatron.lite.runtime.contracts", + } + if name in _lazy: + mod = importlib.import_module(_lazy[name]) + return getattr(mod, name) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/experimental/lite/megatron/lite/model/__init__.py b/experimental/lite/megatron/lite/model/__init__.py new file mode 100644 index 00000000000..71a16072b71 --- /dev/null +++ b/experimental/lite/megatron/lite/model/__init__.py @@ -0,0 +1,18 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Public model-registry helpers for Megatron Lite.""" + +from megatron.lite.model.registry import ( + get_model_package, + get_train_runtime_module, + register_model, + resolve_model_type_from_hf, + resolve_runtime_model_name, +) + +__all__ = [ + "get_model_package", + "get_train_runtime_module", + "register_model", + "resolve_model_type_from_hf", + "resolve_runtime_model_name", +] diff --git a/experimental/lite/megatron/lite/model/deepseek_v4/__init__.py b/experimental/lite/megatron/lite/model/deepseek_v4/__init__.py new file mode 100644 index 00000000000..96bd0214bad --- /dev/null +++ b/experimental/lite/megatron/lite/model/deepseek_v4/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +from .config import DeepseekV4Config diff --git a/experimental/lite/megatron/lite/model/deepseek_v4/config.py b/experimental/lite/megatron/lite/model/deepseek_v4/config.py new file mode 100644 index 00000000000..2b095556448 --- /dev/null +++ b/experimental/lite/megatron/lite/model/deepseek_v4/config.py @@ -0,0 +1,82 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +from __future__ import annotations + +from dataclasses import dataclass, field, fields +from typing import Any + +from megatron.lite.primitive.config import load_hf_config_dict + + +@dataclass +class DeepseekV4Config: + vocab_size: int = 129280 + hidden_size: int = 4096 + moe_intermediate_size: int = 2048 + num_hidden_layers: int = 43 + num_attention_heads: int = 64 + num_key_value_heads: int = 1 + head_dim: int = 128 + qk_rope_head_dim: int = 64 + q_lora_rank: int = 1024 + o_lora_rank: int = 1024 + o_groups: int = 8 + n_routed_experts: int = 256 + n_shared_experts: int = 1 + num_experts_per_tok: int = 6 + routed_scaling_factor: float = 1.5 + norm_topk_prob: bool = True + scoring_func: str = "sqrtsoftplus" + swiglu_limit: float = 10.0 + max_position_embeddings: int = 1_048_576 + rope_theta: float = 10_000.0 + compress_rope_theta: float = 160_000.0 + rotary_scaling_factor: float = 40.0 + original_max_position_embeddings: int = 4096 + beta_fast: float = 32.0 + beta_slow: float = 1.0 + compress_ratios: list[int] = field(default_factory=list) + sliding_window: int = 128 + num_hash_layers: int = 3 + hc_eps: float = 1e-6 + hc_mult: int = 4 + hc_sinkhorn_iters: int = 20 + index_head_dim: int = 128 + index_n_heads: int = 64 + index_topk: int = 512 + num_nextn_predict_layers: int = 1 + mtp_loss_scaling_factor: float = 0.1 + rms_norm_eps: float = 1e-6 + initializer_range: float = 0.02 + + @property + def num_experts(self) -> int: + return self.n_routed_experts + + @classmethod + def from_hf(cls, path: str, **overrides) -> DeepseekV4Config: + return cls._from_hf_dict(load_hf_config_dict(path), **overrides) + + @classmethod + def _from_hf_dict(cls, hf: dict[str, Any], **overrides) -> DeepseekV4Config: + hf_fields = {item.name for item in fields(cls)} + kwargs = {key: value for key, value in hf.items() if key in hf_fields and value is not None} + if "num_nextn_predict_layers" not in kwargs and hf.get("num_nextn_predict") is not None: + kwargs["num_nextn_predict_layers"] = int(hf["num_nextn_predict"]) + rope_parameters = hf.get("rope_parameters") + if isinstance(rope_parameters, dict): + if "rope_theta" not in kwargs: + kwargs["rope_theta"] = float(rope_parameters.get("rope_theta", cls.rope_theta)) + rope_scaling = hf.get("rope_scaling") + if isinstance(rope_scaling, dict): + if rope_scaling.get("factor") is not None: + kwargs["rotary_scaling_factor"] = float(rope_scaling["factor"]) + if rope_scaling.get("original_max_position_embeddings") is not None: + kwargs["original_max_position_embeddings"] = int( + rope_scaling["original_max_position_embeddings"] + ) + if rope_scaling.get("beta_fast") is not None: + kwargs["beta_fast"] = float(rope_scaling["beta_fast"]) + if rope_scaling.get("beta_slow") is not None: + kwargs["beta_slow"] = float(rope_scaling["beta_slow"]) + kwargs.update(overrides) + return cls(**kwargs) diff --git a/experimental/lite/megatron/lite/model/deepseek_v4/lite/__init__.py b/experimental/lite/megatron/lite/model/deepseek_v4/lite/__init__.py new file mode 100644 index 00000000000..12f41bb5a9e --- /dev/null +++ b/experimental/lite/megatron/lite/model/deepseek_v4/lite/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Native DeepSeek V4 (ds4flash) lite implementation.""" + +from megatron.lite.model.deepseek_v4.lite.model import DeepseekV4Model + +__all__ = ["DeepseekV4Model"] diff --git a/experimental/lite/megatron/lite/model/deepseek_v4/lite/checkpoint.py b/experimental/lite/megatron/lite/model/deepseek_v4/lite/checkpoint.py new file mode 100644 index 00000000000..922764c1309 --- /dev/null +++ b/experimental/lite/megatron/lite/model/deepseek_v4/lite/checkpoint.py @@ -0,0 +1,518 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""DeepSeek V4 (ds4flash) lite native <-> HF checkpoint mapping. + +Like kimi_k2 / glm5: ``DeepseekV4WeightSpec`` encodes the per-param native -> HF +name (+ TP/EP shard spec); export/save route through the shared +``primitive/ckpt/hf_weights.py`` exporter (its PP ``all_gather_object`` is +reached by all ranks before any ``rank0_only`` filter, so PP>1 export doesn't +desync). Native names are bare ``DeepseekV4Model`` keys; ``self.layers`` is a +ModuleDict keyed by GLOBAL layer index (kimi uses a local ModuleList), so the +exporter's local->global remap is an identity here. + +HF targets are canonical HF DeepSeek (``model.embed_tokens.weight`` / +``model.norm.weight`` / ``lm_head.weight`` / ``model.layers..self_attn.*`` / +``...mlp.experts..{gate,up,down}_proj.weight``). DS4 extras: + * CSA: ``self_attn.*`` incl. ``compressor.*`` / ``indexer.*``; ``sinks`` -> + ``self_attn.attn_sink``. + * mHC: ``attn_hc`` / ``ffn_hc`` -> ``...self_attn.hc_*`` / ``...mlp.hc_*``; + model-wide ``hc_head`` -> ``model.hc_head.*`` (no HF analogue, kept + model.-rooted; fidelity vs Megatron's latest mHC is a TODO). + * MTP: folded into the decoder namespace at ``model.layers.``. + +CSA is not TP-capable: DS4 runs TP=ETP=1 (only EP shards experts), like GLM-5. +""" + +from __future__ import annotations + +import math +import re + +import torch +import torch.distributed as dist +import torch.nn as nn + +from megatron.lite.model.deepseek_v4.config import DeepseekV4Config +from megatron.lite.primitive.ckpt.hf_weights import ( + SafeTensorReader, + _cast_export_tensor, + _resolve_export_dtype, + parse_expert_idx, + to_global_layer_name, + unwrap_model, +) +from megatron.lite.primitive.parallel import ParallelState +from megatron.lite.primitive.utils import ensure_divisible, log_rank0 + + +def EXPERT_CLASSIFIER(name: str) -> bool: + return ".experts." in name and ".shared_experts." not in name + + +def PLACEMENT_FN(param_name: str) -> list: + # distckpt sharded placement (TP=ETP=1 for ds4; shares kimi/glm5's + # Experts/SwiGLUMLP/VocabParallel structure). EP-sharded experts must carry + # an explicit placement or the dist-opt checkpoint won't restore them + # bit-exactly. The CSA/mHC/MTP-norm params fall through to all-Replicate. + from torch.distributed.tensor import Replicate, Shard + + if ".experts." in param_name and ".shared_experts." not in param_name: + if "fc1" in param_name: + return [Replicate(), Replicate(), Shard(0), Shard(0)] + if "fc2" in param_name: + return [Replicate(), Replicate(), Shard(0), Shard(1)] + return [Replicate(), Replicate(), Replicate(), Replicate()] + if "eh_proj.linear.weight" in param_name: + return [Replicate(), Replicate(), Replicate(), Shard(0)] + if "gate_up" in param_name: + return [Replicate(), Replicate(), Replicate(), Shard(0)] + if "down" in param_name: + return [Replicate(), Replicate(), Replicate(), Shard(1)] + if "embed" in param_name or "head" in param_name: + return [Replicate(), Replicate(), Replicate(), Shard(0)] + return [Replicate(), Replicate(), Replicate(), Replicate()] + + +# Native <-> HF name mapping (shared by export spec and load path). Native +# names are bare DeepseekV4Model state_dict keys with GLOBAL layer indices. +_BLOCK_KEY_RE = re.compile(r"^(layers|mtp)\.(\d+)\.(.+)$") +_GROUPED_EXPERT_RE = re.compile(r"^mlp\.experts\.fc([12])\.weight(\d+)$") +# Native top-level params -> real DeepSeek-V4-Flash release names (NOT DeepSeek-V3 HF +# names; the V4 release uses bare `embed.weight` / `head.weight` / `norm.weight` and a +# `layers.N.attn.* / ffn.* / hc_*` layout). This same mapping drives both the load path +# and the export spec, so MLite round-trips against the real release / vLLM ds4 format. +_TOP_LEVEL = { + "embed_tokens.embedding.weight": "embed.weight", + "norm.weight": "norm.weight", + "hc_head.hc_fn": "hc_head_fn", + "hc_head.hc_base": "hc_head_base", + "hc_head.hc_scale": "hc_head_scale", + "lm_head.col.linear.weight": "head.weight", +} + + +def _map_block_attr(attr: str, block: str) -> str | tuple[str, ...] | None: + """Map a native per-block attr -> real V4-Flash suffix (relative to layers.N / mtp.N).""" + if attr == "input_layernorm.weight": + return "attn_norm.weight" + if attr == "post_attention_layernorm.weight": + return "ffn_norm.weight" + # CSA attention: native `self_attn.self_attn.*` (the SBHD wrapper adds one extra + # `self_attn` level) -> real `attn.*`. Covers compressor.* / indexer.* / wq_a / wkv / ... + sub = None + if attr.startswith("self_attn.self_attn."): + sub = attr.removeprefix("self_attn.self_attn.") + elif attr.startswith("self_attn."): + sub = attr.removeprefix("self_attn.") + if sub is not None: + return "attn.attn_sink" if sub == "sinks" else f"attn.{sub}" + if attr.startswith("mlp.gate."): + suffix = attr.removeprefix("mlp.gate.") + return "ffn.gate." + { + "gate.weight": "weight", + "weight": "weight", + "expert_bias": "bias", + "e_score_correction_bias": "bias", + "tid2eid": "tid2eid", + }.get(suffix, suffix) + if attr.startswith("mlp.shared_experts."): + proj = attr.removeprefix("mlp.shared_experts.").removesuffix(".weight") + if proj == "gate_up": + return "ffn.shared_experts.w1.weight", "ffn.shared_experts.w3.weight" + if proj == "down": + return "ffn.shared_experts.w2.weight" + return f"ffn.shared_experts.{proj}.weight" + # mHC (hyper-connections): native attn_hc/ffn_hc.{base,fn,scale} -> hc_attn_*/hc_ffn_*; + # mtp carries its own hc_head.hc_* -> hc_head_*. + for prefix, target in (("attn_hc", "hc_attn"), ("ffn_hc", "hc_ffn")): + if attr.startswith(f"{prefix}."): + return f"{target}_{attr.rsplit('.', 1)[-1]}" + if attr.startswith("hc_head."): + return f"hc_head_{attr.rsplit('.', 1)[-1].removeprefix('hc_')}" + if block == "mtp" and attr in { + "e_proj.weight", + "h_proj.weight", + "enorm.weight", + "hnorm.weight", + "norm.weight", + }: + return attr + return None + + +def _global_expert_idx_from_local(local_idx: int, config: DeepseekV4Config, ps: ParallelState) -> int: + num_local = ensure_divisible(config.n_routed_experts, ps.ep_size) + return ps.ep_rank * num_local + local_idx + + +def _hf_names_for_state_key(name: str, config: DeepseekV4Config) -> list[str]: + """Map a bare DS4 native key (global layer idx, global expert id) to HF name(s). + + Callers (load path + shared exporter) supply global expert ids first. + """ + mapped = _TOP_LEVEL.get(name) + if mapped is not None: + return [mapped] + match = _BLOCK_KEY_RE.match(name) + if match is None: + return [] + block, index, attr = match.groups() + # Real V4-Flash keeps decoder layers under ``layers.{i}`` and the MTP block under its + # own ``mtp.{i}`` namespace (no ``model.`` prefix, no continued global index). + prefix = f"layers.{index}" if block == "layers" else f"mtp.{index}" + mapped = _map_block_attr(attr, block) + if mapped is not None: + if isinstance(mapped, tuple): + return [f"{prefix}.{part}" for part in mapped] + return [f"{prefix}.{mapped}"] + expert = _GROUPED_EXPERT_RE.match(attr) + if expert is None: + return [] + fc, expert_id = expert.groups() + # native fused gate_up (fc1) -> real w1 (gate) + w3 (up); fc2 -> w2 (down). + expert_prefix = f"{prefix}.ffn.experts.{int(expert_id)}" + if fc == "1": + return [f"{expert_prefix}.w1.weight", f"{expert_prefix}.w3.weight"] + return [f"{expert_prefix}.w2.weight"] + + +# ====================================================================== +# FP4 / scaled-tensor dequant helpers (load path). +# ====================================================================== + +_FP4_E2M1_TABLE = ( + 0.0, + 0.5, + 1.0, + 1.5, + 2.0, + 3.0, + 4.0, + 6.0, + 0.0, + -0.5, + -1.0, + -1.5, + -2.0, + -3.0, + -4.0, + -6.0, +) + + +def _has(reader: SafeTensorReader, name: str) -> bool: + if reader.index: + return name in reader.index + try: + reader.get_tensor(name) + except Exception: + return False + return True + + +def _is_native_metadata_key(name: str) -> bool: + return name.endswith("._extra_state") + + +def _scale_name_for_hf_name(name: str) -> str: + return f"{name[:-7] if name.endswith('.weight') else name}.scale" + + +def _scale_to_float(scale: torch.Tensor) -> torch.Tensor: + if scale.dtype.is_floating_point: + return scale.float() + if scale.dtype == torch.uint8: + return torch.pow(torch.tensor(2.0, dtype=torch.float32), scale.float() - 127.0) + return scale.float() + + +def _expand_block_scale( + scale: torch.Tensor, target_shape: torch.Size | tuple[int, ...] +) -> torch.Tensor: + target = tuple(int(dim) for dim in target_shape) + while scale.ndim > len(target) and scale.shape[0] == 1: + scale = scale.squeeze(0) + while scale.ndim < len(target): + scale = scale.unsqueeze(-1) + if tuple(scale.shape) == target: + return scale + out = scale + for dim, size in enumerate(target): + if out.shape[dim] == size: + continue + repeat = math.ceil(size / out.shape[dim]) + out = out.repeat_interleave(repeat, dim=dim) + slices = tuple(slice(0, size) for size in target) + return out[slices] + + +def _unpack_fp4_e2m1_if_needed( + tensor: torch.Tensor, target_shape: torch.Size | tuple[int, ...] +) -> torch.Tensor: + target = tuple(int(dim) for dim in target_shape) + if ( + tensor.dtype != torch.int8 + or tensor.ndim != len(target) + or tuple(tensor.shape[:-1]) != target[:-1] + or tensor.shape[-1] * 2 != target[-1] + ): + return tensor.float() + + table = torch.tensor(_FP4_E2M1_TABLE, dtype=torch.float32, device=tensor.device) + packed = tensor.view(torch.uint8) + low = packed & 0x0F + high = (packed >> 4) & 0x0F + return torch.stack((table[low.long()], table[high.long()]), dim=-1).flatten(-2) + + +def _dequantize_scaled_tensor( + tensor: torch.Tensor, scale: torch.Tensor, shape: torch.Size +) -> torch.Tensor: + scale_f = _expand_block_scale(_scale_to_float(scale), shape) + return _unpack_fp4_e2m1_if_needed(tensor, shape) * scale_f + + +def _copy_param( + param: nn.Parameter | torch.Tensor, + tensor: torch.Tensor, + *, + scale: torch.Tensor | None = None, +) -> None: + if scale is not None: + tensor = _dequantize_scaled_tensor(tensor, scale, param.shape) + elif param.dtype.is_floating_point and not tensor.dtype.is_floating_point: + raise RuntimeError( + f"Refusing to copy quantized tensor with dtype {tensor.dtype} into {tuple(param.shape)} " + "without a matching .scale tensor." + ) + param.data.copy_(tensor.to(device=param.device, dtype=param.dtype)) + + +def _read_hf_tensor( + reader: SafeTensorReader, hf_name: str, target_shape: torch.Size | tuple[int, ...] +) -> torch.Tensor: + scale_name = _scale_name_for_hf_name(hf_name) + tensor = reader.get_tensor(hf_name) + scale = reader.get_tensor(scale_name) if _has(reader, scale_name) else None + if scale is not None: + return _dequantize_scaled_tensor(tensor, scale, torch.Size(target_shape)) + return tensor + + +def load_hf_weights( + model: nn.Module, path: str, config: DeepseekV4Config, ps: ParallelState +) -> None: + """Load HF safetensors into the DS4 model. + + Kept as DS4's native loader (the inverse of the spec's ``native_to_hf``): + it walks the native ``state_dict`` and resolves each key's HF name(s) via + ``_hf_names_for_state_key`` -- the SAME mapping the export spec uses, so the + round-trip names stay consistent. EP-local expert ids are converted to + global before mapping. CSA is TP=ETP=1, so there is no TP split here. + + Under PP the ``self.layers`` ModuleDict is keyed by LOCAL pipeline position, + so -- like the exporter -- the local layer index is lifted to global before + mapping (identity at PP=1); else a non-first stage reads the wrong layer. + """ + if (ps.tp_size, ps.etp_size) != (1, 1): + raise NotImplementedError("DeepSeek V4 direct HF load currently supports only TP=ETP=1.") + + reader = SafeTensorReader(path) + base_model = unwrap_model(model) + state = base_model.state_dict() + # local pipeline position -> global layer index (identity at PP=1) + layer_map = ( + {i: base_model.layer_indices[i] for i in range(len(base_model.layer_indices))} + if hasattr(base_model, "layer_indices") + else {} + ) + loaded = 0 + missing: list[str] = [] + for name, target in state.items(): + if _is_native_metadata_key(name): + continue + global_name = to_global_layer_name(name, layer_map) + hf_names = _hf_names_for_state_key(_to_global_expert_name(global_name, config, ps), config) + if not hf_names or not all(_has(reader, hf_name) for hf_name in hf_names): + missing.append(name) + continue + if len(hf_names) == 2: + first = target.shape[0] // 2 + tensor = torch.cat( + [ + _read_hf_tensor(reader, hf_names[0], (first, *target.shape[1:])), + _read_hf_tensor( + reader, hf_names[1], (target.shape[0] - first, *target.shape[1:]) + ), + ], + dim=0, + ) + target.data.copy_(tensor.to(device=target.device, dtype=target.dtype)) + else: + scale_name = _scale_name_for_hf_name(hf_names[0]) + scale = reader.get_tensor(scale_name) if _has(reader, scale_name) else None + _copy_param(target, reader.get_tensor(hf_names[0]), scale=scale) + loaded += 1 + + log_rank0(f"DeepSeek V4 native loaded {loaded} tensors from {path}") + for name in missing: + log_rank0(f"WARNING: DeepSeek V4 checkpoint tensor missing: {name}") + + +def _to_global_expert_name(name: str, config: DeepseekV4Config, ps: ParallelState) -> str: + """Rewrite an EP-local expert ``weight`` suffix to its global id. + + The native ``state_dict`` carries the EP-local expert index; the HF target + name uses the global expert id. Non-expert names pass through unchanged. + """ + match = _BLOCK_KEY_RE.match(name) + if match is None: + return name + block, index, attr = match.groups() + expert = _GROUPED_EXPERT_RE.match(attr) + if expert is None: + return name + fc, local_idx = expert.groups() + global_idx = _global_expert_idx_from_local(int(local_idx), config, ps) + return f"{block}.{index}.mlp.experts.fc{fc}.weight{global_idx}" + + +# ====================================================================== +# Export: shared TP/ETP/EP/PP gather via DeepseekV4WeightSpec. +# ====================================================================== + + +class DeepseekV4WeightSpec: + """Export DS4 lite weights to HF DeepSeek-V4 names (CSA / mHC / MTP / MoE). + + Mirrors ``KimiK2WeightSpec`` / ``Glm5WeightSpec`` on DS4's bare native names + with global layer indices. The shared exporter rewrites EP-local expert + ``weight`` ids to global before calling ``native_to_hf``. + """ + + def __init__(self, config: DeepseekV4Config): + self.config = config + + @property + def num_experts(self) -> int: + return self.config.n_routed_experts + + def weight_map(self) -> dict[str, list[str]]: + return {} + + def hf_to_native(self, native_name: str, hf_tensors: list[torch.Tensor]) -> torch.Tensor: + del native_name + return hf_tensors[0] + + def native_to_hf( + self, native_name: str, tensor: torch.Tensor + ) -> list[tuple[str, torch.Tensor]]: + # ``native_name`` is the global native name; experts already carry the + # global expert id (shared exporter rewrote weight -> weight). + hf_names = _hf_names_for_state_key(native_name, self.config) + if not hf_names: + return [] + if len(hf_names) == 1: + return [(hf_names[0], tensor)] + if len(hf_names) == 2: + # 2 targets == fused gate/up split into (w1, w3) for shared/routed + # experts; split the leading dim exactly as the bespoke export did. + first, second = tensor.chunk(2, dim=0) + return [ + (hf_names[0], first.contiguous()), + (hf_names[1], second.contiguous()), + ] + raise AssertionError(f"Unexpected HF name fan-out for {native_name}: {hf_names}") + + def qkv_spec(self, native_name: str) -> tuple[int, int, int] | None: + del native_name + return None + + def tp_spec(self, native_name: str) -> tuple[int, int] | None: + # DS4 is TP=ETP=1 (CSA is not TP-capable); only EP shards experts. The + # expert (split_dim, ETP) entries are declared so the shared ETP path + # would be correct if ETP were ever enabled; embed/head/eh_proj carry + # the vocab split-dim spec for completeness (no-op at TP=1). + if self.is_expert(native_name): + if ".fc1." in native_name: + return (0, 1) + if ".fc2." in native_name: + return (1, 1) + return None + if native_name.endswith(".eh_proj.linear.weight"): + return (0, 0) + if native_name in { + "embed_tokens.embedding.weight", + "lm_head.col.linear.weight", + }: + return (0, 0) + return None + + def is_expert(self, native_name: str) -> bool: + return ".mlp.experts." in native_name and ".shared_experts." not in native_name + + def expert_global_id(self, native_name: str) -> int | None: + if self.is_expert(native_name): + return parse_expert_idx(native_name) + return None + + def expert_local_name(self, native_name: str, local_idx: int) -> str: + prefix = native_name.rsplit(".weight", 1)[0] + return f"{prefix}.weight{local_idx}" + + +def export_hf_weights(model, config: DeepseekV4Config, ps: ParallelState, **kwargs): + """Export DS4 weights as HF (name, tensor) pairs via the SHARED exporter. + + Identical structure to kimi/glm5: delegate to the shared ``_export`` (which + does the TP/ETP/EP/PP gather, including the PP ``all_gather_object`` reached + by ALL ranks before any ``rank0_only`` filter), then append the persistent + router buffers (``tid2eid`` for hash layers, ``expert_bias`` for non-hash + layers) which the parameter-only ``_export`` does not visit. + """ + from megatron.lite.primitive.ckpt.hf_weights import export_hf_weights as _export + + spec = DeepseekV4WeightSpec(config) + rank0_only = bool(kwargs.get("rank0_only", False)) + export_dtype = _resolve_export_dtype(kwargs.get("export_dtype")) + yield from _export(model, spec, ps, vocab_size=config.vocab_size, **kwargs) + + rank = dist.get_rank() if dist.is_initialized() else 0 + if rank0_only and rank != 0: + return + chunks = list(model) if isinstance(model, list | nn.ModuleList) else [model] + for chunk in chunks: + base_chunk = unwrap_model(chunk) + layer_map = ( + {i: base_chunk.layer_indices[i] for i in range(len(base_chunk.layer_indices))} + if hasattr(base_chunk, "layer_indices") + else {} + ) + for name, buffer in base_chunk.named_buffers(): + # Persistent router buffers carried into HF: hash-layer ``tid2eid`` + # and the (made-persistent for non-hash layers) ``expert_bias``. + if not (name.endswith(".mlp.gate.tid2eid") or name.endswith(".mlp.gate.expert_bias")): + continue + global_name = to_global_layer_name(name, layer_map) + for hf_name, hf_tensor in spec.native_to_hf(global_name, buffer.detach().cpu()): + yield hf_name, _cast_export_tensor(hf_tensor, export_dtype) + + +def save_hf_weights(model, path: str, config: DeepseekV4Config, ps: ParallelState, **kwargs) -> None: + from megatron.lite.primitive.ckpt.hf_weights import save_safetensors + + rank = dist.get_rank() if dist.is_initialized() else 0 + out = dict(export_hf_weights(model, config, ps, rank0_only=True, **kwargs)) + if rank == 0 and out: + save_safetensors(out, path) + if dist.is_initialized(): + dist.barrier() + + +__all__ = [ + "EXPERT_CLASSIFIER", + "DeepseekV4WeightSpec", + "PLACEMENT_FN", + "export_hf_weights", + "load_hf_weights", + "save_hf_weights", +] diff --git a/experimental/lite/megatron/lite/model/deepseek_v4/lite/model.py b/experimental/lite/megatron/lite/model/deepseek_v4/lite/model.py new file mode 100644 index 00000000000..e83dd92c544 --- /dev/null +++ b/experimental/lite/megatron/lite/model/deepseek_v4/lite/model.py @@ -0,0 +1,615 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""DeepSeek V4 (ds4flash) lite native model. + +A clone of the Kimi-K2 (deepseek_v3) lite model -- same approach as GLM-5 -- +inheriting Kimi's Megatron plumbing (SBHD ``[S, B, H]`` layout, VocabParallel +embed/head, ``set_input_tensor``, ``build_pipeline_chunk_layout`` boundaries, +MTP via ``layout.has_mtp``, dist-opt/distckpt via the protocol). Three +model-wide DS4 deviations (documented inline at each site): + +1. Attention = CSA, not MLA. The CSA primitive is batch-first ``[B, S, H]`` and + needs explicit ``position_ids``; ``DeepseekV4CSAAttention`` wraps it with an + SBHD<->BSHD shim (like GLM-5's DSA shim). Per-layer behaviour is driven by + ``config.compress_ratios[layer_idx]`` inside CSA. + +2. mHC (multi-head hyper-connection): the hidden carries ``hc_mult`` parallel + residual streams (4-D ``[S, B, hc_mult, H]`` in SBHD), expanded after embed + and contracted before the head, persisting across layers and PP stages; each + layer wraps attn/FFN in a ``HyperConnection``. At PP boundaries the 4-D + hidden folds to 3-D ``[S, B, hc_mult*H]`` to match the P2P buffer + (``_infer_pipeline_tensor_shape`` scales hidden by ``hc_mult``); fold/unfold + live in ``primitive/parallel/mhc.py``. + +3. MoE = hash-routed DeepSeek: first ``num_hash_layers`` use a token-id hash + route, the rest the shared sigmoid-topk router (``DeepseekV4MoE`` over the + shared Experts/Router/Dispatcher). + +CSA is not TP-capable, so DS4 is a documented TP=1 case (protocol gate raises +for TP>1/ETP>1); VPP/PP/EP/CP work, inherited from the Kimi skeleton. +""" + +from __future__ import annotations + +import os +from contextlib import nullcontext + +import torch +import torch.nn as nn +import transformer_engine.pytorch as te + +from megatron.lite.model.deepseek_v4.config import DeepseekV4Config +from megatron.lite.model.deepseek_v4.lite.moe import DeepseekV4MoE +from megatron.lite.primitive.modules.attention.csa import CompressedSparseAttention +from megatron.lite.primitive.modules.attention.hca import HyperConnection +from megatron.lite.primitive.modules.attention.mhc import MultiHeadHyperConnectionHead +from megatron.lite.primitive.modules.mtp import MTPLossAutoScaler +from megatron.lite.primitive.ops.cross_entropy import vocab_parallel_cross_entropy +from megatron.lite.primitive.ops.linear_cross_entropy import linear_cross_entropy +from megatron.lite.primitive.ops.logprob import vocab_parallel_entropy +from megatron.lite.primitive.parallel import ( + ParallelState, + VocabParallelEmbedding, + VocabParallelOutput, + build_pipeline_chunk_layout, + gather_from_sequence_parallel, + scatter_to_sequence_parallel, +) +from megatron.lite.primitive.parallel.mhc import ( + contract_mhc_hidden_for_pipeline, + expand_mhc_hidden_for_pipeline, + fold_mhc_hidden_for_pipeline, + unfold_mhc_hidden_from_pipeline, +) +from megatron.lite.primitive.utils import build_fp8_recipe + + +def _roll_mtp_left( + tensor: torch.Tensor, + *, + dims: int = -1, +) -> tuple[torch.Tensor, torch.Tensor]: + """Shift labels/ids one position left (next-token target for MTP depth d). + + DS4 inputs are dense ``[B, S]`` (TP=1, MTP runs only at CP==1), so -- unlike + Kimi -- there is no packed-THD branch here; a plain ``torch.roll`` on the + sequence dim suffices. + """ + dim = dims if dims >= 0 else tensor.dim() + dims + rolled = torch.roll(tensor, shifts=-1, dims=dim) + rolled.select(dim, -1).zero_() + return rolled, rolled.sum() + + +# -- DS4 ONLY: CSA attention wrapper. Holds ``CompressedSparseAttention`` and +# adapts it to the skeleton's SBHD-in / SBHD-out attention contract, mirroring +# GLM-5's ``Glm5DSAAttention`` DSA shim. Per-layer behaviour (window / compress +# / indexer) is selected inside CSA via ``config.compress_ratios[layer_idx]``. +class DeepseekV4CSAAttention(nn.Module): + """SBHD ``[S, B, H]`` shim around the batch-first CSA primitive. + + The skeleton hands attention a 3-D SBHD tensor ``[S, B, H]`` (the + hyper-connection has already collapsed the ``hc_mult`` streams to a single + pre-mix stream). CSA is hard-wired ``[B, S, H]`` and needs ``position_ids``. + This wrapper transposes ``[S, B, H] -> [B, S, H]``, runs CSA, and transposes + the ``[B, S, H]`` output back to ``[S, B, H]``. The skeleton therefore never + observes the batch-first interior. + """ + + def __init__(self, config: DeepseekV4Config, *, layer_idx: int, ps: ParallelState): + super().__init__() + self.ps = ps + self.self_attn = CompressedSparseAttention(config, layer_idx=layer_idx, ps=ps) + + def forward(self, x: torch.Tensor, *, position_ids: torch.Tensor) -> torch.Tensor: + # Skeleton feeds SBHD [S, B, H]; CSA needs batch-first [B, S, H]. + x_bsh = x.transpose(0, 1).contiguous() + out_bsh = self.self_attn(x_bsh, position_ids=position_ids, attention_mask=None) + # Back to SBHD [S, B, H] for the skeleton. + return out_bsh.transpose(0, 1).contiguous() + + +class DeepseekV4Layer(nn.Module): + """One decoder layer. + + Structurally the Kimi layer, but the plain ``x = x + sublayer(norm(x))`` + residual is replaced by DS4's per-layer ``HyperConnection`` (mHC), and + attention is CSA via the SBHD shim above. The hidden is the 4-D SBHD mHC + tensor ``[S, B, hc_mult, H]`` end-to-end. Attribute names (``self_attn`` / + ``input_layernorm`` / ``post_attention_layernorm`` / ``mlp`` / ``attn_hc`` / + ``ffn_hc``) are preserved from the previous DS4 model so the HF checkpoint + weight names are unchanged. + """ + + def __init__( + self, + config: DeepseekV4Config, + ps: ParallelState, + layer_idx: int, + *, + use_deepep: bool = False, + ): + super().__init__() + self.layer_idx = layer_idx + self.ps = ps + self.input_layernorm = te.RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.post_attention_layernorm = te.RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + # DS4 ONLY: CSA attention behind the SBHD shim (Kimi builds MLA here). + self.self_attn = DeepseekV4CSAAttention(config, layer_idx=layer_idx, ps=ps) + # DS4 ONLY: hash-routed MoE family (shared Experts/Router/dispatcher). + self.mlp = DeepseekV4MoE(config, ps, layer_idx=layer_idx, use_deepep=use_deepep) + # DS4 ONLY: per-layer multi-head hyper-connections wrapping attn + ffn. + self.attn_hc = HyperConnection( + config.hidden_size, config.hc_mult, config.hc_sinkhorn_iters, config.hc_eps + ) + self.ffn_hc = HyperConnection( + config.hidden_size, config.hc_mult, config.hc_sinkhorn_iters, config.hc_eps + ) + + def forward( + self, + x: torch.Tensor, + *, + position_ids: torch.Tensor, + input_ids: torch.Tensor | None = None, + ) -> torch.Tensor: + # x is SBHD mHC [S, B, hc_mult, H]. HyperConnection collapses the + # streams to a 3-D [S, B, H] pre-mix, the sub-block runs SBHD, and + # HyperConnection.post recombines into the 4-D residual streams. + residual = x + attn_in, post, comb = self.attn_hc(x) + attn_out = self.self_attn(self.input_layernorm(attn_in), position_ids=position_ids) + x = HyperConnection.post(attn_out, residual, post, comb) + + residual = x + ffn_in, post, comb = self.ffn_hc(x) + # DS4 hash-routed MoE indexes tid2eid[input_ids.reshape(-1)] and must + # align with the flattened hidden. The skeleton is SBHD, so the FFN + # input flattens in (S, B) order; transpose input_ids [B, S] -> [S, B] + # so its flatten matches. (No-op semantics for non-hash layers.) + mlp_input_ids = None if input_ids is None else input_ids.transpose(0, 1).contiguous() + ffn_out = self.mlp(self.post_attention_layernorm(ffn_in), input_ids=mlp_input_ids) + return HyperConnection.post(ffn_out, residual, post, comb) + + +class DeepseekV4MTPLayer(DeepseekV4Layer): + """MTP depth layer: Kimi's MTP combiner, adapted to DS4's mHC hidden. + + Like Kimi's ``KimiK2MTPLayer`` it owns ``enorm`` / ``hnorm`` and a projection + to fuse the rolled-token embedding with the running hidden, then runs one + transformer layer. DS4 differences: the projection uses DS4's + ``e_proj`` / ``h_proj`` names (preserved for HF export), the embedding / + hidden are lifted into the ``hc_mult`` streams, and ``contract`` collapses + the streams via ``hc_head`` + ``norm`` before the (shared) head. + """ + + def __init__( + self, + config: DeepseekV4Config, + ps: ParallelState, + layer_idx: int, + *, + embedding: VocabParallelEmbedding, + use_deepep: bool, + detach_encoder: bool, + ): + super().__init__(config, ps, layer_idx, use_deepep=use_deepep) + self.config = config + object.__setattr__(self, "embedding", embedding) + self.detach_encoder = detach_encoder + self.e_proj = nn.Linear(config.hidden_size, config.hidden_size, bias=False) + self.h_proj = nn.Linear(config.hidden_size, config.hidden_size, bias=False) + self.enorm = te.RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.hnorm = te.RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.norm = te.RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.hc_head = MultiHeadHyperConnectionHead(config.hidden_size, config.hc_mult, config.hc_eps) + + def forward( + self, + *, + input_ids: torch.Tensor, + hidden_states: torch.Tensor, + position_ids: torch.Tensor, + ) -> torch.Tensor: + # hidden_states is the per-stream mHC source [S, B, hc_mult, H]. + embedded = self.embedding(input_ids) + embedded = scatter_to_sequence_parallel(embedded, self.ps) + if self.detach_encoder: + embedded = embedded.detach() + hidden_states = hidden_states.detach() + embedded = self.enorm(embedded) + # e_proj on the [S, B, H] embedding, broadcast across the hc_mult streams; + # h_proj on the normed mHC hidden keeps the per-stream state. + projected = self.e_proj(embedded).unsqueeze(2) + self.h_proj(self.hnorm(hidden_states)) + return super().forward(projected, position_ids=position_ids, input_ids=input_ids) + + def contract(self, x: torch.Tensor) -> torch.Tensor: + # Collapse the hc_mult streams [S, B, hc_mult, H] -> [S, B, H]. + return self.norm(self.hc_head(x)) + + +def _temperature_to_float(temperature: float | torch.Tensor) -> float: + if isinstance(temperature, torch.Tensor): + if temperature.numel() != 1: + raise ValueError("DeepseekV4Model supports scalar temperature only.") + return float(temperature.detach().float().item()) + return float(temperature) + + +def _apply_attention_backend_override(backend: str | None) -> None: + if backend in (None, "flash"): + backend = "fused" + env = { + "auto": ("1", "1", "1"), + "flash": ("1", "0", "0"), + "fused": ("0", "1", "0"), + "unfused": ("0", "0", "1"), + "local": ("0", "0", "1"), + }.get(backend) + if env is None: + raise ValueError( + "attention_backend_override must be one of " + "{'auto', 'flash', 'fused', 'unfused', 'local'}" + ) + ( + os.environ["NVTE_FLASH_ATTN"], + os.environ["NVTE_FUSED_ATTN"], + os.environ["NVTE_UNFUSED_ATTN"], + ) = env + + +class DeepseekV4Model(nn.Module): + """DS4 model. Mirrors ``KimiK2Model``: ``build_pipeline_chunk_layout`` for + embed/head/layer placement, ``set_input_tensor`` for the PP recv buffer, a + shared embed/head, MTP gated on ``layout.has_mtp``, SBHD scatter at embed, + and a single forward producing the loss / logits / MTP outputs. + + DS4 differences (all documented inline): the hidden is the 4-D SBHD mHC + tensor; it is expanded after embed and contracted before the head; pipeline + stage boundaries fold/unfold the ``hc_mult`` streams to match the 3-D P2P + buffer; attention is CSA; the MoE is hash-routed and so layers receive + ``input_ids``. Attribute names (``embed_tokens`` / ``norm`` / ``lm_head`` / + ``hc_head`` / ``layers`` ModuleDict) are preserved from the previous DS4 so + HF checkpoint names are unchanged. + """ + + def __init__( + self, + config: DeepseekV4Config, + train_config, + ps: ParallelState, + *, + vpp_chunk_id: int | None = None, + use_thd: bool = False, + hf_path: str = "", + attention_backend_override: str | None = None, + mtp_enable: bool = False, + mtp_enable_train: bool = False, + mtp_detach_encoder: bool = False, + use_deepep: bool = False, + ): + super().__init__() + del hf_path, use_thd # DS4 CSA derives its own masking from position_ids. + _apply_attention_backend_override(attention_backend_override) + self.config = config + self.train_config = train_config + self.ps = ps + self.hc_mult = int(config.hc_mult) + self._input_tensor: torch.Tensor | None = None + self.mtp_enable_train = bool(mtp_enable and mtp_enable_train) + self.mtp_loss_scaling_factor = config.mtp_loss_scaling_factor + + layout = build_pipeline_chunk_layout( + config.num_hidden_layers, + ps, + train_config.vpp, + vpp_chunk_id, + num_mtp_layers=config.num_nextn_predict_layers if mtp_enable else 0, + ) + self.layer_indices = layout.layer_indices + self.pre_process = layout.has_embed + self.post_process = layout.has_head + # DS4 does not tie embeddings; the attribute is preserved for the + # dist-opt / distckpt interface (matches the previous DS4 model). + self.share_embeddings_and_output_weights = False + self.vision_model: nn.Module | None = None + + self.embed_tokens: VocabParallelEmbedding | None = None + if layout.has_embed: + self.embed_tokens = VocabParallelEmbedding(config.vocab_size, config.hidden_size, ps) + + # Key by LOCAL pipeline-stage position (0..len-1), not the global layer id, + # so parameter names ("layers.{local}.…") follow the same convention as the + # other lite models (glm5 / kimi_k2 use nn.ModuleList → local names). The + # shared HF weight map (build via enumerate(layer_indices)) is keyed by local + # position; keying this dict by the global id instead double-maps under an + # uneven PP split (a stage owning [1,2] would remap layers.1→layers.2 and + # drop layer 1 on export/load). The global id is still passed to the layer + # for its dense-vs-MoE / per-layer logic. + self.layers = nn.ModuleDict( + { + str(local): DeepseekV4Layer(config, ps, global_idx, use_deepep=use_deepep) + for local, global_idx in enumerate(self.layer_indices) + } + ) + + self.norm: nn.Module | None = None + self.hc_head: MultiHeadHyperConnectionHead | None = None + self.lm_head: VocabParallelOutput | None = None + if layout.has_head: + self.norm = te.RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.hc_head = MultiHeadHyperConnectionHead( + config.hidden_size, config.hc_mult, config.hc_eps + ) + self.lm_head = VocabParallelOutput(config.vocab_size, config.hidden_size, ps) + + self.mtp_embed: VocabParallelEmbedding | None = None + self.mtp: nn.ModuleList = nn.ModuleList() + if mtp_enable and config.num_nextn_predict_layers > 0 and layout.has_mtp: + mtp_embedding = self.embed_tokens + if mtp_embedding is None: + mtp_embedding = VocabParallelEmbedding(config.vocab_size, config.hidden_size, ps) + self.mtp_embed = mtp_embedding + self.mtp = nn.ModuleList( + [ + DeepseekV4MTPLayer( + config, + ps, + config.num_hidden_layers + idx, + embedding=mtp_embedding, + use_deepep=use_deepep, + detach_encoder=mtp_detach_encoder, + ) + for idx in range(config.num_nextn_predict_layers) + ] + ) + + def set_input_tensor(self, input_tensor): + if isinstance(input_tensor, list): + if len(input_tensor) > 1: + raise ValueError("DeepseekV4Model expects a single pipeline input tensor.") + input_tensor = input_tensor[0] if input_tensor else None + self._input_tensor = input_tensor + + def _embed_or_recv( + self, input_ids: torch.Tensor | None, hidden_states: torch.Tensor | None + ) -> tuple[torch.Tensor, torch.Tensor | None, int]: + """Return (mHC hidden [S, B, hc_mult, H], input_ids, seq_len) for this stage.""" + if self.embed_tokens is not None: + assert input_ids is not None + # Embed -> SBHD [S, B, H] -> SP scatter (no-op at TP=1) -> expand to + # the hc_mult parallel residual streams. + h = self.embed_tokens(input_ids) + h = scatter_to_sequence_parallel(h, self.ps) + seq_len = h.size(0) + h = expand_mhc_hidden_for_pipeline(h, hc_mult=self.hc_mult) + return h, input_ids, seq_len + if hidden_states is None: + hidden_states = self._input_tensor + assert hidden_states is not None + # Non-first PP stage: the previous stage folded the streams into the + # hidden dim ([S, B, hc_mult * H]); unfold back to [S, B, hc_mult, H] + # instead of re-expanding (which would discard the cross-stream state). + h = unfold_mhc_hidden_from_pipeline(hidden_states, hc_mult=self.hc_mult) + return h, input_ids, h.size(0) + + def forward( + self, + input_ids: torch.Tensor | None = None, + hidden_states: torch.Tensor | None = None, + position_ids: torch.Tensor | None = None, + labels: torch.Tensor | None = None, + loss_mask: torch.Tensor | None = None, + temperature: float | torch.Tensor = 1.0, + use_fused_kernels: bool = False, + calculate_entropy: bool = False, + enable_mtp: bool = True, + ) -> dict: + # THD-packed inputs may arrive 1-D ([total_tokens]); VocabParallelEmbedding + # and the dense skeleton expect [B, S], so add the batch row (matches the + # previous DS4 and the [1, S] single-sequence convention). + if input_ids is not None and input_ids.dim() == 1: + input_ids = input_ids.unsqueeze(0) + if position_ids is not None and position_ids.dim() == 1: + position_ids = position_ids.unsqueeze(0) + if labels is not None and labels.dim() == 1: + labels = labels.unsqueeze(0) + if loss_mask is not None and loss_mask.dim() == 1: + loss_mask = loss_mask.unsqueeze(0) + h, input_ids, _seq_len = self._embed_or_recv(input_ids, hidden_states) + + # CSA is batch-first and needs [B, S] position ids; build them from the + # SBHD hidden when not supplied. position_ids is the only forward arg + # that crosses into the batch-first CSA interior. + if position_ids is None: + seq_len, batch = h.size(0), h.size(1) + position_ids = ( + torch.arange(seq_len, device=h.device).unsqueeze(0).expand(batch, -1) + ) + + fp8_ctx = ( + te.fp8_autocast(enabled=True, fp8_recipe=build_fp8_recipe(self.train_config)) + if self.train_config.fp8 + else nullcontext() + ) + with fp8_ctx: + for layer in self.layers.values(): + h = layer(h, position_ids=position_ids, input_ids=input_ids) + + output: dict = {"hidden_states": fold_mhc_hidden_for_pipeline(h)} + if self.lm_head is None or self.norm is None or self.hc_head is None: + # Non-last PP stage: fold the hc_mult streams into the hidden dim so + # the pipeline P2P buffer ([S, B, hc_mult * H]) carries the full + # hyper-connection state. + return output + + # Last stage: contract the mHC streams, then run head / MTP / loss + # exactly as the Kimi skeleton does. + mtp_source = h + hidden_for_head = contract_mhc_hidden_for_pipeline(h, norm=self.norm, head=self.hc_head) + + # MTP runs on the head stage (where self.mtp is built with a valid bound + # embedding -- self.embed_tokens when present, else the self.mtp_embed + # fallback, mirroring Kimi). Disabled at CP>1 (the rolled MTP targets are + # not CP-sliced) and when no input_ids are available. + run_mtp = ( + enable_mtp + and input_ids is not None + and len(self.mtp) > 0 + and self.ps.cp_size == 1 + ) + mtp_hidden_states = self._apply_mtp( + mtp_source, input_ids=input_ids, position_ids=position_ids, run_mtp=run_mtp + ) + if mtp_hidden_states is not None: + output["mtp_hidden_states"] = mtp_hidden_states + + if labels is not None: + temperature_value = _temperature_to_float(temperature) + mtp_result = self._apply_mtp_loss( + hidden_for_head, + mtp_hidden_states=mtp_hidden_states, + labels=labels, + loss_mask=loss_mask, + temperature=temperature_value, + use_fused_kernels=use_fused_kernels, + ) + if mtp_result is not None: + hidden_for_head, mtp_loss = mtp_result + output["mtp_loss"] = mtp_loss + labels_sb = labels.transpose(0, 1).contiguous() + if use_fused_kernels: + hidden_full = gather_from_sequence_parallel(hidden_for_head, self.ps) + log_probs, entropy = linear_cross_entropy( + hidden_full, + self._head_weight_for_fused_ce(hidden_full), + labels_sb, + temperature_value, + self.ps.tp_group, + ) + output["loss"] = (-log_probs).mean() + output["log_probs"] = log_probs.transpose(0, 1).contiguous() + if calculate_entropy: + output["entropy"] = entropy.transpose(0, 1).contiguous() + else: + logits = self.lm_head(hidden_for_head) + if temperature_value != 1.0: + logits = logits / temperature_value + loss = vocab_parallel_cross_entropy(logits, labels_sb, self.ps.tp_group) + output["loss"] = loss.mean() + output["log_probs"] = (-loss).transpose(0, 1).contiguous() + if calculate_entropy: + entropy = vocab_parallel_entropy(logits, self.ps.tp_group) + output["entropy"] = entropy.transpose(0, 1).contiguous() + else: + logits = self.lm_head(hidden_for_head) + output["logits"] = self.lm_head.gather(logits).transpose(0, 1).contiguous() + if mtp_hidden_states is not None: + output["mtp_logits"] = [ + self.lm_head.gather(self.lm_head(mtp_hidden)).transpose(0, 1).contiguous() + for mtp_hidden in mtp_hidden_states + ] + return output + + def _apply_mtp( + self, + mtp_source: torch.Tensor, + *, + input_ids: torch.Tensor | None, + position_ids: torch.Tensor | None, + run_mtp: bool, + ) -> list[torch.Tensor] | None: + if not run_mtp: + return None + assert input_ids is not None + # DS4 MTP rolls input_ids per depth and runs each MTP layer on the + # running per-stream source, contracting each depth's output to [S, B, H] + # for the head. Mirrors Kimi's KimiK2MTPBlock loop. + mtp_input_ids = input_ids + source = mtp_source + outputs: list[torch.Tensor] = [] + for mtp_layer in self.mtp: + mtp_input_ids, _ = _roll_mtp_left(mtp_input_ids, dims=-1) + source = mtp_layer( + input_ids=mtp_input_ids, + hidden_states=source, + position_ids=position_ids, + ) + outputs.append(mtp_layer.contract(source)) + return outputs + + def _apply_mtp_loss( + self, + hidden_states: torch.Tensor, + *, + mtp_hidden_states: list[torch.Tensor] | None, + labels: torch.Tensor, + loss_mask: torch.Tensor | None, + temperature: float, + use_fused_kernels: bool, + ) -> tuple[torch.Tensor, torch.Tensor] | None: + if mtp_hidden_states is None or not self.mtp_enable_train: + return None + if loss_mask is None: + mtp_loss_mask = torch.ones_like(labels, dtype=torch.float32) + else: + mtp_loss_mask = loss_mask.to(dtype=torch.float32).clone() + mtp_labels = labels.clone() + + mtp_loss_values = [] + for mtp_hidden in mtp_hidden_states: + mtp_labels, _ = _roll_mtp_left(mtp_labels, dims=-1) + mtp_loss_mask, num_tokens = _roll_mtp_left(mtp_loss_mask, dims=-1) + labels_sb = mtp_labels.transpose(0, 1).contiguous() + mask_sb = mtp_loss_mask.transpose(0, 1).contiguous() + + if use_fused_kernels: + mtp_hidden_full = gather_from_sequence_parallel(mtp_hidden, self.ps) + log_probs, _entropy = linear_cross_entropy( + mtp_hidden_full, + self._head_weight_for_fused_ce(mtp_hidden_full), + labels_sb, + temperature, + self.ps.tp_group, + ) + token_loss = -log_probs + else: + logits = self.lm_head(mtp_hidden) + if temperature != 1.0: + logits = logits / temperature + token_loss = vocab_parallel_cross_entropy(logits, labels_sb, self.ps.tp_group) + + token_loss = token_loss * mask_sb.to(dtype=token_loss.dtype) + num_tokens = num_tokens.to(dtype=token_loss.dtype).clamp_min(1.0) + mtp_loss_values.append(token_loss.sum() / num_tokens) + + mtp_loss_scale = self.mtp_loss_scaling_factor / max(len(mtp_hidden_states), 1) + hidden_states = MTPLossAutoScaler.apply( + hidden_states, + mtp_loss_scale * token_loss / num_tokens, + ) + + if not mtp_loss_values: + return None + return ( + hidden_states, + torch.stack([loss.detach().float() for loss in mtp_loss_values]).mean(), + ) + + def _head_weight_for_fused_ce(self, hidden_states: torch.Tensor) -> torch.Tensor: + assert self.lm_head is not None + weight = self.lm_head.col.linear.weight + return ( + weight if weight.dtype == hidden_states.dtype else weight.to(dtype=hidden_states.dtype) + ) + + +__all__ = [ + "CompressedSparseAttention", + "DeepseekV4CSAAttention", + "DeepseekV4Layer", + "DeepseekV4Model", + "DeepseekV4MTPLayer", + "HyperConnection", + "MTPLossAutoScaler", + "MultiHeadHyperConnectionHead", +] diff --git a/experimental/lite/megatron/lite/model/deepseek_v4/lite/moe.py b/experimental/lite/megatron/lite/model/deepseek_v4/lite/moe.py new file mode 100644 index 00000000000..0ac8f85ad0d --- /dev/null +++ b/experimental/lite/megatron/lite/model/deepseek_v4/lite/moe.py @@ -0,0 +1,95 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +import torch +import torch.nn as nn +import torch.nn.functional as F + +from megatron.lite.model.deepseek_v4.config import DeepseekV4Config +from megatron.lite.primitive.modules.dispatcher import TokenDispatcher +from megatron.lite.primitive.modules.experts import Experts +from megatron.lite.primitive.modules.mlp import SwiGLUMLP +from megatron.lite.primitive.modules.router import SigmoidTopKRouter +from megatron.lite.primitive.parallel.state import ParallelState + + +class DeepseekV4MoE(nn.Module): + """Model-specific assembly over shared router, Experts, dispatcher, and shared MLP. + + Allowlist reason: this owns DS4 hash routing wiring, while expert compute stays shared. + """ + + def __init__( + self, + config: DeepseekV4Config, + ps: ParallelState, + *, + layer_idx: int, + use_deepep: bool = False, + ): + super().__init__() + self.hidden_size = config.hidden_size + self.topk = config.num_experts_per_tok + self.route_scale = config.routed_scaling_factor + self.is_hash_layer = layer_idx < config.num_hash_layers + self.gate = SigmoidTopKRouter(config, ps, compute_aux_loss=False) + if self.is_hash_layer: + self.gate.register_buffer( + "tid2eid", + torch.zeros(config.vocab_size, self.topk, dtype=torch.int64), + persistent=True, + ) + else: + self.gate._non_persistent_buffers_set.discard("expert_bias") + self.experts = Experts(config, ps) + shared_intermediate = config.n_shared_experts * config.moe_intermediate_size + self.shared_experts = ( + SwiGLUMLP( + config.hidden_size, + shared_intermediate, + swiglu_limit=config.swiglu_limit, + ) + if config.n_shared_experts > 0 + else None + ) + self.dispatcher = TokenDispatcher( + config.n_routed_experts, + config.hidden_size, + ps, + use_deepep=use_deepep, + ) + + def _hash_route( + self, + x: torch.Tensor, + input_ids: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + logits = self.gate.gate(x).view(-1, self.gate.num_experts) + if self.gate.score_function == "sqrtsoftplus": + scores = F.softplus(logits.float()).sqrt() + else: + scores = logits.float().sigmoid() + indices = self.gate.tid2eid[input_ids.reshape(-1).to(torch.int64)] + weights = scores.gather(1, indices) + if self.topk > 1: + weights = weights / (weights.sum(dim=-1, keepdim=True) + 1e-20) + return (weights * self.route_scale).to(dtype=x.dtype), indices + + def forward(self, x: torch.Tensor, *, input_ids: torch.Tensor | None = None) -> torch.Tensor: + shape = x.shape + x_flat = x.reshape(-1, self.hidden_size) + if self.is_hash_layer and input_ids is not None: + weights, indices = self._hash_route(x_flat, input_ids) + else: + weights, indices = self.gate(x_flat) + dispatched, tpe, permuted_probs = self.dispatcher.dispatch(x_flat, weights, indices) + del weights, indices + self.dispatcher.wait_dispatch_event() + out = self.experts( + dispatched, + tpe, + permuted_probs, + tokens_per_expert_list=getattr(self.dispatcher, "_local_tpe_list", None), + ) + out = self.dispatcher.combine(out) + if self.shared_experts is not None: + out = out + self.shared_experts(x_flat) + return out.view(shape) diff --git a/experimental/lite/megatron/lite/model/deepseek_v4/lite/protocol.py b/experimental/lite/megatron/lite/model/deepseek_v4/lite/protocol.py new file mode 100644 index 00000000000..d67dc284a7e --- /dev/null +++ b/experimental/lite/megatron/lite/model/deepseek_v4/lite/protocol.py @@ -0,0 +1,492 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +from dataclasses import dataclass, field +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import torch +import torch.nn as nn + +from megatron.lite.model.deepseek_v4.config import DeepseekV4Config +from megatron.lite.model.deepseek_v4.lite.checkpoint import ( + EXPERT_CLASSIFIER, + PLACEMENT_FN, + export_hf_weights as _export_hf_weights_impl, + load_hf_weights as _load_hf_weights_impl, + save_hf_weights as _save_hf_weights_impl, +) +from megatron.lite.model.protocol_utils import add_loss_context_kwargs +from megatron.lite.primitive.bundle import ModelBundle +from megatron.lite.primitive.parallel import ParallelState, init_parallel +from megatron.lite.primitive.parallel.cp import ( + contiguous_position_ids_for_cp, + contiguous_slice_for_cp, + local_position_ids_for_cp, + local_sequence_tensor_for_cp, +) +from megatron.lite.primitive.parallel.thd import ( + pack_nested_thd, + parallel_state_from_model, + thd_pack_meta, + unpack_thd_to_nested, +) +from megatron.lite.primitive.recompute import apply_recompute, parse_recompute_spec +from megatron.lite.runtime.contracts import OptimizerConfig, PackedBatch, ParallelConfig + + +def is_expert_param(name: str) -> bool: + return EXPERT_CLASSIFIER(name) + + +@dataclass(frozen=True) +class ImplConfig: + parallel: ParallelConfig = field(default_factory=ParallelConfig) + optimizer: str | None = "dist_opt" + optimizer_config: OptimizerConfig | None = None + hf_path: str = "" + recompute: list[str] = field(default_factory=list) + offload: list[str] = field(default_factory=list) + use_thd: bool = False + use_deepep: bool = False + attention_backend_override: str | None = None + deterministic: bool = True + mtp_enable: bool = True + mtp_enable_train: bool = False + mtp_detach_encoder: bool = False + mtp_num_layers: int | None = None + num_nextn_predict_layers: int | None = None + mtp_loss_scaling_factor: float = 0.1 + + +MODULE_MAP = { + "attn": lambda layer: layer.self_attn, + "core_attn": lambda layer: layer.self_attn, + "moe": lambda layer: layer.mlp, + "experts": lambda layer: layer.mlp.experts, + "router": lambda layer: layer.mlp.gate, + "attn_norm": lambda layer: layer.input_layernorm, + "ffn_norm": lambda layer: layer.post_attention_layernorm, +} + +# The Kimi-derived model has no ``attention_mask`` arg (CSA derives its causal / +# sliding-window masking from ``position_ids``, as the previous DS4 did with +# attention_mask=None); keep it out of the forward whitelist. +_MODEL_FORWARD_KEYS = ( + "input_ids", + "position_ids", + "labels", + "loss_mask", + "temperature", + "calculate_entropy", + "enable_mtp", +) + + +def build_model_config(source: str | Path | dict, **overrides) -> DeepseekV4Config: + if isinstance(source, dict): + cfg = DeepseekV4Config._from_hf_dict(source) + else: + cfg = DeepseekV4Config.from_hf(str(source)) + for key, value in overrides.items(): + if hasattr(cfg, key): + setattr(cfg, key, value) + return cfg + + +def _normalize_ds4_position_ids(position_ids): + if position_ids is None: + return None + if position_ids.dim() == 3: + if position_ids.size(0) == 3: + position_ids = position_ids[0] + elif position_ids.size(1) == 1: + position_ids = position_ids.squeeze(1) + if position_ids.dim() == 1: + position_ids = position_ids.unsqueeze(0) + return position_ids + + +def _as_batch_row(tensor): + if tensor is not None and tensor.dim() == 1: + return tensor.unsqueeze(0) + return tensor + + +def _infer_cp_local_seq_len( + *, + input_ids, + position_ids, + cp_size, +): + seq_len = input_ids.size(1) + if cp_size <= 1: + return seq_len + if position_ids is not None and position_ids.size(-1) in (seq_len, seq_len * cp_size): + return seq_len + return seq_len // cp_size if seq_len % cp_size == 0 else seq_len + + +def _nested_from_packed_tensor(tensor, seq_lens): + if tensor is None: + return None + if tensor.dim() == 2 and tensor.size(0) == 1: + tensor = tensor.squeeze(0) + if tensor.dim() != 1: + raise ValueError(f"PackedBatch tensor must be 1-D, got {tuple(tensor.shape)}.") + + pieces = [] + offset = 0 + for length_t in seq_lens: + length = int(length_t.item()) + pieces.append(tensor.narrow(0, offset, length)) + offset += length + if offset != tensor.numel(): + raise ValueError(f"PackedBatch sizes sum to {offset}, tensor has {tensor.numel()} tokens.") + return torch.nested.as_nested_tensor(pieces, layout=torch.jagged) + + +def _prepare_packed_batch_kwargs(model, batch: PackedBatch) -> dict[str, Any]: + ps = parallel_state_from_model(model) or ParallelState() + seq_lens = batch.sizes().to(device=batch.input_ids.device) + packed = pack_nested_thd( + _nested_from_packed_tensor(batch.input_ids, seq_lens), + cp_size=ps.cp_size, + cp_rank=ps.cp_rank, + cp_group=ps.cp_group, + split_cp=False, + labels=_nested_from_packed_tensor(batch.labels, seq_lens), + loss_mask=_nested_from_packed_tensor(batch.loss_mask, seq_lens), + ) + kwargs: dict[str, Any] = { + "input_ids": packed.input_ids, + "labels": packed.labels, + "loss_mask": packed.loss_mask, + "position_ids": packed.position_ids, + "packed_seq_params": packed.packed_seq_params, + "enable_mtp": False, + } + add_loss_context_kwargs(kwargs) + _prepare_packed_contiguous_cp_kwargs(model, kwargs) + kwargs.pop("packed_seq_params", None) + return {key: value for key, value in kwargs.items() if key in _MODEL_FORWARD_KEYS} + + +def _base_model_forward_kwargs(batch: PackedBatch): + kwargs: dict[str, Any] = {"input_ids": _as_batch_row(batch.input_ids)} + if batch.labels is not None: + kwargs["labels"] = _as_batch_row(batch.labels) + if batch.loss_mask is not None: + kwargs["loss_mask"] = _as_batch_row(batch.loss_mask) + add_loss_context_kwargs(kwargs) + position_ids = _normalize_ds4_position_ids(getattr(batch, "position_ids", None)) + if position_ids is not None: + kwargs["position_ids"] = position_ids + return kwargs + + +def _prepare_packed_contiguous_cp_kwargs(model, kwargs): + ps = parallel_state_from_model(model) or ParallelState() + if ps.cp_size <= 1: + return kwargs + for key in ("input_ids", "labels", "loss_mask", "position_ids"): + tensor = kwargs.get(key) + if tensor is not None: + kwargs[key] = contiguous_slice_for_cp(tensor, ps.cp_rank, ps.cp_size, seq_dim=1) + return kwargs + + +def _prepare_contiguous_cp_kwargs(model, kwargs): + ps = parallel_state_from_model(model) or ParallelState() + local_seq_len = _infer_cp_local_seq_len( + input_ids=kwargs["input_ids"], + position_ids=kwargs.get("position_ids"), + cp_size=ps.cp_size, + ) + kwargs["input_ids"] = local_sequence_tensor_for_cp( + kwargs["input_ids"], + local_seq_len=local_seq_len, + cp_rank=ps.cp_rank, + cp_size=ps.cp_size, + name="input_ids", + ) + if kwargs.get("position_ids") is None: + full_seq_len = local_seq_len * ps.cp_size + position_ids = contiguous_position_ids_for_cp( + full_seq_len, + cp_rank=ps.cp_rank, + cp_size=ps.cp_size, + device=kwargs["input_ids"].device, + ).expand(kwargs["input_ids"].size(0), -1) + else: + position_ids = local_position_ids_for_cp( + kwargs["position_ids"], + batch=kwargs["input_ids"].size(0), + local_seq_len=kwargs["input_ids"].size(1), + cp_rank=ps.cp_rank, + cp_size=ps.cp_size, + ) + kwargs["position_ids"] = position_ids + for key in ("labels", "loss_mask"): + if kwargs.get(key) is not None: + kwargs[key] = local_sequence_tensor_for_cp( + kwargs[key], + local_seq_len=kwargs["input_ids"].size(1), + cp_rank=ps.cp_rank, + cp_size=ps.cp_size, + name=key, + ) + return kwargs + + +def _prepare_model_forward_kwargs(model, batch: PackedBatch): + # THD-packed inputs (1-D values, or a single padded [1, S] row) carry their own + # cu_seqlens and go through the packed builder. A dense multi-row [B, S] batch is + # split per row under contiguous CP, where contiguous_position_ids_for_cp rebuilds + # the per-rank global position ids. + input_ids = batch.input_ids + is_thd_packed = input_ids.dim() == 1 or (input_ids.dim() == 2 and input_ids.size(0) == 1) + if is_thd_packed: + return _prepare_packed_batch_kwargs(model, batch) + kwargs = _base_model_forward_kwargs(batch) + return _prepare_contiguous_cp_kwargs(model, kwargs) + + +def _forward_step(model: nn.Module, batch: PackedBatch) -> dict: + return model(**_prepare_model_forward_kwargs(model, batch)) + + +def unpack_forward_output(model: nn.Module, batch: PackedBatch, output) -> Any: + # DeepSeek-V4 packs each sequence to the (zigzag) TE alignment but slices CP + # contiguously for the fused DSA indexer, so reconstruct contiguously. + ps = parallel_state_from_model(model) or ParallelState() + meta = thd_pack_meta( + batch.seq_lens, + tp_size=ps.tp_size, + cp_size=ps.cp_size, + cp_group=ps.cp_group if ps.cp_size > 1 else None, + ) + return unpack_thd_to_nested(output, meta, contiguous=True) + + +def _apply_mtp_config(model_cfg: DeepseekV4Config, impl_cfg: ImplConfig) -> None: + override = impl_cfg.num_nextn_predict_layers + if override is None: + override = impl_cfg.mtp_num_layers + if override is not None: + if override < 0: + raise ValueError(f"DeepSeek V4 MTP layer count must be >=0, got {override}.") + model_cfg.num_nextn_predict_layers = int(override) + if impl_cfg.mtp_enable: + if model_cfg.num_nextn_predict_layers <= 0: + raise ValueError("mtp_enable=True but DeepSeek V4 config has no MTP layers.") + model_cfg.mtp_loss_scaling_factor = impl_cfg.mtp_loss_scaling_factor + else: + model_cfg.num_nextn_predict_layers = 0 + + +def _make_aux_loss_hook(): + """Per-step hook that syncs the MTP auxiliary-loss backward scale to the main + loss scale (DP size / gradient accumulation), mirroring the sibling protocols + (kimi_k2 / glm5 / qwen3_5 / qwen3_moe). + + DS4 only injects an MTP auxiliary loss: its MoE router is aux-loss-free + (``SigmoidTopKRouter(..., compute_aux_loss=False)``) and its CSA indexer runs + with ``sparse_loss=False``, so -- unlike GLM-5, which also scales the MoE-aux + and DSA-indexer losses -- only ``MTPLossAutoScaler`` needs scaling here. + Without this hook the injected MTP gradient keeps ``MTPLossAutoScaler``'s + class-default scale of 1.0 and is mis-weighted relative to the main loss. + """ + from megatron.lite.primitive.modules.mtp import MTPLossAutoScaler + + def hook(scale: torch.Tensor) -> None: + MTPLossAutoScaler.set_loss_scale(scale) + + return hook + + +def _optimizer_backend_name(optimizer: Any) -> str | None: + if isinstance(optimizer, dict) or isinstance(optimizer, OptimizerConfig): + return "dist_opt" + return optimizer + + +def _configure_attention_backend(chunks: list[nn.Module], *, backend: str | None) -> None: + backend_name = backend or "torch" + for chunk in chunks: + for module in chunk.modules(): + if hasattr(module, "attention_backend"): + module.attention_backend = backend_name + + +def _iter_transformer_units(chunk: nn.Module) -> list[nn.Module]: + model = getattr(chunk, "model", None) + if model is None: + return [] + layers = list(getattr(model, "layers", {}).values()) + mtp_layers = list(getattr(model, "mtp", [])) + return [*layers, *mtp_layers] + + +def _validate_parallel_scope(p: ParallelConfig) -> None: + """DS4 CSA attention is not tensor-parallel-capable (documented TP=1 case). + + PP / VPP / EP / CP are inherited from the Kimi skeleton and work; only + TP>1 / ETP>1 are unsupported. Mirrors GLM-5's gate. + """ + etp = 1 if p.etp is None else p.etp + if p.tp > 1: + raise NotImplementedError( + "DeepSeek V4 native CSA attention does not support tensor parallelism; " + f"got tp={p.tp}. Use tp=1 (PP/VPP/EP/CP are supported)." + ) + if etp > 1: + raise NotImplementedError( + "DeepSeek V4 native CSA attention does not support expert tensor parallelism; " + f"got etp={etp}. Use etp=1 (EP is supported)." + ) + + +def build_model(model_cfg: DeepseekV4Config, *, impl_cfg: ImplConfig) -> ModelBundle: + from megatron.lite.model.deepseek_v4.lite.model import DeepseekV4Model + + p = impl_cfg.parallel + _validate_parallel_scope(p) + _apply_mtp_config(model_cfg, impl_cfg) + mtp_enable = bool(impl_cfg.mtp_enable) and model_cfg.num_nextn_predict_layers > 0 + mtp_enable_train = mtp_enable and bool(impl_cfg.mtp_enable_train) + ps = init_parallel(impl_cfg.parallel) + vpp = None if p.vpp == 1 else p.vpp + train_cfg = SimpleNamespace( + tp=ps.tp_size, + ep=ps.ep_size, + etp=ps.etp_size, + pp=ps.pp_size, + cp=ps.cp_size, + vpp=vpp, + fp8=False, + use_deepep=impl_cfg.use_deepep, + ) + + def _chunk(i: int | None = None): + return ( + DeepseekV4Model( + model_cfg, + train_cfg, + ps, + vpp_chunk_id=i, + use_deepep=impl_cfg.use_deepep, + use_thd=impl_cfg.use_thd, + hf_path=impl_cfg.hf_path, + attention_backend_override=impl_cfg.attention_backend_override, + mtp_enable=mtp_enable, + mtp_enable_train=mtp_enable_train, + mtp_detach_encoder=impl_cfg.mtp_detach_encoder, + ) + .to(torch.bfloat16) + .cuda() + ) + + chunks = [_chunk(i) for i in range(vpp)] if vpp is not None else [_chunk()] + _configure_attention_backend(chunks, backend=impl_cfg.attention_backend_override) + + recompute_spec = parse_recompute_spec(impl_cfg.recompute) + if recompute_spec: + for chunk in chunks: + apply_recompute(_iter_transformer_units(chunk), recompute_spec, MODULE_MAP) + + if impl_cfg.offload: + from megatron.lite.primitive.recompute import apply_offload + + for chunk in chunks: + apply_offload(_iter_transformer_units(chunk), impl_cfg.offload, MODULE_MAP) + + optimizer = None + finalize_grads = None + post_model_load_hook = None + optimizer_backend = "none" + optimizer_name = _optimizer_backend_name(impl_cfg.optimizer) + if optimizer_name == "dist_opt": + from megatron.lite.primitive.ckpt import attach_model_sharded_state_dict + from megatron.lite.primitive.optimizers.megatron_wrap import ( + build_dist_opt_training_optimizer, + ) + from megatron.lite.runtime.megatron_utils import register_training_hooks + + optimizer, finalize_grads = build_dist_opt_training_optimizer( + chunks, + model_cfg=model_cfg, + impl_cfg=impl_cfg, + ps=ps, + model_name="deepseek_v4", + is_expert=is_expert_param, + deterministic=impl_cfg.deterministic, + ) + attach_model_sharded_state_dict( + chunks, ps, get_placements=PLACEMENT_FN, is_expert=is_expert_param + ) + register_training_hooks(chunks, optimizer) + optimizer_backend = "dist_opt" + elif optimizer_name == "fsdp2": + optimizer_backend = "fsdp2" + + def _post_model_load_hook(): + from megatron.lite.model.deepseek_v4.lite.model import DeepseekV4Layer + from megatron.lite.primitive.optimizers.fsdp2 import build_fsdp2_training_optimizer + + return { + "optimizer": build_fsdp2_training_optimizer( + chunks, + impl_cfg.optimizer_config, + ps, + unit_modules=(DeepseekV4Layer,), + expert_classifier=is_expert_param, + deterministic=impl_cfg.deterministic, + vpp=impl_cfg.parallel.vpp, + leaf_module_names=(), + use_fp32_shards=False, + ) + } + + post_model_load_hook = _post_model_load_hook + elif optimizer_name is None: + optimizer_backend = "none" + else: + raise ValueError(f"Unknown DeepSeek V4 lite optimizer: {impl_cfg.optimizer!r}.") + + return ModelBundle( + chunks=chunks, + parallel_state=ps, + optimizer=optimizer, + finalize_grads=finalize_grads, + forward_step=_forward_step, + extras={ + "model_cfg": model_cfg, + "optimizer_backend": optimizer_backend, + "post_model_load_hook": post_model_load_hook, + "pre_forward_hook": _make_aux_loss_hook(), + }, + ) + + +def load_hf_weights( + chunk: nn.Module, hf_path: str, model_cfg: DeepseekV4Config, ps: ParallelState +) -> None: + if not hf_path: + return + _load_hf_weights_impl(chunk, hf_path, model_cfg, ps) + + +def export_hf_weights( + chunks: list[nn.Module], model_cfg: DeepseekV4Config, ps: ParallelState, **kwargs +): + yield from _export_hf_weights_impl(chunks, model_cfg, ps, **kwargs) + + +def save_hf_weights( + chunks: list[nn.Module], path: str, model_cfg: DeepseekV4Config, ps: ParallelState, **kwargs +) -> None: + _save_hf_weights_impl(chunks, path, model_cfg, ps, **kwargs) + + +def vocab_size(model_cfg: DeepseekV4Config) -> int | None: + return model_cfg.vocab_size diff --git a/experimental/lite/megatron/lite/model/glm5/__init__.py b/experimental/lite/megatron/lite/model/glm5/__init__.py new file mode 100644 index 00000000000..de6ed48164b --- /dev/null +++ b/experimental/lite/megatron/lite/model/glm5/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""GLM-5 model family.""" + +from megatron.lite.model.glm5.config import Glm5Config + +__all__ = ["Glm5Config"] diff --git a/experimental/lite/megatron/lite/model/glm5/config.py b/experimental/lite/megatron/lite/model/glm5/config.py new file mode 100644 index 00000000000..76d794f3c11 --- /dev/null +++ b/experimental/lite/megatron/lite/model/glm5/config.py @@ -0,0 +1,201 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""GLM-5 architecture config.""" + +from __future__ import annotations + +from dataclasses import dataclass +from dataclasses import fields as dc_fields +from typing import Any + +from megatron.lite.primitive.config import load_hf_config_dict + +_HF_FIELDS = frozenset( + { + "attention_dropout", + "calculate_per_token_loss", + "dsa_indexer_loss_coeff", + "dsa_indexer_use_sparse_loss", + "first_k_dense_replace", + "head_dim", + "hidden_size", + "index_head_dim", + "index_n_heads", + "index_topk", + "indexer_layer_norm_eps", + "indexer_rope_interleave", + "indexer_rope_first", + "indexer_use_hadamard", + "initializer_range", + "intermediate_size", + "kv_lora_rank", + "max_position_embeddings", + "mlp_layer_types", + "moe_intermediate_size", + "n_group", + "n_routed_experts", + "n_shared_experts", + "norm_topk_prob", + "mtp_loss_scaling_factor", + "mtp_use_repeated_layer", + "num_attention_heads", + "num_experts_per_tok", + "num_hidden_layers", + "num_key_value_heads", + "num_nextn_predict_layers", + "q_lora_rank", + "qk_head_dim", + "qk_nope_head_dim", + "qk_rope_head_dim", + "latent_rms_norm_eps", + "rms_norm_eps", + "rope_interleave", + "rope_theta", + "routed_scaling_factor", + "topk_group", + "v_head_dim", + "vocab_size", + } +) + + +@dataclass +class Glm5Config: + """Pure GLM-5 MoE + MLA + DSA architecture parameters.""" + + num_hidden_layers: int = 78 + hidden_size: int = 6144 + num_attention_heads: int = 64 + num_key_value_heads: int = 64 + head_dim: int = 64 + vocab_size: int = 154880 + max_position_embeddings: int = 202752 + rms_norm_eps: float = 1e-5 + attention_dropout: float = 0.0 + initializer_range: float = 0.02 + + q_lora_rank: int = 2048 + kv_lora_rank: int = 512 + qk_head_dim: int = 256 + qk_nope_head_dim: int = 192 + qk_rope_head_dim: int = 64 + v_head_dim: int = 256 + + index_head_dim: int = 128 + index_n_heads: int = 32 + index_topk: int = 2048 + indexer_layer_norm_eps: float = 1e-6 + indexer_rope_interleave: bool = False + indexer_rope_first: bool = True + indexer_use_hadamard: bool = False + dsa_indexer_loss_coeff: float = 0.0 + dsa_indexer_use_sparse_loss: bool = False + calculate_per_token_loss: bool = False + rope_interleave: bool = False + rope_theta: float = 1_000_000.0 + latent_rms_norm_eps: float = 1e-6 + + intermediate_size: int = 12288 + moe_intermediate_size: int = 2048 + first_k_dense_replace: int = 3 + n_routed_experts: int = 256 + n_shared_experts: int = 1 + num_experts_per_tok: int = 8 + n_group: int = 1 + topk_group: int = 1 + routed_scaling_factor: float = 2.5 + norm_topk_prob: bool = True + num_nextn_predict_layers: int = 1 + mtp_loss_scaling_factor: float = 0.1 + mtp_use_repeated_layer: bool = False + mlp_layer_types: list[str] | None = None + + def __post_init__(self): + self._validate() + + @property + def num_experts(self) -> int: + return self.n_routed_experts + + def is_moe_layer(self, layer_idx: int) -> bool: + if self.mlp_layer_types is not None and layer_idx < len(self.mlp_layer_types): + return self.mlp_layer_types[layer_idx] == "sparse" + return layer_idx >= self.first_k_dense_replace + + def _validate(self) -> None: + errors: list[str] = [] + + def check(cond: bool, message: str) -> None: + if not cond: + errors.append(message) + + check(self.num_hidden_layers >= 1, "num_hidden_layers must be >= 1") + check(self.hidden_size > 0, "hidden_size must be > 0") + check(self.q_lora_rank > 0, "q_lora_rank must be > 0") + check(self.kv_lora_rank > 0, "kv_lora_rank must be > 0") + check( + self.qk_head_dim == self.qk_nope_head_dim + self.qk_rope_head_dim, + "qk_head_dim must equal qk_nope_head_dim + qk_rope_head_dim", + ) + check( + self.index_head_dim >= self.qk_rope_head_dim, + "index_head_dim must be >= qk_rope_head_dim", + ) + check(self.dsa_indexer_loss_coeff >= 0.0, "dsa_indexer_loss_coeff must be >= 0") + check( + self.num_key_value_heads == self.num_attention_heads, + "initial GLM5 native path expects MLA heads to be ungrouped", + ) + check(self.vocab_size > 0, "vocab_size must be > 0") + check(self.num_nextn_predict_layers >= 0, "num_nextn_predict_layers must be >= 0") + check(self.n_routed_experts >= 1, "n_routed_experts must be >= 1") + check( + 1 <= self.num_experts_per_tok <= self.n_routed_experts, + "num_experts_per_tok must be in [1, n_routed_experts]", + ) + check(1 <= self.topk_group <= self.n_group, "topk_group must be in [1, n_group]") + if self.mlp_layer_types is not None: + expected_layer_type_lengths = { + self.num_hidden_layers, + self.num_hidden_layers + self.num_nextn_predict_layers, + } + check( + len(self.mlp_layer_types) in expected_layer_type_lengths, + "len(mlp_layer_types) must equal num_hidden_layers or " + "num_hidden_layers + num_nextn_predict_layers", + ) + for idx, layer_type in enumerate(self.mlp_layer_types): + check( + layer_type in {"dense", "sparse"}, + f"mlp_layer_types[{idx}] must be 'dense' or 'sparse'", + ) + + if errors: + raise ValueError( + f"Invalid Glm5Config ({len(errors)} error" + f"{'s' if len(errors) != 1 else ''}):\n " + "\n ".join(errors) + ) + + def to_dict(self) -> dict[str, Any]: + return {f.name: getattr(self, f.name) for f in dc_fields(self)} + + @classmethod + def from_hf(cls, path_or_name: str, **overrides) -> Glm5Config: + return cls._from_hf_dict(load_hf_config_dict(path_or_name), **overrides) + + @classmethod + def from_hf_config(cls, hf_config, **overrides) -> Glm5Config: + hf = hf_config.to_dict() if hasattr(hf_config, "to_dict") else vars(hf_config) + return cls._from_hf_dict(hf, **overrides) + + @classmethod + def _from_hf_dict(cls, hf: dict[str, Any], **overrides) -> Glm5Config: + kwargs = { + key: value for key, value in hf.items() if key in _HF_FIELDS and value is not None + } + if "num_nextn_predict_layers" not in kwargs and hf.get("num_nextn_predict") is not None: + kwargs["num_nextn_predict_layers"] = int(hf["num_nextn_predict"]) + rope_parameters = hf.get("rope_parameters") + if isinstance(rope_parameters, dict) and "rope_theta" not in kwargs: + kwargs["rope_theta"] = float(rope_parameters.get("rope_theta", cls.rope_theta)) + kwargs.update(overrides) + return cls(**kwargs) diff --git a/experimental/lite/megatron/lite/model/glm5/lite/__init__.py b/experimental/lite/megatron/lite/model/glm5/lite/__init__.py new file mode 100644 index 00000000000..406ca1c4854 --- /dev/null +++ b/experimental/lite/megatron/lite/model/glm5/lite/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Native GLM-5 lite implementation.""" + +from megatron.lite.model.glm5.lite.model import Glm5Model + +__all__ = ["Glm5Model"] diff --git a/experimental/lite/megatron/lite/model/glm5/lite/checkpoint.py b/experimental/lite/megatron/lite/model/glm5/lite/checkpoint.py new file mode 100644 index 00000000000..9f8a78a2a88 --- /dev/null +++ b/experimental/lite/megatron/lite/model/glm5/lite/checkpoint.py @@ -0,0 +1,672 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""GLM-5 (deepseek_v3_2) lite native checkpoint mapping. + +Cloned from ``megatron/lite/model/kimi_k2/lite/checkpoint.py`` (deepseek_v3) +and renamed KimiK2 -> Glm5. The MoE / dense-MLP / embed / head / MTP weight +mapping is structurally identical to Kimi. The ONLY adaptations are in the +attention block: + + * GLM-5's attention is the DSA primitive ``DynamicSparseAttention`` (wrapped + by ``Glm5DSAAttention``), whose native parameters live under + ``self_attention.self_attention.*`` -- i.e. ``q_a_proj`` / ``q_a_layernorm`` + / ``q_b_proj`` / ``kv_a_proj_with_mqa`` / ``kv_a_layernorm`` / ``kv_b_proj`` + / ``o_proj`` plus the indexer (``indexer.wq_b`` / ``indexer.wk`` / + ``indexer.k_norm`` / ``indexer.weights_proj``). Kimi's MLA names + (``linear_q_down_proj`` etc.) are replaced accordingly. + * The HF weight NAMES match GLM-5's HF model: the DSA submodule names map 1:1 + to ``model.layers.{i}.self_attn.*`` (and ``...self_attn.indexer.*``), + preserving the names the previous GLM-5 checkpoint used. + +GLM-5 is TP=1 (DSA is not tensor-parallel-capable), so the ``_tp`` helpers are +no-ops here; they are kept for structural parity with Kimi. +""" + +from __future__ import annotations + +import torch +import torch.distributed as dist +import torch.nn as nn +from torch.distributed.tensor import Replicate, Shard + +from megatron.lite.model.glm5.config import Glm5Config +from megatron.lite.primitive.ckpt.hf_weights import ( + SafeTensorReader, + _cast_export_tensor, + _resolve_export_dtype, + parse_expert_idx, + to_global_layer_name, + unwrap_model, +) +from megatron.lite.primitive.parallel import ParallelState +from megatron.lite.primitive.utils import ensure_divisible, log_rank0 + + +def EXPERT_CLASSIFIER(name: str) -> bool: + return "experts" in name and "router" not in name and "shared" not in name + + +def PLACEMENT_FN(param_name: str) -> list: + if "experts" in param_name and "router" not in param_name and "shared" not in param_name: + if "fc1" in param_name: + return [Replicate(), Replicate(), Shard(0), Shard(0)] + if "fc2" in param_name: + return [Replicate(), Replicate(), Shard(0), Shard(1)] + return [Replicate(), Replicate(), Replicate(), Replicate()] + if "eh_proj.linear.weight" in param_name: + return [Replicate(), Replicate(), Replicate(), Shard(0)] + # GLM-5 DSA is TP=1 (no head sharding of attention projections). + if "gate_up" in param_name: + return [Replicate(), Replicate(), Replicate(), Shard(0)] + if "down" in param_name: + return [Replicate(), Replicate(), Replicate(), Shard(1)] + if "embed" in param_name or "head" in param_name: + return [Replicate(), Replicate(), Replicate(), Shard(0)] + return [Replicate(), Replicate(), Replicate(), Replicate()] + + +def _tp(tensor: torch.Tensor, rank: int, size: int, dim: int = 0) -> torch.Tensor: + return tensor if size <= 1 else tensor.chunk(size, dim=dim)[rank].contiguous() + + +def _split_gate_up(tensor: torch.Tensor, rank: int, size: int) -> torch.Tensor: + if size <= 1: + return tensor + ffn = tensor.shape[0] // 2 + gate = tensor[:ffn].chunk(size, dim=0)[rank] + up = tensor[ffn:].chunk(size, dim=0)[rank] + return torch.cat([gate, up], dim=0).contiguous() + + +def _has(reader: SafeTensorReader, name: str) -> bool: + if reader.index: + return name in reader.index + try: + reader.get_tensor(name) + except Exception: + return False + return True + + +def _dequant_fp8_weight(reader: SafeTensorReader, name: str, weight: torch.Tensor) -> torch.Tensor: + scale_name = f"{name}_scale_inv" + if weight.dim() != 2 or weight.element_size() != 1 or not _has(reader, scale_name): + return weight + scale = reader.get_tensor(scale_name).float() + rows, cols = weight.shape + expanded = scale.repeat_interleave(128, dim=0).repeat_interleave(128, dim=1) + expanded = expanded[:rows, :cols] + return weight.float() * expanded + + +def _unpack_int4_from_int32(packed: torch.Tensor, shape: torch.Size) -> torch.Tensor: + if packed.dtype != torch.int32: + raise ValueError(f"Expected packed int4 tensor to be int32, got {packed.dtype}.") + pack_factor = 8 + mask = 0xF + rows, cols = int(shape[0]), int(shape[1]) + unpacked = torch.empty((packed.shape[0], packed.shape[1] * pack_factor), dtype=torch.int32) + for offset in range(pack_factor): + unpacked[:, offset::pack_factor] = (packed >> (4 * offset)) & mask + return (unpacked[:rows, :cols] - 8).to(torch.int8) + + +def _dequant_int4_weight(reader: SafeTensorReader, name: str) -> torch.Tensor: + packed = reader.get_tensor(f"{name}_packed") + scale = reader.get_tensor(f"{name}_scale") + shape_tensor = reader.get_tensor(f"{name}_shape") + shape = torch.Size(int(x) for x in shape_tensor.tolist()) + + unpacked = _unpack_int4_from_int32(packed, shape).to(scale.dtype) + if scale.dim() != 2 or unpacked.dim() != 2: + raise ValueError( + f"Expected groupwise int4 tensors for {name}, got " + f"weight={tuple(unpacked.shape)} scale={tuple(scale.shape)}." + ) + if scale.shape[0] not in (1, unpacked.shape[0]): + raise ValueError( + f"Unsupported int4 scale rows for {name}: " + f"weight={tuple(unpacked.shape)} scale={tuple(scale.shape)}." + ) + if unpacked.shape[1] % scale.shape[1] != 0: + raise ValueError( + f"Unsupported int4 group layout for {name}: " + f"weight={tuple(unpacked.shape)} scale={tuple(scale.shape)}." + ) + + group_size = unpacked.shape[1] // scale.shape[1] + return (unpacked.unflatten(-1, (scale.shape[1], group_size)) * scale.unsqueeze(-1)).flatten( + start_dim=-2 + ) + + +def _get(reader: SafeTensorReader, name: str) -> torch.Tensor: + if not _has(reader, name) and _has(reader, f"{name}_packed"): + return _dequant_int4_weight(reader, name) + tensor = reader.get_tensor(name) + return _dequant_fp8_weight(reader, name, tensor) + + +def _text_prefix(reader: SafeTensorReader) -> str: + for prefix in ("model", "language_model.model", "model.language_model"): + if _has(reader, f"{prefix}.embed_tokens.weight"): + return prefix + raise KeyError("Could not find GLM-5 text model prefix in HF checkpoint.") + + +def _lm_head_name(reader: SafeTensorReader, text_prefix: str) -> str: + candidates = [ + "lm_head.weight", + "language_model.lm_head.weight", + f"{text_prefix}.lm_head.weight", + ] + for name in candidates: + if _has(reader, name): + return name + raise KeyError("Could not find GLM-5 lm_head.weight in HF checkpoint.") + + +def _load_vocab( + reader: SafeTensorReader, name: str, cfg: Glm5Config, ps: ParallelState +) -> torch.Tensor: + from megatron.lite.primitive.parallel import pad_vocab_for_tp + + tensor = _get(reader, name) + padded = pad_vocab_for_tp(cfg.vocab_size, ps.tp_size) + if tensor.size(0) < padded: + pad = torch.zeros(padded - tensor.size(0), tensor.size(1), dtype=tensor.dtype) + tensor = torch.cat([tensor, pad], dim=0) + return _tp(tensor, ps.tp_rank, ps.tp_size) + + +def _load_attention( + out: dict[str, torch.Tensor], + *, + local_prefix: str, + hf_prefix: str, + reader: SafeTensorReader, + ps: ParallelState, +) -> None: + # GLM-5 ONLY: DSA attention. Native params live under + # `.self_attention.self_attention.*` (the wrapper adds one + # `self_attention` level around the DSA module). HF names are GLM-5's + # `.{...}` (DSA submodule names map 1:1 to the HF model). + ap = f"{local_prefix}.self_attention.self_attention" + out[f"{ap}.q_a_proj.weight"] = _get(reader, f"{hf_prefix}.q_a_proj.weight") + out[f"{ap}.q_a_layernorm.weight"] = _get(reader, f"{hf_prefix}.q_a_layernorm.weight") + out[f"{ap}.q_b_proj.weight"] = _get(reader, f"{hf_prefix}.q_b_proj.weight") + out[f"{ap}.kv_a_proj_with_mqa.weight"] = _get( + reader, f"{hf_prefix}.kv_a_proj_with_mqa.weight" + ) + out[f"{ap}.kv_a_layernorm.weight"] = _get(reader, f"{hf_prefix}.kv_a_layernorm.weight") + out[f"{ap}.kv_b_proj.weight"] = _get(reader, f"{hf_prefix}.kv_b_proj.weight") + out[f"{ap}.o_proj.weight"] = _get(reader, f"{hf_prefix}.o_proj.weight") + # DSA indexer. + ip = f"{ap}.indexer" + hip = f"{hf_prefix}.indexer" + out[f"{ip}.wq_b.weight"] = _get(reader, f"{hip}.wq_b.weight") + out[f"{ip}.wk.weight"] = _get(reader, f"{hip}.wk.weight") + out[f"{ip}.k_norm.weight"] = _get(reader, f"{hip}.k_norm.weight") + if _has(reader, f"{hip}.k_norm.bias"): + out[f"{ip}.k_norm.bias"] = _get(reader, f"{hip}.k_norm.bias") + out[f"{ip}.weights_proj.weight"] = _get(reader, f"{hip}.weights_proj.weight") + + +def _load_dense_mlp( + out: dict[str, torch.Tensor], + *, + local_prefix: str, + hf_mlp_prefix: str, + hf_layer_prefix: str, + reader: SafeTensorReader, + ps: ParallelState, +) -> None: + out[f"{local_prefix}.mlp.gate_up.linear.layer_norm_weight"] = _get( + reader, + f"{hf_layer_prefix}.post_attention_layernorm.weight", + ) + gate_up = torch.cat( + [ + _get(reader, f"{hf_mlp_prefix}.gate_proj.weight"), + _get(reader, f"{hf_mlp_prefix}.up_proj.weight"), + ], + dim=0, + ) + out[f"{local_prefix}.mlp.gate_up.linear.weight"] = _split_gate_up( + gate_up, + ps.tp_rank, + ps.tp_size, + ) + out[f"{local_prefix}.mlp.down.linear.weight"] = _tp( + _get(reader, f"{hf_mlp_prefix}.down_proj.weight"), + ps.tp_rank, + ps.tp_size, + dim=1, + ) + + +def _load_shared_expert( + out: dict[str, torch.Tensor], + *, + local_prefix: str, + hf_mlp_prefix: str, + reader: SafeTensorReader, + ps: ParallelState, +) -> None: + prefixes = [f"{hf_mlp_prefix}.shared_experts", f"{hf_mlp_prefix}.shared_expert"] + shared = next(prefix for prefix in prefixes if _has(reader, f"{prefix}.down_proj.weight")) + gate_up = torch.cat( + [ + _get(reader, f"{shared}.gate_proj.weight"), + _get(reader, f"{shared}.up_proj.weight"), + ], + dim=0, + ) + out[f"{local_prefix}.moe.shared_expert.gate_up.linear.weight"] = _split_gate_up( + gate_up, + ps.tp_rank, + ps.tp_size, + ) + out[f"{local_prefix}.moe.shared_expert.down.linear.weight"] = _tp( + _get(reader, f"{shared}.down_proj.weight"), + ps.tp_rank, + ps.tp_size, + dim=1, + ) + + +def _load_experts( + out: dict[str, torch.Tensor], + *, + local_prefix: str, + hf_mlp_prefix: str, + cfg: Glm5Config, + ps: ParallelState, + reader: SafeTensorReader, +) -> None: + num_local = ensure_divisible(cfg.num_experts, ps.ep_size) + local_start = ps.ep_rank * num_local + for local_idx in range(num_local): + global_idx = local_start + local_idx + ep = f"{hf_mlp_prefix}.experts.{global_idx}" + fc1 = torch.cat( + [ + _get(reader, f"{ep}.gate_proj.weight"), + _get(reader, f"{ep}.up_proj.weight"), + ], + dim=0, + ) + fc2 = _get(reader, f"{ep}.down_proj.weight") + if ps.etp_size > 1: + fc1 = _split_gate_up(fc1, ps.etp_rank, ps.etp_size) + fc2 = _tp(fc2, ps.etp_rank, ps.etp_size, dim=1) + out[f"{local_prefix}.moe.experts.fc1.weight{local_idx}"] = fc1 + out[f"{local_prefix}.moe.experts.fc2.weight{local_idx}"] = fc2 + + +def _copy_loaded_state(model: nn.Module, loaded: dict[str, torch.Tensor]) -> None: + state = model.state_dict() + resolved: dict[str, torch.Tensor] = {} + for name, tensor in loaded.items(): + actual = name if name in state else None + if actual is None: + for key in state: + if name in key: + actual = key + break + if actual is not None: + resolved[actual] = tensor + else: + log_rank0(f"WARNING: glm5 checkpoint tensor has no target param: {name}") + + for name, target in model.named_parameters(): + if name not in resolved: + log_rank0(f"WARNING: {name} not loaded from checkpoint") + continue + tensor = resolved[name].to(device=target.device) + target.data.copy_(tensor.to(dtype=target.dtype)) + + for name, target in model.named_buffers(): + if name not in resolved: + continue + tensor = resolved[name].to(device=target.device) + target.data.copy_(tensor.to(dtype=target.dtype) if target.is_floating_point() else tensor) + + +class Glm5WeightSpec: + """Export GLM-5 lite weights to HF GLM-5 (deepseek_v3_2) names.""" + + def __init__(self, config: Glm5Config): + self.config = config + + @property + def num_experts(self) -> int: + return self.config.num_experts + + def weight_map(self) -> dict[str, list[str]]: + return {} + + def hf_to_native(self, native_name: str, hf_tensors: list[torch.Tensor]) -> torch.Tensor: + del native_name + return hf_tensors[0] + + # GLM-5 ONLY: DSA attention native suffix -> HF suffix. The wrapper places + # the DSA module under `self_attention.self_attention.*`; the HF model uses + # `self_attn.*` with identical submodule names. + _ATTN_SUFFIX_MAP: dict[str, str] = { + "self_attention.self_attention.q_a_proj.weight": "q_a_proj.weight", + "self_attention.self_attention.q_a_layernorm.weight": "q_a_layernorm.weight", + "self_attention.self_attention.q_b_proj.weight": "q_b_proj.weight", + "self_attention.self_attention.kv_a_proj_with_mqa.weight": "kv_a_proj_with_mqa.weight", + "self_attention.self_attention.kv_a_layernorm.weight": "kv_a_layernorm.weight", + "self_attention.self_attention.kv_b_proj.weight": "kv_b_proj.weight", + "self_attention.self_attention.o_proj.weight": "o_proj.weight", + "self_attention.self_attention.indexer.wq_b.weight": "indexer.wq_b.weight", + "self_attention.self_attention.indexer.wk.weight": "indexer.wk.weight", + "self_attention.self_attention.indexer.k_norm.weight": "indexer.k_norm.weight", + "self_attention.self_attention.indexer.k_norm.bias": "indexer.k_norm.bias", + "self_attention.self_attention.indexer.weights_proj.weight": "indexer.weights_proj.weight", + } + + def native_to_hf( + self, native_name: str, tensor: torch.Tensor + ) -> list[tuple[str, torch.Tensor]]: + if native_name == "mtp_embed.embedding.weight": + return [] + if native_name.startswith("mtp.layers."): + parts = native_name.split(".") + mtp_idx = int(parts[2]) + hf_layer_idx = self.config.num_hidden_layers + mtp_idx + hp = f"model.layers.{hf_layer_idx}" + if native_name.endswith(".enorm.weight"): + return [(f"{hp}.enorm.weight", tensor)] + if native_name.endswith(".hnorm.weight"): + return [(f"{hp}.hnorm.weight", tensor)] + if native_name.endswith(".eh_proj.linear.weight"): + return [(f"{hp}.eh_proj.weight", tensor)] + if native_name.endswith(".final_layernorm.weight"): + return [(f"{hp}.shared_head.norm.weight", tensor)] + proxy = native_name.replace( + f"mtp.layers.{mtp_idx}.transformer_layer", + f"layers.{hf_layer_idx}", + ) + return self.native_to_hf(proxy, tensor) + if native_name == "embed.embedding.weight": + return [("model.embed_tokens.weight", tensor)] + if native_name == "norm.weight": + return [("model.norm.weight", tensor)] + if native_name == "head.col.linear.weight": + return [("lm_head.weight", tensor)] + + parts = native_name.split(".") + if len(parts) < 3 or parts[0] != "layers": + return [] + layer_idx = int(parts[1]) + suffix = ".".join(parts[2:]) + hp = f"model.layers.{layer_idx}" + ap = f"{hp}.self_attn" + mp = f"{hp}.mlp" + + if suffix == "input_layernorm.weight": + return [(f"{hp}.input_layernorm.weight", tensor)] + + # GLM-5 ONLY: DSA attention (incl. indexer) maps 1:1 onto self_attn.*. + attn_hf = self._ATTN_SUFFIX_MAP.get(suffix) + if attn_hf is not None: + return [(f"{ap}.{attn_hf}", tensor)] + + if suffix == "mlp.gate_up.linear.layer_norm_weight": + return [(f"{hp}.post_attention_layernorm.weight", tensor)] + if suffix == "mlp.gate_up.linear.weight": + gate, up = tensor.chunk(2, dim=0) + return [ + (f"{mp}.gate_proj.weight", gate.contiguous()), + (f"{mp}.up_proj.weight", up.contiguous()), + ] + if suffix == "mlp.down.linear.weight": + return [(f"{mp}.down_proj.weight", tensor)] + + if suffix == "mlp_norm.weight": + return [(f"{hp}.post_attention_layernorm.weight", tensor)] + if suffix == "moe.router.gate.weight": + return [(f"{mp}.gate.weight", tensor)] + if suffix == "moe.router.expert_bias": + return [(f"{mp}.gate.e_score_correction_bias", tensor.float())] + if suffix == "moe.shared_expert.gate_up.linear.weight": + gate, up = tensor.chunk(2, dim=0) + return [ + (f"{mp}.shared_experts.gate_proj.weight", gate.contiguous()), + (f"{mp}.shared_experts.up_proj.weight", up.contiguous()), + ] + if suffix == "moe.shared_expert.down.linear.weight": + return [(f"{mp}.shared_experts.down_proj.weight", tensor)] + + if ".moe.experts.fc1.weight" in native_name: + expert_idx = parse_expert_idx(native_name) + gate, up = tensor.chunk(2, dim=0) + return [ + (f"{mp}.experts.{expert_idx}.gate_proj.weight", gate.contiguous()), + (f"{mp}.experts.{expert_idx}.up_proj.weight", up.contiguous()), + ] + if ".moe.experts.fc2.weight" in native_name: + expert_idx = parse_expert_idx(native_name) + return [(f"{mp}.experts.{expert_idx}.down_proj.weight", tensor)] + + return [] + + def qkv_spec(self, native_name: str) -> tuple[int, int, int] | None: + del native_name + return None + + def tp_spec(self, native_name: str) -> tuple[int, int] | None: + # GLM-5 is TP=1 (DSA not TP-capable); only EP / ETP shard tensors. + if native_name.startswith("mtp.layers.") and ".transformer_layer." in native_name: + proxy = native_name.replace(".transformer_layer.", ".") + return self.tp_spec(proxy) + if native_name.endswith(".eh_proj.linear.weight"): + return (0, 0) + if self.is_expert(native_name): + if ".fc1." in native_name: + return (0, 1) + if ".fc2." in native_name: + return (1, 1) + return None + if native_name in {"embed.embedding.weight", "head.col.linear.weight"}: + return (0, 0) + if native_name.endswith(".mlp.gate_up.linear.weight"): + return (0, 0) + if native_name.endswith(".mlp.down.linear.weight"): + return (1, 0) + if native_name.endswith(".moe.shared_expert.gate_up.linear.weight"): + return (0, 0) + if native_name.endswith(".moe.shared_expert.down.linear.weight"): + return (1, 0) + return None + + def is_expert(self, native_name: str) -> bool: + return ".moe.experts." in native_name and ".router." not in native_name + + def expert_global_id(self, native_name: str) -> int | None: + if self.is_expert(native_name): + return parse_expert_idx(native_name) + return None + + def expert_local_name(self, native_name: str, local_idx: int) -> str: + prefix = native_name.rsplit(".weight", 1)[0] + return f"{prefix}.weight{local_idx}" + + +def load_hf_weights(model: nn.Module, path: str, config: Glm5Config, ps: ParallelState) -> None: + base_model = unwrap_model(model) + reader = SafeTensorReader(path) + out: dict[str, torch.Tensor] = {} + + prefix = _text_prefix(reader) + + if getattr(base_model, "embed", None) is not None: + out["embed.embedding.weight"] = _load_vocab( + reader, f"{prefix}.embed_tokens.weight", config, ps + ) + if getattr(base_model, "mtp_embed", None) is not None: + out["mtp_embed.embedding.weight"] = _load_vocab( + reader, + f"{prefix}.embed_tokens.weight", + config, + ps, + ) + if getattr(base_model, "norm", None) is not None: + out["norm.weight"] = _get(reader, f"{prefix}.norm.weight") + if getattr(base_model, "head", None) is not None: + out["head.col.linear.weight"] = _load_vocab( + reader, _lm_head_name(reader, prefix), config, ps + ) + + for local_idx, global_idx in enumerate(base_model.layer_indices): + lp = f"layers.{local_idx}" + hp = f"{prefix}.layers.{global_idx}" + out[f"{lp}.input_layernorm.weight"] = _get(reader, f"{hp}.input_layernorm.weight") + _load_attention( + out, + local_prefix=lp, + hf_prefix=f"{hp}.self_attn", + reader=reader, + ps=ps, + ) + if config.is_moe_layer(global_idx): + out[f"{lp}.mlp_norm.weight"] = _get(reader, f"{hp}.post_attention_layernorm.weight") + out[f"{lp}.moe.router.gate.weight"] = _get(reader, f"{hp}.mlp.gate.weight") + bias_name = f"{hp}.mlp.gate.e_score_correction_bias" + if _has(reader, bias_name): + out[f"{lp}.moe.router.expert_bias"] = _get(reader, bias_name).float() + _load_shared_expert( + out, local_prefix=lp, hf_mlp_prefix=f"{hp}.mlp", reader=reader, ps=ps + ) + _load_experts( + out, + local_prefix=lp, + hf_mlp_prefix=f"{hp}.mlp", + cfg=config, + ps=ps, + reader=reader, + ) + else: + _load_dense_mlp( + out, + local_prefix=lp, + hf_mlp_prefix=f"{hp}.mlp", + hf_layer_prefix=hp, + reader=reader, + ps=ps, + ) + + mtp = getattr(base_model, "mtp", None) + if mtp is not None: + for local_idx, _mtp_layer in enumerate(mtp.layers): + global_idx = config.num_hidden_layers + local_idx + lp = f"mtp.layers.{local_idx}" + hp = f"{prefix}.layers.{global_idx}" + tlp = f"{lp}.transformer_layer" + out[f"{lp}.enorm.weight"] = _get(reader, f"{hp}.enorm.weight") + out[f"{lp}.hnorm.weight"] = _get(reader, f"{hp}.hnorm.weight") + out[f"{lp}.eh_proj.linear.weight"] = _tp( + _get(reader, f"{hp}.eh_proj.weight"), + ps.tp_rank, + ps.tp_size, + ) + shared_head_norm = f"{hp}.shared_head.norm.weight" + final_norm = ( + shared_head_norm + if _has(reader, shared_head_norm) + else f"{hp}.final_layernorm.weight" + ) + out[f"{lp}.final_layernorm.weight"] = _get(reader, final_norm) + out[f"{tlp}.input_layernorm.weight"] = _get(reader, f"{hp}.input_layernorm.weight") + _load_attention( + out, + local_prefix=tlp, + hf_prefix=f"{hp}.self_attn", + reader=reader, + ps=ps, + ) + if config.is_moe_layer(global_idx): + out[f"{tlp}.mlp_norm.weight"] = _get( + reader, f"{hp}.post_attention_layernorm.weight" + ) + out[f"{tlp}.moe.router.gate.weight"] = _get(reader, f"{hp}.mlp.gate.weight") + bias_name = f"{hp}.mlp.gate.e_score_correction_bias" + if _has(reader, bias_name): + out[f"{tlp}.moe.router.expert_bias"] = _get(reader, bias_name).float() + _load_shared_expert( + out, + local_prefix=tlp, + hf_mlp_prefix=f"{hp}.mlp", + reader=reader, + ps=ps, + ) + _load_experts( + out, + local_prefix=tlp, + hf_mlp_prefix=f"{hp}.mlp", + cfg=config, + ps=ps, + reader=reader, + ) + else: + _load_dense_mlp( + out, + local_prefix=tlp, + hf_mlp_prefix=f"{hp}.mlp", + hf_layer_prefix=hp, + reader=reader, + ps=ps, + ) + + _copy_loaded_state(base_model, out) + + +def export_hf_weights(model, config: Glm5Config, ps: ParallelState, **kwargs): + from megatron.lite.primitive.ckpt.hf_weights import export_hf_weights as _export + + spec = Glm5WeightSpec(config) + rank0_only = bool(kwargs.get("rank0_only", False)) + export_dtype = _resolve_export_dtype(kwargs.get("export_dtype")) + yield from _export(model, spec, ps, vocab_size=config.vocab_size, **kwargs) + rank = dist.get_rank() if dist.is_initialized() else 0 + if rank0_only and rank != 0: + return + chunks = list(model) if isinstance(model, list | nn.ModuleList) else [model] + for chunk in chunks: + base_chunk = unwrap_model(chunk) + layer_map = ( + {i: base_chunk.layer_indices[i] for i in range(len(base_chunk.layer_indices))} + if hasattr(base_chunk, "layer_indices") + else {} + ) + for name, buffer in base_chunk.named_buffers(): + if not name.endswith(".moe.router.expert_bias"): + continue + global_name = to_global_layer_name(name, layer_map) + for hf_name, hf_tensor in spec.native_to_hf(global_name, buffer.detach().cpu()): + yield hf_name, _cast_export_tensor(hf_tensor, export_dtype) + + +def save_hf_weights(model, path: str, config: Glm5Config, ps: ParallelState, **kwargs) -> None: + from megatron.lite.primitive.ckpt.hf_weights import save_safetensors + + rank = dist.get_rank() if dist.is_initialized() else 0 + out = dict(export_hf_weights(model, config, ps, rank0_only=True, **kwargs)) + if rank == 0 and out: + save_safetensors(out, path) + if dist.is_initialized(): + dist.barrier() + + +__all__ = [ + "EXPERT_CLASSIFIER", + "Glm5WeightSpec", + "PLACEMENT_FN", + "_dequant_int4_weight", + "_dequant_fp8_weight", + "export_hf_weights", + "load_hf_weights", + "save_hf_weights", +] diff --git a/experimental/lite/megatron/lite/model/glm5/lite/model.py b/experimental/lite/megatron/lite/model/glm5/lite/model.py new file mode 100644 index 00000000000..ffbd30aa0e2 --- /dev/null +++ b/experimental/lite/megatron/lite/model/glm5/lite/model.py @@ -0,0 +1,966 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""GLM-5 (deepseek_v3_2) lite native model. + +This model is a near-verbatim clone of the Kimi-K2 (deepseek_v3) lite model +(``megatron/lite/model/kimi_k2/lite/model.py``). It inherits ALL of Kimi's +Megatron plumbing unchanged: SBHD ``[S, B, H]`` layout, sequence-parallel +scatter/gather, ``VocabParallelEmbedding`` / ``VocabParallelOutput``, +``share_embeddings_and_output_weights``, ``set_input_tensor``, +``build_pipeline_chunk_layout``-based pipeline boundaries, MTP, and the +dist-opt / distckpt integration. + +The ONLY functional difference from Kimi is the attention module: where Kimi +uses ``MultiLatentAttention`` (MLA), GLM-5 uses Dynamic Sparse Attention (DSA). +The DSA primitive is hard-wired batch-first ``[B, S, H]`` and expects explicit +``cos`` / ``sin`` / ``position_ids`` + ``packed_seq_params``, so it is wrapped +by ``Glm5DSAAttention`` which transposes ``[S, B, H] -> [B, S, H]`` before DSA +and back after, and builds the rotary embeddings / position ids locally. The +surrounding Kimi skeleton therefore stays byte-for-byte SBHD and untouched. + +NOTE: DSA is NOT tensor-parallel-capable, so GLM-5 is a documented TP=1 special +case (the protocol gate raises for TP>1 / ETP>1). VPP / PP / EP / CP work, +inherited from Kimi. +""" + +from __future__ import annotations + +import inspect +import os +from contextlib import nullcontext + +import torch +import torch.distributed as dist +import torch.nn as nn +import torch.nn.functional as F +import transformer_engine.pytorch as te + +from megatron.lite.model.glm5.config import Glm5Config +from megatron.lite.primitive.modules.attention import ( + DynamicSparseAttention, + build_rotary_embeddings, +) +from megatron.lite.primitive.modules.dispatcher import TokenDispatcher +from megatron.lite.primitive.modules.experts import Experts +from megatron.lite.primitive.modules.moe import MoEAuxLossAutoScaler +from megatron.lite.primitive.modules.mtp import MTPLossAutoScaler +from megatron.lite.primitive.ops.cross_entropy import vocab_parallel_cross_entropy +from megatron.lite.primitive.ops.linear_cross_entropy import linear_cross_entropy +from megatron.lite.primitive.ops.logprob import vocab_parallel_entropy +from megatron.lite.primitive.ops.sp_ops import ReduceScatterDim0 +from megatron.lite.primitive.kernels.swiglu import bias_swiglu_impl +from megatron.lite.primitive.parallel import ( + ColumnParallelLinear, + ParallelState, + RowParallelLinear, + VanillaColumnParallelLinear, + VocabParallelEmbedding, + VocabParallelOutput, + build_pipeline_chunk_layout, + gather_from_sequence_parallel, + roll_packed_thd_left, + scatter_to_sequence_parallel, +) +from megatron.lite.primitive.utils import build_fp8_recipe +from megatron.lite.primitive.utils.moe import ( + compute_routing_scores_for_aux_loss, + router_gating_linear, + switch_load_balancing_loss_func, + topk_routing_with_score_function, +) + +# -- GLM-5 ONLY: SP-grad parameter suffixes for the DSA attention (Kimi's MLA +# suffixes -- linear_q_down_proj / linear_kv_down_proj / *_up_proj.linear.* -- +# are replaced by the DSA module's parameter names). These tag the +# layernorm-fronted columns whose grads must be all-reduced across SP ranks. +# (At GLM-5's enforced TP=1 this list is unused, but is kept structurally so +# the model stays a faithful Kimi clone.) +_SP_GRAD_SUFFIXES: tuple[str, ...] = ( + ".input_layernorm.weight", + ".self_attention.q_a_proj.weight", + ".self_attention.q_a_layernorm.weight", + ".self_attention.kv_a_proj_with_mqa.weight", + ".self_attention.kv_a_layernorm.weight", + ".mlp_norm.weight", + ".mlp.gate_up.linear.layer_norm_weight", + ".moe.router.gate.weight", + ".norm.weight", +) + + +def _collect_sp_grad_params(model: nn.Module) -> list[nn.Parameter]: + return [ + param + for name, param in model.named_parameters() + if any(name.endswith(suffix) for suffix in _SP_GRAD_SUFFIXES) or name == "norm.weight" + ] + + +def _swiglu(x: torch.Tensor) -> torch.Tensor: + if x.is_cuda: + return bias_swiglu_impl(x, None, False, False) + x1, x2 = torch.chunk(x, 2, dim=-1) + return F.silu(x1) * x2 + + +def _reduce_scatter_to_sequence_parallel(x: torch.Tensor, ps: ParallelState) -> torch.Tensor: + if ps.tp_size == 1: + return x + return ReduceScatterDim0.apply(x, ps.tp_size, ps.tp_rank, ps.tp_group) + + +def _ordered_topk_from_routing_map( + probs_dense: torch.Tensor, routing_map: torch.Tensor, topk: int +) -> tuple[torch.Tensor, torch.Tensor]: + expert_ids = torch.arange( + probs_dense.size(-1), device=probs_dense.device, dtype=torch.long + ).expand_as(routing_map) + masked_ids = torch.where( + routing_map, expert_ids, torch.full_like(expert_ids, probs_dense.size(-1)) + ) + topk_indices = torch.sort(masked_ids, dim=-1).values[:, :topk] + topk_scores = torch.gather(probs_dense, dim=-1, index=topk_indices) + return topk_scores, topk_indices + + +def _router_linear( + x: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor | None, + router_dtype: torch.dtype, +) -> torch.Tensor: + if x.is_cuda: + return router_gating_linear(x, weight, bias, router_dtype) + return F.linear( + x.to(router_dtype), + weight.to(router_dtype), + None if bias is None else bias.to(router_dtype), + ) + + +def _topk_routing_supports_groups() -> bool: + params = inspect.signature(topk_routing_with_score_function).parameters + return "num_groups" in params and "group_topk" in params + + +# -- GLM-5 ONLY: DSA attention wrapper. Holds ``DynamicSparseAttention`` and +# adapts it to Kimi's SBHD-in / SBHD-out attention contract. Everything outside +# this class (norms, residual, MoE, MTP, embed, head, SP scatter/gather) is +# identical to Kimi. +class Glm5DSAAttention(nn.Module): + """SBHD ``[S, B, H]`` shim around the batch-first DSA primitive. + + Kimi's decoder layer calls ``self.self_attention(x_sbhd, packed_seq_params=)`` + where ``x_sbhd`` is ``[S, B, H]``. DSA is hard-wired ``[B, S, H]`` and needs + explicit ``cos`` / ``sin`` / ``position_ids``. This wrapper: + 1. transposes ``[S, B, H] -> [B, S, H]``, + 2. builds ``position_ids`` (local sequence) and the rotary ``cos`` / ``sin``, + 3. runs DSA, + 4. transposes the ``[B, S, H]`` output back to ``[S, B, H]``. + The Kimi skeleton therefore never observes the batch-first interior. + """ + + def __init__(self, config: Glm5Config, ps: ParallelState): + super().__init__() + self.ps = ps + self.qk_rope_head_dim = config.qk_rope_head_dim + self.rope_theta = config.rope_theta + self.self_attention = DynamicSparseAttention( + hidden_size=config.hidden_size, + num_attention_heads=config.num_attention_heads, + q_lora_rank=config.q_lora_rank, + kv_lora_rank=config.kv_lora_rank, + qk_nope_head_dim=config.qk_nope_head_dim, + qk_rope_head_dim=config.qk_rope_head_dim, + v_head_dim=config.v_head_dim, + index_n_heads=config.index_n_heads, + index_head_dim=config.index_head_dim, + index_topk=config.index_topk, + rms_norm_eps=config.rms_norm_eps, + rope_interleaved=config.rope_interleave, + latent_rms_norm_eps=config.latent_rms_norm_eps, + indexer_layer_norm_eps=config.indexer_layer_norm_eps, + indexer_rope_interleaved=config.indexer_rope_interleave, + indexer_rope_first=config.indexer_rope_first, + indexer_use_hadamard=config.indexer_use_hadamard, + indexer_loss_coeff=config.dsa_indexer_loss_coeff, + indexer_use_sparse_loss=config.dsa_indexer_use_sparse_loss, + calculate_per_token_loss=config.calculate_per_token_loss, + cp_size=ps.cp_size, + cp_rank=ps.cp_rank, + cp_group=ps.cp_group, + ) + + def forward(self, x: torch.Tensor, packed_seq_params=None) -> torch.Tensor: + # Kimi feeds SBHD [S, B, H]; DSA needs batch-first [B, S, H]. + x_bsh = x.transpose(0, 1).contiguous() + batch, seq_len, _ = x_bsh.shape + # Local (this-rank) position ids; DSA reconstructs the full sequence + # itself when CP > 1. + position_ids = ( + torch.arange(seq_len, device=x_bsh.device, dtype=torch.long) + .unsqueeze(0) + .expand(batch, -1) + ) + cos, sin = build_rotary_embeddings( + position_ids=position_ids, + dim=self.qk_rope_head_dim, + rope_theta=self.rope_theta, + dtype=x_bsh.dtype, + ) + out_bsh = self.self_attention( + x_bsh, + cos=cos, + sin=sin, + position_ids=position_ids, + attention_mask=None, + packed_seq_params=packed_seq_params, + ) + # Back to SBHD [S, B, H] for the Kimi skeleton. + return out_bsh.transpose(0, 1).contiguous() + + +class Glm5SigmoidTopKRouter(nn.Module): + """GLM-5 sigmoid router with group-limited routing and persistent expert bias. + + Byte-identical to Kimi's ``KimiK2SigmoidTopKRouter`` except for the config + type and that GLM-5 has no ``aux_loss_alpha`` HF field -- the aux coefficient + therefore defaults to 0 (aux loss contributes 0). + """ + + def __init__( + self, + config: Glm5Config, + ps: ParallelState, + *, + router_bias_rate: float = 0.0, + compute_aux_loss: bool = True, + use_pre_softmax: bool = False, + moe_router_fusion: bool = False, + ): + super().__init__() + if router_bias_rate > 0: + raise NotImplementedError( + "GLM-5 expert-bias EMA update is not implemented in lite yet." + ) + self.topk = config.num_experts_per_tok + self.num_experts = config.n_routed_experts + # GLM-5 has no aux_loss_alpha HF field; default the coefficient to 0. + self.aux_loss_coeff = getattr(config, "aux_loss_alpha", 0.0) + self.scaling_factor = config.routed_scaling_factor + self.num_groups = config.n_group if (config.n_group and config.n_group > 1) else None + self.group_topk = config.topk_group if self.num_groups is not None else None + self.router_bias_rate = router_bias_rate + self.compute_aux_loss = compute_aux_loss + self.use_pre_softmax = use_pre_softmax + self.moe_router_fusion = moe_router_fusion + + self.gate = nn.Linear(config.hidden_size, config.n_routed_experts, bias=False) + self.register_buffer( + "expert_bias", + torch.zeros(config.n_routed_experts, dtype=torch.float32), + persistent=True, + ) + self.register_buffer( + "local_tokens_per_expert", + torch.zeros(config.n_routed_experts, dtype=torch.float32), + persistent=False, + ) + self._aux_loss_group = ps.tp_group if ps.tp_size > 1 else None + + def _apply(self, fn): + super()._apply(fn) + self.expert_bias.data = self.expert_bias.data.float() + self.local_tokens_per_expert.data = self.local_tokens_per_expert.data.float() + return self + + def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + logits = _router_linear(x, self.gate.weight, None, torch.float32) + logits = logits.view(-1, self.num_experts) + num_tokens = logits.size(0) + routing_kwargs = {} + if self.num_groups is not None and self.group_topk is not None: + if not _topk_routing_supports_groups(): + raise NotImplementedError( + "topk_routing_with_score_function does not support group-limited routing." + ) + routing_kwargs = dict(num_groups=self.num_groups, group_topk=self.group_topk) + probs_dense, routing_map = topk_routing_with_score_function( + logits, + self.topk, + use_pre_softmax=self.use_pre_softmax, + score_function="sigmoid", + expert_bias=self.expert_bias.to(logits.dtype), + scaling_factor=(self.scaling_factor or None), + fused=self.moe_router_fusion, + **routing_kwargs, + ) + if torch.is_grad_enabled(): + with torch.no_grad(): + self.local_tokens_per_expert += routing_map.sum(dim=0) + topk_scores, topk_indices = _ordered_topk_from_routing_map( + probs_dense, routing_map, self.topk + ) + topk_scores = topk_scores.to(logits.dtype) + + if self.compute_aux_loss and self.training and torch.is_grad_enabled(): + _, aux_scores = compute_routing_scores_for_aux_loss( + logits, self.topk, score_function="sigmoid", fused=self.moe_router_fusion + ) + tokens_per_expert = routing_map.sum(dim=0).to(torch.int64) + total_num_tokens = num_tokens + if self._aux_loss_group is not None: + dist.all_reduce(tokens_per_expert, group=self._aux_loss_group) + total_num_tokens = num_tokens * dist.get_world_size(group=self._aux_loss_group) + aux_loss = switch_load_balancing_loss_func( + aux_scores, + tokens_per_expert, + total_num_tokens, + self.topk, + self.num_experts, + self.aux_loss_coeff, + fused=False, + ) + topk_scores = MoEAuxLossAutoScaler.apply(topk_scores, aux_loss) + + return topk_scores, topk_indices + + +class DenseMLP(nn.Module): + def __init__(self, config: Glm5Config, ps: ParallelState): + super().__init__() + self.gate_up = ColumnParallelLinear( + config.hidden_size, + config.intermediate_size * 2, + ps, + bias=False, + normalization="RMSNorm", + eps=config.rms_norm_eps, + ) + self.down = RowParallelLinear(config.intermediate_size, config.hidden_size, ps, bias=False) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.down(_swiglu(self.gate_up(x))) + + +class SharedExpert(nn.Module): + def __init__(self, config: Glm5Config, ps: ParallelState): + super().__init__() + self.ps = ps + # GLM-5 has no shared_expert_intermediate_size property; compute it from + # n_shared_experts * moe_intermediate_size (matches Kimi's property). + ffn = config.n_shared_experts * config.moe_intermediate_size + self.gate_up = _LocalLinear(config.hidden_size, ffn * 2 // ps.tp_size) + self.down = _LocalLinear(ffn // ps.tp_size, config.hidden_size) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + squeeze_batch = x.dim() == 2 + if squeeze_batch: + x = x.unsqueeze(1) + full_x = gather_from_sequence_parallel(x, self.ps) + partial_out = self.down(_swiglu(self.gate_up(full_x))) + out = _reduce_scatter_to_sequence_parallel(partial_out, self.ps) + return out.squeeze(1) if squeeze_batch else out + + +class _LocalLinear(nn.Module): + """TE linear without built-in TP collectives; weight remains TP-local.""" + + def __init__(self, in_features: int, out_features: int): + super().__init__() + self.linear = te.Linear( + in_features, + out_features, + bias=False, + params_dtype=torch.bfloat16, + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.linear(x) + + +class MoELayer(nn.Module): + def __init__( + self, + config: Glm5Config, + ps: ParallelState, + *, + use_deepep: bool, + router_bias_rate: float, + fp8: bool, + moe_act_recompute: bool, + ): + super().__init__() + if fp8: + raise NotImplementedError("GLM-5 lite MoE fp8 training is not implemented yet.") + self.router = Glm5SigmoidTopKRouter( + config, + ps, + router_bias_rate=router_bias_rate, + compute_aux_loss=True, + use_pre_softmax=True, + ) + self.experts = Experts( + config, + ps, + fp8=fp8, + moe_act_recompute=moe_act_recompute, + ) + self.dispatcher = TokenDispatcher( + config.num_experts, + config.hidden_size, + ps, + use_deepep=use_deepep, + ) + self.shared_expert = SharedExpert(config, ps) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + input_shape = x.shape + + flat_x = x.view(-1, x.size(-1)) + scores, indices = self.router(flat_x) + dispatched, tpe, permuted_probs = self.dispatcher.dispatch(flat_x, scores, indices) + del scores, indices + self.dispatcher.wait_dispatch_event() + expert_out = self.experts( + dispatched, + tpe, + permuted_probs, + tokens_per_expert_list=getattr(self.dispatcher, "_local_tpe_list", None), + ) + routed_out = self.dispatcher.combine(expert_out) + shared_out = self.shared_expert(x) + output = routed_out.view(input_shape) + output += shared_out + return output.to(x.dtype) + + +class Glm5Layer(nn.Module): + def __init__( + self, + config: Glm5Config, + ps: ParallelState, + layer_idx: int, + *, + use_deepep: bool = False, + router_bias_rate: float = 0.0, + fp8: bool = False, + moe_act_recompute: bool = False, + use_thd: bool = False, + ): + super().__init__() + self.layer_idx = layer_idx + self.input_layernorm = te.RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + # GLM-5 ONLY: DSA attention (Kimi builds MultiLatentAttention here). + # The wrapper preserves the SBHD self_attention(x, packed_seq_params=) + # contract so this layer's forward stays identical to Kimi's. + del use_thd # DSA derives its own THD handling from packed_seq_params. + self.self_attention = Glm5DSAAttention(config, ps) + if config.is_moe_layer(layer_idx): + self.mlp_norm: nn.Module | None = te.RMSNorm( + config.hidden_size, eps=config.rms_norm_eps + ) + self.moe: MoELayer | None = MoELayer( + config, + ps, + use_deepep=use_deepep, + router_bias_rate=router_bias_rate, + fp8=fp8, + moe_act_recompute=moe_act_recompute, + ) + self.mlp: DenseMLP | None = None + else: + self.mlp_norm = None + self.moe = None + self.mlp = DenseMLP(config, ps) + + def forward(self, x: torch.Tensor, packed_seq_params=None) -> torch.Tensor: + x = x + self.self_attention(self.input_layernorm(x), packed_seq_params=packed_seq_params) + if self.moe is not None: + assert self.mlp_norm is not None + mlp_input = self.mlp_norm(x) + return x + self.moe(mlp_input) + assert self.mlp is not None + return x + self.mlp(x) + + +def _roll_mtp_left( + tensor: torch.Tensor, + *, + packed_seq_params=None, + dims: int = -1, +) -> tuple[torch.Tensor, torch.Tensor]: + if packed_seq_params is not None: + return roll_packed_thd_left(tensor, packed_seq_params=packed_seq_params, dims=dims) + dim = dims if dims >= 0 else tensor.dim() + dims + rolled = torch.roll(tensor, shifts=-1, dims=dim) + rolled.select(dim, -1).zero_() + return rolled, rolled.sum() + + +class Glm5MTPLayer(nn.Module): + def __init__( + self, + config: Glm5Config, + ps: ParallelState, + layer_idx: int, + *, + embedding: VocabParallelEmbedding, + use_deepep: bool, + router_bias_rate: float, + fp8: bool, + moe_act_recompute: bool, + use_thd: bool, + detach_encoder: bool, + ): + super().__init__() + self.ps = ps + object.__setattr__(self, "embedding", embedding) + self.detach_encoder = detach_encoder + self.enorm = te.RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.hnorm = te.RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.eh_proj = VanillaColumnParallelLinear( + config.hidden_size * 2, + config.hidden_size, + ps, + sp=ps.tp_size > 1, + gather_output=True, + ) + self.transformer_layer = Glm5Layer( + config, + ps, + config.num_hidden_layers + layer_idx, + use_deepep=use_deepep, + router_bias_rate=router_bias_rate, + fp8=fp8, + moe_act_recompute=moe_act_recompute, + use_thd=use_thd, + ) + self.final_layernorm = te.RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + def forward( + self, + *, + input_ids: torch.Tensor, + position_ids: torch.Tensor | None, + hidden_states: torch.Tensor, + rotary_position_ids: torch.Tensor | None = None, + packed_seq_params=None, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]: + del rotary_position_ids + input_ids, _ = _roll_mtp_left(input_ids, packed_seq_params=packed_seq_params, dims=-1) + if position_ids is not None: + position_ids, _ = _roll_mtp_left( + position_ids, packed_seq_params=packed_seq_params, dims=-1 + ) + decoder_input = self.embedding(input_ids) + decoder_input = scatter_to_sequence_parallel(decoder_input, self.ps) + + if self.detach_encoder: + decoder_input = decoder_input.detach() + hidden_states = hidden_states.detach() + + decoder_input = self.enorm(decoder_input) + hidden_states = self.hnorm(hidden_states) + hidden_states = torch.cat((decoder_input, hidden_states), dim=-1) + hidden_states = self.eh_proj(hidden_states) + hidden_states = scatter_to_sequence_parallel(hidden_states, self.ps) + hidden_states = self.transformer_layer(hidden_states, packed_seq_params=packed_seq_params) + hidden_states = self.final_layernorm(hidden_states) + return hidden_states, input_ids, position_ids + + +class Glm5MTPBlock(nn.Module): + def __init__( + self, + config: Glm5Config, + ps: ParallelState, + *, + embedding: VocabParallelEmbedding, + use_deepep: bool, + router_bias_rate: float, + fp8: bool, + moe_act_recompute: bool, + use_thd: bool, + detach_encoder: bool, + repeated_layer: bool, + ): + super().__init__() + self.num_layers = config.num_nextn_predict_layers + self.repeated_layer = repeated_layer + layers_to_build = 1 if repeated_layer else self.num_layers + self.layers = nn.ModuleList( + [ + Glm5MTPLayer( + config, + ps, + idx, + embedding=embedding, + use_deepep=use_deepep, + router_bias_rate=router_bias_rate, + fp8=fp8, + moe_act_recompute=moe_act_recompute, + use_thd=use_thd, + detach_encoder=detach_encoder, + ) + for idx in range(layers_to_build) + ] + ) + + def forward( + self, + *, + input_ids: torch.Tensor, + position_ids: torch.Tensor | None, + hidden_states: torch.Tensor, + packed_seq_params=None, + ) -> list[torch.Tensor]: + outputs: list[torch.Tensor] = [] + rotary_position_ids = position_ids + for depth in range(self.num_layers): + layer = self.layers[0] if self.repeated_layer else self.layers[depth] + hidden_states, input_ids, position_ids = layer( + input_ids=input_ids, + position_ids=position_ids, + hidden_states=hidden_states, + rotary_position_ids=rotary_position_ids, + packed_seq_params=packed_seq_params, + ) + outputs.append(hidden_states) + return outputs + + +def _temperature_to_float(temperature: float | torch.Tensor) -> float: + if isinstance(temperature, torch.Tensor): + if temperature.numel() != 1: + raise ValueError("Glm5Model supports scalar temperature only.") + return float(temperature.detach().float().item()) + return float(temperature) + + +def _apply_attention_backend_override(backend: str | None) -> None: + if backend in (None, "flash"): + backend = "fused" + env = { + "auto": ("1", "1", "1"), + "flash": ("1", "0", "0"), + "fused": ("0", "1", "0"), + "unfused": ("0", "0", "1"), + "local": ("0", "0", "1"), + }.get(backend) + if env is None: + raise ValueError( + "attention_backend_override must be one of " + "{'auto', 'flash', 'fused', 'unfused', 'local'}" + ) + ( + os.environ["NVTE_FLASH_ATTN"], + os.environ["NVTE_FUSED_ATTN"], + os.environ["NVTE_UNFUSED_ATTN"], + ) = env + + +class Glm5Model(nn.Module): + def __init__( + self, + config: Glm5Config, + train_config, + ps: ParallelState, + *, + vpp_chunk_id: int | None = None, + router_bias_rate: float = 0.0, + use_thd: bool = False, + hf_path: str = "", + attention_backend_override: str | None = None, + mtp_enable: bool = False, + mtp_enable_train: bool = False, + mtp_detach_encoder: bool = False, + ): + super().__init__() + del hf_path + _apply_attention_backend_override(attention_backend_override) + self.config = config + self.train_config = train_config + self.ps = ps + self._input_tensor: torch.Tensor | None = None + self.mtp_enable_train = bool(mtp_enable and mtp_enable_train) + self.mtp_loss_scaling_factor = config.mtp_loss_scaling_factor + + layout = build_pipeline_chunk_layout( + config.num_hidden_layers, + ps, + train_config.vpp, + vpp_chunk_id, + num_mtp_layers=config.num_nextn_predict_layers if mtp_enable else 0, + ) + self.layer_indices = layout.layer_indices + self.pre_process = layout.has_embed + self.post_process = layout.has_head + # GLM-5 does not tie embeddings (no tie_word_embeddings HF field); the + # attribute is preserved for the dist-opt / distckpt interface. + self.share_embeddings_and_output_weights = bool( + getattr(config, "tie_word_embeddings", False) + ) + self.vision_model: nn.Module | None = None + + self.embed: VocabParallelEmbedding | None = None + if layout.has_embed: + self.embed = VocabParallelEmbedding(config.vocab_size, config.hidden_size, ps) + + recompute_modules = getattr(train_config, "recompute_modules", []) + moe_act_recompute = "moe_act" in recompute_modules and "moe" not in recompute_modules + self.layers = nn.ModuleList( + [ + Glm5Layer( + config, + ps, + idx, + use_deepep=train_config.use_deepep, + router_bias_rate=router_bias_rate, + fp8=train_config.fp8, + moe_act_recompute=moe_act_recompute, + use_thd=use_thd, + ) + for idx in self.layer_indices + ] + ) + + self.norm: nn.Module | None = None + self.head: VocabParallelOutput | None = None + if layout.has_head: + self.norm = te.RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.head = VocabParallelOutput(config.vocab_size, config.hidden_size, ps) + + self.mtp_embed: VocabParallelEmbedding | None = None + self.mtp: Glm5MTPBlock | None = None + if mtp_enable and config.num_nextn_predict_layers > 0 and layout.has_mtp: + mtp_embedding = self.embed + if mtp_embedding is None: + mtp_embedding = VocabParallelEmbedding(config.vocab_size, config.hidden_size, ps) + self.mtp_embed = mtp_embedding + self.mtp = Glm5MTPBlock( + config, + ps, + embedding=mtp_embedding, + use_deepep=train_config.use_deepep, + router_bias_rate=router_bias_rate, + fp8=train_config.fp8, + moe_act_recompute=moe_act_recompute, + use_thd=use_thd, + detach_encoder=mtp_detach_encoder, + repeated_layer=config.mtp_use_repeated_layer, + ) + + self.sp_params: list[nn.Parameter] = [] + if ps.tp_size > 1: + self.sp_params = _collect_sp_grad_params(self) + + def set_input_tensor(self, input_tensor): + if isinstance(input_tensor, list): + if len(input_tensor) > 1: + raise ValueError("Glm5Model expects a single pipeline input tensor.") + input_tensor = input_tensor[0] if input_tensor else None + self._input_tensor = input_tensor + + def forward( + self, + input_ids: torch.Tensor | None = None, + hidden_states: torch.Tensor | None = None, + position_ids: torch.Tensor | None = None, + packed_seq_params=None, + labels: torch.Tensor | None = None, + loss_mask: torch.Tensor | None = None, + temperature: float | torch.Tensor = 1.0, + use_fused_kernels: bool = False, + calculate_entropy: bool = False, + ) -> dict: + if self.embed is not None: + assert input_ids is not None + h = self.embed(input_ids) + else: + if hidden_states is None: + hidden_states = self._input_tensor + assert hidden_states is not None + h = hidden_states + + fp8_ctx = ( + te.fp8_autocast(enabled=True, fp8_recipe=build_fp8_recipe(self.train_config)) + if self.train_config.fp8 + else nullcontext() + ) + with fp8_ctx: + if self.embed is not None: + h = scatter_to_sequence_parallel(h, self.ps) + for layer in self.layers: + h = layer(h, packed_seq_params=packed_seq_params) + + output = {"hidden_states": h} + if self.head is not None: + assert self.norm is not None + hidden_for_head = self.norm(h) + mtp_hidden_states = self._apply_mtp( + hidden_for_head, + input_ids=input_ids, + position_ids=position_ids, + packed_seq_params=packed_seq_params, + ) + if mtp_hidden_states is not None: + output["mtp_hidden_states"] = mtp_hidden_states + if labels is not None: + temperature_value = _temperature_to_float(temperature) + mtp_result = self._apply_mtp_loss( + hidden_for_head, + mtp_hidden_states=mtp_hidden_states, + labels=labels, + loss_mask=loss_mask, + packed_seq_params=packed_seq_params, + temperature=temperature_value, + use_fused_kernels=use_fused_kernels, + ) + if mtp_result is not None: + hidden_for_head, mtp_loss = mtp_result + output["mtp_loss"] = mtp_loss + labels_sb = labels.transpose(0, 1).contiguous() + if use_fused_kernels: + hidden_full = gather_from_sequence_parallel(hidden_for_head, self.ps) + log_probs, entropy = linear_cross_entropy( + hidden_full, + self._head_weight_for_fused_ce(hidden_full), + labels_sb, + temperature_value, + self.ps.tp_group, + ) + output["loss"] = (-log_probs).mean() + output["log_probs"] = log_probs.transpose(0, 1).contiguous() + if calculate_entropy: + output["entropy"] = entropy.transpose(0, 1).contiguous() + else: + logits = self.head(hidden_for_head) + if temperature_value != 1.0: + logits = logits / temperature_value + loss = vocab_parallel_cross_entropy(logits, labels_sb, self.ps.tp_group) + output["loss"] = loss.mean() + output["log_probs"] = (-loss).transpose(0, 1).contiguous() + if calculate_entropy: + entropy = vocab_parallel_entropy(logits, self.ps.tp_group) + output["entropy"] = entropy.transpose(0, 1).contiguous() + else: + logits = self.head(hidden_for_head) + output["logits"] = self.head.gather(logits).transpose(0, 1).contiguous() + if mtp_hidden_states is not None: + output["mtp_logits"] = [ + self.head.gather(self.head(mtp_hidden)).transpose(0, 1).contiguous() + for mtp_hidden in mtp_hidden_states + ] + return output + + def _apply_mtp( + self, + hidden_states: torch.Tensor, + *, + input_ids: torch.Tensor | None, + position_ids: torch.Tensor | None, + packed_seq_params, + ) -> list[torch.Tensor] | None: + if self.mtp is None: + return None + if input_ids is None: + if self.mtp_enable_train: + raise ValueError("MTP training requires input_ids.") + return None + return self.mtp( + input_ids=input_ids, + position_ids=position_ids, + hidden_states=hidden_states, + packed_seq_params=packed_seq_params, + ) + + def _apply_mtp_loss( + self, + hidden_states: torch.Tensor, + *, + mtp_hidden_states: list[torch.Tensor] | None, + labels: torch.Tensor, + loss_mask: torch.Tensor | None, + packed_seq_params, + temperature: float, + use_fused_kernels: bool, + ) -> tuple[torch.Tensor, torch.Tensor] | None: + if mtp_hidden_states is None: + return None + if not self.mtp_enable_train: + return None + if loss_mask is None: + mtp_loss_mask = torch.ones_like(labels, dtype=torch.float32) + else: + mtp_loss_mask = loss_mask.to(dtype=torch.float32).clone() + mtp_labels = labels.clone() + + mtp_loss_values = [] + for mtp_hidden in mtp_hidden_states: + mtp_labels, _ = _roll_mtp_left( + mtp_labels, + packed_seq_params=packed_seq_params, + dims=-1, + ) + mtp_loss_mask, num_tokens = _roll_mtp_left( + mtp_loss_mask, + packed_seq_params=packed_seq_params, + dims=-1, + ) + labels_sb = mtp_labels.transpose(0, 1).contiguous() + mask_sb = mtp_loss_mask.transpose(0, 1).contiguous() + + if use_fused_kernels: + mtp_hidden_full = gather_from_sequence_parallel(mtp_hidden, self.ps) + log_probs, _entropy = linear_cross_entropy( + mtp_hidden_full, + self._head_weight_for_fused_ce(mtp_hidden_full), + labels_sb, + temperature, + self.ps.tp_group, + ) + token_loss = -log_probs + else: + logits = self.head(mtp_hidden) + if temperature != 1.0: + logits = logits / temperature + token_loss = vocab_parallel_cross_entropy(logits, labels_sb, self.ps.tp_group) + + token_loss = token_loss * mask_sb.to(dtype=token_loss.dtype) + num_tokens = num_tokens.to(dtype=token_loss.dtype).clamp_min(1.0) + mtp_loss_values.append(token_loss.sum() / num_tokens) + + mtp_loss_scale = self.mtp_loss_scaling_factor / max(len(mtp_hidden_states), 1) + hidden_states = MTPLossAutoScaler.apply( + hidden_states, + mtp_loss_scale * token_loss / num_tokens, + ) + + if not mtp_loss_values: + return None + return ( + hidden_states, + torch.stack([loss.detach().float() for loss in mtp_loss_values]).mean(), + ) + + def _head_weight_for_fused_ce(self, hidden_states: torch.Tensor) -> torch.Tensor: + assert self.head is not None + weight = self.head.col.linear.weight + return ( + weight if weight.dtype == hidden_states.dtype else weight.to(dtype=hidden_states.dtype) + ) + + +__all__ = [ + "DenseMLP", + "DynamicSparseAttention", + "Glm5DSAAttention", + "Glm5Layer", + "Glm5MTPBlock", + "Glm5MTPLayer", + "Glm5Model", + "Glm5SigmoidTopKRouter", + "MoELayer", + "MTPLossAutoScaler", + "SharedExpert", +] diff --git a/experimental/lite/megatron/lite/model/glm5/lite/protocol.py b/experimental/lite/megatron/lite/model/glm5/lite/protocol.py new file mode 100644 index 00000000000..4a0f3d9d0ea --- /dev/null +++ b/experimental/lite/megatron/lite/model/glm5/lite/protocol.py @@ -0,0 +1,343 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""GLM-5 (deepseek_v3_2) lite native model protocol for Megatron Lite runtime. + +Cloned from ``megatron/lite/model/kimi_k2/lite/protocol.py`` (deepseek_v3) and +renamed KimiK2 -> Glm5. The build / optimizer / checkpoint plumbing is +identical to Kimi; the only adaptations are: + * the config type (``Glm5Config``), + * the DSA indexer-loss pre-forward hook (``DSAIndexerLossAutoScaler`` in + addition to the MoE aux-loss scaler), + * the attention ``MODULE_MAP`` entries point at the DSA ``self_attention`` + instead of MLA, and + * a ``_validate_parallel_scope`` gate: GLM-5's DSA attention is NOT + tensor-parallel-capable, so TP>1 / ETP>1 raise ``NotImplementedError``. + PP / VPP / EP / CP all work (inherited from Kimi). +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import torch +import torch.nn as nn +from megatron.lite.model.glm5.config import Glm5Config +from megatron.lite.model.protocol_utils import ( + add_cross_entropy_fusion, + add_loss_context_kwargs, + pack_thd_forward_kwargs, + set_cross_entropy_fusion, + unpack_thd_forward_output, +) +from megatron.lite.primitive.bundle import ModelBundle +from megatron.lite.primitive.parallel import ParallelState, init_parallel +from megatron.lite.primitive.recompute import apply_recompute, parse_recompute_spec +from megatron.lite.runtime.contracts import OptimizerConfig, ParallelConfig +from megatron.lite.runtime.contracts.data import PackedBatch + + +def EXPERT_CLASSIFIER(name: str) -> bool: + return "experts" in name and "router" not in name and "shared" not in name + + +def PLACEMENT_FN(param_name: str) -> list: + from megatron.lite.model.glm5.lite.checkpoint import PLACEMENT_FN as placement_fn + + return placement_fn(param_name) + + +def is_expert_param(name: str) -> bool: + return EXPERT_CLASSIFIER(name) + + +def _maybe(module_name: str): + def getter(layer): + module = getattr(layer, module_name, None) + return module + + return getter + + +def _moe_module(name: str): + def getter(layer): + moe = getattr(layer, "moe", None) + return getattr(moe, name, None) if moe is not None else None + + return getter + + +# GLM-5 ONLY: attention entries point at the DSA wrapper / module instead of +# Kimi's MLA. Everything else mirrors Kimi's MODULE_MAP. +MODULE_MAP = { + "core_attn": lambda layer: layer.self_attention.self_attention, + "experts": _moe_module("experts"), + "moe": _maybe("moe"), + "router": _moe_module("router"), + "mlp": _maybe("mlp"), + "mlp_norm": lambda layer: layer.mlp_norm, + "attn_proj": lambda layer: layer.self_attention.self_attention.o_proj, + "self_attn": lambda layer: layer.self_attention, + "dsa": lambda layer: layer.self_attention.self_attention, +} + + +@dataclass(frozen=True) +class ImplConfig: + parallel: ParallelConfig = field(default_factory=ParallelConfig) + optimizer: str | None = "dist_opt" + recompute: list[str] = field(default_factory=list) + offload: list[str] = field(default_factory=list) + use_deepep: bool = False + use_thd: bool = False + cross_entropy_fusion: bool = False + hf_path: str = "" + attention_backend_override: str | None = None + router_aux_loss_coef: float | None = None + router_bias_rate: float = 0.0 + deterministic: bool = True + optimizer_config: OptimizerConfig | None = None + mtp_enable: bool = False + mtp_enable_train: bool = False + mtp_detach_encoder: bool = False + mtp_loss_scaling_factor: float = 0.1 + mtp_use_repeated_layer: bool | None = None + + +def build_model_config(source: str | Path | dict, **overrides) -> Glm5Config: + if isinstance(source, dict): + cfg = Glm5Config._from_hf_dict(source) + else: + cfg = Glm5Config.from_hf(str(source)) + for key, value in overrides.items(): + if hasattr(cfg, key): + setattr(cfg, key, value) + return cfg + + +def _forward_step(model: nn.Module, batch: PackedBatch) -> dict: + kwargs = pack_thd_forward_kwargs(model, batch) + add_loss_context_kwargs(kwargs) + add_cross_entropy_fusion(kwargs, model) + return model(**kwargs) + + +def unpack_forward_output(model: nn.Module, batch: PackedBatch, output) -> Any: + return unpack_thd_forward_output(model, batch, output) + + +def _make_aux_loss_hook(): + # GLM-5 ONLY: also scale the DSA indexer auxiliary loss (Kimi has no indexer). + from megatron.lite.primitive.modules.attention.dsa import DSAIndexerLossAutoScaler + from megatron.lite.primitive.modules.moe import MoEAuxLossAutoScaler + from megatron.lite.primitive.modules.mtp import MTPLossAutoScaler + + def hook(scale: torch.Tensor) -> None: + MoEAuxLossAutoScaler.set_loss_scale(scale) + MTPLossAutoScaler.set_loss_scale(scale) + DSAIndexerLossAutoScaler.set_loss_scale(scale) + + return hook + + +def _validate_parallel_scope(p: ParallelConfig) -> None: + """GLM-5 DSA attention is not tensor-parallel-capable (documented TP=1 case). + + PP / VPP / EP / CP are inherited from the Kimi skeleton and work; only + TP>1 / ETP>1 are unsupported. + """ + etp = 1 if p.etp is None else p.etp + if p.tp > 1: + raise NotImplementedError( + "GLM-5 native DSA attention does not support tensor parallelism; " + f"got tp={p.tp}. Use tp=1 (PP/VPP/EP/CP are supported)." + ) + if etp > 1: + raise NotImplementedError( + "GLM-5 native DSA attention does not support expert tensor parallelism; " + f"got etp={etp}. Use etp=1 (EP is supported)." + ) + + +def _build_dist_opt_optimizer( + chunks, model_cfg: Glm5Config, impl_cfg: ImplConfig, ps: ParallelState +): + from megatron.lite.primitive.optimizers.megatron_wrap import build_dist_opt_training_optimizer + + return build_dist_opt_training_optimizer( + chunks, + model_cfg=model_cfg, + impl_cfg=impl_cfg, + ps=ps, + model_name="glm5", + is_expert=is_expert_param, + deterministic=impl_cfg.deterministic, + ) + + +def build_model(model_cfg: Glm5Config, *, impl_cfg: ImplConfig) -> ModelBundle: + p = impl_cfg.parallel + _validate_parallel_scope(p) + if impl_cfg.use_deepep and (p.etp is not None and p.etp > 1): + raise ValueError("use_deepep and etp>1 are mutually exclusive") + if impl_cfg.router_aux_loss_coef is not None: + # GLM-5 has no aux_loss_alpha HF field; honour an explicit override only. + model_cfg.aux_loss_alpha = impl_cfg.router_aux_loss_coef + mtp_enable = bool(impl_cfg.mtp_enable) + mtp_enable_train = mtp_enable and bool(impl_cfg.mtp_enable_train) + if mtp_enable: + if model_cfg.num_nextn_predict_layers <= 0: + raise ValueError("mtp_enable=True but HF config has no num_nextn_predict_layers.") + model_cfg.mtp_loss_scaling_factor = impl_cfg.mtp_loss_scaling_factor + if impl_cfg.mtp_use_repeated_layer is not None: + model_cfg.mtp_use_repeated_layer = impl_cfg.mtp_use_repeated_layer + elif hasattr(model_cfg, "num_nextn_predict_layers"): + model_cfg.num_nextn_predict_layers = 0 + + from megatron.lite.model.glm5.lite.model import Glm5Model + + ps = init_parallel(p) + recompute_spec = parse_recompute_spec(impl_cfg.recompute) + vpp = None if p.vpp == 1 else p.vpp + train_cfg = SimpleNamespace( + tp=ps.tp_size, + ep=ps.ep_size, + etp=ps.etp_size, + pp=ps.pp_size, + cp=ps.cp_size, + vpp=vpp, + use_deepep=impl_cfg.use_deepep, + fp8=False, + recompute_modules=recompute_spec, + deterministic=impl_cfg.deterministic, + ) + model_kwargs: dict[str, Any] = dict( + router_bias_rate=impl_cfg.router_bias_rate, + use_thd=impl_cfg.use_thd, + hf_path=impl_cfg.hf_path, + attention_backend_override=impl_cfg.attention_backend_override, + mtp_enable=mtp_enable, + mtp_enable_train=mtp_enable_train, + mtp_detach_encoder=impl_cfg.mtp_detach_encoder, + ) + + if vpp is None: + chunks = [Glm5Model(model_cfg, train_cfg, ps, **model_kwargs).to(torch.bfloat16).cuda()] + else: + chunks = [ + Glm5Model( + model_cfg, + train_cfg, + ps, + vpp_chunk_id=i, + **model_kwargs, + ) + .to(torch.bfloat16) + .cuda() + for i in range(vpp) + ] + set_cross_entropy_fusion(chunks, impl_cfg.cross_entropy_fusion) + + if recompute_spec: + for chunk in chunks: + apply_recompute(chunk.layers, recompute_spec, MODULE_MAP) + + if impl_cfg.offload: + from megatron.lite.primitive.recompute import apply_offload + + for chunk in chunks: + apply_offload(chunk.layers, impl_cfg.offload, MODULE_MAP) + + optimizer = None + finalize_grads = None + post_model_load_hook = None + optimizer_backend = "none" + if impl_cfg.optimizer == "dist_opt": + optimizer, finalize_grads = _build_dist_opt_optimizer(chunks, model_cfg, impl_cfg, ps) + from megatron.lite.primitive.ckpt import attach_model_sharded_state_dict + from megatron.lite.runtime.megatron_utils import register_training_hooks + + attach_model_sharded_state_dict( + chunks, ps, get_placements=PLACEMENT_FN, is_expert=is_expert_param + ) + register_training_hooks(chunks, optimizer) + optimizer_backend = "dist_opt" + elif impl_cfg.optimizer == "fsdp2": + optimizer_backend = "fsdp2" + + def _post_model_load_hook(): + from megatron.lite.model.glm5.lite.model import Glm5Layer + from megatron.lite.primitive.optimizers.fsdp2 import build_fsdp2_training_optimizer + + return { + "optimizer": build_fsdp2_training_optimizer( + chunks, + impl_cfg.optimizer_config, + ps, + unit_modules=(Glm5Layer,), + expert_classifier=is_expert_param, + deterministic=impl_cfg.deterministic, + vpp=impl_cfg.parallel.vpp, + leaf_module_names=(), + ) + } + + post_model_load_hook = _post_model_load_hook + elif impl_cfg.optimizer is not None: + raise ValueError(f"Unknown glm5 lite optimizer: {impl_cfg.optimizer!r}.") + + return ModelBundle( + chunks=chunks, + parallel_state=ps, + optimizer=optimizer, + finalize_grads=finalize_grads, + forward_step=_forward_step, + extras={ + "model_cfg": model_cfg, + "optimizer_backend": optimizer_backend, + "post_model_load_hook": post_model_load_hook, + "pre_forward_hook": _make_aux_loss_hook(), + }, + ) + + +def load_hf_weights( + chunk: nn.Module, hf_path: str, model_cfg: Glm5Config, ps: ParallelState +) -> None: + if not hf_path: + return + from megatron.lite.model.glm5.lite.checkpoint import load_hf_weights as load_impl + + load_impl(chunk, hf_path, model_cfg, ps) + + +def export_hf_weights(chunks, model_cfg: Glm5Config, ps: ParallelState, **kwargs): + from megatron.lite.model.glm5.lite.checkpoint import export_hf_weights as export_impl + + yield from export_impl(chunks, model_cfg, ps, **kwargs) + + +def save_hf_weights(chunks, path: str, model_cfg: Glm5Config, ps: ParallelState, **kwargs): + from megatron.lite.model.glm5.lite.checkpoint import save_hf_weights as save_impl + + save_impl(chunks, path, model_cfg, ps, **kwargs) + + +def vocab_size(model_cfg) -> int | None: + cfg = getattr(model_cfg, "text_config", model_cfg) + return getattr(cfg, "vocab_size", None) + + +__all__ = [ + "EXPERT_CLASSIFIER", + "ImplConfig", + "PLACEMENT_FN", + "build_model", + "build_model_config", + "export_hf_weights", + "is_expert_param", + "load_hf_weights", + "save_hf_weights", + "vocab_size", +] diff --git a/experimental/lite/megatron/lite/model/kimi_k2/__init__.py b/experimental/lite/megatron/lite/model/kimi_k2/__init__.py new file mode 100644 index 00000000000..b71cb95d9c0 --- /dev/null +++ b/experimental/lite/megatron/lite/model/kimi_k2/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Kimi K2 model package.""" diff --git a/experimental/lite/megatron/lite/model/kimi_k2/config.py b/experimental/lite/megatron/lite/model/kimi_k2/config.py new file mode 100644 index 00000000000..bc4f7a9e1c5 --- /dev/null +++ b/experimental/lite/megatron/lite/model/kimi_k2/config.py @@ -0,0 +1,175 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Kimi K2 / DeepSeekV3-style model configuration.""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +from megatron.lite.primitive.config import load_hf_config_dict + +_HF_FIELDS = frozenset( + { + "num_hidden_layers", + "hidden_size", + "num_attention_heads", + "num_key_value_heads", + "vocab_size", + "rms_norm_eps", + "max_position_embeddings", + "intermediate_size", + "moe_intermediate_size", + "n_routed_experts", + "n_shared_experts", + "num_experts_per_tok", + "n_group", + "topk_group", + "topk_method", + "norm_topk_prob", + "scoring_func", + "seq_aux", + "first_k_dense_replace", + "moe_layer_freq", + "aux_loss_alpha", + "routed_scaling_factor", + "q_lora_rank", + "kv_lora_rank", + "qk_nope_head_dim", + "qk_rope_head_dim", + "v_head_dim", + "rope_theta", + "rope_scaling", + "tie_word_embeddings", + "num_nextn_predict_layers", + "mtp_loss_scaling_factor", + "mtp_use_repeated_layer", + } +) + + +@dataclass +class KimiK2Config: + """Pure architecture parameters for Kimi K2 Instruct.""" + + num_hidden_layers: int = 61 + hidden_size: int = 7168 + num_attention_heads: int = 64 + num_key_value_heads: int = 64 + vocab_size: int = 163840 + rms_norm_eps: float = 1e-6 + max_position_embeddings: int = 131072 + intermediate_size: int = 18432 + moe_intermediate_size: int = 2048 + n_routed_experts: int = 384 + n_shared_experts: int = 1 + num_experts_per_tok: int = 8 + n_group: int | None = 1 + topk_group: int | None = 1 + topk_method: str = "noaux_tc" + norm_topk_prob: bool = True + scoring_func: str = "sigmoid" + seq_aux: bool = True + first_k_dense_replace: int = 1 + moe_layer_freq: int = 1 + aux_loss_alpha: float = 0.001 + routed_scaling_factor: float = 2.827 + q_lora_rank: int = 1536 + kv_lora_rank: int = 512 + qk_nope_head_dim: int = 128 + qk_rope_head_dim: int = 64 + v_head_dim: int = 128 + rope_theta: float = 50_000.0 + rope_scaling: dict = field( + default_factory=lambda: { + "type": "yarn", + "factor": 32.0, + "original_max_position_embeddings": 4096, + "beta_fast": 1.0, + "beta_slow": 1.0, + "mscale": 1.0, + "mscale_all_dim": 1.0, + } + ) + tie_word_embeddings: bool = False + num_nextn_predict_layers: int = 0 + mtp_loss_scaling_factor: float = 0.1 + mtp_use_repeated_layer: bool = False + + @property + def num_experts(self) -> int: + return self.n_routed_experts + + @property + def shared_expert_intermediate_size(self) -> int: + return self.moe_intermediate_size * self.n_shared_experts + + @property + def q_head_dim(self) -> int: + return self.qk_nope_head_dim + self.qk_rope_head_dim + + def __post_init__(self) -> None: + errors: list[str] = [] + + def _check(cond: bool, msg: str) -> None: + if not cond: + errors.append(msg) + + _check(self.num_hidden_layers >= 1, "num_hidden_layers must be >= 1") + _check(self.hidden_size > 0, "hidden_size must be > 0") + _check(self.num_attention_heads >= 1, "num_attention_heads must be >= 1") + _check( + self.num_attention_heads % self.num_key_value_heads == 0, + "num_attention_heads must be divisible by num_key_value_heads", + ) + _check(self.q_lora_rank > 0, "q_lora_rank must be > 0") + _check(self.kv_lora_rank > 0, "kv_lora_rank must be > 0") + _check(self.qk_nope_head_dim > 0, "qk_nope_head_dim must be > 0") + _check(self.qk_rope_head_dim > 0, "qk_rope_head_dim must be > 0") + _check(self.v_head_dim > 0, "v_head_dim must be > 0") + _check(self.n_routed_experts >= 1, "n_routed_experts must be >= 1") + _check( + 1 <= self.num_experts_per_tok <= self.n_routed_experts, + "num_experts_per_tok must be in [1, n_routed_experts]", + ) + if self.n_group is not None or self.topk_group is not None: + _check( + self.n_group is not None and self.topk_group is not None, + "n_group and topk_group must be set together", + ) + if self.n_group is not None and self.topk_group is not None: + _check(self.n_group >= 1, "n_group must be >= 1") + _check(1 <= self.topk_group <= self.n_group, "topk_group must be in [1, n_group]") + _check( + self.n_routed_experts % self.n_group == 0, + "n_routed_experts must be divisible by n_group", + ) + _check(0 <= self.first_k_dense_replace <= self.num_hidden_layers, "bad dense prefix") + _check(self.num_nextn_predict_layers >= 0, "num_nextn_predict_layers must be >= 0") + if errors: + raise ValueError("Invalid KimiK2Config:\n " + "\n ".join(errors)) + + @classmethod + def from_hf(cls, path: str, **overrides) -> KimiK2Config: + return cls._from_hf_dict(load_hf_config_dict(path), **overrides) + + @classmethod + def from_hf_config(cls, hf_config, **overrides) -> KimiK2Config: + return cls._from_hf_dict(hf_config.to_dict(), **overrides) + + @classmethod + def _from_hf_dict(cls, hf: dict, **overrides) -> KimiK2Config: + if "text_config" in hf and isinstance(hf["text_config"], dict): + hf = hf["text_config"] + kwargs = {k: v for k, v in hf.items() if k in _HF_FIELDS} + if "rope_scaling" in kwargs and kwargs["rope_scaling"] is None: + kwargs.pop("rope_scaling") + kwargs.update(overrides) + return cls(**kwargs) + + def is_moe_layer(self, layer_idx: int) -> bool: + if layer_idx < self.first_k_dense_replace: + return False + freq = int(self.moe_layer_freq or 1) + return (layer_idx - self.first_k_dense_replace) % freq == 0 + + +__all__ = ["KimiK2Config"] diff --git a/experimental/lite/megatron/lite/model/kimi_k2/lite/__init__.py b/experimental/lite/megatron/lite/model/kimi_k2/lite/__init__.py new file mode 100644 index 00000000000..51099e78ca6 --- /dev/null +++ b/experimental/lite/megatron/lite/model/kimi_k2/lite/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Kimi K2 lite native sub-package.""" diff --git a/experimental/lite/megatron/lite/model/kimi_k2/lite/checkpoint.py b/experimental/lite/megatron/lite/model/kimi_k2/lite/checkpoint.py new file mode 100644 index 00000000000..078e0652f37 --- /dev/null +++ b/experimental/lite/megatron/lite/model/kimi_k2/lite/checkpoint.py @@ -0,0 +1,662 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Kimi K2 lite native checkpoint mapping.""" + +from __future__ import annotations + +import torch +import torch.distributed as dist +import torch.nn as nn +from torch.distributed.tensor import Replicate, Shard + +from megatron.lite.model.kimi_k2.config import KimiK2Config +from megatron.lite.primitive.ckpt.hf_weights import ( + SafeTensorReader, + _cast_export_tensor, + _resolve_export_dtype, + parse_expert_idx, + to_global_layer_name, + unwrap_model, +) +from megatron.lite.primitive.parallel import ParallelState +from megatron.lite.primitive.utils import ensure_divisible, log_rank0 + + +def EXPERT_CLASSIFIER(name: str) -> bool: + return "experts" in name and "router" not in name and "shared" not in name + + +def PLACEMENT_FN(param_name: str) -> list: + if "experts" in param_name and "router" not in param_name and "shared" not in param_name: + if "fc1" in param_name: + return [Replicate(), Replicate(), Shard(0), Shard(0)] + if "fc2" in param_name: + return [Replicate(), Replicate(), Shard(0), Shard(1)] + return [Replicate(), Replicate(), Replicate(), Replicate()] + if "eh_proj.linear.weight" in param_name: + return [Replicate(), Replicate(), Replicate(), Shard(0)] + if "linear_q_up_proj.linear.weight" in param_name: + return [Replicate(), Replicate(), Replicate(), Shard(0)] + if "linear_kv_up_proj.linear.weight" in param_name: + return [Replicate(), Replicate(), Replicate(), Shard(0)] + if "linear_proj.linear.weight" in param_name: + return [Replicate(), Replicate(), Replicate(), Shard(1)] + if "gate_up" in param_name: + return [Replicate(), Replicate(), Replicate(), Shard(0)] + if "down" in param_name: + return [Replicate(), Replicate(), Replicate(), Shard(1)] + if "embed" in param_name or "head" in param_name: + return [Replicate(), Replicate(), Replicate(), Shard(0)] + return [Replicate(), Replicate(), Replicate(), Replicate()] + + +def _tp(tensor: torch.Tensor, rank: int, size: int, dim: int = 0) -> torch.Tensor: + return tensor if size <= 1 else tensor.chunk(size, dim=dim)[rank].contiguous() + + +def _split_gate_up(tensor: torch.Tensor, rank: int, size: int) -> torch.Tensor: + if size <= 1: + return tensor + ffn = tensor.shape[0] // 2 + gate = tensor[:ffn].chunk(size, dim=0)[rank] + up = tensor[ffn:].chunk(size, dim=0)[rank] + return torch.cat([gate, up], dim=0).contiguous() + + +def _has(reader: SafeTensorReader, name: str) -> bool: + if reader.index: + return name in reader.index + try: + reader.get_tensor(name) + except Exception: + return False + return True + + +def _dequant_fp8_weight(reader: SafeTensorReader, name: str, weight: torch.Tensor) -> torch.Tensor: + scale_name = f"{name}_scale_inv" + if weight.dim() != 2 or weight.element_size() != 1 or not _has(reader, scale_name): + return weight + scale = reader.get_tensor(scale_name).float() + rows, cols = weight.shape + expanded = scale.repeat_interleave(128, dim=0).repeat_interleave(128, dim=1) + expanded = expanded[:rows, :cols] + return weight.float() * expanded + + +def _unpack_int4_from_int32(packed: torch.Tensor, shape: torch.Size) -> torch.Tensor: + if packed.dtype != torch.int32: + raise ValueError(f"Expected packed int4 tensor to be int32, got {packed.dtype}.") + pack_factor = 8 + mask = 0xF + rows, cols = int(shape[0]), int(shape[1]) + unpacked = torch.empty((packed.shape[0], packed.shape[1] * pack_factor), dtype=torch.int32) + for offset in range(pack_factor): + unpacked[:, offset::pack_factor] = (packed >> (4 * offset)) & mask + return (unpacked[:rows, :cols] - 8).to(torch.int8) + + +def _dequant_int4_weight(reader: SafeTensorReader, name: str) -> torch.Tensor: + packed = reader.get_tensor(f"{name}_packed") + scale = reader.get_tensor(f"{name}_scale") + shape_tensor = reader.get_tensor(f"{name}_shape") + shape = torch.Size(int(x) for x in shape_tensor.tolist()) + + unpacked = _unpack_int4_from_int32(packed, shape).to(scale.dtype) + if scale.dim() != 2 or unpacked.dim() != 2: + raise ValueError( + f"Expected groupwise int4 tensors for {name}, got " + f"weight={tuple(unpacked.shape)} scale={tuple(scale.shape)}." + ) + if scale.shape[0] not in (1, unpacked.shape[0]): + raise ValueError( + f"Unsupported int4 scale rows for {name}: " + f"weight={tuple(unpacked.shape)} scale={tuple(scale.shape)}." + ) + if unpacked.shape[1] % scale.shape[1] != 0: + raise ValueError( + f"Unsupported int4 group layout for {name}: " + f"weight={tuple(unpacked.shape)} scale={tuple(scale.shape)}." + ) + + group_size = unpacked.shape[1] // scale.shape[1] + return (unpacked.unflatten(-1, (scale.shape[1], group_size)) * scale.unsqueeze(-1)).flatten( + start_dim=-2 + ) + + +def _get(reader: SafeTensorReader, name: str) -> torch.Tensor: + if not _has(reader, name) and _has(reader, f"{name}_packed"): + return _dequant_int4_weight(reader, name) + tensor = reader.get_tensor(name) + return _dequant_fp8_weight(reader, name, tensor) + + +def _text_prefix(reader: SafeTensorReader) -> str: + for prefix in ("model", "language_model.model", "model.language_model"): + if _has(reader, f"{prefix}.embed_tokens.weight"): + return prefix + raise KeyError("Could not find Kimi K2 text model prefix in HF checkpoint.") + + +def _lm_head_name(reader: SafeTensorReader, text_prefix: str) -> str: + candidates = [ + "lm_head.weight", + "language_model.lm_head.weight", + f"{text_prefix}.lm_head.weight", + ] + for name in candidates: + if _has(reader, name): + return name + raise KeyError("Could not find Kimi K2 lm_head.weight in HF checkpoint.") + + +def _load_vocab( + reader: SafeTensorReader, name: str, cfg: KimiK2Config, ps: ParallelState +) -> torch.Tensor: + from megatron.lite.primitive.parallel import pad_vocab_for_tp + + tensor = _get(reader, name) + padded = pad_vocab_for_tp(cfg.vocab_size, ps.tp_size) + if tensor.size(0) < padded: + pad = torch.zeros(padded - tensor.size(0), tensor.size(1), dtype=tensor.dtype) + tensor = torch.cat([tensor, pad], dim=0) + return _tp(tensor, ps.tp_rank, ps.tp_size) + + +def _load_attention( + out: dict[str, torch.Tensor], + *, + local_prefix: str, + hf_prefix: str, + reader: SafeTensorReader, + ps: ParallelState, +) -> None: + out[f"{local_prefix}.self_attention.linear_q_down_proj.weight"] = _get( + reader, + f"{hf_prefix}.q_a_proj.weight", + ) + out[f"{local_prefix}.self_attention.linear_q_up_proj.linear.layer_norm_weight"] = _get( + reader, + f"{hf_prefix}.q_a_layernorm.weight", + ) + out[f"{local_prefix}.self_attention.linear_q_up_proj.linear.weight"] = _tp( + _get(reader, f"{hf_prefix}.q_b_proj.weight"), + ps.tp_rank, + ps.tp_size, + ) + out[f"{local_prefix}.self_attention.linear_kv_down_proj.weight"] = _get( + reader, + f"{hf_prefix}.kv_a_proj_with_mqa.weight", + ) + out[f"{local_prefix}.self_attention.linear_kv_up_proj.linear.layer_norm_weight"] = _get( + reader, + f"{hf_prefix}.kv_a_layernorm.weight", + ) + out[f"{local_prefix}.self_attention.linear_kv_up_proj.linear.weight"] = _tp( + _get(reader, f"{hf_prefix}.kv_b_proj.weight"), + ps.tp_rank, + ps.tp_size, + ) + out[f"{local_prefix}.self_attention.linear_proj.linear.weight"] = _tp( + _get(reader, f"{hf_prefix}.o_proj.weight"), + ps.tp_rank, + ps.tp_size, + dim=1, + ) + + +def _load_dense_mlp( + out: dict[str, torch.Tensor], + *, + local_prefix: str, + hf_mlp_prefix: str, + hf_layer_prefix: str, + reader: SafeTensorReader, + ps: ParallelState, +) -> None: + out[f"{local_prefix}.mlp.gate_up.linear.layer_norm_weight"] = _get( + reader, + f"{hf_layer_prefix}.post_attention_layernorm.weight", + ) + gate_up = torch.cat( + [ + _get(reader, f"{hf_mlp_prefix}.gate_proj.weight"), + _get(reader, f"{hf_mlp_prefix}.up_proj.weight"), + ], + dim=0, + ) + out[f"{local_prefix}.mlp.gate_up.linear.weight"] = _split_gate_up( + gate_up, + ps.tp_rank, + ps.tp_size, + ) + out[f"{local_prefix}.mlp.down.linear.weight"] = _tp( + _get(reader, f"{hf_mlp_prefix}.down_proj.weight"), + ps.tp_rank, + ps.tp_size, + dim=1, + ) + + +def _load_shared_expert( + out: dict[str, torch.Tensor], + *, + local_prefix: str, + hf_mlp_prefix: str, + reader: SafeTensorReader, + ps: ParallelState, +) -> None: + prefixes = [f"{hf_mlp_prefix}.shared_experts", f"{hf_mlp_prefix}.shared_expert"] + shared = next(prefix for prefix in prefixes if _has(reader, f"{prefix}.down_proj.weight")) + gate_up = torch.cat( + [ + _get(reader, f"{shared}.gate_proj.weight"), + _get(reader, f"{shared}.up_proj.weight"), + ], + dim=0, + ) + out[f"{local_prefix}.moe.shared_expert.gate_up.linear.weight"] = _split_gate_up( + gate_up, + ps.tp_rank, + ps.tp_size, + ) + out[f"{local_prefix}.moe.shared_expert.down.linear.weight"] = _tp( + _get(reader, f"{shared}.down_proj.weight"), + ps.tp_rank, + ps.tp_size, + dim=1, + ) + + +def _load_experts( + out: dict[str, torch.Tensor], + *, + local_prefix: str, + hf_mlp_prefix: str, + cfg: KimiK2Config, + ps: ParallelState, + reader: SafeTensorReader, +) -> None: + num_local = ensure_divisible(cfg.num_experts, ps.ep_size) + local_start = ps.ep_rank * num_local + for local_idx in range(num_local): + global_idx = local_start + local_idx + ep = f"{hf_mlp_prefix}.experts.{global_idx}" + fc1 = torch.cat( + [ + _get(reader, f"{ep}.gate_proj.weight"), + _get(reader, f"{ep}.up_proj.weight"), + ], + dim=0, + ) + fc2 = _get(reader, f"{ep}.down_proj.weight") + if ps.etp_size > 1: + fc1 = _split_gate_up(fc1, ps.etp_rank, ps.etp_size) + fc2 = _tp(fc2, ps.etp_rank, ps.etp_size, dim=1) + out[f"{local_prefix}.moe.experts.fc1.weight{local_idx}"] = fc1 + out[f"{local_prefix}.moe.experts.fc2.weight{local_idx}"] = fc2 + + +def _copy_loaded_state(model: nn.Module, loaded: dict[str, torch.Tensor]) -> None: + state = model.state_dict() + resolved: dict[str, torch.Tensor] = {} + for name, tensor in loaded.items(): + actual = name if name in state else None + if actual is None: + for key in state: + if name in key: + actual = key + break + if actual is not None: + resolved[actual] = tensor + else: + log_rank0(f"WARNING: kimi_k2 checkpoint tensor has no target param: {name}") + + for name, target in model.named_parameters(): + if name not in resolved: + log_rank0(f"WARNING: {name} not loaded from checkpoint") + continue + tensor = resolved[name].to(device=target.device) + target.data.copy_(tensor.to(dtype=target.dtype)) + + for name, target in model.named_buffers(): + if name not in resolved: + continue + tensor = resolved[name].to(device=target.device) + target.data.copy_(tensor.to(dtype=target.dtype) if target.is_floating_point() else tensor) + + +class KimiK2WeightSpec: + """Export Kimi K2 lite weights to HF DeepSeekV3/Kimi-style names.""" + + def __init__(self, config: KimiK2Config): + self.config = config + + @property + def num_experts(self) -> int: + return self.config.num_experts + + def weight_map(self) -> dict[str, list[str]]: + return {} + + def hf_to_native(self, native_name: str, hf_tensors: list[torch.Tensor]) -> torch.Tensor: + del native_name + return hf_tensors[0] + + def native_to_hf( + self, native_name: str, tensor: torch.Tensor + ) -> list[tuple[str, torch.Tensor]]: + if native_name == "mtp_embed.embedding.weight": + return [] + if native_name.startswith("mtp.layers."): + parts = native_name.split(".") + mtp_idx = int(parts[2]) + hf_layer_idx = self.config.num_hidden_layers + mtp_idx + hp = f"model.layers.{hf_layer_idx}" + if native_name.endswith(".enorm.weight"): + return [(f"{hp}.enorm.weight", tensor)] + if native_name.endswith(".hnorm.weight"): + return [(f"{hp}.hnorm.weight", tensor)] + if native_name.endswith(".eh_proj.linear.weight"): + return [(f"{hp}.eh_proj.weight", tensor)] + if native_name.endswith(".final_layernorm.weight"): + return [(f"{hp}.shared_head.norm.weight", tensor)] + proxy = native_name.replace( + f"mtp.layers.{mtp_idx}.transformer_layer", + f"layers.{hf_layer_idx}", + ) + return self.native_to_hf(proxy, tensor) + if native_name == "embed.embedding.weight": + return [("model.embed_tokens.weight", tensor)] + if native_name == "norm.weight": + return [("model.norm.weight", tensor)] + if native_name == "head.col.linear.weight": + return [("lm_head.weight", tensor)] + + parts = native_name.split(".") + if len(parts) < 3 or parts[0] != "layers": + return [] + layer_idx = int(parts[1]) + suffix = ".".join(parts[2:]) + hp = f"model.layers.{layer_idx}" + ap = f"{hp}.self_attn" + mp = f"{hp}.mlp" + + if suffix == "input_layernorm.weight": + return [(f"{hp}.input_layernorm.weight", tensor)] + if suffix == "self_attention.linear_q_down_proj.weight": + return [(f"{ap}.q_a_proj.weight", tensor)] + if suffix == "self_attention.linear_q_up_proj.linear.layer_norm_weight": + return [(f"{ap}.q_a_layernorm.weight", tensor)] + if suffix == "self_attention.linear_q_up_proj.linear.weight": + return [(f"{ap}.q_b_proj.weight", tensor)] + if suffix == "self_attention.linear_kv_down_proj.weight": + return [(f"{ap}.kv_a_proj_with_mqa.weight", tensor)] + if suffix == "self_attention.linear_kv_up_proj.linear.layer_norm_weight": + return [(f"{ap}.kv_a_layernorm.weight", tensor)] + if suffix == "self_attention.linear_kv_up_proj.linear.weight": + return [(f"{ap}.kv_b_proj.weight", tensor)] + if suffix == "self_attention.linear_proj.linear.weight": + return [(f"{ap}.o_proj.weight", tensor)] + + if suffix == "mlp.gate_up.linear.layer_norm_weight": + return [(f"{hp}.post_attention_layernorm.weight", tensor)] + if suffix == "mlp.gate_up.linear.weight": + gate, up = tensor.chunk(2, dim=0) + return [ + (f"{mp}.gate_proj.weight", gate.contiguous()), + (f"{mp}.up_proj.weight", up.contiguous()), + ] + if suffix == "mlp.down.linear.weight": + return [(f"{mp}.down_proj.weight", tensor)] + + if suffix == "mlp_norm.weight": + return [(f"{hp}.post_attention_layernorm.weight", tensor)] + if suffix == "moe.router.gate.weight": + return [(f"{mp}.gate.weight", tensor)] + if suffix == "moe.router.expert_bias": + return [(f"{mp}.gate.e_score_correction_bias", tensor.float())] + if suffix == "moe.shared_expert.gate_up.linear.weight": + gate, up = tensor.chunk(2, dim=0) + return [ + (f"{mp}.shared_experts.gate_proj.weight", gate.contiguous()), + (f"{mp}.shared_experts.up_proj.weight", up.contiguous()), + ] + if suffix == "moe.shared_expert.down.linear.weight": + return [(f"{mp}.shared_experts.down_proj.weight", tensor)] + + if ".moe.experts.fc1.weight" in native_name: + expert_idx = parse_expert_idx(native_name) + gate, up = tensor.chunk(2, dim=0) + return [ + (f"{mp}.experts.{expert_idx}.gate_proj.weight", gate.contiguous()), + (f"{mp}.experts.{expert_idx}.up_proj.weight", up.contiguous()), + ] + if ".moe.experts.fc2.weight" in native_name: + expert_idx = parse_expert_idx(native_name) + return [(f"{mp}.experts.{expert_idx}.down_proj.weight", tensor)] + + return [] + + def qkv_spec(self, native_name: str) -> tuple[int, int, int] | None: + del native_name + return None + + def tp_spec(self, native_name: str) -> tuple[int, int] | None: + if native_name.startswith("mtp.layers.") and ".transformer_layer." in native_name: + proxy = native_name.replace(".transformer_layer.", ".") + return self.tp_spec(proxy) + if native_name.endswith(".eh_proj.linear.weight"): + return (0, 0) + if self.is_expert(native_name): + if ".fc1." in native_name: + return (0, 1) + if ".fc2." in native_name: + return (1, 1) + return None + if native_name in {"embed.embedding.weight", "head.col.linear.weight"}: + return (0, 0) + if native_name.endswith(".self_attention.linear_q_up_proj.linear.weight"): + return (0, 0) + if native_name.endswith(".self_attention.linear_kv_up_proj.linear.weight"): + return (0, 0) + if native_name.endswith(".self_attention.linear_proj.linear.weight"): + return (1, 0) + if native_name.endswith(".mlp.gate_up.linear.weight"): + return (0, 0) + if native_name.endswith(".mlp.down.linear.weight"): + return (1, 0) + if native_name.endswith(".moe.shared_expert.gate_up.linear.weight"): + return (0, 0) + if native_name.endswith(".moe.shared_expert.down.linear.weight"): + return (1, 0) + return None + + def is_expert(self, native_name: str) -> bool: + return ".moe.experts." in native_name and ".router." not in native_name + + def expert_global_id(self, native_name: str) -> int | None: + if self.is_expert(native_name): + return parse_expert_idx(native_name) + return None + + def expert_local_name(self, native_name: str, local_idx: int) -> str: + prefix = native_name.rsplit(".weight", 1)[0] + return f"{prefix}.weight{local_idx}" + + +def load_hf_weights(model: nn.Module, path: str, config: KimiK2Config, ps: ParallelState) -> None: + base_model = unwrap_model(model) + reader = SafeTensorReader(path) + out: dict[str, torch.Tensor] = {} + + prefix = _text_prefix(reader) + + if getattr(base_model, "embed", None) is not None: + out["embed.embedding.weight"] = _load_vocab( + reader, f"{prefix}.embed_tokens.weight", config, ps + ) + if getattr(base_model, "mtp_embed", None) is not None: + out["mtp_embed.embedding.weight"] = _load_vocab( + reader, + f"{prefix}.embed_tokens.weight", + config, + ps, + ) + if getattr(base_model, "norm", None) is not None: + out["norm.weight"] = _get(reader, f"{prefix}.norm.weight") + if getattr(base_model, "head", None) is not None: + out["head.col.linear.weight"] = _load_vocab( + reader, _lm_head_name(reader, prefix), config, ps + ) + + for local_idx, global_idx in enumerate(base_model.layer_indices): + lp = f"layers.{local_idx}" + hp = f"{prefix}.layers.{global_idx}" + out[f"{lp}.input_layernorm.weight"] = _get(reader, f"{hp}.input_layernorm.weight") + _load_attention( + out, + local_prefix=lp, + hf_prefix=f"{hp}.self_attn", + reader=reader, + ps=ps, + ) + if config.is_moe_layer(global_idx): + out[f"{lp}.mlp_norm.weight"] = _get(reader, f"{hp}.post_attention_layernorm.weight") + out[f"{lp}.moe.router.gate.weight"] = _get(reader, f"{hp}.mlp.gate.weight") + bias_name = f"{hp}.mlp.gate.e_score_correction_bias" + if _has(reader, bias_name): + out[f"{lp}.moe.router.expert_bias"] = _get(reader, bias_name).float() + _load_shared_expert( + out, local_prefix=lp, hf_mlp_prefix=f"{hp}.mlp", reader=reader, ps=ps + ) + _load_experts( + out, + local_prefix=lp, + hf_mlp_prefix=f"{hp}.mlp", + cfg=config, + ps=ps, + reader=reader, + ) + else: + _load_dense_mlp( + out, + local_prefix=lp, + hf_mlp_prefix=f"{hp}.mlp", + hf_layer_prefix=hp, + reader=reader, + ps=ps, + ) + + mtp = getattr(base_model, "mtp", None) + if mtp is not None: + for local_idx, _mtp_layer in enumerate(mtp.layers): + global_idx = config.num_hidden_layers + local_idx + lp = f"mtp.layers.{local_idx}" + hp = f"{prefix}.layers.{global_idx}" + tlp = f"{lp}.transformer_layer" + out[f"{lp}.enorm.weight"] = _get(reader, f"{hp}.enorm.weight") + out[f"{lp}.hnorm.weight"] = _get(reader, f"{hp}.hnorm.weight") + out[f"{lp}.eh_proj.linear.weight"] = _tp( + _get(reader, f"{hp}.eh_proj.weight"), + ps.tp_rank, + ps.tp_size, + ) + shared_head_norm = f"{hp}.shared_head.norm.weight" + final_norm = ( + shared_head_norm + if _has(reader, shared_head_norm) + else f"{hp}.final_layernorm.weight" + ) + out[f"{lp}.final_layernorm.weight"] = _get(reader, final_norm) + out[f"{tlp}.input_layernorm.weight"] = _get(reader, f"{hp}.input_layernorm.weight") + _load_attention( + out, + local_prefix=tlp, + hf_prefix=f"{hp}.self_attn", + reader=reader, + ps=ps, + ) + if config.is_moe_layer(global_idx): + out[f"{tlp}.mlp_norm.weight"] = _get( + reader, f"{hp}.post_attention_layernorm.weight" + ) + out[f"{tlp}.moe.router.gate.weight"] = _get(reader, f"{hp}.mlp.gate.weight") + bias_name = f"{hp}.mlp.gate.e_score_correction_bias" + if _has(reader, bias_name): + out[f"{tlp}.moe.router.expert_bias"] = _get(reader, bias_name).float() + _load_shared_expert( + out, + local_prefix=tlp, + hf_mlp_prefix=f"{hp}.mlp", + reader=reader, + ps=ps, + ) + _load_experts( + out, + local_prefix=tlp, + hf_mlp_prefix=f"{hp}.mlp", + cfg=config, + ps=ps, + reader=reader, + ) + else: + _load_dense_mlp( + out, + local_prefix=tlp, + hf_mlp_prefix=f"{hp}.mlp", + hf_layer_prefix=hp, + reader=reader, + ps=ps, + ) + + _copy_loaded_state(base_model, out) + + +def export_hf_weights(model, config: KimiK2Config, ps: ParallelState, **kwargs): + from megatron.lite.primitive.ckpt.hf_weights import export_hf_weights as _export + + spec = KimiK2WeightSpec(config) + rank0_only = bool(kwargs.get("rank0_only", False)) + export_dtype = _resolve_export_dtype(kwargs.get("export_dtype")) + yield from _export(model, spec, ps, vocab_size=config.vocab_size, **kwargs) + rank = dist.get_rank() if dist.is_initialized() else 0 + if rank0_only and rank != 0: + return + chunks = list(model) if isinstance(model, list | nn.ModuleList) else [model] + for chunk in chunks: + base_chunk = unwrap_model(chunk) + layer_map = ( + {i: base_chunk.layer_indices[i] for i in range(len(base_chunk.layer_indices))} + if hasattr(base_chunk, "layer_indices") + else {} + ) + for name, buffer in base_chunk.named_buffers(): + if not name.endswith(".moe.router.expert_bias"): + continue + global_name = to_global_layer_name(name, layer_map) + for hf_name, hf_tensor in spec.native_to_hf(global_name, buffer.detach().cpu()): + yield hf_name, _cast_export_tensor(hf_tensor, export_dtype) + + +def save_hf_weights(model, path: str, config: KimiK2Config, ps: ParallelState) -> None: + from megatron.lite.primitive.ckpt.hf_weights import save_safetensors + + rank = dist.get_rank() if dist.is_initialized() else 0 + out = dict(export_hf_weights(model, config, ps, rank0_only=True)) + if rank == 0 and out: + save_safetensors(out, path) + if dist.is_initialized(): + dist.barrier() + + +__all__ = [ + "EXPERT_CLASSIFIER", + "KimiK2WeightSpec", + "PLACEMENT_FN", + "_dequant_int4_weight", + "_dequant_fp8_weight", + "export_hf_weights", + "load_hf_weights", + "save_hf_weights", +] diff --git a/experimental/lite/megatron/lite/model/kimi_k2/lite/model.py b/experimental/lite/megatron/lite/model/kimi_k2/lite/model.py new file mode 100644 index 00000000000..4abcbf60183 --- /dev/null +++ b/experimental/lite/megatron/lite/model/kimi_k2/lite/model.py @@ -0,0 +1,854 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Kimi K2 lite native model.""" + +from __future__ import annotations + +import inspect +import os +from contextlib import nullcontext + +import torch +import torch.distributed as dist +import torch.nn as nn +import torch.nn.functional as F +import transformer_engine.pytorch as te + +from megatron.lite.model.kimi_k2.config import KimiK2Config +from megatron.lite.primitive.modules.attention import MultiLatentAttention +from megatron.lite.primitive.modules.dispatcher import TokenDispatcher +from megatron.lite.primitive.modules.experts import Experts +from megatron.lite.primitive.modules.moe import MoEAuxLossAutoScaler +from megatron.lite.primitive.modules.mtp import MTPLossAutoScaler +from megatron.lite.primitive.ops.cross_entropy import vocab_parallel_cross_entropy +from megatron.lite.primitive.ops.linear_cross_entropy import linear_cross_entropy +from megatron.lite.primitive.ops.logprob import vocab_parallel_entropy +from megatron.lite.primitive.ops.sp_ops import ReduceScatterDim0 +from megatron.lite.primitive.kernels.swiglu import bias_swiglu_impl +from megatron.lite.primitive.parallel import ( + ColumnParallelLinear, + ParallelState, + RowParallelLinear, + VanillaColumnParallelLinear, + VocabParallelEmbedding, + VocabParallelOutput, + build_pipeline_chunk_layout, + gather_from_sequence_parallel, + roll_packed_thd_left, + scatter_to_sequence_parallel, +) +from megatron.lite.primitive.utils import build_fp8_recipe +from megatron.lite.primitive.utils.moe import ( + compute_routing_scores_for_aux_loss, + router_gating_linear, + switch_load_balancing_loss_func, + topk_routing_with_score_function, +) + +_SP_GRAD_SUFFIXES: tuple[str, ...] = ( + ".input_layernorm.weight", + ".self_attention.linear_q_down_proj.weight", + ".self_attention.linear_q_up_proj.linear.layer_norm_weight", + ".self_attention.linear_kv_down_proj.weight", + ".self_attention.linear_kv_up_proj.linear.layer_norm_weight", + ".mlp_norm.weight", + ".mlp.gate_up.linear.layer_norm_weight", + ".moe.router.gate.weight", + ".norm.weight", +) + + +def _collect_sp_grad_params(model: nn.Module) -> list[nn.Parameter]: + return [ + param + for name, param in model.named_parameters() + if any(name.endswith(suffix) for suffix in _SP_GRAD_SUFFIXES) or name == "norm.weight" + ] + + +def _swiglu(x: torch.Tensor) -> torch.Tensor: + if x.is_cuda: + return bias_swiglu_impl(x, None, False, False) + x1, x2 = torch.chunk(x, 2, dim=-1) + return F.silu(x1) * x2 + + +def _reduce_scatter_to_sequence_parallel(x: torch.Tensor, ps: ParallelState) -> torch.Tensor: + if ps.tp_size == 1: + return x + return ReduceScatterDim0.apply(x, ps.tp_size, ps.tp_rank, ps.tp_group) + + +def _ordered_topk_from_routing_map( + probs_dense: torch.Tensor, routing_map: torch.Tensor, topk: int +) -> tuple[torch.Tensor, torch.Tensor]: + expert_ids = torch.arange( + probs_dense.size(-1), device=probs_dense.device, dtype=torch.long + ).expand_as(routing_map) + masked_ids = torch.where( + routing_map, expert_ids, torch.full_like(expert_ids, probs_dense.size(-1)) + ) + topk_indices = torch.sort(masked_ids, dim=-1).values[:, :topk] + topk_scores = torch.gather(probs_dense, dim=-1, index=topk_indices) + return topk_scores, topk_indices + + +def _router_linear( + x: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor | None, + router_dtype: torch.dtype, +) -> torch.Tensor: + if x.is_cuda: + return router_gating_linear(x, weight, bias, router_dtype) + return F.linear( + x.to(router_dtype), + weight.to(router_dtype), + None if bias is None else bias.to(router_dtype), + ) + + +def _topk_routing_supports_groups() -> bool: + params = inspect.signature(topk_routing_with_score_function).parameters + return "num_groups" in params and "group_topk" in params + + +class KimiK2SigmoidTopKRouter(nn.Module): + """Kimi K2 sigmoid router with group-limited routing and persistent expert bias.""" + + def __init__( + self, + config: KimiK2Config, + ps: ParallelState, + *, + router_bias_rate: float = 0.0, + compute_aux_loss: bool = True, + use_pre_softmax: bool = False, + moe_router_fusion: bool = False, + ): + super().__init__() + if router_bias_rate > 0: + raise NotImplementedError( + "Kimi K2 expert-bias EMA update is not implemented in lite yet." + ) + self.topk = config.num_experts_per_tok + self.num_experts = config.n_routed_experts + self.aux_loss_coeff = config.aux_loss_alpha + self.scaling_factor = config.routed_scaling_factor + self.num_groups = config.n_group + self.group_topk = config.topk_group + self.router_bias_rate = router_bias_rate + self.compute_aux_loss = compute_aux_loss + self.use_pre_softmax = use_pre_softmax + self.moe_router_fusion = moe_router_fusion + + self.gate = nn.Linear(config.hidden_size, config.n_routed_experts, bias=False) + self.register_buffer( + "expert_bias", + torch.zeros(config.n_routed_experts, dtype=torch.float32), + persistent=True, + ) + self.register_buffer( + "local_tokens_per_expert", + torch.zeros(config.n_routed_experts, dtype=torch.float32), + persistent=False, + ) + self._aux_loss_group = ps.tp_group if ps.tp_size > 1 else None + + def _apply(self, fn): + super()._apply(fn) + self.expert_bias.data = self.expert_bias.data.float() + self.local_tokens_per_expert.data = self.local_tokens_per_expert.data.float() + return self + + def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + logits = _router_linear(x, self.gate.weight, None, torch.float32) + logits = logits.view(-1, self.num_experts) + num_tokens = logits.size(0) + routing_kwargs = {} + if self.num_groups is not None and self.group_topk is not None: + if not _topk_routing_supports_groups(): + raise NotImplementedError( + "topk_routing_with_score_function does not support group-limited routing." + ) + routing_kwargs = dict(num_groups=self.num_groups, group_topk=self.group_topk) + probs_dense, routing_map = topk_routing_with_score_function( + logits, + self.topk, + use_pre_softmax=self.use_pre_softmax, + score_function="sigmoid", + expert_bias=self.expert_bias.to(logits.dtype), + scaling_factor=(self.scaling_factor or None), + fused=self.moe_router_fusion, + **routing_kwargs, + ) + if torch.is_grad_enabled(): + with torch.no_grad(): + self.local_tokens_per_expert += routing_map.sum(dim=0) + topk_scores, topk_indices = _ordered_topk_from_routing_map( + probs_dense, routing_map, self.topk + ) + topk_scores = topk_scores.to(logits.dtype) + + if self.compute_aux_loss and self.training and torch.is_grad_enabled(): + _, aux_scores = compute_routing_scores_for_aux_loss( + logits, self.topk, score_function="sigmoid", fused=self.moe_router_fusion + ) + tokens_per_expert = routing_map.sum(dim=0).to(torch.int64) + total_num_tokens = num_tokens + if self._aux_loss_group is not None: + dist.all_reduce(tokens_per_expert, group=self._aux_loss_group) + total_num_tokens = num_tokens * dist.get_world_size(group=self._aux_loss_group) + aux_loss = switch_load_balancing_loss_func( + aux_scores, + tokens_per_expert, + total_num_tokens, + self.topk, + self.num_experts, + self.aux_loss_coeff, + fused=False, + ) + topk_scores = MoEAuxLossAutoScaler.apply(topk_scores, aux_loss) + + return topk_scores, topk_indices + + +class DenseMLP(nn.Module): + def __init__(self, config: KimiK2Config, ps: ParallelState): + super().__init__() + self.gate_up = ColumnParallelLinear( + config.hidden_size, + config.intermediate_size * 2, + ps, + bias=False, + normalization="RMSNorm", + eps=config.rms_norm_eps, + ) + self.down = RowParallelLinear(config.intermediate_size, config.hidden_size, ps, bias=False) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.down(_swiglu(self.gate_up(x))) + + +class SharedExpert(nn.Module): + def __init__(self, config: KimiK2Config, ps: ParallelState): + super().__init__() + self.ps = ps + ffn = config.shared_expert_intermediate_size + self.gate_up = _LocalLinear(config.hidden_size, ffn * 2 // ps.tp_size) + self.down = _LocalLinear(ffn // ps.tp_size, config.hidden_size) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + squeeze_batch = x.dim() == 2 + if squeeze_batch: + x = x.unsqueeze(1) + full_x = gather_from_sequence_parallel(x, self.ps) + partial_out = self.down(_swiglu(self.gate_up(full_x))) + out = _reduce_scatter_to_sequence_parallel(partial_out, self.ps) + return out.squeeze(1) if squeeze_batch else out + + +class _LocalLinear(nn.Module): + """TE linear without built-in TP collectives; weight remains TP-local.""" + + def __init__(self, in_features: int, out_features: int): + super().__init__() + self.linear = te.Linear( + in_features, + out_features, + bias=False, + params_dtype=torch.bfloat16, + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.linear(x) + + +class MoELayer(nn.Module): + def __init__( + self, + config: KimiK2Config, + ps: ParallelState, + *, + use_deepep: bool, + router_bias_rate: float, + fp8: bool, + moe_act_recompute: bool, + ): + super().__init__() + if fp8: + raise NotImplementedError("Kimi K2 lite MoE fp8 training is not implemented yet.") + self.router = KimiK2SigmoidTopKRouter( + config, + ps, + router_bias_rate=router_bias_rate, + compute_aux_loss=True, + use_pre_softmax=True, + ) + self.experts = Experts( + config, + ps, + fp8=fp8, + moe_act_recompute=moe_act_recompute, + ) + self.dispatcher = TokenDispatcher( + config.num_experts, + config.hidden_size, + ps, + use_deepep=use_deepep, + ) + self.shared_expert = SharedExpert(config, ps) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + input_shape = x.shape + + flat_x = x.view(-1, x.size(-1)) + scores, indices = self.router(flat_x) + dispatched, tpe, permuted_probs = self.dispatcher.dispatch(flat_x, scores, indices) + del scores, indices + self.dispatcher.wait_dispatch_event() + expert_out = self.experts( + dispatched, + tpe, + permuted_probs, + tokens_per_expert_list=getattr(self.dispatcher, "_local_tpe_list", None), + ) + routed_out = self.dispatcher.combine(expert_out) + shared_out = self.shared_expert(x) + output = routed_out.view(input_shape) + output += shared_out + return output.to(x.dtype) + + +class KimiK2Layer(nn.Module): + def __init__( + self, + config: KimiK2Config, + ps: ParallelState, + layer_idx: int, + *, + use_deepep: bool = False, + router_bias_rate: float = 0.0, + fp8: bool = False, + moe_act_recompute: bool = False, + use_thd: bool = False, + ): + super().__init__() + self.layer_idx = layer_idx + self.input_layernorm = te.RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.self_attention = MultiLatentAttention( + hidden_size=config.hidden_size, + num_attention_heads=config.num_attention_heads, + q_lora_rank=config.q_lora_rank, + kv_lora_rank=config.kv_lora_rank, + qk_nope_head_dim=config.qk_nope_head_dim, + qk_rope_head_dim=config.qk_rope_head_dim, + v_head_dim=config.v_head_dim, + ps=ps, + rms_norm_eps=config.rms_norm_eps, + rope_theta=config.rope_theta, + rope_scaling=config.rope_scaling, + use_thd=use_thd, + ) + if config.is_moe_layer(layer_idx): + self.mlp_norm: nn.Module | None = te.RMSNorm( + config.hidden_size, eps=config.rms_norm_eps + ) + self.moe: MoELayer | None = MoELayer( + config, + ps, + use_deepep=use_deepep, + router_bias_rate=router_bias_rate, + fp8=fp8, + moe_act_recompute=moe_act_recompute, + ) + self.mlp: DenseMLP | None = None + else: + self.mlp_norm = None + self.moe = None + self.mlp = DenseMLP(config, ps) + + def forward(self, x: torch.Tensor, packed_seq_params=None) -> torch.Tensor: + x = x + self.self_attention(self.input_layernorm(x), packed_seq_params=packed_seq_params) + if self.moe is not None: + assert self.mlp_norm is not None + mlp_input = self.mlp_norm(x) + return x + self.moe(mlp_input) + assert self.mlp is not None + return x + self.mlp(x) + + +def _roll_mtp_left( + tensor: torch.Tensor, + *, + packed_seq_params=None, + dims: int = -1, +) -> tuple[torch.Tensor, torch.Tensor]: + if packed_seq_params is not None: + return roll_packed_thd_left(tensor, packed_seq_params=packed_seq_params, dims=dims) + dim = dims if dims >= 0 else tensor.dim() + dims + rolled = torch.roll(tensor, shifts=-1, dims=dim) + rolled.select(dim, -1).zero_() + return rolled, rolled.sum() + + +class KimiK2MTPLayer(nn.Module): + def __init__( + self, + config: KimiK2Config, + ps: ParallelState, + layer_idx: int, + *, + embedding: VocabParallelEmbedding, + use_deepep: bool, + router_bias_rate: float, + fp8: bool, + moe_act_recompute: bool, + use_thd: bool, + detach_encoder: bool, + ): + super().__init__() + self.ps = ps + object.__setattr__(self, "embedding", embedding) + self.detach_encoder = detach_encoder + self.enorm = te.RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.hnorm = te.RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.eh_proj = VanillaColumnParallelLinear( + config.hidden_size * 2, + config.hidden_size, + ps, + sp=ps.tp_size > 1, + gather_output=True, + ) + self.transformer_layer = KimiK2Layer( + config, + ps, + config.num_hidden_layers + layer_idx, + use_deepep=use_deepep, + router_bias_rate=router_bias_rate, + fp8=fp8, + moe_act_recompute=moe_act_recompute, + use_thd=use_thd, + ) + self.final_layernorm = te.RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + def forward( + self, + *, + input_ids: torch.Tensor, + position_ids: torch.Tensor | None, + hidden_states: torch.Tensor, + rotary_position_ids: torch.Tensor | None = None, + packed_seq_params=None, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]: + del rotary_position_ids + input_ids, _ = _roll_mtp_left(input_ids, packed_seq_params=packed_seq_params, dims=-1) + if position_ids is not None: + position_ids, _ = _roll_mtp_left( + position_ids, packed_seq_params=packed_seq_params, dims=-1 + ) + decoder_input = self.embedding(input_ids) + decoder_input = scatter_to_sequence_parallel(decoder_input, self.ps) + + if self.detach_encoder: + decoder_input = decoder_input.detach() + hidden_states = hidden_states.detach() + + decoder_input = self.enorm(decoder_input) + hidden_states = self.hnorm(hidden_states) + hidden_states = torch.cat((decoder_input, hidden_states), dim=-1) + hidden_states = self.eh_proj(hidden_states) + hidden_states = scatter_to_sequence_parallel(hidden_states, self.ps) + hidden_states = self.transformer_layer(hidden_states, packed_seq_params=packed_seq_params) + hidden_states = self.final_layernorm(hidden_states) + return hidden_states, input_ids, position_ids + + +class KimiK2MTPBlock(nn.Module): + def __init__( + self, + config: KimiK2Config, + ps: ParallelState, + *, + embedding: VocabParallelEmbedding, + use_deepep: bool, + router_bias_rate: float, + fp8: bool, + moe_act_recompute: bool, + use_thd: bool, + detach_encoder: bool, + repeated_layer: bool, + ): + super().__init__() + self.num_layers = config.num_nextn_predict_layers + self.repeated_layer = repeated_layer + layers_to_build = 1 if repeated_layer else self.num_layers + self.layers = nn.ModuleList( + [ + KimiK2MTPLayer( + config, + ps, + idx, + embedding=embedding, + use_deepep=use_deepep, + router_bias_rate=router_bias_rate, + fp8=fp8, + moe_act_recompute=moe_act_recompute, + use_thd=use_thd, + detach_encoder=detach_encoder, + ) + for idx in range(layers_to_build) + ] + ) + + def forward( + self, + *, + input_ids: torch.Tensor, + position_ids: torch.Tensor | None, + hidden_states: torch.Tensor, + packed_seq_params=None, + ) -> list[torch.Tensor]: + outputs: list[torch.Tensor] = [] + rotary_position_ids = position_ids + for depth in range(self.num_layers): + layer = self.layers[0] if self.repeated_layer else self.layers[depth] + hidden_states, input_ids, position_ids = layer( + input_ids=input_ids, + position_ids=position_ids, + hidden_states=hidden_states, + rotary_position_ids=rotary_position_ids, + packed_seq_params=packed_seq_params, + ) + outputs.append(hidden_states) + return outputs + + +def _temperature_to_float(temperature: float | torch.Tensor) -> float: + if isinstance(temperature, torch.Tensor): + if temperature.numel() != 1: + raise ValueError("KimiK2Model supports scalar temperature only.") + return float(temperature.detach().float().item()) + return float(temperature) + + +def _apply_attention_backend_override(backend: str | None) -> None: + if backend in (None, "flash"): + backend = "fused" + env = { + "auto": ("1", "1", "1"), + "flash": ("1", "0", "0"), + "fused": ("0", "1", "0"), + "unfused": ("0", "0", "1"), + "local": ("0", "0", "1"), + }.get(backend) + if env is None: + raise ValueError( + "attention_backend_override must be one of " + "{'auto', 'flash', 'fused', 'unfused', 'local'}" + ) + ( + os.environ["NVTE_FLASH_ATTN"], + os.environ["NVTE_FUSED_ATTN"], + os.environ["NVTE_UNFUSED_ATTN"], + ) = env + + +class KimiK2Model(nn.Module): + def __init__( + self, + config: KimiK2Config, + train_config, + ps: ParallelState, + *, + vpp_chunk_id: int | None = None, + router_bias_rate: float = 0.0, + use_thd: bool = False, + hf_path: str = "", + attention_backend_override: str | None = None, + mtp_enable: bool = False, + mtp_enable_train: bool = False, + mtp_detach_encoder: bool = False, + ): + super().__init__() + del hf_path + _apply_attention_backend_override(attention_backend_override) + self.config = config + self.train_config = train_config + self.ps = ps + self._input_tensor: torch.Tensor | None = None + self.mtp_enable_train = bool(mtp_enable and mtp_enable_train) + self.mtp_loss_scaling_factor = config.mtp_loss_scaling_factor + + layout = build_pipeline_chunk_layout( + config.num_hidden_layers, + ps, + train_config.vpp, + vpp_chunk_id, + num_mtp_layers=config.num_nextn_predict_layers if mtp_enable else 0, + ) + self.layer_indices = layout.layer_indices + self.pre_process = layout.has_embed + self.post_process = layout.has_head + self.share_embeddings_and_output_weights = bool(config.tie_word_embeddings) + self.vision_model: nn.Module | None = None + + self.embed: VocabParallelEmbedding | None = None + if layout.has_embed: + self.embed = VocabParallelEmbedding(config.vocab_size, config.hidden_size, ps) + + recompute_modules = getattr(train_config, "recompute_modules", []) + moe_act_recompute = "moe_act" in recompute_modules and "moe" not in recompute_modules + self.layers = nn.ModuleList( + [ + KimiK2Layer( + config, + ps, + idx, + use_deepep=train_config.use_deepep, + router_bias_rate=router_bias_rate, + fp8=train_config.fp8, + moe_act_recompute=moe_act_recompute, + use_thd=use_thd, + ) + for idx in self.layer_indices + ] + ) + + self.norm: nn.Module | None = None + self.head: VocabParallelOutput | None = None + if layout.has_head: + self.norm = te.RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.head = VocabParallelOutput(config.vocab_size, config.hidden_size, ps) + + self.mtp_embed: VocabParallelEmbedding | None = None + self.mtp: KimiK2MTPBlock | None = None + if mtp_enable and config.num_nextn_predict_layers > 0 and layout.has_mtp: + mtp_embedding = self.embed + if mtp_embedding is None: + mtp_embedding = VocabParallelEmbedding(config.vocab_size, config.hidden_size, ps) + self.mtp_embed = mtp_embedding + self.mtp = KimiK2MTPBlock( + config, + ps, + embedding=mtp_embedding, + use_deepep=train_config.use_deepep, + router_bias_rate=router_bias_rate, + fp8=train_config.fp8, + moe_act_recompute=moe_act_recompute, + use_thd=use_thd, + detach_encoder=mtp_detach_encoder, + repeated_layer=config.mtp_use_repeated_layer, + ) + + self.sp_params: list[nn.Parameter] = [] + if ps.tp_size > 1: + self.sp_params = _collect_sp_grad_params(self) + + def set_input_tensor(self, input_tensor): + if isinstance(input_tensor, list): + if len(input_tensor) > 1: + raise ValueError("KimiK2Model expects a single pipeline input tensor.") + input_tensor = input_tensor[0] if input_tensor else None + self._input_tensor = input_tensor + + def forward( + self, + input_ids: torch.Tensor | None = None, + hidden_states: torch.Tensor | None = None, + position_ids: torch.Tensor | None = None, + packed_seq_params=None, + labels: torch.Tensor | None = None, + loss_mask: torch.Tensor | None = None, + temperature: float | torch.Tensor = 1.0, + use_fused_kernels: bool = False, + calculate_entropy: bool = False, + ) -> dict: + if self.embed is not None: + assert input_ids is not None + h = self.embed(input_ids) + else: + if hidden_states is None: + hidden_states = self._input_tensor + assert hidden_states is not None + h = hidden_states + + fp8_ctx = ( + te.fp8_autocast(enabled=True, fp8_recipe=build_fp8_recipe(self.train_config)) + if self.train_config.fp8 + else nullcontext() + ) + with fp8_ctx: + if self.embed is not None: + h = scatter_to_sequence_parallel(h, self.ps) + for layer in self.layers: + h = layer(h, packed_seq_params=packed_seq_params) + + output = {"hidden_states": h} + if self.head is not None: + assert self.norm is not None + hidden_for_head = self.norm(h) + mtp_hidden_states = self._apply_mtp( + hidden_for_head, + input_ids=input_ids, + position_ids=position_ids, + packed_seq_params=packed_seq_params, + ) + if mtp_hidden_states is not None: + output["mtp_hidden_states"] = mtp_hidden_states + if labels is not None: + temperature_value = _temperature_to_float(temperature) + mtp_result = self._apply_mtp_loss( + hidden_for_head, + mtp_hidden_states=mtp_hidden_states, + labels=labels, + loss_mask=loss_mask, + packed_seq_params=packed_seq_params, + temperature=temperature_value, + use_fused_kernels=use_fused_kernels, + ) + if mtp_result is not None: + hidden_for_head, mtp_loss = mtp_result + output["mtp_loss"] = mtp_loss + labels_sb = labels.transpose(0, 1).contiguous() + if use_fused_kernels: + hidden_full = gather_from_sequence_parallel(hidden_for_head, self.ps) + log_probs, entropy = linear_cross_entropy( + hidden_full, + self._head_weight_for_fused_ce(hidden_full), + labels_sb, + temperature_value, + self.ps.tp_group, + ) + output["loss"] = (-log_probs).mean() + output["log_probs"] = log_probs.transpose(0, 1).contiguous() + if calculate_entropy: + output["entropy"] = entropy.transpose(0, 1).contiguous() + else: + logits = self.head(hidden_for_head) + if temperature_value != 1.0: + logits = logits / temperature_value + loss = vocab_parallel_cross_entropy(logits, labels_sb, self.ps.tp_group) + output["loss"] = loss.mean() + output["log_probs"] = (-loss).transpose(0, 1).contiguous() + if calculate_entropy: + entropy = vocab_parallel_entropy(logits, self.ps.tp_group) + output["entropy"] = entropy.transpose(0, 1).contiguous() + else: + logits = self.head(hidden_for_head) + output["logits"] = self.head.gather(logits).transpose(0, 1).contiguous() + if mtp_hidden_states is not None: + output["mtp_logits"] = [ + self.head.gather(self.head(mtp_hidden)).transpose(0, 1).contiguous() + for mtp_hidden in mtp_hidden_states + ] + return output + + def _apply_mtp( + self, + hidden_states: torch.Tensor, + *, + input_ids: torch.Tensor | None, + position_ids: torch.Tensor | None, + packed_seq_params, + ) -> list[torch.Tensor] | None: + if self.mtp is None: + return None + if input_ids is None: + if self.mtp_enable_train: + raise ValueError("MTP training requires input_ids.") + return None + return self.mtp( + input_ids=input_ids, + position_ids=position_ids, + hidden_states=hidden_states, + packed_seq_params=packed_seq_params, + ) + + def _apply_mtp_loss( + self, + hidden_states: torch.Tensor, + *, + mtp_hidden_states: list[torch.Tensor] | None, + labels: torch.Tensor, + loss_mask: torch.Tensor | None, + packed_seq_params, + temperature: float, + use_fused_kernels: bool, + ) -> tuple[torch.Tensor, torch.Tensor] | None: + if mtp_hidden_states is None: + return None + if not self.mtp_enable_train: + return None + if loss_mask is None: + mtp_loss_mask = torch.ones_like(labels, dtype=torch.float32) + else: + mtp_loss_mask = loss_mask.to(dtype=torch.float32).clone() + mtp_labels = labels.clone() + + mtp_loss_values = [] + for mtp_hidden in mtp_hidden_states: + mtp_labels, _ = _roll_mtp_left( + mtp_labels, + packed_seq_params=packed_seq_params, + dims=-1, + ) + mtp_loss_mask, num_tokens = _roll_mtp_left( + mtp_loss_mask, + packed_seq_params=packed_seq_params, + dims=-1, + ) + labels_sb = mtp_labels.transpose(0, 1).contiguous() + mask_sb = mtp_loss_mask.transpose(0, 1).contiguous() + + if use_fused_kernels: + mtp_hidden_full = gather_from_sequence_parallel(mtp_hidden, self.ps) + log_probs, _entropy = linear_cross_entropy( + mtp_hidden_full, + self._head_weight_for_fused_ce(mtp_hidden_full), + labels_sb, + temperature, + self.ps.tp_group, + ) + token_loss = -log_probs + else: + logits = self.head(mtp_hidden) + if temperature != 1.0: + logits = logits / temperature + token_loss = vocab_parallel_cross_entropy(logits, labels_sb, self.ps.tp_group) + + token_loss = token_loss * mask_sb.to(dtype=token_loss.dtype) + num_tokens = num_tokens.to(dtype=token_loss.dtype).clamp_min(1.0) + mtp_loss_values.append(token_loss.sum() / num_tokens) + + mtp_loss_scale = self.mtp_loss_scaling_factor / max(len(mtp_hidden_states), 1) + hidden_states = MTPLossAutoScaler.apply( + hidden_states, + mtp_loss_scale * token_loss / num_tokens, + ) + + if not mtp_loss_values: + return None + return ( + hidden_states, + torch.stack([loss.detach().float() for loss in mtp_loss_values]).mean(), + ) + + def _head_weight_for_fused_ce(self, hidden_states: torch.Tensor) -> torch.Tensor: + assert self.head is not None + weight = self.head.col.linear.weight + return ( + weight if weight.dtype == hidden_states.dtype else weight.to(dtype=hidden_states.dtype) + ) + + +__all__ = [ + "DenseMLP", + "KimiK2MTPBlock", + "KimiK2MTPLayer", + "KimiK2Layer", + "KimiK2Model", + "MoELayer", + "MTPLossAutoScaler", + "MultiLatentAttention", + "SharedExpert", +] diff --git a/experimental/lite/megatron/lite/model/kimi_k2/lite/protocol.py b/experimental/lite/megatron/lite/model/kimi_k2/lite/protocol.py new file mode 100644 index 00000000000..be69ad58f0b --- /dev/null +++ b/experimental/lite/megatron/lite/model/kimi_k2/lite/protocol.py @@ -0,0 +1,296 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Kimi K2 lite impl - native model protocol for Megatron Lite runtime.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import torch +import torch.nn as nn +from megatron.lite.model.kimi_k2.config import KimiK2Config +from megatron.lite.model.protocol_utils import ( + add_cross_entropy_fusion, + add_loss_context_kwargs, + pack_thd_forward_kwargs, + set_cross_entropy_fusion, + unpack_thd_forward_output, +) +from megatron.lite.primitive.bundle import ModelBundle +from megatron.lite.primitive.parallel import ParallelState, init_parallel +from megatron.lite.primitive.recompute import apply_recompute, parse_recompute_spec +from megatron.lite.runtime.contracts import OptimizerConfig, ParallelConfig +from megatron.lite.runtime.contracts.data import PackedBatch + + +def EXPERT_CLASSIFIER(name: str) -> bool: + return "experts" in name and "router" not in name and "shared" not in name + + +def PLACEMENT_FN(param_name: str) -> list: + from megatron.lite.model.kimi_k2.lite.checkpoint import PLACEMENT_FN as placement_fn + + return placement_fn(param_name) + + +def is_expert_param(name: str) -> bool: + return EXPERT_CLASSIFIER(name) + + +def _maybe(module_name: str): + def getter(layer): + module = getattr(layer, module_name, None) + return module + + return getter + + +def _moe_module(name: str): + def getter(layer): + moe = getattr(layer, "moe", None) + return getattr(moe, name, None) if moe is not None else None + + return getter + + +MODULE_MAP = { + "core_attn": lambda layer: layer.self_attention.core_attn, + "experts": _moe_module("experts"), + "moe": _maybe("moe"), + "router": _moe_module("router"), + "mlp": _maybe("mlp"), + "mlp_norm": lambda layer: layer.mlp_norm, + "attn_proj": lambda layer: layer.self_attention.linear_proj, + "mla": lambda layer: layer.self_attention, +} + + +@dataclass(frozen=True) +class ImplConfig: + parallel: ParallelConfig = field(default_factory=ParallelConfig) + optimizer: str | None = "dist_opt" + recompute: list[str] = field(default_factory=list) + offload: list[str] = field(default_factory=list) + use_deepep: bool = False + use_thd: bool = False + cross_entropy_fusion: bool = False + hf_path: str = "" + attention_backend_override: str | None = None + router_aux_loss_coef: float | None = None + router_bias_rate: float = 0.0 + deterministic: bool = True + optimizer_config: OptimizerConfig | None = None + mtp_enable: bool = False + mtp_enable_train: bool = False + mtp_detach_encoder: bool = False + mtp_loss_scaling_factor: float = 0.1 + mtp_use_repeated_layer: bool | None = None + + +def build_model_config(source: str | Path | dict, **overrides) -> KimiK2Config: + if isinstance(source, dict): + cfg = KimiK2Config._from_hf_dict(source) + else: + cfg = KimiK2Config.from_hf(str(source)) + for key, value in overrides.items(): + if hasattr(cfg, key): + setattr(cfg, key, value) + return cfg + + +def _forward_step(model: nn.Module, batch: PackedBatch) -> dict: + kwargs = pack_thd_forward_kwargs(model, batch) + add_loss_context_kwargs(kwargs) + add_cross_entropy_fusion(kwargs, model) + return model(**kwargs) + + +def unpack_forward_output(model: nn.Module, batch: PackedBatch, output) -> Any: + return unpack_thd_forward_output(model, batch, output) + + +def _make_aux_loss_hook(): + from megatron.lite.primitive.modules.moe import MoEAuxLossAutoScaler + from megatron.lite.primitive.modules.mtp import MTPLossAutoScaler + + def hook(scale: torch.Tensor) -> None: + MoEAuxLossAutoScaler.set_loss_scale(scale) + MTPLossAutoScaler.set_loss_scale(scale) + + return hook + + +def _build_dist_opt_optimizer( + chunks, model_cfg: KimiK2Config, impl_cfg: ImplConfig, ps: ParallelState +): + from megatron.lite.primitive.optimizers.megatron_wrap import build_dist_opt_training_optimizer + + return build_dist_opt_training_optimizer( + chunks, + model_cfg=model_cfg, + impl_cfg=impl_cfg, + ps=ps, + model_name="kimi_k2", + is_expert=is_expert_param, + deterministic=impl_cfg.deterministic, + ) + + +def build_model(model_cfg: KimiK2Config, *, impl_cfg: ImplConfig) -> ModelBundle: + p = impl_cfg.parallel + if impl_cfg.use_deepep and (p.etp is not None and p.etp > 1): + raise ValueError("use_deepep and etp>1 are mutually exclusive") + if impl_cfg.router_aux_loss_coef is not None: + model_cfg.aux_loss_alpha = impl_cfg.router_aux_loss_coef + mtp_enable = bool(impl_cfg.mtp_enable) + mtp_enable_train = mtp_enable and bool(impl_cfg.mtp_enable_train) + if mtp_enable: + if model_cfg.num_nextn_predict_layers <= 0: + raise ValueError("mtp_enable=True but HF config has no num_nextn_predict_layers.") + model_cfg.mtp_loss_scaling_factor = impl_cfg.mtp_loss_scaling_factor + if impl_cfg.mtp_use_repeated_layer is not None: + model_cfg.mtp_use_repeated_layer = impl_cfg.mtp_use_repeated_layer + elif hasattr(model_cfg, "num_nextn_predict_layers"): + model_cfg.num_nextn_predict_layers = 0 + + from megatron.lite.model.kimi_k2.lite.model import KimiK2Model + + ps = init_parallel(p) + recompute_spec = parse_recompute_spec(impl_cfg.recompute) + vpp = None if p.vpp == 1 else p.vpp + train_cfg = SimpleNamespace( + tp=ps.tp_size, + ep=ps.ep_size, + etp=ps.etp_size, + pp=ps.pp_size, + cp=ps.cp_size, + vpp=vpp, + use_deepep=impl_cfg.use_deepep, + fp8=False, + recompute_modules=recompute_spec, + deterministic=impl_cfg.deterministic, + ) + model_kwargs: dict[str, Any] = dict( + router_bias_rate=impl_cfg.router_bias_rate, + use_thd=impl_cfg.use_thd, + hf_path=impl_cfg.hf_path, + attention_backend_override=impl_cfg.attention_backend_override, + mtp_enable=mtp_enable, + mtp_enable_train=mtp_enable_train, + mtp_detach_encoder=impl_cfg.mtp_detach_encoder, + ) + + if vpp is None: + chunks = [KimiK2Model(model_cfg, train_cfg, ps, **model_kwargs).to(torch.bfloat16).cuda()] + else: + chunks = [ + KimiK2Model( + model_cfg, + train_cfg, + ps, + vpp_chunk_id=i, + **model_kwargs, + ) + .to(torch.bfloat16) + .cuda() + for i in range(vpp) + ] + set_cross_entropy_fusion(chunks, impl_cfg.cross_entropy_fusion) + + if recompute_spec: + for chunk in chunks: + apply_recompute(chunk.layers, recompute_spec, MODULE_MAP) + + if impl_cfg.offload: + from megatron.lite.primitive.recompute import apply_offload + + for chunk in chunks: + apply_offload(chunk.layers, impl_cfg.offload, MODULE_MAP) + + optimizer = None + finalize_grads = None + post_model_load_hook = None + optimizer_backend = "none" + if impl_cfg.optimizer == "dist_opt": + optimizer, finalize_grads = _build_dist_opt_optimizer(chunks, model_cfg, impl_cfg, ps) + from megatron.lite.primitive.ckpt import attach_model_sharded_state_dict + from megatron.lite.runtime.megatron_utils import register_training_hooks + + attach_model_sharded_state_dict( + chunks, ps, get_placements=PLACEMENT_FN, is_expert=is_expert_param + ) + register_training_hooks(chunks, optimizer) + optimizer_backend = "dist_opt" + elif impl_cfg.optimizer == "fsdp2": + optimizer_backend = "fsdp2" + + def _post_model_load_hook(): + from megatron.lite.model.kimi_k2.lite.model import KimiK2Layer + from megatron.lite.primitive.optimizers.fsdp2 import build_fsdp2_training_optimizer + + return { + "optimizer": build_fsdp2_training_optimizer( + chunks, + impl_cfg.optimizer_config, + ps, + unit_modules=(KimiK2Layer,), + expert_classifier=is_expert_param, + deterministic=impl_cfg.deterministic, + vpp=impl_cfg.parallel.vpp, + leaf_module_names=(), + ) + } + + post_model_load_hook = _post_model_load_hook + elif impl_cfg.optimizer is not None: + raise ValueError(f"Unknown kimi_k2 lite optimizer: {impl_cfg.optimizer!r}.") + + return ModelBundle( + chunks=chunks, + parallel_state=ps, + optimizer=optimizer, + finalize_grads=finalize_grads, + forward_step=_forward_step, + extras={ + "model_cfg": model_cfg, + "optimizer_backend": optimizer_backend, + "post_model_load_hook": post_model_load_hook, + "pre_forward_hook": _make_aux_loss_hook(), + }, + ) + + +def load_hf_weights( + chunk: nn.Module, hf_path: str, model_cfg: KimiK2Config, ps: ParallelState +) -> None: + if not hf_path: + return + from megatron.lite.model.kimi_k2.lite.checkpoint import load_hf_weights as load_impl + + load_impl(chunk, hf_path, model_cfg, ps) + + +def export_hf_weights(chunks, model_cfg: KimiK2Config, ps: ParallelState, **kwargs): + from megatron.lite.model.kimi_k2.lite.checkpoint import export_hf_weights as export_impl + + yield from export_impl(chunks, model_cfg, ps, **kwargs) + + +def vocab_size(model_cfg) -> int | None: + cfg = getattr(model_cfg, "text_config", model_cfg) + return getattr(cfg, "vocab_size", None) + + +__all__ = [ + "EXPERT_CLASSIFIER", + "ImplConfig", + "PLACEMENT_FN", + "build_model", + "build_model_config", + "export_hf_weights", + "is_expert_param", + "load_hf_weights", + "vocab_size", +] diff --git a/experimental/lite/megatron/lite/model/protocol_utils.py b/experimental/lite/megatron/lite/model/protocol_utils.py new file mode 100644 index 00000000000..f1c4b53d3d5 --- /dev/null +++ b/experimental/lite/megatron/lite/model/protocol_utils.py @@ -0,0 +1,129 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Helpers shared by model protocol forward steps. + +The verl/runtime layers hand each protocol a raw, model-agnostic ``PackedBatch`` +(true per-sequence lengths, no padding, no ``PackedSeqParams``). Each model owns +its pack/unpack pair: ``pack_thd_forward_kwargs`` pads + CP-splits the batch into +model forward kwargs, and ``unpack_thd_forward_output`` reverses a model output +back to jagged true-length form. THD models share the zigzag-CP pair below; +models with a different CP layout (e.g. DeepSeek-V4 contiguous DSA) provide their +own pair. +""" + +from __future__ import annotations + +from typing import Any + +import torch +from megatron.lite.primitive.parallel import ParallelState +from megatron.lite.primitive.parallel.thd import ( + pack_nested_thd, + parallel_state_from_model, + prepare_packed_thd_kwargs_for_context_parallel, + thd_pack_meta, + unpack_thd_to_nested, +) +from megatron.lite.primitive.utils.packed_seq import PackedSeqParams +from megatron.lite.runtime.contracts.data import PackedBatch +from megatron.lite.runtime.contracts.loss import get_loss_context + + +def _parallel_state(model) -> ParallelState: + return parallel_state_from_model(model) or ParallelState() + + +def nested_from_packed(tensor: torch.Tensor | None, seq_lens: torch.Tensor): + """Split a 1-D packed (true, unpadded) tensor back into a jagged nested tensor.""" + if tensor is None: + return None + if tensor.dim() == 2 and tensor.size(0) == 1: + tensor = tensor.squeeze(0) + if tensor.dim() != 1: + raise ValueError(f"PackedBatch tensor must be 1-D, got {tuple(tensor.shape)}.") + pieces = [] + offset = 0 + for length_t in seq_lens: + length = int(length_t.item()) + pieces.append(tensor.narrow(0, offset, length)) + offset += length + if offset != tensor.numel(): + raise ValueError(f"PackedBatch sizes sum to {offset}, tensor has {tensor.numel()} tokens.") + return torch.nested.as_nested_tensor(pieces, layout=torch.jagged) + + +def pack_thd_forward_kwargs(model, batch: PackedBatch) -> dict[str, Any]: + """Pad + zigzag-CP-split a raw THD batch into model forward kwargs. + + Pads each sequence to the TE/zigzag alignment, then CP-splits tokens, + labels, masks and position ids through the shared THD primitive — the same + layout the model was validated against, now produced inside the protocol + rather than the connector. + """ + ps = _parallel_state(model) + seq_lens = batch.seq_lens + packed = pack_nested_thd( + nested_from_packed(batch.input_ids, seq_lens), + tp_size=ps.tp_size, + cp_size=ps.cp_size, + cp_rank=ps.cp_rank, + cp_group=ps.cp_group if ps.cp_size > 1 else None, + split_cp=False, + labels=nested_from_packed(batch.labels, seq_lens), + roll_labels=batch.labels is not None, + loss_mask=nested_from_packed(batch.loss_mask, seq_lens), + roll_loss_mask=batch.loss_mask is not None, + ) + max_seqlen = int(packed.padded_lengths.max().item()) if packed.padded_lengths.numel() else 0 + # pack_nested_thd already returns [1, T] token rows; do not unsqueeze again. + kwargs: dict[str, Any] = { + "input_ids": packed.input_ids, + "labels": packed.labels, + "loss_mask": packed.loss_mask, + "position_ids": packed.position_ids, + "packed_seq_params": PackedSeqParams.from_cu_seqlens( + packed.cu_seqlens_padded, max_seqlen=max_seqlen + ), + } + prepare_packed_thd_kwargs_for_context_parallel(model, kwargs) + return kwargs + + +def unpack_thd_forward_output(model, batch: PackedBatch, output: torch.Tensor) -> torch.Tensor: + """Reverse a zigzag-CP THD model output back to jagged true-length form.""" + ps = _parallel_state(model) + meta = thd_pack_meta( + batch.seq_lens, + tp_size=ps.tp_size, + cp_size=ps.cp_size, + cp_group=ps.cp_group if ps.cp_size > 1 else None, + ) + return unpack_thd_to_nested(output, meta, contiguous=False) + + +def add_loss_context_kwargs(kwargs: dict[str, Any], *, include_return_log_probs: bool = False) -> None: + loss_context = get_loss_context() + if loss_context is None: + return + kwargs["temperature"] = loss_context.temperature + kwargs["calculate_entropy"] = loss_context.calculate_entropy + if include_return_log_probs: + kwargs["return_log_probs"] = loss_context.return_log_probs + + +def add_cross_entropy_fusion(kwargs: dict[str, Any], model) -> None: + kwargs["use_fused_kernels"] = bool(getattr(model, "cross_entropy_fusion", False)) + + +def set_cross_entropy_fusion(chunks: list, enabled: bool) -> None: + for chunk in chunks: + chunk.cross_entropy_fusion = bool(enabled) + + +__all__ = [ + "add_cross_entropy_fusion", + "add_loss_context_kwargs", + "nested_from_packed", + "pack_thd_forward_kwargs", + "set_cross_entropy_fusion", + "unpack_thd_forward_output", +] diff --git a/experimental/lite/megatron/lite/model/qwen3_5/__init__.py b/experimental/lite/megatron/lite/model/qwen3_5/__init__.py new file mode 100644 index 00000000000..d009e1c6f13 --- /dev/null +++ b/experimental/lite/megatron/lite/model/qwen3_5/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Qwen3.5 model package.""" diff --git a/experimental/lite/megatron/lite/model/qwen3_5/config.py b/experimental/lite/megatron/lite/model/qwen3_5/config.py new file mode 100644 index 00000000000..246ca33dc54 --- /dev/null +++ b/experimental/lite/megatron/lite/model/qwen3_5/config.py @@ -0,0 +1,262 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Qwen3.5 model configuration.""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +from megatron.lite.primitive.config import load_hf_config_dict + +_HF_FIELDS = frozenset( + { + "num_hidden_layers", + "hidden_size", + "num_attention_heads", + "num_key_value_heads", + "head_dim", + "vocab_size", + "rms_norm_eps", + "max_position_embeddings", + "router_aux_loss_coef", + "num_experts", + "num_experts_per_tok", + "moe_intermediate_size", + "shared_expert_intermediate_size", + "linear_num_key_heads", + "linear_key_head_dim", + "linear_num_value_heads", + "linear_value_head_dim", + "linear_conv_kernel_dim", + "layer_types", + "partial_rotary_factor", + "mrope_section", + "mtp_num_hidden_layers", + "mtp_use_dedicated_embeddings", + "num_nextn_predict_layers", + "mtp_loss_scaling_factor", + "mtp_use_repeated_layer", + "mtp_layer_types", + } +) + + +@dataclass +class Qwen35Config: + """Pure Qwen3.5-35B-A3B architecture parameters.""" + + num_hidden_layers: int = 40 + hidden_size: int = 2048 + num_attention_heads: int = 16 + num_key_value_heads: int = 2 + head_dim: int = 256 + vocab_size: int = 248320 + rms_norm_eps: float = 1e-6 + max_position_embeddings: int = 262144 + router_aux_loss_coef: float = 0.001 + num_experts: int = 256 + num_experts_per_tok: int = 8 + moe_intermediate_size: int = 512 + shared_expert_intermediate_size: int = 512 + linear_num_key_heads: int = 16 + linear_key_head_dim: int = 128 + linear_num_value_heads: int = 32 + linear_value_head_dim: int = 128 + linear_conv_kernel_dim: int = 4 + layer_types: list[str] = field( + default_factory=lambda: ( + ["linear_attention", "linear_attention", "linear_attention", "full_attention"] * 10 + ) + ) + partial_rotary_factor: float = 0.25 + rope_theta: float = 10_000_000.0 + mrope_section: list[int] | None = None + num_nextn_predict_layers: int = 0 + mtp_loss_scaling_factor: float = 0.1 + mtp_use_dedicated_embeddings: bool = False + mtp_use_repeated_layer: bool = False + mtp_layer_types: list[str] = field(default_factory=list) + + # ------------------------------------------------------------------ + # Derived properties + # ------------------------------------------------------------------ + + @property + def rotary_dim(self) -> int: + return int(self.head_dim * self.partial_rotary_factor) + + @property + def full_attn_qkv_size(self) -> int: + return (self.num_attention_heads + 2 * self.num_key_value_heads) * self.head_dim + + @property + def linear_conv_dim(self) -> int: + return self.linear_num_key_heads * self.linear_key_head_dim + + # ------------------------------------------------------------------ + # Validation + # ------------------------------------------------------------------ + + def __post_init__(self): + if ( + self.num_nextn_predict_layers > 0 + and not self.mtp_layer_types + and len(self.layer_types) == self.num_hidden_layers + self.num_nextn_predict_layers + ): + self.mtp_layer_types = self.layer_types[self.num_hidden_layers :] + self.layer_types = self.layer_types[: self.num_hidden_layers] + if self.num_nextn_predict_layers > 0 and not self.mtp_layer_types: + self.mtp_layer_types = ["full_attention"] * self.num_nextn_predict_layers + if len(self.layer_types) != self.num_hidden_layers: + raise ValueError( + f"len(layer_types)={len(self.layer_types)} != " + f"num_hidden_layers={self.num_hidden_layers}" + ) + if ( + self.num_nextn_predict_layers > 0 + and len(self.mtp_layer_types) != self.num_nextn_predict_layers + ): + raise ValueError( + f"len(mtp_layer_types)={len(self.mtp_layer_types)} != " + f"num_nextn_predict_layers={self.num_nextn_predict_layers}" + ) + self._validate() + + def _validate(self): + errors: list[str] = [] + + def _check(cond: bool, msg: str): + if not cond: + errors.append(msg) + + _check( + self.num_hidden_layers >= 1, + f"num_hidden_layers must be >= 1, got {self.num_hidden_layers}", + ) + _check( + self.num_nextn_predict_layers >= 0, + f"num_nextn_predict_layers must be >= 0, got {self.num_nextn_predict_layers}", + ) + _check( + not self.mtp_use_dedicated_embeddings, + "Qwen35Config only supports shared embeddings for MTP " + "(mtp_use_dedicated_embeddings=False)", + ) + _check(self.hidden_size > 0, f"hidden_size must be > 0, got {self.hidden_size}") + _check(self.head_dim > 0, f"head_dim must be > 0, got {self.head_dim}") + _check(self.vocab_size > 0, f"vocab_size must be > 0, got {self.vocab_size}") + _check( + self.num_attention_heads >= 1, + f"num_attention_heads must be >= 1, got {self.num_attention_heads}", + ) + _check( + self.num_attention_heads % self.num_key_value_heads == 0, + f"num_attention_heads({self.num_attention_heads}) must be divisible by " + f"num_key_value_heads({self.num_key_value_heads})", + ) + _check(self.num_experts >= 1, f"num_experts must be >= 1, got {self.num_experts}") + _check( + 1 <= self.num_experts_per_tok <= self.num_experts, + f"num_experts_per_tok({self.num_experts_per_tok}) must be in " + f"[1, num_experts({self.num_experts})]", + ) + _check( + self.moe_intermediate_size > 0, + f"moe_intermediate_size must be > 0, got {self.moe_intermediate_size}", + ) + _check( + self.shared_expert_intermediate_size > 0, + f"shared_expert_intermediate_size must be > 0, got {self.shared_expert_intermediate_size}", + ) + _check( + self.linear_num_key_heads >= 1, + f"linear_num_key_heads must be >= 1, got {self.linear_num_key_heads}", + ) + _check( + self.linear_key_head_dim > 0, + f"linear_key_head_dim must be > 0, got {self.linear_key_head_dim}", + ) + _check( + self.linear_num_value_heads >= 1, + f"linear_num_value_heads must be >= 1, got {self.linear_num_value_heads}", + ) + _check( + self.linear_value_head_dim > 0, + f"linear_value_head_dim must be > 0, got {self.linear_value_head_dim}", + ) + _check( + 0.0 < self.partial_rotary_factor <= 1.0, + f"partial_rotary_factor must be in (0, 1], got {self.partial_rotary_factor}", + ) + if self.mrope_section is not None: + _check( + len(self.mrope_section) == 3, + f"mrope_section must have three entries, got {self.mrope_section}", + ) + _check( + all(section >= 0 for section in self.mrope_section), + f"mrope_section entries must be non-negative, got {self.mrope_section}", + ) + _check( + 2 * sum(self.mrope_section) == self.rotary_dim, + f"sum(mrope_section)*2 must equal rotary_dim({self.rotary_dim}), " + f"got {self.mrope_section}", + ) + valid_types = {"linear_attention", "full_attention"} + for i, lt in enumerate(self.layer_types): + _check(lt in valid_types, f"layer_types[{i}] must be one of {valid_types}, got '{lt}'") + for i, lt in enumerate(self.mtp_layer_types): + _check( + lt in valid_types, f"mtp_layer_types[{i}] must be one of {valid_types}, got '{lt}'" + ) + + if errors: + raise ValueError( + f"Invalid Qwen35Config ({len(errors)} error" + f"{'s' if len(errors) > 1 else ''}):\n " + "\n ".join(errors) + ) + + # ------------------------------------------------------------------ + # Constructors + # ------------------------------------------------------------------ + + @classmethod + def from_hf(cls, path: str, **overrides) -> Qwen35Config: + hf = load_hf_config_dict(path) + return cls._from_hf_dict(hf, **overrides) + + @classmethod + def from_hf_config(cls, hf_config, **overrides) -> Qwen35Config: + return cls._from_hf_dict(hf_config.to_dict(), **overrides) + + @classmethod + def _from_hf_dict(cls, hf: dict, **overrides) -> Qwen35Config: + if "text_config" in hf and isinstance(hf["text_config"], dict): + hf = hf["text_config"] + kwargs = {k: v for k, v in hf.items() if k in _HF_FIELDS} + mtp_num_hidden_layers = kwargs.pop("mtp_num_hidden_layers", None) + if kwargs.get("num_nextn_predict_layers") is None and mtp_num_hidden_layers is not None: + kwargs["num_nextn_predict_layers"] = int(mtp_num_hidden_layers) + if "rope_parameters" in hf and isinstance(hf["rope_parameters"], dict): + rp = hf["rope_parameters"] + if "rope_theta" not in kwargs: + kwargs["rope_theta"] = float(rp.get("rope_theta", 10_000_000.0)) + if "partial_rotary_factor" not in kwargs and "partial_rotary_factor" in rp: + kwargs["partial_rotary_factor"] = float(rp["partial_rotary_factor"]) + if "mrope_section" not in kwargs and "mrope_section" in rp: + kwargs["mrope_section"] = list(rp["mrope_section"]) + if "head_dim" not in kwargs or kwargs.get("head_dim") is None: + kwargs["head_dim"] = kwargs.get("hidden_size", 2048) // kwargs.get( + "num_attention_heads", 16 + ) + if kwargs.get("num_nextn_predict_layers") is None: + kwargs["num_nextn_predict_layers"] = 0 + kwargs.update(overrides) + return cls(**kwargs) + + def layer_type_at(self, layer_idx: int) -> str: + if layer_idx < self.num_hidden_layers: + return self.layer_types[layer_idx] + mtp_idx = layer_idx - self.num_hidden_layers + if 0 <= mtp_idx < len(self.mtp_layer_types): + return self.mtp_layer_types[mtp_idx] + return "full_attention" diff --git a/experimental/lite/megatron/lite/model/qwen3_5/lite/__init__.py b/experimental/lite/megatron/lite/model/qwen3_5/lite/__init__.py new file mode 100644 index 00000000000..acffccfba76 --- /dev/null +++ b/experimental/lite/megatron/lite/model/qwen3_5/lite/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Qwen3.5 lite native sub-package.""" diff --git a/experimental/lite/megatron/lite/model/qwen3_5/lite/checkpoint.py b/experimental/lite/megatron/lite/model/qwen3_5/lite/checkpoint.py new file mode 100644 index 00000000000..ec6efdf8890 --- /dev/null +++ b/experimental/lite/megatron/lite/model/qwen3_5/lite/checkpoint.py @@ -0,0 +1,782 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Qwen3.5 lite native checkpoint mapping. + +The loader reads HF safetensors directly into lite's native module names. +It intentionally does not require wrapper-specific state on the model. +""" + +from __future__ import annotations + +import re + +import torch +import torch.distributed as dist +import torch.nn as nn +from torch.distributed.tensor import Replicate, Shard + +from megatron.lite.model.qwen3_5.config import Qwen35Config +from megatron.lite.primitive.ckpt.hf_weights import SafeTensorReader, unwrap_model +from megatron.lite.primitive.parallel import ParallelState +from megatron.lite.primitive.utils import ensure_divisible, log_rank0 + + +def EXPERT_CLASSIFIER(name: str) -> bool: + return "experts" in name and "router" not in name and "shared" not in name + + +def PLACEMENT_FN(param_name: str) -> list: + if "experts" in param_name and "router" not in param_name and "shared" not in param_name: + if "fc1" in param_name: + return [Replicate(), Replicate(), Shard(0), Shard(0)] + if "fc2" in param_name: + return [Replicate(), Replicate(), Shard(0), Shard(1)] + return [Replicate(), Replicate(), Replicate(), Replicate()] + if "in_proj" in param_name and "layer_norm" not in param_name: + return [Replicate(), Replicate(), Replicate(), Shard(0)] + if "qkv" in param_name and "layer_norm" not in param_name: + return [Replicate(), Replicate(), Replicate(), Shard(0)] + if ( + ("proj" in param_name or "o_proj" in param_name) + and ("full_attn" in param_name or "linear_attn" in param_name) + and "layer_norm" not in param_name + ): + # Row-parallel output proj weight: TP-shard on dim 1. Exclude layer_norm_weight (1-D, replicated + # under TP) which otherwise matches here ("in_proj" contains "proj") and gets an invalid Shard(1). + return [Replicate(), Replicate(), Replicate(), Shard(1)] + if "gate_up" in param_name and "shared" in param_name: + return [Replicate(), Replicate(), Replicate(), Shard(0)] + if "down" in param_name and "shared" in param_name: + return [Replicate(), Replicate(), Replicate(), Shard(1)] + if "embed" in param_name or "head" in param_name: + return [Replicate(), Replicate(), Replicate(), Shard(0)] + if "conv1d" in param_name or "dt_bias" in param_name or "A_log" in param_name: + return [Replicate(), Replicate(), Replicate(), Shard(0)] + return [Replicate(), Replicate(), Replicate(), Replicate()] + + +def _tp(tensor: torch.Tensor, rank: int, size: int, dim: int = 0) -> torch.Tensor: + return tensor if size <= 1 else tensor.chunk(size, dim=dim)[rank].contiguous() + + +def _split_gate_up(tensor: torch.Tensor, rank: int, size: int) -> torch.Tensor: + if size <= 1: + return tensor + ffn = tensor.shape[0] // 2 + gate = tensor[:ffn].chunk(size, dim=0)[rank] + up = tensor[ffn:].chunk(size, dim=0)[rank] + return torch.cat([gate, up], dim=0).contiguous() + + +def _tp_linear_attn_in_proj( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + z: torch.Tensor, + b: torch.Tensor, + a: torch.Tensor, + *, + ps: ParallelState, +) -> torch.Tensor: + """Shard each GDN projection independently, then pack the local layout.""" + return torch.cat( + [ + _tp(q, ps.tp_rank, ps.tp_size), + _tp(k, ps.tp_rank, ps.tp_size), + _tp(v, ps.tp_rank, ps.tp_size), + _tp(z, ps.tp_rank, ps.tp_size), + _tp(b, ps.tp_rank, ps.tp_size), + _tp(a, ps.tp_rank, ps.tp_size), + ], + dim=0, + ).contiguous() + + +def _tp_linear_attn_conv1d( + tensor: torch.Tensor, *, cfg: Qwen35Config, ps: ParallelState +) -> torch.Tensor: + qk_dim = cfg.linear_num_key_heads * cfg.linear_key_head_dim + v_dim = cfg.linear_num_value_heads * cfg.linear_value_head_dim + q, k, v = tensor.split([qk_dim, qk_dim, v_dim], dim=0) + return torch.cat( + [ + _tp(q, ps.tp_rank, ps.tp_size), + _tp(k, ps.tp_rank, ps.tp_size), + _tp(v, ps.tp_rank, ps.tp_size), + ], + dim=0, + ).contiguous() + + +def _zero_centered_gamma_from_hf(tensor: torch.Tensor) -> torch.Tensor: + return tensor - 1 + + +def _get(reader: SafeTensorReader, name: str) -> torch.Tensor: + return reader.get_tensor(name) + + +def _has(reader: SafeTensorReader, name: str) -> bool: + if reader.index: + return name in reader.index + try: + reader.get_tensor(name) + except Exception: + return False + return True + + +def _load_vocab( + reader: SafeTensorReader, name: str, cfg: Qwen35Config, ps: ParallelState +) -> torch.Tensor: + from megatron.lite.primitive.parallel import pad_vocab_for_tp + + tensor = _get(reader, name) + padded = pad_vocab_for_tp(cfg.vocab_size, ps.tp_size) + if tensor.size(0) < padded: + pad = torch.zeros(padded - tensor.size(0), tensor.size(1), dtype=tensor.dtype) + tensor = torch.cat([tensor, pad], dim=0) + return _tp(tensor, ps.tp_rank, ps.tp_size) + + +def _load_full_attn( + out: dict[str, torch.Tensor], + *, + local_prefix: str, + hf_prefix: str, + input_ln: torch.Tensor, + cfg: Qwen35Config, + ps: ParallelState, + reader: SafeTensorReader, +) -> None: + out[f"{local_prefix}.full_attn.qkv.linear.layer_norm_weight"] = input_ln + q = _get(reader, f"{hf_prefix}.q_proj.weight") + k = _get(reader, f"{hf_prefix}.k_proj.weight") + v = _get(reader, f"{hf_prefix}.v_proj.weight") + out[f"{local_prefix}.full_attn.qkv.linear.weight"] = _tp( + _merge_full_attn_qkvg(q, k, v, cfg=cfg), ps.tp_rank, ps.tp_size + ) + out[f"{local_prefix}.full_attn.q_norm.weight"] = _get(reader, f"{hf_prefix}.q_norm.weight") + out[f"{local_prefix}.full_attn.k_norm.weight"] = _get(reader, f"{hf_prefix}.k_norm.weight") + out[f"{local_prefix}.full_attn.proj.linear.weight"] = _tp( + _get(reader, f"{hf_prefix}.o_proj.weight"), ps.tp_rank, ps.tp_size, dim=1 + ) + + +def _merge_full_attn_qkvg( + q_gate: torch.Tensor, key: torch.Tensor, value: torch.Tensor, *, cfg: Qwen35Config +) -> torch.Tensor: + kv_heads = cfg.num_key_value_heads + head_dim = cfg.head_dim + hidden = q_gate.shape[1] + q_gate = q_gate.reshape(cfg.num_attention_heads, 2 * head_dim, hidden) + query = q_gate.narrow(1, 0, head_dim).reshape(cfg.num_attention_heads * head_dim, hidden) + gate = q_gate.narrow(1, head_dim, head_dim).reshape(cfg.num_attention_heads * head_dim, hidden) + q_heads_per_group = ensure_divisible(cfg.num_attention_heads, cfg.num_key_value_heads) + q_group_width = q_heads_per_group * head_dim + query = query.reshape(kv_heads, q_group_width, hidden) + gate = gate.reshape(kv_heads, q_group_width, hidden) + key = key.reshape(kv_heads, head_dim, hidden) + value = value.reshape(kv_heads, head_dim, hidden) + return torch.cat([query, gate, key, value], dim=1).reshape(-1, hidden).contiguous() + + +def _unmerge_full_attn_qkvg( + tensor: torch.Tensor, *, cfg: Qwen35Config +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Invert Qwen35 lite's full-attention q/g/k/v packing.""" + q_heads_per_group = ensure_divisible(cfg.num_attention_heads, cfg.num_key_value_heads) + group_width = (2 * q_heads_per_group + 2) * cfg.head_dim + hidden = tensor.shape[-1] + packed = tensor.reshape(cfg.num_key_value_heads, group_width, hidden) + query, gate, key, value = packed.split( + [ + q_heads_per_group * cfg.head_dim, + q_heads_per_group * cfg.head_dim, + cfg.head_dim, + cfg.head_dim, + ], + dim=1, + ) + query = query.reshape(cfg.num_attention_heads, cfg.head_dim, hidden) + gate = gate.reshape(cfg.num_attention_heads, cfg.head_dim, hidden) + q_gate = torch.cat([query, gate], dim=1).reshape( + cfg.num_attention_heads * 2 * cfg.head_dim, hidden + ) + key = key.reshape(cfg.num_key_value_heads * cfg.head_dim, hidden) + value = value.reshape(cfg.num_key_value_heads * cfg.head_dim, hidden) + return q_gate.contiguous(), key.contiguous(), value.contiguous() + + +def _split_linear_attn_in_proj( + tensor: torch.Tensor, *, cfg: Qwen35Config +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + qk_dim = cfg.linear_num_key_heads * cfg.linear_key_head_dim + v_dim = cfg.linear_num_value_heads * cfg.linear_value_head_dim + return tensor.split( + [qk_dim, qk_dim, v_dim, v_dim, cfg.linear_num_value_heads, cfg.linear_num_value_heads], + dim=0, + ) + + +def _merge_linear_attn_in_proj_tp_shards( + shards: list[torch.Tensor], *, cfg: Qwen35Config +) -> torch.Tensor: + world_size = len(shards) + qk_dim = ensure_divisible(cfg.linear_num_key_heads * cfg.linear_key_head_dim, world_size) + v_dim = ensure_divisible(cfg.linear_num_value_heads * cfg.linear_value_head_dim, world_size) + value_heads = ensure_divisible(cfg.linear_num_value_heads, world_size) + + parts: list[list[torch.Tensor]] = [[] for _ in range(6)] + for shard in shards: + for bucket, part in zip( + parts, + shard.split([qk_dim, qk_dim, v_dim, v_dim, value_heads, value_heads], dim=0), + strict=True, + ): + bucket.append(part) + + return torch.cat([torch.cat(bucket, dim=0) for bucket in parts], dim=0).contiguous() + + +def _merge_linear_attn_conv1d_tp_shards( + shards: list[torch.Tensor], *, cfg: Qwen35Config +) -> torch.Tensor: + world_size = len(shards) + qk_dim = ensure_divisible(cfg.linear_num_key_heads * cfg.linear_key_head_dim, world_size) + v_dim = ensure_divisible(cfg.linear_num_value_heads * cfg.linear_value_head_dim, world_size) + + parts: list[list[torch.Tensor]] = [[] for _ in range(3)] + for shard in shards: + for bucket, part in zip(parts, shard.split([qk_dim, qk_dim, v_dim], dim=0), strict=True): + bucket.append(part) + + return torch.cat([torch.cat(bucket, dim=0) for bucket in parts], dim=0).contiguous() + + +def _merge_gate_up_tp_shards(shards: list[torch.Tensor]) -> torch.Tensor: + gates: list[torch.Tensor] = [] + ups: list[torch.Tensor] = [] + for shard in shards: + gate, up = shard.chunk(2, dim=0) + gates.append(gate) + ups.append(up) + return torch.cat([torch.cat(gates, dim=0), torch.cat(ups, dim=0)], dim=0).contiguous() + + +def _allgather_tp_shards(tensor: torch.Tensor, ps: ParallelState) -> list[torch.Tensor]: + shards = [torch.empty_like(tensor) for _ in range(ps.tp_size)] + dist.all_gather(shards, tensor.contiguous(), group=ps.tp_group) + return shards + + +class Qwen35WeightSpec: + """Export Qwen35 lite weights to HF checkpoint or vLLM runtime names.""" + + def __init__(self, config: Qwen35Config, target: str = "hf"): + if target not in {"hf", "vllm"}: + raise ValueError(f"Unsupported Qwen3.5 export target: {target!r}") + self.config = config + self.target = target + self._expert_export_buffers: dict[tuple[int, str], dict[int, torch.Tensor]] = {} + + @property + def num_experts(self) -> int: + return self.config.num_experts + + def weight_map(self) -> dict[str, list[str]]: + return {} + + def hf_to_native(self, native_name: str, hf_tensors: list[torch.Tensor]) -> torch.Tensor: + del native_name + return hf_tensors[0] + + def gather_dense( + self, native_name: str, tensor: torch.Tensor, ps: ParallelState + ) -> torch.Tensor | None: + if ps.tp_size <= 1: + return None + if native_name.endswith(".linear_attn.in_proj.linear.weight"): + return _merge_linear_attn_in_proj_tp_shards( + _allgather_tp_shards(tensor, ps), cfg=self.config + ) + if native_name.endswith(".linear_attn.conv1d.weight"): + return _merge_linear_attn_conv1d_tp_shards( + _allgather_tp_shards(tensor, ps), cfg=self.config + ) + if native_name.endswith(".moe.shared_expert.gate_up.linear.weight"): + return _merge_gate_up_tp_shards(_allgather_tp_shards(tensor, ps)) + return None + + def packed_expert_group_name(self, native_name: str) -> str | None: + if re.fullmatch(r"layers\.\d+\.moe\.experts\.fc[12]\.weight\d+", native_name) is None: + return None + return re.sub(r"\.weight\d+$", ".packed", native_name) + + def native_to_hf( + self, native_name: str, tensor: torch.Tensor + ) -> list[tuple[str, torch.Tensor]]: + if self.target == "vllm": + return self._native_to_vllm(native_name, tensor) + + if native_name == "embed.embedding.weight": + return [("model.language_model.embed_tokens.weight", tensor)] + if native_name == "norm.weight": + return [("model.language_model.norm.weight", tensor)] + if native_name == "head.col.linear.weight": + return [("lm_head.weight", tensor)] + if native_name == "mtp_embed.embedding.weight" or native_name.startswith("mtp."): + return [] + + match = re.match(r"layers\.(\d+)\.(.*)", native_name) + if match is None: + return [] + + layer_idx = int(match.group(1)) + suffix = match.group(2) + prefix = f"model.language_model.layers.{layer_idx}" + + if suffix == "full_attn.qkv.linear.layer_norm_weight": + return [(f"{prefix}.input_layernorm.weight", tensor)] + if suffix == "full_attn.qkv.linear.weight": + q_gate, key, value = _unmerge_full_attn_qkvg(tensor, cfg=self.config) + return [ + (f"{prefix}.self_attn.q_proj.weight", q_gate), + (f"{prefix}.self_attn.k_proj.weight", key), + (f"{prefix}.self_attn.v_proj.weight", value), + ] + if suffix == "full_attn.q_norm.weight": + return [(f"{prefix}.self_attn.q_norm.weight", tensor)] + if suffix == "full_attn.k_norm.weight": + return [(f"{prefix}.self_attn.k_norm.weight", tensor)] + if suffix == "full_attn.proj.linear.weight": + return [(f"{prefix}.self_attn.o_proj.weight", tensor)] + + if suffix == "linear_attn.in_proj.linear.layer_norm_weight": + return [(f"{prefix}.input_layernorm.weight", tensor)] + if suffix == "linear_attn.in_proj.linear.weight": + q, k, value, z, b, a = _split_linear_attn_in_proj(tensor, cfg=self.config) + return [ + ( + f"{prefix}.linear_attn.in_proj_qkv.weight", + torch.cat([q, k, value], dim=0).contiguous(), + ), + (f"{prefix}.linear_attn.in_proj_z.weight", z.contiguous()), + (f"{prefix}.linear_attn.in_proj_b.weight", b.contiguous()), + (f"{prefix}.linear_attn.in_proj_a.weight", a.contiguous()), + ] + if suffix == "linear_attn.conv1d.weight": + return [(f"{prefix}.linear_attn.conv1d.weight", tensor)] + if suffix == "linear_attn.dt_bias": + return [(f"{prefix}.linear_attn.dt_bias", tensor)] + if suffix == "linear_attn.A_log": + return [(f"{prefix}.linear_attn.A_log", tensor)] + if suffix == "linear_attn.norm.weight": + return [(f"{prefix}.linear_attn.norm.weight", tensor + 1)] + if suffix == "linear_attn.o_proj.linear.weight": + return [(f"{prefix}.linear_attn.out_proj.weight", tensor)] + + if suffix == "mlp_norm.weight": + return [(f"{prefix}.post_attention_layernorm.weight", tensor)] + if suffix == "moe.router.gate.weight": + return [(f"{prefix}.mlp.gate.weight", tensor)] + if suffix == "moe.shared_expert.gate_up.linear.weight": + gate, up = tensor.chunk(2, dim=0) + return [ + (f"{prefix}.mlp.shared_expert.gate_proj.weight", gate.contiguous()), + (f"{prefix}.mlp.shared_expert.up_proj.weight", up.contiguous()), + ] + if suffix == "moe.shared_expert.down.linear.weight": + return [(f"{prefix}.mlp.shared_expert.down_proj.weight", tensor)] + if suffix == "moe.shared_expert.shared_gate.weight": + return [(f"{prefix}.mlp.shared_expert_gate.weight", tensor)] + + expert_match = re.fullmatch(r"moe\.experts\.fc([12])\.weight(\d+)", suffix) + if expert_match is not None: + kind, expert_idx = expert_match.groups() + buffer_key = (layer_idx, "gate_up" if kind == "1" else "down") + buffer = self._expert_export_buffers.setdefault(buffer_key, {}) + buffer[int(expert_idx)] = tensor.contiguous() + if len(buffer) < self.config.num_experts: + return [] + packed = torch.stack( + [buffer[i] for i in range(self.config.num_experts)], dim=0 + ).contiguous() + del self._expert_export_buffers[buffer_key] + if kind == "1": + return [(f"{prefix}.mlp.experts.gate_up_proj", packed)] + return [(f"{prefix}.mlp.experts.down_proj", packed)] + + packed_expert_match = re.fullmatch(r"moe\.experts\.fc([12])\.packed", suffix) + if packed_expert_match is not None: + kind = packed_expert_match.group(1) + if kind == "1": + return [(f"{prefix}.mlp.experts.gate_up_proj", tensor.contiguous())] + return [(f"{prefix}.mlp.experts.down_proj", tensor.contiguous())] + + return [] + + def _native_to_vllm( + self, native_name: str, tensor: torch.Tensor + ) -> list[tuple[str, torch.Tensor]]: + if native_name == "embed.embedding.weight": + return [("language_model.model.embed_tokens.weight", tensor)] + if native_name == "norm.weight": + return [("language_model.model.norm.weight", tensor)] + if native_name == "head.col.linear.weight": + return [("language_model.lm_head.weight", tensor)] + if native_name == "mtp_embed.embedding.weight" or native_name.startswith("mtp."): + return [] + + match = re.match(r"layers\.(\d+)\.(.*)", native_name) + if match is None: + return [] + + layer_idx = int(match.group(1)) + suffix = match.group(2) + prefix = f"language_model.model.layers.{layer_idx}" + + if suffix == "full_attn.qkv.linear.layer_norm_weight": + return [(f"{prefix}.input_layernorm.weight", tensor)] + if suffix == "full_attn.qkv.linear.weight": + q_gate, key, value = _unmerge_full_attn_qkvg(tensor, cfg=self.config) + return [ + (f"{prefix}.self_attn.q_proj.weight", q_gate), + (f"{prefix}.self_attn.k_proj.weight", key), + (f"{prefix}.self_attn.v_proj.weight", value), + ] + if suffix == "full_attn.q_norm.weight": + return [(f"{prefix}.self_attn.q_norm.weight", tensor)] + if suffix == "full_attn.k_norm.weight": + return [(f"{prefix}.self_attn.k_norm.weight", tensor)] + if suffix == "full_attn.proj.linear.weight": + return [(f"{prefix}.self_attn.o_proj.weight", tensor)] + + if suffix == "linear_attn.in_proj.linear.layer_norm_weight": + return [(f"{prefix}.input_layernorm.weight", tensor)] + if suffix == "linear_attn.in_proj.linear.weight": + q, k, value, z, b, a = _split_linear_attn_in_proj(tensor, cfg=self.config) + return [ + ( + f"{prefix}.linear_attn.in_proj_qkv.weight", + torch.cat([q, k, value], dim=0).contiguous(), + ), + (f"{prefix}.linear_attn.in_proj_z.weight", z.contiguous()), + (f"{prefix}.linear_attn.in_proj_b.weight", b.contiguous()), + (f"{prefix}.linear_attn.in_proj_a.weight", a.contiguous()), + ] + if suffix == "linear_attn.conv1d.weight": + return [(f"{prefix}.linear_attn.conv1d.weight", tensor)] + if suffix == "linear_attn.dt_bias": + return [(f"{prefix}.linear_attn.dt_bias", tensor)] + if suffix == "linear_attn.A_log": + return [(f"{prefix}.linear_attn.A_log", tensor)] + if suffix == "linear_attn.norm.weight": + return [(f"{prefix}.linear_attn.norm.weight", tensor + 1)] + if suffix == "linear_attn.o_proj.linear.weight": + return [(f"{prefix}.linear_attn.out_proj.weight", tensor)] + + if suffix == "mlp_norm.weight": + return [(f"{prefix}.post_attention_layernorm.weight", tensor)] + if suffix == "moe.router.gate.weight": + return [(f"{prefix}.mlp.gate.weight", tensor)] + if suffix == "moe.shared_expert.gate_up.linear.weight": + gate, up = tensor.chunk(2, dim=0) + return [ + (f"{prefix}.mlp.shared_expert.gate_proj.weight", gate.contiguous()), + (f"{prefix}.mlp.shared_expert.up_proj.weight", up.contiguous()), + ] + if suffix == "moe.shared_expert.down.linear.weight": + return [(f"{prefix}.mlp.shared_expert.down_proj.weight", tensor)] + if suffix == "moe.shared_expert.shared_gate.weight": + return [(f"{prefix}.mlp.shared_expert_gate.weight", tensor)] + + expert_match = re.fullmatch(r"moe\.experts\.fc([12])\.weight(\d+)", suffix) + if expert_match is not None: + kind, expert_idx = expert_match.groups() + buffer_key = (layer_idx, "vllm_gate_up" if kind == "1" else "vllm_down") + buffer = self._expert_export_buffers.setdefault(buffer_key, {}) + buffer[int(expert_idx)] = tensor.contiguous() + if len(buffer) < self.config.num_experts: + return [] + packed = torch.stack( + [buffer[i] for i in range(self.config.num_experts)], dim=0 + ).contiguous() + del self._expert_export_buffers[buffer_key] + if kind == "1": + return [(f"{prefix}.mlp.experts.gate_up_proj", packed)] + return [(f"{prefix}.mlp.experts.down_proj", packed)] + + packed_expert_match = re.fullmatch(r"moe\.experts\.fc([12])\.packed", suffix) + if packed_expert_match is not None: + kind = packed_expert_match.group(1) + if kind == "1": + return [(f"{prefix}.mlp.experts.gate_up_proj", tensor.contiguous())] + return [(f"{prefix}.mlp.experts.down_proj", tensor.contiguous())] + + return [] + + def qkv_spec(self, native_name: str) -> tuple[int, int, int] | None: + del native_name + return None + + def tp_spec(self, native_name: str) -> tuple[int, int] | None: + if self.is_expert(native_name): + if ".fc1." in native_name: + return (0, 1) + if ".fc2." in native_name: + return (1, 1) + return None + if native_name in {"embed.embedding.weight", "head.col.linear.weight"}: + return (0, 0) + if native_name.endswith(".full_attn.qkv.linear.weight"): + return (0, 0) + if native_name.endswith(".full_attn.proj.linear.weight"): + return (1, 0) + if native_name.endswith(".linear_attn.in_proj.linear.weight"): + return (0, 0) + if native_name.endswith(".linear_attn.o_proj.linear.weight"): + return (1, 0) + if native_name.endswith(".moe.shared_expert.gate_up.linear.weight"): + return (0, 0) + if native_name.endswith(".moe.shared_expert.down.linear.weight"): + return (1, 0) + if any( + native_name.endswith(suffix) + for suffix in ( + ".linear_attn.conv1d.weight", + ".linear_attn.dt_bias", + ".linear_attn.A_log", + ) + ): + return (0, 0) + return None + + def is_expert(self, native_name: str) -> bool: + return ( + ".moe.experts." in native_name + and ".router." not in native_name + and ".shared" not in native_name + ) + + def expert_global_id(self, native_name: str) -> int | None: + match = re.search(r"\.weight(\d+)$", native_name) + return int(match.group(1)) if match is not None else None + + def expert_local_name(self, native_name: str, local_idx: int) -> str: + return re.sub(r"\.weight\d+$", f".weight{local_idx}", native_name) + + +def _load_linear_attn( + out: dict[str, torch.Tensor], + *, + local_prefix: str, + hf_prefix: str, + input_ln: torch.Tensor, + cfg: Qwen35Config, + ps: ParallelState, + reader: SafeTensorReader, +) -> None: + dk, dv = cfg.linear_key_head_dim, cfg.linear_value_head_dim + nk, nv = cfg.linear_num_key_heads, cfg.linear_num_value_heads + qk_dim, v_dim = nk * dk, nv * dv + qkv = _get(reader, f"{hf_prefix}.in_proj_qkv.weight") + q, k, v = qkv.split([qk_dim, qk_dim, v_dim], dim=0) + z = _get(reader, f"{hf_prefix}.in_proj_z.weight") + b = _get(reader, f"{hf_prefix}.in_proj_b.weight") + a = _get(reader, f"{hf_prefix}.in_proj_a.weight") + + out[f"{local_prefix}.linear_attn.in_proj.linear.weight"] = _tp_linear_attn_in_proj( + q, k, v, z, b, a, ps=ps + ) + out[f"{local_prefix}.linear_attn.in_proj.linear.layer_norm_weight"] = input_ln + out[f"{local_prefix}.linear_attn.conv1d.weight"] = _tp_linear_attn_conv1d( + _get(reader, f"{hf_prefix}.conv1d.weight"), cfg=cfg, ps=ps + ) + out[f"{local_prefix}.linear_attn.dt_bias"] = _tp( + _get(reader, f"{hf_prefix}.dt_bias"), ps.tp_rank, ps.tp_size + ) + out[f"{local_prefix}.linear_attn.A_log"] = _tp( + _get(reader, f"{hf_prefix}.A_log"), ps.tp_rank, ps.tp_size + ) + out[f"{local_prefix}.linear_attn.norm.weight"] = _zero_centered_gamma_from_hf( + _get(reader, f"{hf_prefix}.norm.weight") + ) + out[f"{local_prefix}.linear_attn.o_proj.linear.weight"] = _tp( + _get(reader, f"{hf_prefix}.out_proj.weight"), ps.tp_rank, ps.tp_size, dim=1 + ) + + +def _load_shared_expert( + out: dict[str, torch.Tensor], + *, + local_prefix: str, + hf_mlp_prefix: str, + ps: ParallelState, + reader: SafeTensorReader, +) -> None: + shared = f"{hf_mlp_prefix}.shared_expert" + gate_up = torch.cat( + [_get(reader, f"{shared}.gate_proj.weight"), _get(reader, f"{shared}.up_proj.weight")], + dim=0, + ) + out[f"{local_prefix}.moe.shared_expert.gate_up.linear.weight"] = _split_gate_up( + gate_up, ps.tp_rank, ps.tp_size + ) + out[f"{local_prefix}.moe.shared_expert.down.linear.weight"] = _tp( + _get(reader, f"{shared}.down_proj.weight"), ps.tp_rank, ps.tp_size, dim=1 + ) + out[f"{local_prefix}.moe.shared_expert.shared_gate.weight"] = _get( + reader, f"{hf_mlp_prefix}.shared_expert_gate.weight" + ) + + +def _load_experts( + out: dict[str, torch.Tensor], + *, + local_prefix: str, + hf_mlp_prefix: str, + cfg: Qwen35Config, + ps: ParallelState, + reader: SafeTensorReader, +) -> None: + num_local = ensure_divisible(cfg.num_experts, ps.ep_size) + local_start = ps.ep_rank * num_local + packed_gate_up = f"{hf_mlp_prefix}.experts.gate_up_proj" + packed_down = f"{hf_mlp_prefix}.experts.down_proj" + if _has(reader, packed_gate_up) and _has(reader, packed_down): + gate_up_all = _get(reader, packed_gate_up) + down_all = _get(reader, packed_down) + for local_idx in range(num_local): + global_idx = local_start + local_idx + fc1 = gate_up_all[global_idx] + fc2 = down_all[global_idx] + if ps.etp_size > 1: + fc1 = _split_gate_up(fc1, ps.etp_rank, ps.etp_size) + fc2 = _tp(fc2, ps.etp_rank, ps.etp_size, dim=1) + out[f"{local_prefix}.moe.experts.fc1.weight{local_idx}"] = fc1 + out[f"{local_prefix}.moe.experts.fc2.weight{local_idx}"] = fc2 + return + + for local_idx in range(num_local): + global_idx = local_start + local_idx + ep = f"{hf_mlp_prefix}.experts.{global_idx}" + fc1 = torch.cat( + [_get(reader, f"{ep}.gate_proj.weight"), _get(reader, f"{ep}.up_proj.weight")], dim=0 + ) + fc2 = _get(reader, f"{ep}.down_proj.weight") + if ps.etp_size > 1: + fc1 = _split_gate_up(fc1, ps.etp_rank, ps.etp_size) + fc2 = _tp(fc2, ps.etp_rank, ps.etp_size, dim=1) + out[f"{local_prefix}.moe.experts.fc1.weight{local_idx}"] = fc1 + out[f"{local_prefix}.moe.experts.fc2.weight{local_idx}"] = fc2 + + +def _copy_loaded_state(model: nn.Module, loaded: dict[str, torch.Tensor]) -> None: + state = model.state_dict() + resolved: dict[str, torch.Tensor] = {} + for name, tensor in loaded.items(): + actual = name if name in state else None + if actual is None: + for key in state: + if name in key: + actual = key + break + if actual is not None: + resolved[actual] = tensor + else: + log_rank0(f"WARNING: lite checkpoint tensor has no target param: {name}") + + fp32_names = ("A_log", "dt_bias") + for name, param in model.named_parameters(): + if name not in resolved: + log_rank0(f"WARNING: {name} not loaded from checkpoint") + continue + tensor = resolved[name].to(device=param.device) + if any(k in name for k in fp32_names): + param.data.copy_(tensor.float()) + else: + param.data.copy_(tensor.to(dtype=param.dtype)) + + +def load_hf_weights(model: nn.Module, path: str, config: Qwen35Config, ps: ParallelState) -> None: + base_model = unwrap_model(model) + reader = SafeTensorReader(path) + out: dict[str, torch.Tensor] = {} + + prefix = "model.language_model" + if getattr(base_model, "embed", None) is not None: + out["embed.embedding.weight"] = _load_vocab( + reader, f"{prefix}.embed_tokens.weight", config, ps + ) + if getattr(base_model, "norm", None) is not None: + out["norm.weight"] = _get(reader, f"{prefix}.norm.weight") + if getattr(base_model, "head", None) is not None: + out["head.col.linear.weight"] = _load_vocab(reader, "lm_head.weight", config, ps) + + for local_idx, global_idx in enumerate(base_model.layer_indices): + lp = f"layers.{local_idx}" + hp = f"{prefix}.layers.{global_idx}" + input_ln = _get(reader, f"{hp}.input_layernorm.weight") + if config.layer_type_at(global_idx) == "full_attention": + _load_full_attn( + out, + local_prefix=lp, + hf_prefix=f"{hp}.self_attn", + input_ln=input_ln, + cfg=config, + ps=ps, + reader=reader, + ) + else: + _load_linear_attn( + out, + local_prefix=lp, + hf_prefix=f"{hp}.linear_attn", + input_ln=input_ln, + cfg=config, + ps=ps, + reader=reader, + ) + out[f"{lp}.mlp_norm.weight"] = _get(reader, f"{hp}.post_attention_layernorm.weight") + out[f"{lp}.moe.router.gate.weight"] = _get(reader, f"{hp}.mlp.gate.weight")[ + : config.num_experts + ] + _load_shared_expert(out, local_prefix=lp, hf_mlp_prefix=f"{hp}.mlp", ps=ps, reader=reader) + _load_experts( + out, local_prefix=lp, hf_mlp_prefix=f"{hp}.mlp", cfg=config, ps=ps, reader=reader + ) + + _copy_loaded_state(base_model, out) + + +def export_hf_weights( + model: nn.Module | list[nn.Module], config: Qwen35Config, ps: ParallelState, **kwargs +): + from megatron.lite.primitive.ckpt.hf_weights import export_hf_weights as _export + + include_mtp_only = kwargs.pop("include_mtp_only", False) + kwargs.pop("include_local_prefixes", None) + target = kwargs.pop("target", "hf") + if include_mtp_only: + return + yield from _export( + model, Qwen35WeightSpec(config, target=target), ps, vocab_size=config.vocab_size, **kwargs + ) + + +def save_hf_weights( + model: nn.Module | list[nn.Module], path: str, config: Qwen35Config, ps: ParallelState +) -> None: + from megatron.lite.primitive.ckpt.hf_weights import save_hf_weights as _save + + _save(model, path, Qwen35WeightSpec(config), ps, vocab_size=config.vocab_size) + + +__all__ = [ + "EXPERT_CLASSIFIER", + "PLACEMENT_FN", + "Qwen35WeightSpec", + "export_hf_weights", + "load_hf_weights", + "save_hf_weights", +] diff --git a/experimental/lite/megatron/lite/model/qwen3_5/lite/model.py b/experimental/lite/megatron/lite/model/qwen3_5/lite/model.py new file mode 100644 index 00000000000..cb1f7dd6454 --- /dev/null +++ b/experimental/lite/megatron/lite/model/qwen3_5/lite/model.py @@ -0,0 +1,709 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Qwen3.5 lite native model. + +This implementation keeps the lightweight qwen3_moe/lite composition style +and uses Megatron Lite parallel, TE, RoPE, and MoE primitives directly. +""" + +from __future__ import annotations + +from contextlib import nullcontext + +import torch +import torch.distributed as dist +import torch.nn as nn +import transformer_engine.pytorch as te + +from megatron.lite.model.qwen3_5.config import Qwen35Config +from megatron.lite.primitive.modules.dispatcher import TokenDispatcher +from megatron.lite.primitive.modules.experts import Experts, swiglu_with_probs +from megatron.lite.primitive.modules.gated_delta_net import GatedDeltaNet +from megatron.lite.primitive.modules.gqa import GQAttention as FullAttention +from megatron.lite.primitive.modules.gqa import split_grouped_qkvg as _split_grouped_qkvg +from megatron.lite.primitive.modules.mrope import MultimodalRotaryEmbedding as Qwen35MRoPE +from megatron.lite.primitive.modules.mtp import ( + MTPBlock, + MTPDecoderLayer, + MTPLossAutoScaler, + roll_mtp_tensor_left, +) +from megatron.lite.primitive.modules.router import TopKRouter +from megatron.lite.primitive.ops.cross_entropy import vocab_parallel_cross_entropy +from megatron.lite.primitive.ops.linear_cross_entropy import linear_cross_entropy +from megatron.lite.primitive.ops.logprob import vocab_parallel_entropy +from megatron.lite.primitive.parallel import ( + ColumnParallelLinear, + ParallelState, + RowParallelLinear, + VocabParallelEmbedding, + VocabParallelOutput, + build_pipeline_chunk_layout, + gather_from_sequence_parallel, + scatter_to_sequence_parallel, +) +from megatron.lite.primitive.utils import build_fp8_recipe + +_SP_GRAD_SUFFIXES: tuple[str, ...] = ( + ".full_attn.qkv.linear.layer_norm_weight", + ".full_attn.q_norm.weight", + ".full_attn.k_norm.weight", + ".linear_attn.in_proj.linear.layer_norm_weight", + ".linear_attn.norm.weight", + ".mlp_norm.weight", + ".moe.router.gate.weight", + ".moe.shared_expert.shared_gate.weight", + ".enorm.weight", + ".hnorm.weight", + ".final_layernorm.weight", +) + + +def _collect_sp_grad_params(model: nn.Module) -> list[nn.Parameter]: + params = [] + for name, param in model.named_parameters(): + if any(name.endswith(s) for s in _SP_GRAD_SUFFIXES) or name == "norm.weight": + params.append(param) + return params + + +def _swiglu(x: torch.Tensor) -> torch.Tensor: + return swiglu_with_probs(x, probs=None) + + +def _qwen_mrope_section(config: Qwen35Config) -> list[int]: + section = getattr(config, "mrope_section", None) + if section is not None: + return list(section) + rotary_half = max(int(config.rotary_dim // 2), 1) + base = rotary_half // 3 + return [base, base, rotary_half - 2 * base] + + +class SharedExpert(nn.Module): + _stream: torch.cuda.Stream | None = None + + class _CopyToTPRegion(torch.autograd.Function): + @staticmethod + def forward(ctx, x, group): + ctx.group = group + return x + + @staticmethod + def backward(ctx, grad): + group = ctx.group + if group is not None and dist.get_world_size(group) > 1: + dist.all_reduce(grad, group=group) + return grad, None + + class _PlainTELinear(nn.Module): + def __init__(self, in_features: int, out_features: int): + super().__init__() + self.linear = te.Linear( + in_features, + out_features, + bias=False, + params_dtype=torch.bfloat16, + parallel_mode=None, + sequence_parallel=False, + tp_group=None, + tp_size=1, + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.linear(x) + + def __init__( + self, config: Qwen35Config, ps: ParallelState, *, use_plain_te_linear: bool = False + ): + super().__init__() + ffn = config.shared_expert_intermediate_size + if use_plain_te_linear and ps.tp_size == 1: + self.gate_up = self._PlainTELinear(config.hidden_size, ffn * 2) + self.down = self._PlainTELinear(ffn, config.hidden_size) + else: + self.gate_up = ColumnParallelLinear(config.hidden_size, ffn * 2, ps, bias=False) + self.down = RowParallelLinear(ffn, config.hidden_size, ps, bias=False) + self.shared_gate = nn.Linear(config.hidden_size, 1, bias=False) + self.tp_group = ps.tp_group + self.use_mcore_overlap_graph = bool(use_plain_te_linear and ps.tp_size == 1) + + @staticmethod + def _get_stream() -> torch.cuda.Stream: + if SharedExpert._stream is None: + SharedExpert._stream = torch.cuda.Stream() + return SharedExpert._stream + + @staticmethod + def _set_grad_fn_sequence_sr(tensor: torch.Tensor) -> None: + grad_fn = getattr(tensor, "grad_fn", None) + if grad_fn is not None and hasattr(grad_fn, "_set_sequence_nr"): + grad_fn._set_sequence_nr(torch.iinfo(torch.int).max) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + gate_val = self.shared_gate(x).sigmoid() + fc1_input = x + if self.use_mcore_overlap_graph: + fc1_input = self._CopyToTPRegion.apply(x, self.tp_group) + self._set_grad_fn_sequence_sr(fc1_input) + output = self.down(_swiglu(self.gate_up(fc1_input))) + if self.use_mcore_overlap_graph: + self._set_grad_fn_sequence_sr(output) + return output * gate_val + + +class MoELayer(nn.Module): + def __init__( + self, + config: Qwen35Config, + ps: ParallelState, + *, + use_deepep: bool, + router_bias_rate: float, + fp8: bool, + moe_act_recompute: bool, + router_dtype: torch.dtype | None = None, + preserve_3d_graph: bool = False, + shared_expert_plain_te: bool = False, + ): + super().__init__() + if fp8: + raise NotImplementedError("lite qwen35 MoE fp8 is not implemented yet.") + self.router = TopKRouter( + config, + ps, + router_bias_rate=router_bias_rate, + compute_aux_loss=True, + router_dtype=router_dtype, + ) + self.experts = Experts(config, ps, fp8=fp8, moe_act_recompute=moe_act_recompute) + self.dispatcher = TokenDispatcher( + config.num_experts, config.hidden_size, ps, use_deepep=use_deepep + ) + self.shared_expert = SharedExpert(config, ps, use_plain_te_linear=shared_expert_plain_te) + self.preserve_3d_graph = bool(preserve_3d_graph) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + input_shape = x.shape + x_2d = x.reshape(-1, x.size(-1)) + shared_input = x_2d.view(input_shape) if self.preserve_3d_graph else x_2d + router_input = x if self.preserve_3d_graph else x_2d + + shared_out = None + side_stream = None + if x_2d.is_cuda: + side_stream = SharedExpert._get_stream() + side_stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(side_stream): + shared_out = self.shared_expert(shared_input) + scores, indices = self.router(router_input) + dispatched, tpe, permuted_probs = self.dispatcher.dispatch(x_2d, scores, indices) + del scores, indices + self.dispatcher.wait_dispatch_event() + expert_out = self.experts( + dispatched, + tpe, + permuted_probs, + tokens_per_expert_list=getattr(self.dispatcher, "_local_tpe_list", None), + ) + routed_out = self.dispatcher.combine(expert_out) + + if shared_out is None: + shared_out = self.shared_expert(shared_input) + else: + assert side_stream is not None + torch.cuda.current_stream().wait_stream(side_stream) + output = routed_out.view(input_shape) + if shared_out.shape != input_shape: + shared_out = shared_out.view(input_shape) + output += shared_out + return output.to(x.dtype) + + +class Qwen35Layer(nn.Module): + def __init__( + self, + config: Qwen35Config, + ps: ParallelState, + layer_idx: int, + *, + use_deepep: bool = False, + router_bias_rate: float = 0.0, + fp8: bool = False, + moe_act_recompute: bool = False, + use_thd: bool = False, + deterministic: bool = False, + gdn_cp_mode: str = "replicated", + ): + super().__init__() + self.layer_idx = layer_idx + self._layer_type = config.layer_type_at(layer_idx) + self.full_attn: FullAttention | None = None + self.linear_attn: GatedDeltaNet | None = None + if self._layer_type == "full_attention": + self.full_attn = FullAttention( + hidden_size=config.hidden_size, + num_attention_heads=config.num_attention_heads, + num_key_value_heads=config.num_key_value_heads, + head_dim=config.head_dim, + ps=ps, + rms_norm_eps=config.rms_norm_eps, + rope_theta=config.rope_theta, + rotary_percent=config.partial_rotary_factor, + use_thd=use_thd, + output_gate=True, + zero_centered_gamma=True, + qkv_layout="mcore", + mrope_section=_qwen_mrope_section(config), + ) + else: + self.linear_attn = GatedDeltaNet( + hidden_size=config.hidden_size, + linear_num_key_heads=config.linear_num_key_heads, + linear_key_head_dim=config.linear_key_head_dim, + linear_num_value_heads=config.linear_num_value_heads, + linear_value_head_dim=config.linear_value_head_dim, + linear_conv_kernel_dim=config.linear_conv_kernel_dim, + rms_norm_eps=config.rms_norm_eps, + ps=ps, + deterministic=deterministic, + cp_mode=gdn_cp_mode, + ) + self.mlp_norm = te.RMSNorm( + config.hidden_size, eps=config.rms_norm_eps, zero_centered_gamma=True + ) + self.moe = MoELayer( + config, + ps, + use_deepep=use_deepep, + router_bias_rate=router_bias_rate, + fp8=fp8, + moe_act_recompute=moe_act_recompute, + router_dtype=torch.float32 if deterministic else None, + preserve_3d_graph=deterministic, + shared_expert_plain_te=deterministic, + ) + + def forward( + self, x: torch.Tensor, position_ids: torch.Tensor | None = None, packed_seq_params=None + ) -> torch.Tensor: + residual = x + if self.full_attn is not None: + h = self.full_attn(x, position_ids=position_ids, packed_seq_params=packed_seq_params) + else: + assert self.linear_attn is not None + h = self.linear_attn(x, position_ids=position_ids, packed_seq_params=packed_seq_params) + x = residual + h + residual = x + x = residual + self.moe(self.mlp_norm(x)) + return x + + +def _temperature_to_float(temperature: float | torch.Tensor) -> float: + if isinstance(temperature, torch.Tensor): + if temperature.numel() != 1: + raise ValueError("Qwen35Model fused/MTP SFT supports scalar temperature only.") + return float(temperature.detach().float().item()) + return float(temperature) + + +def _ensure_mrope_position_ids(position_ids: torch.Tensor | None) -> torch.Tensor | None: + if position_ids is None: + return None + if position_ids.dim() == 2: + return position_ids.unsqueeze(0).expand(3, -1, -1).contiguous() + if position_ids.dim() == 3: + if position_ids.shape[0] == 1: + return position_ids.expand(3, -1, -1).contiguous() + if position_ids.shape[0] == 3: + return position_ids + raise ValueError("Qwen3.5 MRoPE expects position_ids shape (B,S), (1,B,S), or (3,B,S).") + + +class Qwen35Model(nn.Module): + def __init__( + self, + config: Qwen35Config, + train_config, + ps: ParallelState, + *, + vpp_chunk_id: int | None = None, + router_bias_rate: float = 0.0, + use_thd: bool = False, + hf_path: str = "", + attention_backend_override: str | None = None, + mtp_enable: bool = False, + mtp_enable_train: bool = False, + mtp_detach_encoder: bool = False, + mount_vision_model: bool = False, + gdn_cp_mode: str = "replicated", + ): + super().__init__() + del attention_backend_override + self.config = config + self.train_config = train_config + self.ps = ps + self.mtp_enable_train = bool(mtp_enable and mtp_enable_train) + self.mtp_loss_scaling_factor = config.mtp_loss_scaling_factor + self._input_tensor: torch.Tensor | None = None + + layout = build_pipeline_chunk_layout( + config.num_hidden_layers, ps, train_config.vpp, vpp_chunk_id + ) + self.layer_indices = layout.layer_indices + has_embed = layout.has_embed + has_head = layout.has_head + self.pre_process = has_embed + self.post_process = has_head + self.share_embeddings_and_output_weights = False + self.vision_model: nn.Module | None = ( + _build_native_vision_model(hf_path) if has_embed and mount_vision_model else None + ) + + self.embed: VocabParallelEmbedding | None = None + if has_embed: + self.embed = VocabParallelEmbedding(config.vocab_size, config.hidden_size, ps) + + recompute_modules = getattr(train_config, "recompute_modules", []) + moe_act_recompute = "moe_act" in recompute_modules and "moe" not in recompute_modules + self.layers = nn.ModuleList( + [ + Qwen35Layer( + config, + ps, + idx, + use_deepep=train_config.use_deepep, + router_bias_rate=router_bias_rate, + fp8=train_config.fp8, + moe_act_recompute=moe_act_recompute, + use_thd=use_thd, + deterministic=getattr(train_config, "deterministic", False), + gdn_cp_mode=gdn_cp_mode, + ) + for idx in self.layer_indices + ] + ) + + self.norm: nn.Module | None = None + self.head: VocabParallelOutput | None = None + if has_head: + self.norm = te.RMSNorm( + config.hidden_size, eps=config.rms_norm_eps, zero_centered_gamma=True + ) + self.head = VocabParallelOutput(config.vocab_size, config.hidden_size, ps) + + self.mtp_embed: VocabParallelEmbedding | None = None + self.mtp: MTPBlock | None = None + if mtp_enable and config.num_nextn_predict_layers > 0 and self.head is not None: + mtp_embedding = self.embed + if mtp_embedding is None: + mtp_embedding = VocabParallelEmbedding(config.vocab_size, config.hidden_size, ps) + self.mtp_embed = mtp_embedding + + def make_mtp_layer(layer_idx: int) -> MTPDecoderLayer: + return MTPDecoderLayer( + hidden_size=config.hidden_size, + rms_norm_eps=config.rms_norm_eps, + ps=ps, + embedding=mtp_embedding, + transformer_layer=Qwen35Layer( + config, + ps, + config.num_hidden_layers + layer_idx, + use_deepep=train_config.use_deepep, + router_bias_rate=router_bias_rate, + fp8=train_config.fp8, + moe_act_recompute=moe_act_recompute, + use_thd=use_thd, + deterministic=getattr(train_config, "deterministic", False), + gdn_cp_mode=gdn_cp_mode, + ), + detach_encoder=mtp_detach_encoder, + ) + + self.mtp = MTPBlock( + num_layers=config.num_nextn_predict_layers, + repeated_layer=config.mtp_use_repeated_layer, + layer_factory=make_mtp_layer, + ) + + self.sp_params: list[nn.Parameter] = [] + if ps.tp_size > 1: + self.sp_params = _collect_sp_grad_params(self) + + def set_input_tensor(self, input_tensor): + if isinstance(input_tensor, list): + if len(input_tensor) > 1: + raise ValueError("Qwen35Model expects a single pipeline input tensor.") + input_tensor = input_tensor[0] if input_tensor else None + self._input_tensor = input_tensor + + def forward( + self, + input_ids: torch.Tensor | None = None, + hidden_states: torch.Tensor | None = None, + position_ids: torch.Tensor | None = None, + packed_seq_params=None, + labels: torch.Tensor | None = None, + loss_mask: torch.Tensor | None = None, + temperature: float | torch.Tensor = 1.0, + use_fused_kernels: bool = False, + calculate_entropy: bool = False, + ) -> dict: + if self.embed is not None: + assert input_ids is not None + h = self.embed(input_ids) + else: + if hidden_states is None: + hidden_states = self._input_tensor + assert hidden_states is not None + h = hidden_states + + if packed_seq_params is not None: + assert position_ids is not None, "THD path requires caller-supplied MRoPE position_ids." + elif position_ids is None and input_ids is not None: + if self.ps.cp_size > 1: + raise ValueError("CP>1 requires caller-supplied FULL position_ids (3,B,S).") + batch, seq = input_ids.shape + pos = torch.arange(seq, device=input_ids.device, dtype=torch.long) + position_ids = pos.unsqueeze(0).unsqueeze(0).expand(3, batch, seq).contiguous() + position_ids = _ensure_mrope_position_ids(position_ids) + + fp8_ctx = ( + te.fp8_autocast(enabled=True, fp8_recipe=build_fp8_recipe(self.train_config)) + if self.train_config.fp8 + else nullcontext() + ) + with fp8_ctx: + if self.embed is not None: + h = scatter_to_sequence_parallel(h, self.ps) + for layer in self.layers: + h = layer(h, position_ids=position_ids, packed_seq_params=packed_seq_params) + + output = {"hidden_states": h} + if self.head is not None: + assert self.norm is not None + hidden_for_head = self.norm(h) + if labels is not None: + temperature_value = _temperature_to_float(temperature) + mtp_result = self._apply_mtp_loss( + hidden_for_head, + input_ids=input_ids, + position_ids=position_ids, + labels=labels, + loss_mask=loss_mask, + packed_seq_params=packed_seq_params, + temperature=temperature_value, + use_fused_kernels=use_fused_kernels, + ) + if mtp_result is not None: + hidden_for_head, mtp_loss = mtp_result + output["mtp_loss"] = mtp_loss + labels_sb = labels.transpose(0, 1).contiguous() + if use_fused_kernels: + hidden_full = gather_from_sequence_parallel(hidden_for_head, self.ps) + log_probs, entropy = linear_cross_entropy( + hidden_full, + self._head_weight_for_fused_ce(hidden_full), + labels_sb, + temperature_value, + self.ps.tp_group, + ) + output["loss"] = (-log_probs).mean() + output["log_probs"] = log_probs.transpose(0, 1).contiguous() + if calculate_entropy: + output["entropy"] = entropy.transpose(0, 1).contiguous() + else: + logits = self.head(hidden_for_head) + if temperature_value != 1.0: + logits = logits / temperature_value + loss = vocab_parallel_cross_entropy(logits, labels_sb, self.ps.tp_group) + output["loss"] = loss.mean() + output["log_probs"] = (-loss).transpose(0, 1).contiguous() + if calculate_entropy: + entropy = vocab_parallel_entropy(logits, self.ps.tp_group) + output["entropy"] = entropy.transpose(0, 1).contiguous() + else: + logits = self.head(hidden_for_head) + output["logits"] = self.head.gather(logits).transpose(0, 1).contiguous() + return output + + def _apply_mtp_loss( + self, + hidden_states: torch.Tensor, + *, + input_ids: torch.Tensor | None, + position_ids: torch.Tensor | None, + labels: torch.Tensor, + loss_mask: torch.Tensor | None, + packed_seq_params, + temperature: float, + use_fused_kernels: bool, + ) -> tuple[torch.Tensor, torch.Tensor] | None: + if self.mtp is None or not self.mtp_enable_train: + return None + if input_ids is None: + raise ValueError("MTP training requires input_ids.") + if loss_mask is None: + loss_mask = torch.ones_like(labels, dtype=torch.float32) + else: + loss_mask = loss_mask.to(dtype=torch.float32) + + mtp_hidden_states = self.mtp( + input_ids=input_ids, + position_ids=position_ids, + hidden_states=hidden_states, + packed_seq_params=packed_seq_params, + ) + mtp_labels = labels.clone() + mtp_loss_mask = loss_mask.clone() + mtp_loss_values = [] + for mtp_hidden in mtp_hidden_states: + mtp_labels, _ = roll_mtp_tensor_left( + mtp_labels, packed_seq_params=packed_seq_params, dims=-1 + ) + mtp_loss_mask, num_tokens = roll_mtp_tensor_left( + mtp_loss_mask, packed_seq_params=packed_seq_params, dims=-1 + ) + labels_sb = mtp_labels.transpose(0, 1).contiguous() + mask_sb = mtp_loss_mask.transpose(0, 1).contiguous() + if use_fused_kernels: + mtp_hidden_full = gather_from_sequence_parallel(mtp_hidden, self.ps) + log_probs, _entropy = linear_cross_entropy( + mtp_hidden_full, + self._head_weight_for_fused_ce(mtp_hidden_full), + labels_sb, + temperature, + self.ps.tp_group, + ) + token_loss = -log_probs + else: + assert self.head is not None + logits = self.head(mtp_hidden) + if temperature != 1.0: + logits = logits / temperature + token_loss = vocab_parallel_cross_entropy(logits, labels_sb, self.ps.tp_group) + token_loss = token_loss * mask_sb.to(dtype=token_loss.dtype) + num_tokens = num_tokens.to(dtype=token_loss.dtype).clamp_min(1.0) + mtp_loss_values.append(token_loss.sum() / num_tokens) + mtp_loss_scale = self.mtp_loss_scaling_factor / max(len(mtp_hidden_states), 1) + hidden_states = MTPLossAutoScaler.apply( + hidden_states, mtp_loss_scale * token_loss / num_tokens + ) + + if not mtp_loss_values: + return None + return ( + hidden_states, + torch.stack([loss.detach().float() for loss in mtp_loss_values]).mean(), + ) + + def _head_weight_for_fused_ce(self, hidden_states: torch.Tensor) -> torch.Tensor: + assert self.head is not None + weight = self.head.col.linear.weight + return ( + weight if weight.dtype == hidden_states.dtype else weight.to(dtype=hidden_states.dtype) + ) + + +def _iter_auto_model_class_refs(hf_config) -> list[str]: + auto_map = getattr(hf_config, "auto_map", None) or {} + preferred_keys = ( + "AutoModelForCausalLM", + "AutoModelForImageTextToText", + "AutoModelForVision2Seq", + "AutoModel", + ) + refs: list[str] = [] + for key in preferred_keys: + ref = auto_map.get(key) + if isinstance(ref, (list, tuple)): + ref = ref[0] if ref else None + if isinstance(ref, str) and ref not in refs: + refs.append(ref) + return refs + + +def _resolve_hf_vision_cls(hf_config, hf_path: str) -> type: + try: + from transformers.dynamic_module_utils import get_class_from_dynamic_module + except ImportError as exc: + raise ImportError( + "mount_vision_model=True requires transformers with dynamic module support." + ) from exc + + errors: list[str] = [] + for class_ref in _iter_auto_model_class_refs(hf_config): + try: + model_cls = get_class_from_dynamic_module(class_ref, hf_path, trust_remote_code=True) + except Exception as exc: + errors.append(f"{class_ref}: {exc}") + continue + vision_cls = getattr(model_cls, "HfVisionClass", None) + if vision_cls is not None: + return vision_cls + errors.append(f"{class_ref}: missing HfVisionClass") + detail = "; ".join(errors) if errors else "config auto_map has no supported AutoModel entry" + raise RuntimeError(f"Cannot resolve native HF vision class for Qwen3.5: {detail}.") + + +def _hook_fp32_rotary_emb(module: nn.Module) -> None: + for submodule in module.modules(): + if hasattr(submodule, "inv_freq") and submodule.inv_freq is not None: + submodule._inv_freq_fp32_original = submodule.inv_freq.detach().clone().float() + + def _hook(mod, args): + del args + if hasattr(mod, "_inv_freq_fp32_original"): + mod.inv_freq = mod._inv_freq_fp32_original.to(device=mod.inv_freq.device) + + submodule.register_forward_pre_hook(_hook) + + +def _hook_vision_params_avg_grad_across_tp(module: nn.Module) -> None: + for param in module.parameters(recurse=True): + param.average_gradients_across_tp_domain = True # type: ignore[assignment] + + +def _build_native_vision_model(hf_path: str) -> nn.Module | None: + if not hf_path: + raise ValueError("mount_vision_model requires hf_path.") + try: + from transformers import AutoConfig + except ImportError as exc: + raise ImportError("mount_vision_model=True requires transformers.") from exc + + hf_config = AutoConfig.from_pretrained(hf_path, trust_remote_code=True) + vision_config = getattr(hf_config, "vision_config", None) + if vision_config is None: + # Text-only checkpoint (no vision tower): nothing to mount -> graceful skip. + return None + auto_map = getattr(hf_config, "auto_map", None) or {} + if auto_map: + # Remote-code checkpoint: resolve the vision class from its dynamic module. + hf_vision_cls = _resolve_hf_vision_cls(hf_config, hf_path) + if hasattr(hf_vision_cls, "_from_config"): + vision = hf_vision_cls._from_config(vision_config) + else: + vision = hf_vision_cls(vision_config) + else: + # Native-transformers checkpoint (auto_map=None, e.g. Qwen3.5-35B-A3B): build the vision tower + # directly from its vision_config via the native AutoModel registry (no remote code). + from transformers import AutoModel + + vision = AutoModel.from_config(vision_config) + _hook_fp32_rotary_emb(vision) + _hook_vision_params_avg_grad_across_tp(vision) + return vision.to(torch.bfloat16) + + +__all__ = [ + "FullAttention", + "GatedDeltaNet", + "MoELayer", + "MTPLossAutoScaler", + "Qwen35MRoPE", + "Qwen35Layer", + "Qwen35Model", + "SharedExpert", + "_split_grouped_qkvg", +] diff --git a/experimental/lite/megatron/lite/model/qwen3_5/lite/protocol.py b/experimental/lite/megatron/lite/model/qwen3_5/lite/protocol.py new file mode 100644 index 00000000000..0c06d75c690 --- /dev/null +++ b/experimental/lite/megatron/lite/model/qwen3_5/lite/protocol.py @@ -0,0 +1,301 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Qwen3.5 lite impl — native model protocol for Megatron Lite runtime.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import torch +import torch.nn as nn +from megatron.lite.model.protocol_utils import ( + add_cross_entropy_fusion, + add_loss_context_kwargs, + pack_thd_forward_kwargs, + set_cross_entropy_fusion, + unpack_thd_forward_output, +) +from megatron.lite.model.qwen3_5.config import Qwen35Config +from megatron.lite.model.qwen3_5.lite.checkpoint import EXPERT_CLASSIFIER, PLACEMENT_FN +from megatron.lite.model.qwen3_5.lite.checkpoint import export_hf_weights as _export_hf_weights_impl +from megatron.lite.model.qwen3_5.lite.checkpoint import load_hf_weights as _load_hf_weights_impl +from megatron.lite.model.qwen3_5.lite.checkpoint import save_hf_weights as _save_hf_weights_impl +from megatron.lite.primitive.bundle import ModelBundle +from megatron.lite.primitive.parallel import ParallelState, init_parallel +from megatron.lite.primitive.recompute import apply_recompute, parse_recompute_spec +from megatron.lite.runtime.contracts import OptimizerConfig, ParallelConfig +from megatron.lite.runtime.contracts.data import PackedBatch + +__all__ = [ + "EXPERT_CLASSIFIER", + "ImplConfig", + "PLACEMENT_FN", + "build_model", + "build_model_config", + "export_hf_weights", + "load_hf_weights", + "save_hf_weights", + "vocab_size", +] + + +def is_expert_param(name: str) -> bool: + return "experts" in name and "router" not in name and "shared" not in name + + +@dataclass(frozen=True) +class ImplConfig: + parallel: ParallelConfig = field(default_factory=ParallelConfig) + optimizer: str | None = "dist_opt" + recompute: list[str] = field(default_factory=list) + offload: list[str] = field(default_factory=list) + use_deepep: bool = False + use_thd: bool = False + cross_entropy_fusion: bool = False + hf_path: str = "" + attention_backend_override: str | None = None + router_aux_loss_coef: float | None = None + router_bias_rate: float = 0.0 + deterministic: bool = True + optimizer_config: OptimizerConfig | None = None + mtp_enable: bool = False + mtp_enable_train: bool = False + mtp_detach_encoder: bool = False + mtp_loss_scaling_factor: float = 0.1 + mtp_use_repeated_layer: bool | None = None + mount_vision_model: bool = False + gdn_cp_mode: str = "replicated" + + +def _full_attn_module(layer, name: str): + full_attn = getattr(layer, "full_attn", None) + return getattr(full_attn, name, None) if full_attn is not None else None + + +MODULE_MAP = { + "core_attn": lambda layer: _full_attn_module(layer, "core_attn"), + "experts": lambda layer: layer.moe.experts, + "moe": lambda layer: layer.moe, + "router": lambda layer: layer.moe.router, + "mlp_norm": lambda layer: layer.mlp_norm, + "attn_proj": lambda layer: _full_attn_module(layer, "proj"), + "linear_attn": lambda layer: layer.linear_attn, +} + + +def build_model_config(source: str | Path | dict, **overrides) -> Qwen35Config: + """Build Qwen3.5 architecture config from HF source.""" + if isinstance(source, dict): + cfg = Qwen35Config._from_hf_dict(source) + else: + cfg = Qwen35Config.from_hf(str(source)) + for k, v in overrides.items(): + if hasattr(cfg, k): + setattr(cfg, k, v) + return cfg + + +def _forward_step(model: nn.Module, batch: PackedBatch) -> dict: + kwargs = pack_thd_forward_kwargs(model, batch) + add_loss_context_kwargs(kwargs) + add_cross_entropy_fusion(kwargs, model) + return model(**kwargs) + + +def _forward_step_bshd(model: nn.Module, batch: PackedBatch) -> dict: + """Dense [b=1, s] forward for a single packed sequence (no THD packing). + + Used for deterministic parity comparison vs a dense Megatron-Core reference: + the THD GatedDeltaNet kernel is non-deterministic, whereas the dense path is + deterministic. CP=1 only (single unpadded sequence => dense == THD tokens). + """ + input_ids = batch.input_ids.reshape(1, -1) + labels = batch.labels.reshape(1, -1) if batch.labels is not None else None + kwargs: dict[str, Any] = {"input_ids": input_ids, "labels": labels, "packed_seq_params": None} + add_cross_entropy_fusion(kwargs, model) + return model(**kwargs) + + +def unpack_forward_output(model: nn.Module, batch: PackedBatch, output) -> Any: + return unpack_thd_forward_output(model, batch, output) + + +def _make_aux_loss_hook(): + from megatron.lite.primitive.modules.moe import MoEAuxLossAutoScaler + from megatron.lite.primitive.modules.mtp import MTPLossAutoScaler + + def hook(scale: torch.Tensor) -> None: + MoEAuxLossAutoScaler.set_loss_scale(scale) + MTPLossAutoScaler.set_loss_scale(scale) + + return hook + + +def _build_dist_opt_optimizer( + chunks, model_cfg: Qwen35Config, impl_cfg: ImplConfig, ps: ParallelState +): + from megatron.lite.primitive.optimizers.megatron_wrap import ( + build_dist_opt_training_optimizer, + ) + + return build_dist_opt_training_optimizer( + chunks, + model_cfg=model_cfg, + impl_cfg=impl_cfg, + ps=ps, + is_expert=is_expert_param, + model_name="qwen3_5", + deterministic=impl_cfg.deterministic, + ) + + +def build_model(model_cfg: Qwen35Config, *, impl_cfg: ImplConfig) -> ModelBundle: + from megatron.lite.model.qwen3_5.lite.model import Qwen35Model + + p = impl_cfg.parallel + + if impl_cfg.use_deepep and (p.etp is not None and p.etp > 1): + raise ValueError("use_deepep and etp>1 are mutually exclusive") + + if impl_cfg.router_aux_loss_coef is not None: + model_cfg.router_aux_loss_coef = impl_cfg.router_aux_loss_coef + mtp_enable = bool(impl_cfg.mtp_enable) + mtp_enable_train = mtp_enable and bool(impl_cfg.mtp_enable_train) + if mtp_enable: + if model_cfg.num_nextn_predict_layers <= 0: + raise ValueError("mtp_enable=True but HF config has no num_nextn_predict_layers.") + model_cfg.mtp_loss_scaling_factor = impl_cfg.mtp_loss_scaling_factor + if impl_cfg.mtp_use_repeated_layer is not None: + model_cfg.mtp_use_repeated_layer = impl_cfg.mtp_use_repeated_layer + else: + model_cfg.num_nextn_predict_layers = 0 + + ps = init_parallel(p) + recompute_spec = parse_recompute_spec(impl_cfg.recompute) + vpp = None if p.vpp == 1 else p.vpp + deterministic = impl_cfg.deterministic + if impl_cfg.use_thd and deterministic and "linear_attention" in model_cfg.layer_types: + deterministic = False + train_cfg = SimpleNamespace( + tp=ps.tp_size, + ep=ps.ep_size, + etp=ps.etp_size, + pp=ps.pp_size, + cp=ps.cp_size, + vpp=vpp, + use_deepep=impl_cfg.use_deepep, + fp8=False, + recompute_modules=recompute_spec, + deterministic=deterministic, + ) + model_kwargs: dict[str, Any] = dict( + router_bias_rate=impl_cfg.router_bias_rate, + use_thd=impl_cfg.use_thd, + hf_path=impl_cfg.hf_path, + attention_backend_override=impl_cfg.attention_backend_override, + mtp_enable=mtp_enable, + mtp_enable_train=mtp_enable_train, + mtp_detach_encoder=impl_cfg.mtp_detach_encoder, + mount_vision_model=impl_cfg.mount_vision_model, + gdn_cp_mode=impl_cfg.gdn_cp_mode, + ) + + if vpp is None: + chunks = [Qwen35Model(model_cfg, train_cfg, ps, **model_kwargs).to(torch.bfloat16).cuda()] + else: + chunks = [ + Qwen35Model(model_cfg, train_cfg, ps, vpp_chunk_id=i, **model_kwargs) + .to(torch.bfloat16) + .cuda() + for i in range(vpp) + ] + set_cross_entropy_fusion(chunks, impl_cfg.cross_entropy_fusion) + + if recompute_spec: + for chunk in chunks: + apply_recompute(chunk.layers, recompute_spec, MODULE_MAP) + + if impl_cfg.offload: + from megatron.lite.primitive.recompute import apply_offload + + for chunk in chunks: + apply_offload(chunk.layers, impl_cfg.offload, MODULE_MAP) + + optimizer = None + finalize_grads = None + post_model_load_hook = None + optimizer_backend = "none" + if impl_cfg.optimizer == "dist_opt": + optimizer, finalize_grads = _build_dist_opt_optimizer(chunks, model_cfg, impl_cfg, ps) + from megatron.lite.primitive.ckpt import attach_model_sharded_state_dict + from megatron.lite.runtime.megatron_utils import register_training_hooks + + attach_model_sharded_state_dict( + chunks, ps, get_placements=PLACEMENT_FN, is_expert=is_expert_param + ) + register_training_hooks(chunks, optimizer) + optimizer_backend = "dist_opt" + elif impl_cfg.optimizer == "fsdp2": + optimizer_backend = "fsdp2" + + def _post_model_load_hook(): + from megatron.lite.model.qwen3_5.lite.model import Qwen35Layer + from megatron.lite.primitive.optimizers.fsdp2 import build_fsdp2_training_optimizer + + return { + "optimizer": build_fsdp2_training_optimizer( + chunks, + impl_cfg.optimizer_config, + ps, + unit_modules=(Qwen35Layer,), + expert_classifier=is_expert_param, + deterministic=deterministic, + vpp=impl_cfg.parallel.vpp, + leaf_module_names=(), + ) + } + + post_model_load_hook = _post_model_load_hook + elif impl_cfg.optimizer is not None: + raise ValueError(f"Unknown qwen3_5 lite optimizer: {impl_cfg.optimizer!r}.") + + return ModelBundle( + chunks=chunks, + parallel_state=ps, + optimizer=optimizer, + finalize_grads=finalize_grads, + forward_step=_forward_step if impl_cfg.use_thd else _forward_step_bshd, + extras={ + "model_cfg": model_cfg, + "optimizer_backend": optimizer_backend, + "post_model_load_hook": post_model_load_hook, + "pre_forward_hook": _make_aux_loss_hook(), + }, + ) + + +def load_hf_weights( + chunk: nn.Module, hf_path: str, model_cfg: Qwen35Config, ps: ParallelState +) -> None: + if not hf_path: + return + _load_hf_weights_impl(chunk, hf_path, model_cfg, ps) + + +def export_hf_weights( + chunks: list[nn.Module], model_cfg: Qwen35Config, ps: ParallelState, **kwargs +): + yield from _export_hf_weights_impl(chunks, model_cfg, ps, **kwargs) + + +def save_hf_weights( + chunks: list[nn.Module], path: str, model_cfg: Qwen35Config, ps: ParallelState +) -> None: + _save_hf_weights_impl(chunks, path, model_cfg, ps) + + +def vocab_size(model_cfg) -> int | None: + cfg = getattr(model_cfg, "text_config", model_cfg) + return getattr(cfg, "vocab_size", None) diff --git a/experimental/lite/megatron/lite/model/qwen3_5/stats.py b/experimental/lite/megatron/lite/model/qwen3_5/stats.py new file mode 100644 index 00000000000..a22e3f64fcf --- /dev/null +++ b/experimental/lite/megatron/lite/model/qwen3_5/stats.py @@ -0,0 +1,197 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Model-level Qwen3.5 benchmark statistics.""" + +from __future__ import annotations + +from megatron.lite.model.qwen3_5.config import Qwen35Config + + +def num_floating_point_operations( + model_cfg: Qwen35Config, *, seq_len: int, global_batch_size: int, tp_size: int = 1 +) -> int | None: + """Megatron-aligned FLOPs estimate for one training step.""" + + cfg = getattr(model_cfg, "text_config", model_cfg) + + def _get(name: str): + if isinstance(cfg, dict): + return cfg.get(name) + return getattr(cfg, name, None) + + try: + if seq_len <= 0 or global_batch_size <= 0: + return None + + hidden_size = int(_get("hidden_size")) + num_hidden_layers = int(_get("num_hidden_layers")) + if num_hidden_layers <= 0: + return None + + layer_types_obj = _get("layer_types") + if layer_types_obj is not None: + layer_types = list(layer_types_obj) + if len(layer_types) < num_hidden_layers: + return None + layer_types = layer_types[:num_hidden_layers] + else: + full_attention_interval = int(_get("full_attention_interval")) + if full_attention_interval <= 0: + return None + layer_types = [ + "full_attention" if (i + 1) % full_attention_interval == 0 else "linear_attention" + for i in range(num_hidden_layers) + ] + + num_full_attention_layers = sum( + layer_type == "full_attention" for layer_type in layer_types + ) + num_linear_attention_layers = sum( + layer_type == "linear_attention" for layer_type in layer_types + ) + if num_full_attention_layers + num_linear_attention_layers != num_hidden_layers: + return None + + total_tokens = seq_len * global_batch_size + total_tokens_squared = seq_len * seq_len * global_batch_size + + num_attention_heads = int(_get("num_attention_heads")) + num_key_value_heads = int(_get("num_key_value_heads")) + head_dim = int(_get("head_dim")) + query_projection_size = head_dim * num_attention_heads + key_projection_size = head_dim * num_key_value_heads + value_projection_size = head_dim * num_key_value_heads + gate_projection_size = query_projection_size + standard_attention_flops = 6 * ( + total_tokens + * hidden_size + * ( + query_projection_size + + key_projection_size + + value_projection_size + + gate_projection_size + ) + + query_projection_size * total_tokens_squared + + total_tokens * query_projection_size * hidden_size + ) + + linear_num_key_heads = int(_get("linear_num_key_heads")) + linear_key_head_dim = int(_get("linear_key_head_dim")) + linear_num_value_heads = int(_get("linear_num_value_heads")) + linear_value_head_dim = int(_get("linear_value_head_dim")) + linear_conv_kernel_dim = int(_get("linear_conv_kernel_dim")) + qk_dim = linear_key_head_dim * linear_num_key_heads + v_dim = linear_value_head_dim * linear_num_value_heads + linear_attention_flops = ( + 6 + * total_tokens + * ( + hidden_size * (2 * qk_dim + 2 * v_dim + 2 * linear_num_value_heads) + + linear_conv_kernel_dim * (2 * qk_dim + v_dim) + + linear_num_value_heads * (linear_value_head_dim**2) * 4 + + hidden_size * v_dim + ) + ) + + moe_intermediate_size = int(_get("moe_intermediate_size")) + num_experts_per_tok = int(_get("num_experts_per_tok")) + shared_expert_intermediate_size = int(_get("shared_expert_intermediate_size")) + moe_flops = ( + 18 + * total_tokens + * hidden_size + * (moe_intermediate_size * num_experts_per_tok + shared_expert_intermediate_size) + * num_hidden_layers + ) + + from megatron.lite.primitive.parallel.linear import pad_vocab_for_tp + + padded_vocab_size = pad_vocab_for_tp(int(_get("vocab_size")), max(tp_size, 1)) + logits_flops = 6 * total_tokens * hidden_size * padded_vocab_size + + return int( + standard_attention_flops * num_full_attention_layers + + linear_attention_flops * num_linear_attention_layers + + moe_flops + + logits_flops + ) + except (TypeError, ValueError): + return None + + +def activated_params(model_cfg: Qwen35Config) -> int | None: + """Approximate active params/token for benchmark TFLOPS reporting.""" + + cfg = getattr(model_cfg, "text_config", model_cfg) + + def _get(name: str): + if isinstance(cfg, dict): + return cfg.get(name) + return getattr(cfg, name, None) + + try: + hidden_size = int(_get("hidden_size")) + num_hidden_layers = int(_get("num_hidden_layers")) + layer_types_obj = _get("layer_types") + if layer_types_obj is None: + return None + layer_types = list(layer_types_obj) + if not layer_types: + return None + if len(layer_types) < num_hidden_layers: + return None + layer_types = layer_types[:num_hidden_layers] + + num_attention_heads = int(_get("num_attention_heads")) + num_key_value_heads = int(_get("num_key_value_heads")) + head_dim = int(_get("head_dim")) + full_qkv_dim = (num_attention_heads + 2 * num_key_value_heads) * head_dim + full_attention = hidden_size * full_qkv_dim + (num_attention_heads * head_dim) * hidden_size + + linear_num_key_heads = int(_get("linear_num_key_heads")) + linear_key_head_dim = int(_get("linear_key_head_dim")) + linear_num_value_heads = int(_get("linear_num_value_heads")) + linear_value_head_dim = int(_get("linear_value_head_dim")) + linear_conv_kernel_dim = int(_get("linear_conv_kernel_dim")) + qk_dim = linear_num_key_heads * linear_key_head_dim + v_dim = linear_num_value_heads * linear_value_head_dim + gdn_in_proj_dim = 2 * qk_dim + 2 * v_dim + 2 * linear_num_value_heads + gdn_conv_dim = 2 * qk_dim + v_dim + linear_attention = ( + hidden_size * gdn_in_proj_dim + + gdn_conv_dim * linear_conv_kernel_dim + + 2 * linear_num_value_heads + + linear_value_head_dim + + v_dim * hidden_size + ) + + num_experts = int(_get("num_experts")) + num_experts_per_tok = int(_get("num_experts_per_tok")) + moe_intermediate_size = int(_get("moe_intermediate_size")) + shared_expert_intermediate_size = int(_get("shared_expert_intermediate_size")) + router = hidden_size * num_experts + routed_expert = ( + hidden_size * (2 * moe_intermediate_size) + moe_intermediate_size * hidden_size + ) + shared_expert = ( + hidden_size * (2 * shared_expert_intermediate_size) + + shared_expert_intermediate_size * hidden_size + + hidden_size + ) + moe = router + num_experts_per_tok * routed_expert + shared_expert + + total = 0 + for layer_type in layer_types: + if layer_type == "full_attention": + total += full_attention + elif layer_type == "linear_attention": + total += linear_attention + else: + return None + total += moe + + return int(total) + except (TypeError, ValueError): + return None + + +__all__ = ["activated_params", "num_floating_point_operations"] diff --git a/experimental/lite/megatron/lite/model/qwen3_moe/__init__.py b/experimental/lite/megatron/lite/model/qwen3_moe/__init__.py new file mode 100644 index 00000000000..1d6ee75b62b --- /dev/null +++ b/experimental/lite/megatron/lite/model/qwen3_moe/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""qwen3_moe model package for Megatron Lite.""" diff --git a/experimental/lite/megatron/lite/model/qwen3_moe/common.py b/experimental/lite/megatron/lite/model/qwen3_moe/common.py new file mode 100644 index 00000000000..ad291bb5dbb --- /dev/null +++ b/experimental/lite/megatron/lite/model/qwen3_moe/common.py @@ -0,0 +1,11 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Shared Qwen3MoE model helpers.""" + +from __future__ import annotations + + +def is_expert_param(name: str) -> bool: + return "experts" in name and "router" not in name + + +__all__ = ["is_expert_param"] diff --git a/experimental/lite/megatron/lite/model/qwen3_moe/config.py b/experimental/lite/megatron/lite/model/qwen3_moe/config.py new file mode 100644 index 00000000000..7395c2497c5 --- /dev/null +++ b/experimental/lite/megatron/lite/model/qwen3_moe/config.py @@ -0,0 +1,117 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Qwen3MoE model configuration — pure architecture parameters. + +Like HuggingFace's model config: only describes the model architecture. +Impl-specific knobs live in protocol.py. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from dataclasses import fields as dc_fields +from typing import Any + +from megatron.lite.primitive.config import load_hf_config_dict + + +@dataclass +class Qwen3MoEConfig: + """Pure Qwen3MoE architecture parameters (Qwen3-30B-A3B defaults).""" + + num_hidden_layers: int = 48 + hidden_size: int = 2048 + num_attention_heads: int = 32 + num_key_value_heads: int = 4 + head_dim: int = 128 + vocab_size: int = 151936 + num_experts: int = 128 + num_experts_per_tok: int = 8 + moe_intermediate_size: int = 768 + rope_theta: float = 1_000_000.0 + rms_norm_eps: float = 1e-6 + max_position_embeddings: int = 32768 + router_aux_loss_coef: float = 0.001 + num_nextn_predict_layers: int = 0 + mtp_loss_scaling_factor: float = 0.1 + mtp_use_repeated_layer: bool = False + layer_types: list[str] = field(default_factory=lambda: ["full_attention"] * 48) + + def __post_init__(self): + self._validate() + + def _validate(self): + errors: list[str] = [] + + def _check(cond: bool, msg: str): + if not cond: + errors.append(msg) + + _check( + self.hidden_size % self.num_attention_heads == 0, + f"hidden_size({self.hidden_size}) % num_attention_heads({self.num_attention_heads}) != 0", + ) + _check( + self.num_attention_heads % self.num_key_value_heads == 0, + f"num_attention_heads({self.num_attention_heads}) % num_key_value_heads({self.num_key_value_heads}) != 0", + ) + _check(self.head_dim > 0, f"head_dim must be > 0, got {self.head_dim}") + _check(self.num_experts >= 1, f"num_experts must be >= 1, got {self.num_experts}") + _check( + 1 <= self.num_experts_per_tok <= self.num_experts, + f"num_experts_per_tok({self.num_experts_per_tok}) not in [1, {self.num_experts}]", + ) + _check(self.moe_intermediate_size > 0, "moe_intermediate_size must be > 0") + _check(self.vocab_size > 0, "vocab_size must be > 0") + _check(self.num_hidden_layers >= 1, "num_hidden_layers must be >= 1") + _check(self.num_nextn_predict_layers >= 0, "num_nextn_predict_layers must be >= 0") + _check( + len(self.layer_types) == self.num_hidden_layers, + f"len(layer_types)={len(self.layer_types)} != " + f"num_hidden_layers={self.num_hidden_layers}", + ) + valid_types = {"full_attention"} + for i, lt in enumerate(self.layer_types): + _check(lt in valid_types, f"layer_types[{i}] must be one of {valid_types}, got '{lt}'") + + if errors: + raise ValueError( + f"Invalid Qwen3MoEConfig ({len(errors)} errors):\n " + "\n ".join(errors) + ) + + @property + def qkv_size(self) -> int: + return (self.num_attention_heads + 2 * self.num_key_value_heads) * self.head_dim + + def to_dict(self) -> dict[str, Any]: + return {f.name: getattr(self, f.name) for f in dc_fields(self)} + + @classmethod + def from_hf(cls, path_or_name: str, **overrides) -> Qwen3MoEConfig: + hf_dict = load_hf_config_dict(path_or_name) + return cls._from_hf_dict(hf_dict, **overrides) + + @classmethod + def from_hf_config(cls, hf_config, **overrides) -> Qwen3MoEConfig: + hf_dict = hf_config.to_dict() if hasattr(hf_config, "to_dict") else vars(hf_config) + return cls._from_hf_dict(hf_dict, **overrides) + + @classmethod + def _from_hf_dict(cls, hf: dict[str, Any], **overrides) -> Qwen3MoEConfig: + valid_fields = {f.name for f in dc_fields(cls)} + kwargs = {k: v for k, v in hf.items() if k in valid_fields} + + if "rope_theta" not in kwargs: + if "rope_parameters" in hf and isinstance(hf["rope_parameters"], dict): + kwargs["rope_theta"] = float(hf["rope_parameters"].get("rope_theta", 1_000_000.0)) + + if "head_dim" not in kwargs or kwargs["head_dim"] is None: + hs = kwargs.get("hidden_size", 2048) + nh = kwargs.get("num_attention_heads", 32) + kwargs["head_dim"] = hs // nh + if kwargs.get("num_nextn_predict_layers") is None: + kwargs["num_nextn_predict_layers"] = 0 + + if "layer_types" not in kwargs: + kwargs["layer_types"] = ["full_attention"] * kwargs.get("num_hidden_layers", 48) + kwargs.update(overrides) + return cls(**kwargs) diff --git a/experimental/lite/megatron/lite/model/qwen3_moe/lite/__init__.py b/experimental/lite/megatron/lite/model/qwen3_moe/lite/__init__.py new file mode 100644 index 00000000000..2764e0f5880 --- /dev/null +++ b/experimental/lite/megatron/lite/model/qwen3_moe/lite/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Native Qwen3MoE implementation.""" diff --git a/experimental/lite/megatron/lite/model/qwen3_moe/lite/checkpoint.py b/experimental/lite/megatron/lite/model/qwen3_moe/lite/checkpoint.py new file mode 100644 index 00000000000..9f72e67a05b --- /dev/null +++ b/experimental/lite/megatron/lite/model/qwen3_moe/lite/checkpoint.py @@ -0,0 +1,294 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Qwen3MoE WeightSpec — name mapping + format conversion. + +Orchestration (TP scatter, EP sharding, PP remap) lives in +primitive/ckpt/weight_loader.py. This file only defines what's +model-specific: the weight map and tensor conversions. +""" + +from __future__ import annotations + +import torch +from torch.distributed.tensor import Replicate, Shard + +from megatron.lite.model.qwen3_moe.config import Qwen3MoEConfig +from megatron.lite.primitive.ckpt.dcp import ( # noqa: F401 — re-export + canonicalize_fc1_for_dcp, + canonicalize_qkv_for_dcp, + decanon_fc1_after_dcp, + decanon_qkv_after_dcp, +) +from megatron.lite.primitive.ckpt.hf_weights import extract_layer_idx, parse_expert_idx + + +def _pack_mcore_qkv( + q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, config: Qwen3MoEConfig +) -> torch.Tensor: + q_per_group = config.num_attention_heads // config.num_key_value_heads + q = q.view(config.num_key_value_heads, q_per_group * config.head_dim, -1) + k = k.view(config.num_key_value_heads, config.head_dim, -1) + v = v.view(config.num_key_value_heads, config.head_dim, -1) + return torch.cat([q, k, v], dim=1).reshape(-1, q.shape[-1]).contiguous() + + +def _unpack_mcore_qkv( + tensor: torch.Tensor, config: Qwen3MoEConfig +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + q_per_group = config.num_attention_heads // config.num_key_value_heads + group_width = (q_per_group + 2) * config.head_dim + packed = tensor.view(config.num_key_value_heads, group_width, -1) + q_end = q_per_group * config.head_dim + k_end = q_end + config.head_dim + q = packed[:, :q_end].reshape(config.num_attention_heads * config.head_dim, -1) + k = packed[:, q_end:k_end].reshape(config.num_key_value_heads * config.head_dim, -1) + v = packed[:, k_end:].reshape(config.num_key_value_heads * config.head_dim, -1) + return q, k, v + + +class Qwen3MoEWeightSpec: + """WeightSpec for Qwen3MoE native impl.""" + + def __init__(self, config: Qwen3MoEConfig): + self.config = config + + @property + def num_experts(self) -> int: + return self.config.num_experts + + def weight_map(self) -> dict[str, list[str]]: + c = self.config + wm: dict[str, list[str]] = { + "embed.embedding.weight": ["model.embed_tokens.weight"], + "mtp_embed.embedding.weight": ["model.embed_tokens.weight"], + "norm.weight": ["model.norm.weight"], + "head.col.linear.weight": ["lm_head.weight"], + } + for li in range(c.num_hidden_layers): + ap = f"model.layers.{li}.self_attn" + mp = f"model.layers.{li}.mlp" + lp = f"layers.{li}" + wm.update( + { + f"{lp}.attn.qkv.linear.layer_norm_weight": [ + f"model.layers.{li}.input_layernorm.weight" + ], + f"{lp}.attn.qkv.linear.weight": [ + f"{ap}.q_proj.weight", + f"{ap}.k_proj.weight", + f"{ap}.v_proj.weight", + ], + f"{lp}.attn.q_norm.weight": [f"{ap}.q_norm.weight"], + f"{lp}.attn.k_norm.weight": [f"{ap}.k_norm.weight"], + f"{lp}.attn.proj.linear.weight": [f"{ap}.o_proj.weight"], + f"{lp}.mlp_norm.weight": [f"model.layers.{li}.post_attention_layernorm.weight"], + f"{lp}.moe.router.gate.weight": [f"{mp}.gate.weight"], + } + ) + for e in range(c.num_experts): + wm[f"{lp}.moe.experts._fc1_weight_{e}"] = [ + f"{mp}.experts.{e}.gate_proj.weight", + f"{mp}.experts.{e}.up_proj.weight", + ] + wm[f"{lp}.moe.experts._fc2_weight_{e}"] = [f"{mp}.experts.{e}.down_proj.weight"] + for mi in range(c.num_nextn_predict_layers): + hf_li = c.num_hidden_layers + mi + hp = f"model.layers.{hf_li}" + ap = f"{hp}.self_attn" + mp = f"{hp}.mlp" + lp = f"mtp.layers.{mi}" + tlp = f"{lp}.transformer_layer" + wm.update( + { + f"{lp}.enorm.weight": [f"{hp}.enorm.weight"], + f"{lp}.hnorm.weight": [f"{hp}.hnorm.weight"], + f"{lp}.eh_proj.linear.weight": [f"{hp}.eh_proj.weight"], + f"{lp}.final_layernorm.weight": [f"{hp}.shared_head.norm.weight"], + f"{tlp}.attn.qkv.linear.layer_norm_weight": [f"{hp}.input_layernorm.weight"], + f"{tlp}.attn.qkv.linear.weight": [ + f"{ap}.q_proj.weight", + f"{ap}.k_proj.weight", + f"{ap}.v_proj.weight", + ], + f"{tlp}.attn.q_norm.weight": [f"{ap}.q_norm.weight"], + f"{tlp}.attn.k_norm.weight": [f"{ap}.k_norm.weight"], + f"{tlp}.attn.proj.linear.weight": [f"{ap}.o_proj.weight"], + f"{tlp}.mlp_norm.weight": [f"{hp}.post_attention_layernorm.weight"], + f"{tlp}.moe.router.gate.weight": [f"{mp}.gate.weight"], + } + ) + for e in range(c.num_experts): + wm[f"{tlp}.moe.experts._fc1_weight_{e}"] = [ + f"{mp}.experts.{e}.gate_proj.weight", + f"{mp}.experts.{e}.up_proj.weight", + ] + wm[f"{tlp}.moe.experts._fc2_weight_{e}"] = [f"{mp}.experts.{e}.down_proj.weight"] + return wm + + def hf_to_native(self, native_name: str, hf_tensors: list[torch.Tensor]) -> torch.Tensor: + if len(hf_tensors) == 3 and "qkv" in native_name: + # Match MCore SelfAttention's local qkv packing: + # [q heads for kv-group 0, k0, v0, q heads for kv-group 1, k1, v1, ...]. + return _pack_mcore_qkv(*hf_tensors, self.config) + if len(hf_tensors) == 2: + # gate + up → concat + return torch.cat(hf_tensors, dim=0) + t = hf_tensors[0] + if "router.gate.weight" in native_name: + return t[: self.config.num_experts] + return t + + def native_to_hf( + self, native_name: str, tensor: torch.Tensor + ) -> list[tuple[str, torch.Tensor]]: + c = self.config + if native_name == "mtp_embed.embedding.weight": + return [] + if native_name.startswith("mtp.layers."): + parts = native_name.split(".") + mtp_idx = int(parts[2]) + hf_li = c.num_hidden_layers + mtp_idx + hp = f"model.layers.{hf_li}" + if native_name.endswith(".enorm.weight"): + return [(f"{hp}.enorm.weight", tensor)] + if native_name.endswith(".hnorm.weight"): + return [(f"{hp}.hnorm.weight", tensor)] + if native_name.endswith(".eh_proj.linear.weight"): + return [(f"{hp}.eh_proj.weight", tensor)] + if native_name.endswith(".final_layernorm.weight"): + return [(f"{hp}.shared_head.norm.weight", tensor)] + proxy = native_name.replace( + f"mtp.layers.{mtp_idx}.transformer_layer", f"layers.{hf_li}" + ) + return self.native_to_hf(proxy, tensor) + if "embed" in native_name and "embedding" in native_name: + return [("model.embed_tokens.weight", tensor)] + if ( + native_name.endswith("norm.weight") + and "layers" not in native_name + and "attn" not in native_name + and "mlp" not in native_name + ): + return [("model.norm.weight", tensor)] + if "head" in native_name: + return [("lm_head.weight", tensor)] + if "layer_norm_weight" in native_name and "qkv" in native_name: + li = extract_layer_idx(native_name) + return [(f"model.layers.{li}.input_layernorm.weight", tensor)] + if "mlp_norm" in native_name: + li = extract_layer_idx(native_name) + return [(f"model.layers.{li}.post_attention_layernorm.weight", tensor)] + if "qkv" in native_name and "layer_norm" not in native_name: + li = extract_layer_idx(native_name) + ap = f"model.layers.{li}.self_attn" + q, k, v = _unpack_mcore_qkv(tensor, c) + return [ + (f"{ap}.q_proj.weight", q), + (f"{ap}.k_proj.weight", k), + (f"{ap}.v_proj.weight", v), + ] + if "q_norm" in native_name: + li = extract_layer_idx(native_name) + return [(f"model.layers.{li}.self_attn.q_norm.weight", tensor)] + if "k_norm" in native_name: + li = extract_layer_idx(native_name) + return [(f"model.layers.{li}.self_attn.k_norm.weight", tensor)] + if "proj.linear" in native_name: + li = extract_layer_idx(native_name) + return [(f"model.layers.{li}.self_attn.o_proj.weight", tensor)] + if "router.gate" in native_name: + li = extract_layer_idx(native_name) + return [(f"model.layers.{li}.mlp.gate.weight", tensor)] + if "experts" in native_name and "fc1" in native_name: + li = extract_layer_idx(native_name) + ei = parse_expert_idx(native_name) + mp = f"model.layers.{li}.mlp" + gate, up = tensor.chunk(2, dim=0) + return [ + (f"{mp}.experts.{ei}.gate_proj.weight", gate), + (f"{mp}.experts.{ei}.up_proj.weight", up), + ] + if "experts" in native_name and "fc2" in native_name: + li = extract_layer_idx(native_name) + ei = parse_expert_idx(native_name) + return [(f"model.layers.{li}.mlp.experts.{ei}.down_proj.weight", tensor)] + return [(native_name, tensor)] + + def qkv_spec(self, native_name: str) -> tuple[int, int, int] | None: + return None + + def tp_spec(self, native_name: str) -> tuple[int, int] | None: + if self.is_expert(native_name): + if "fc1" in native_name: + return (0, 1) # ETP dim 0 + if "fc2" in native_name: + return (1, 1) # ETP dim 1 + return None + if "eh_proj" in native_name: + return (0, 0) + if "qkv" in native_name and "layer_norm" not in native_name: + return (0, 0) + if "proj" in native_name and "attn" in native_name: + return (1, 0) + if "embed" in native_name or "head" in native_name: + return (0, 0) + return None + + def is_expert(self, native_name: str) -> bool: + return "experts" in native_name and "router" not in native_name + + def expert_global_id(self, native_name: str) -> int | None: + if "_fc1_weight_" in native_name or "_fc2_weight_" in native_name: + return int(native_name.split("_")[-1]) + return None + + def expert_local_name(self, native_name: str, local_idx: int) -> str: + prefix = native_name.rsplit("._fc", 1)[0] + fc_tag = "fc1" if "_fc1_weight_" in native_name else "fc2" + return f"{prefix}.{fc_tag}.weight{local_idx}" + + +# --------------------------------------------------------------------------- +# Convenience: standalone functions wrapping WeightSpec + generic loader +# --------------------------------------------------------------------------- + + +def load_hf_weights(model, path: str, config: Qwen3MoEConfig, ps) -> None: + from megatron.lite.primitive.ckpt.hf_weights import load_hf_weights as _load + + _load(model, path, Qwen3MoEWeightSpec(config), ps, vocab_size=config.vocab_size) + + +def export_hf_weights(model, config: Qwen3MoEConfig, ps, **kwargs): + from megatron.lite.primitive.ckpt.hf_weights import export_hf_weights as _export + + yield from _export( + model, Qwen3MoEWeightSpec(config), ps, vocab_size=config.vocab_size, **kwargs + ) + + +def save_hf_weights(model, path: str, config: Qwen3MoEConfig, ps) -> None: + from megatron.lite.primitive.ckpt.hf_weights import save_hf_weights as _save + + _save(model, path, Qwen3MoEWeightSpec(config), ps, vocab_size=config.vocab_size) + + +def EXPERT_CLASSIFIER(name: str) -> bool: + return "experts" in name and "router" not in name + + +def PLACEMENT_FN(param_name: str) -> list: + if "experts" in param_name and "router" not in param_name: + if "fc1" in param_name: + return [Replicate(), Replicate(), Shard(0), Shard(0)] + if "fc2" in param_name: + return [Replicate(), Replicate(), Shard(0), Shard(1)] + return [Replicate(), Replicate(), Replicate(), Replicate()] + if "eh_proj" in param_name: + return [Replicate(), Replicate(), Replicate(), Shard(0)] + if "qkv" in param_name and "layer_norm" not in param_name: + return [Replicate(), Replicate(), Replicate(), Shard(0)] + if "proj" in param_name and "attn" in param_name: + return [Replicate(), Replicate(), Replicate(), Shard(1)] + if "embed" in param_name or "head" in param_name: + return [Replicate(), Replicate(), Replicate(), Shard(0)] + return [Replicate(), Replicate(), Replicate(), Replicate()] diff --git a/experimental/lite/megatron/lite/model/qwen3_moe/lite/lora_adapter.py b/experimental/lite/megatron/lite/model/qwen3_moe/lite/lora_adapter.py new file mode 100644 index 00000000000..ce839108aa8 --- /dev/null +++ b/experimental/lite/megatron/lite/model/qwen3_moe/lite/lora_adapter.py @@ -0,0 +1,803 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""PEFT adapter import/export for Qwen3-MoE lite native LoRA.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import torch +import torch.distributed as dist +import torch.nn as nn + +from megatron.lite.model.qwen3_moe.config import Qwen3MoEConfig +from megatron.lite.primitive.modules.lora import LoraConfig, normalize_lora_config +from megatron.lite.primitive.parallel import ParallelState + +_PEFT_PREFIX = "base_model.model.model" + + +def _rank() -> int: + return dist.get_rank() if dist.is_available() and dist.is_initialized() else 0 + + +def _world_size(group=None) -> int: + if not dist.is_available() or not dist.is_initialized(): + return 1 + return dist.get_world_size(group) + + +def _unwrap_model(module: nn.Module) -> nn.Module: + current = module + seen: set[int] = set() + while hasattr(current, "module") and id(current) not in seen: + seen.add(id(current)) + inner = getattr(current, "module") + if isinstance(inner, list): + if len(inner) != 1: + break + inner = inner[0] + if inner is current or not isinstance(inner, nn.Module): + break + current = inner + return current + + +def _iter_qwen_chunks(chunks: list[nn.Module] | tuple[nn.Module, ...]) -> list[nn.Module]: + return [_unwrap_model(chunk) for chunk in chunks] + + +def _all_gather_cat(tensor: torch.Tensor, group, dim: int) -> torch.Tensor: + if _world_size(group) == 1: + return tensor.contiguous() + gathered = [torch.empty_like(tensor) for _ in range(_world_size(group))] + dist.all_gather(gathered, tensor.contiguous(), group=group) + return torch.cat(gathered, dim=dim).contiguous() + + +def _select_tp_replicated(tensor: torch.Tensor, ps: ParallelState) -> torch.Tensor: + if _world_size(ps.tp_group) == 1: + return tensor.contiguous() + gathered = [torch.empty_like(tensor) for _ in range(_world_size(ps.tp_group))] + dist.all_gather(gathered, tensor.contiguous(), group=ps.tp_group) + return gathered[0].contiguous() + + +def _is_rank_partitioned_lora_a(lora: Any, ps: ParallelState) -> bool: + if getattr(lora, "rank_partitioned_a", False): + return True + rank = int(getattr(lora, "rank", lora.lora_b.shape[1])) + return ps.tp_size > 1 and lora.lora_a.shape[0] != rank + + +def _gather_lora_rank_partition(tensor: torch.Tensor, ps: ParallelState) -> torch.Tensor: + return _all_gather_cat(tensor, ps.tp_group, dim=0) + + +def _slice_lora_rank_partition(tensor: torch.Tensor, ps: ParallelState) -> torch.Tensor: + if ps.tp_size == 1: + return tensor.contiguous() + if tensor.shape[0] % ps.tp_size != 0: + raise ValueError(f"Cannot shard LoRA rank dim {tensor.shape[0]} over TP={ps.tp_size}.") + local_rank = tensor.shape[0] // ps.tp_size + start = ps.tp_rank * local_rank + return tensor[start : start + local_rank].contiguous() + + +def _is_output_partitioned_lora_b(lora: Any, ps: ParallelState) -> bool: + if getattr(lora, "output_partitioned_b", False): + return True + return ps.tp_size > 1 and lora.lora_b.shape[0] * ps.tp_size == getattr(lora, "out_features", -1) + + +def _expert_lora_is_shared(lora: Any) -> bool: + return bool(getattr(lora, "shared_across_experts", False)) or lora.lora_a.dim() == 2 + + +def _expand_shared_expert_lora( + lora: Any, num_local_experts: int +) -> tuple[torch.Tensor, torch.Tensor]: + if _expert_lora_is_shared(lora): + return ( + lora.lora_a.detach().unsqueeze(0).expand(num_local_experts, -1, -1).contiguous(), + lora.lora_b.detach().unsqueeze(0).expand(num_local_experts, -1, -1).contiguous(), + ) + return lora.lora_a.detach(), lora.lora_b.detach() + + +def _split_local_mcore_qkv_b( + qkv_b: torch.Tensor, *, num_heads_local: int, num_kv_heads_local: int, head_dim: int +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + q_per_group = num_heads_local // num_kv_heads_local + group_width = (q_per_group + 2) * head_dim + packed = qkv_b.view(num_kv_heads_local, group_width, -1) + q_end = q_per_group * head_dim + k_end = q_end + head_dim + q = packed[:, :q_end].reshape(num_heads_local * head_dim, -1) + k = packed[:, q_end:k_end].reshape(num_kv_heads_local * head_dim, -1) + v = packed[:, k_end:].reshape(num_kv_heads_local * head_dim, -1) + return q.contiguous(), k.contiguous(), v.contiguous() + + +def _pack_local_mcore_qkv_b( + q_b: torch.Tensor, + k_b: torch.Tensor, + v_b: torch.Tensor, + *, + num_heads_local: int, + num_kv_heads_local: int, + head_dim: int, +) -> torch.Tensor: + q_per_group = num_heads_local // num_kv_heads_local + q = q_b.view(num_kv_heads_local, q_per_group * head_dim, -1) + k = k_b.view(num_kv_heads_local, head_dim, -1) + v = v_b.view(num_kv_heads_local, head_dim, -1) + return torch.cat([q, k, v], dim=1).reshape(-1, q_b.shape[-1]).contiguous() + + +def _layer_prefix(layer_idx: int) -> str: + return f"{_PEFT_PREFIX}.layers.{layer_idx}" + + +def _attn_key(layer_idx: int, module: str, suffix: str) -> str: + return f"{_layer_prefix(layer_idx)}.self_attn.{module}.{suffix}.weight" + + +def _expert_key(layer_idx: int, expert_idx: int, module: str, suffix: str) -> str: + return f"{_layer_prefix(layer_idx)}.mlp.experts.{expert_idx}.{module}.{suffix}.weight" + + +def _target_modules_from_lora_config(lora_config: LoraConfig) -> list[str]: + targets = lora_config.targets() + out: list[str] = [] + if "linear_qkv" in targets: + out += ["q_proj", "k_proj", "v_proj"] + if "linear_proj" in targets: + out.append("o_proj") + if "linear_fc1" in targets: + out += ["gate_proj", "up_proj"] + if "linear_fc2" in targets: + out.append("down_proj") + return out + + +def _state_target_modules(state: dict[str, torch.Tensor]) -> set[str]: + out: set[str] = set() + for key in state: + if ".q_proj." in key: + out.add("q_proj") + elif ".k_proj." in key: + out.add("k_proj") + elif ".v_proj." in key: + out.add("v_proj") + elif ".o_proj." in key: + out.add("o_proj") + elif ".gate_proj." in key: + out.add("gate_proj") + elif ".up_proj." in key: + out.add("up_proj") + elif ".down_proj." in key: + out.add("down_proj") + return out + + +def _effective_lora_alpha(lora_config: LoraConfig) -> int: + return lora_config.rank if lora_config.alpha is None else int(lora_config.alpha) + + +def _peft_target_set(value: Any) -> set[str] | None: + if value is None: + return None + if isinstance(value, str): + return {value} + return {str(item) for item in value} + + +def _infer_state_rank(state: dict[str, torch.Tensor]) -> int | None: + ranks = { + int(tensor.shape[0]) + for key, tensor in state.items() + if key.endswith(".lora_A.weight") and tensor.ndim >= 2 + } + if not ranks: + return None + if len(ranks) != 1: + raise ValueError(f"Adapter contains inconsistent LoRA ranks: {sorted(ranks)}.") + return next(iter(ranks)) + + +def _iter_native_lora_modules(chunks: list[nn.Module] | tuple[nn.Module, ...]): + for chunk in _iter_qwen_chunks(list(chunks)): + for layer in chunk.layers: + attn = layer.attn + if attn.qkv_lora is not None: + yield attn.qkv_lora + if attn.proj_lora is not None: + yield attn.proj_lora + experts = layer.moe.experts + if experts.fc1_lora is not None: + yield experts.fc1_lora + if experts.fc2_lora is not None: + yield experts.fc2_lora + + +def _infer_native_alpha(chunks: list[nn.Module] | tuple[nn.Module, ...]) -> int | None: + alphas: set[int] = set() + for module in _iter_native_lora_modules(chunks): + rank = getattr(module, "rank", None) + scale = getattr(module, "scale", None) + if rank is None or scale is None: + continue + alphas.add(int(round(float(scale) * int(rank)))) + if not alphas: + return None + if len(alphas) != 1: + raise ValueError( + f"Native LoRA modules have inconsistent effective alpha values: {sorted(alphas)}." + ) + return next(iter(alphas)) + + +def _validate_adapter_config( + chunks: list[nn.Module] | tuple[nn.Module, ...], + state: dict[str, torch.Tensor], + adapter_config: dict[str, Any], + *, + lora_config: LoraConfig | dict[str, Any] | None = None, +) -> None: + peft_type = adapter_config.get("peft_type") + if peft_type is not None and str(peft_type).upper() != "LORA": + raise ValueError(f"Expected PEFT adapter_config peft_type='LORA', got {peft_type!r}.") + + state_rank = _infer_state_rank(state) + config_rank = adapter_config.get("r") + if config_rank is not None and state_rank is not None and int(config_rank) != state_rank: + raise ValueError( + f"Adapter config rank r={config_rank} does not match tensor rank {state_rank}." + ) + + config_targets = _peft_target_set(adapter_config.get("target_modules")) + state_targets = _state_target_modules(state) + if config_targets is not None and config_targets != state_targets: + raise ValueError( + "Adapter config target_modules do not match adapter tensors: " + f"config={sorted(config_targets)}, tensors={sorted(state_targets)}." + ) + + config_alpha = adapter_config.get("lora_alpha") + native_alpha = _infer_native_alpha(chunks) + if config_alpha is not None and native_alpha is not None and int(config_alpha) != native_alpha: + raise ValueError( + f"Adapter config lora_alpha={config_alpha} does not match native model alpha={native_alpha}." + ) + + if lora_config is not None: + expected = normalize_lora_config(lora_config) + expected_targets = set(_target_modules_from_lora_config(expected)) + if config_rank is not None and int(config_rank) != expected.rank: + raise ValueError( + f"Adapter config rank r={config_rank} does not match expected rank {expected.rank}." + ) + if config_alpha is not None and int(config_alpha) != _effective_lora_alpha(expected): + raise ValueError( + "Adapter config lora_alpha=" + f"{config_alpha} does not match expected alpha {_effective_lora_alpha(expected)}." + ) + if config_targets is not None and config_targets != expected_targets: + raise ValueError( + "Adapter config target_modules do not match expected LoRA config: " + f"config={sorted(config_targets)}, expected={sorted(expected_targets)}." + ) + + +def _validate_attention_tp(model_cfg: Qwen3MoEConfig, ps: ParallelState) -> None: + if ps.tp_size <= 0: + raise ValueError(f"TP size must be positive, got {ps.tp_size}.") + if model_cfg.num_attention_heads % ps.tp_size != 0: + raise ValueError( + "LoRA adapter import/export requires num_attention_heads " + f"({model_cfg.num_attention_heads}) to be divisible by TP={ps.tp_size}." + ) + if model_cfg.num_key_value_heads % ps.tp_size != 0: + raise ValueError( + "LoRA adapter import/export requires num_key_value_heads " + f"({model_cfg.num_key_value_heads}) to be divisible by TP={ps.tp_size}." + ) + q_heads_local = model_cfg.num_attention_heads // ps.tp_size + kv_heads_local = model_cfg.num_key_value_heads // ps.tp_size + if kv_heads_local <= 0: + raise ValueError( + "LoRA adapter import/export requires at least one local KV head; " + f"got num_key_value_heads={model_cfg.num_key_value_heads}, TP={ps.tp_size}." + ) + if q_heads_local % kv_heads_local != 0: + raise ValueError( + "LoRA adapter import/export requires local query heads to be divisible " + f"by local KV heads, got q={q_heads_local}, kv={kv_heads_local}." + ) + + +def export_lora_adapter_state( + chunks: list[nn.Module] | tuple[nn.Module, ...], + model_cfg: Qwen3MoEConfig, + ps: ParallelState, + *, + cpu: bool = True, +) -> dict[str, torch.Tensor]: + """Export native LoRA tensors to a full PEFT-style adapter state dict. + + All distributed ranks must call this function. Rank 0 returns the full + adapter state; other ranks return an empty dict. + """ + + if ps.pp_size != 1: + raise NotImplementedError("LoRA adapter export currently supports pp=1.") + if ps.etp_size != 1: + raise NotImplementedError("LoRA adapter export currently supports etp=1.") + + _validate_attention_tp(model_cfg, ps) + state: dict[str, torch.Tensor] = {} + q_heads_local = model_cfg.num_attention_heads // ps.tp_size + kv_heads_local = model_cfg.num_key_value_heads // ps.tp_size + + for chunk in _iter_qwen_chunks(list(chunks)): + for layer in chunk.layers: + layer_idx = int(layer.layer_idx) + attn = layer.attn + + if attn.qkv_lora is not None: + if _is_rank_partitioned_lora_a(attn.qkv_lora, ps): + qkv_a = _gather_lora_rank_partition(attn.qkv_lora.lora_a.detach(), ps) + else: + qkv_a = _select_tp_replicated(attn.qkv_lora.lora_a.detach(), ps) + q_b_local, k_b_local, v_b_local = _split_local_mcore_qkv_b( + attn.qkv_lora.lora_b.detach(), + num_heads_local=q_heads_local, + num_kv_heads_local=kv_heads_local, + head_dim=model_cfg.head_dim, + ) + q_b = _all_gather_cat(q_b_local, ps.tp_group, dim=0) + k_b = _all_gather_cat(k_b_local, ps.tp_group, dim=0) + v_b = _all_gather_cat(v_b_local, ps.tp_group, dim=0) + if _rank() == 0: + state[_attn_key(layer_idx, "q_proj", "lora_A")] = qkv_a + state[_attn_key(layer_idx, "q_proj", "lora_B")] = q_b + state[_attn_key(layer_idx, "k_proj", "lora_A")] = qkv_a.clone() + state[_attn_key(layer_idx, "k_proj", "lora_B")] = k_b + state[_attn_key(layer_idx, "v_proj", "lora_A")] = qkv_a.clone() + state[_attn_key(layer_idx, "v_proj", "lora_B")] = v_b + + if attn.proj_lora is not None: + proj_a = _all_gather_cat(attn.proj_lora.lora_a.detach(), ps.tp_group, dim=1) + if _is_output_partitioned_lora_b(attn.proj_lora, ps): + proj_b = _all_gather_cat(attn.proj_lora.lora_b.detach(), ps.tp_group, dim=0) + else: + proj_b = _select_tp_replicated(attn.proj_lora.lora_b.detach(), ps) + if _rank() == 0: + state[_attn_key(layer_idx, "o_proj", "lora_A")] = proj_a + state[_attn_key(layer_idx, "o_proj", "lora_B")] = proj_b + + experts = layer.moe.experts + if experts.fc1_lora is not None: + fc1_a_local, fc1_b_local = _expand_shared_expert_lora( + experts.fc1_lora, experts.num_local_experts + ) + fc1_a = _all_gather_cat(fc1_a_local, ps.ep_group, dim=0) + fc1_b = _all_gather_cat(fc1_b_local, ps.ep_group, dim=0) + gate_b, up_b = fc1_b.chunk(2, dim=1) + if _rank() == 0: + for expert_idx in range(model_cfg.num_experts): + state[_expert_key(layer_idx, expert_idx, "gate_proj", "lora_A")] = fc1_a[ + expert_idx + ] + state[_expert_key(layer_idx, expert_idx, "gate_proj", "lora_B")] = gate_b[ + expert_idx + ] + state[_expert_key(layer_idx, expert_idx, "up_proj", "lora_A")] = fc1_a[ + expert_idx + ].clone() + state[_expert_key(layer_idx, expert_idx, "up_proj", "lora_B")] = up_b[ + expert_idx + ] + + if experts.fc2_lora is not None: + fc2_a_local, fc2_b_local = _expand_shared_expert_lora( + experts.fc2_lora, experts.num_local_experts + ) + fc2_a = _all_gather_cat(fc2_a_local, ps.ep_group, dim=0) + fc2_b = _all_gather_cat(fc2_b_local, ps.ep_group, dim=0) + if _rank() == 0: + for expert_idx in range(model_cfg.num_experts): + state[_expert_key(layer_idx, expert_idx, "down_proj", "lora_A")] = fc2_a[ + expert_idx + ] + state[_expert_key(layer_idx, expert_idx, "down_proj", "lora_B")] = fc2_b[ + expert_idx + ] + + if _rank() != 0: + return {} + if cpu: + return {name: tensor.detach().cpu().contiguous() for name, tensor in state.items()} + return {name: tensor.detach().contiguous() for name, tensor in state.items()} + + +def save_lora_adapter( + chunks: list[nn.Module] | tuple[nn.Module, ...], + model_cfg: Qwen3MoEConfig, + ps: ParallelState, + output_dir: str | Path, + *, + base_model_name_or_path: str = "", + lora_config: LoraConfig | dict[str, Any] | None = None, + metadata: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Save a PEFT/Mint-compatible LoRA adapter directory.""" + + from safetensors.torch import save_file + + if lora_config is None: + raise ValueError("save_lora_adapter requires the LoRA config used to build the model.") + lora = normalize_lora_config(lora_config) + if not lora.enabled: + raise ValueError("save_lora_adapter requires an enabled LoRA config.") + state = export_lora_adapter_state(chunks, model_cfg, ps, cpu=True) + output = Path(output_dir) + if _rank() == 0: + output.mkdir(parents=True, exist_ok=True) + save_file(state, str(output / "adapter_model.safetensors")) + config = { + "peft_type": "LORA", + "task_type": "CAUSAL_LM", + "base_model_name_or_path": base_model_name_or_path, + "inference_mode": False, + "r": lora.rank, + "lora_alpha": _effective_lora_alpha(lora), + "lora_dropout": lora.dropout, + "target_modules": _target_modules_from_lora_config(lora), + "bias": "none", + "fan_in_fan_out": False, + "init_lora_weights": True, + "modules_to_save": None, + } + (output / "adapter_config.json").write_text(json.dumps(config, indent=2) + "\n") + meta = { + "format": "megatron.lite_qwen3_moe_lora_peft_v1", + "expert_lora_representation": ( + "shared_local_expert_group" + if any( + _expert_lora_is_shared(getattr(layer.moe.experts, attr)) + for chunk in _iter_qwen_chunks(list(chunks)) + for layer in chunk.layers + for attr in ("fc1_lora", "fc2_lora") + if getattr(layer.moe.experts, attr) is not None + ) + else "per_expert" + ), + "num_tensors": len(state), + "num_parameters": int(sum(t.numel() for t in state.values())), + "parallel": {"tp": ps.tp_size, "ep": ps.ep_size, "etp": ps.etp_size, "pp": ps.pp_size}, + "model": { + "num_hidden_layers": model_cfg.num_hidden_layers, + "hidden_size": model_cfg.hidden_size, + "num_attention_heads": model_cfg.num_attention_heads, + "num_key_value_heads": model_cfg.num_key_value_heads, + "head_dim": model_cfg.head_dim, + "num_experts": model_cfg.num_experts, + "moe_intermediate_size": model_cfg.moe_intermediate_size, + }, + "metadata": metadata or {}, + } + (output / "megatron.lite_adapter_meta.json").write_text(json.dumps(meta, indent=2) + "\n") + result = { + "path": str(output), + "adapter_model": str(output / "adapter_model.safetensors"), + "adapter_config": str(output / "adapter_config.json"), + **meta, + } + else: + result = {} + if dist.is_available() and dist.is_initialized(): + dist.barrier() + return result + + +def _require_tensor(state: dict[str, torch.Tensor], key: str) -> torch.Tensor: + try: + return state[key] + except KeyError as exc: + raise KeyError(f"Missing adapter tensor {key!r}") from exc + + +def _slice_tp_output(tensor: torch.Tensor, local_width: int, ps: ParallelState) -> torch.Tensor: + start = ps.tp_rank * local_width + return tensor[start : start + local_width].contiguous() + + +def _slice_tp_input(tensor: torch.Tensor, local_width: int, ps: ParallelState) -> torch.Tensor: + start = ps.tp_rank * local_width + return tensor[:, start : start + local_width].contiguous() + + +def load_lora_adapter_state( + chunks: list[nn.Module] | tuple[nn.Module, ...], + state: dict[str, torch.Tensor], + model_cfg: Qwen3MoEConfig, + ps: ParallelState, + *, + strict: bool = True, +) -> dict[str, Any]: + """Load a PEFT adapter state into this rank's native LoRA shards.""" + + if ps.pp_size != 1: + raise NotImplementedError("LoRA adapter import currently supports pp=1.") + if ps.etp_size != 1: + raise NotImplementedError("LoRA adapter import currently supports etp=1.") + + _validate_attention_tp(model_cfg, ps) + q_heads_local = model_cfg.num_attention_heads // ps.tp_size + kv_heads_local = model_cfg.num_key_value_heads // ps.tp_size + q_width_local = q_heads_local * model_cfg.head_dim + kv_width_local = kv_heads_local * model_cfg.head_dim + attn_in_width_local = q_width_local + + loaded = 0 + for chunk in _iter_qwen_chunks(list(chunks)): + for layer in chunk.layers: + layer_idx = int(layer.layer_idx) + attn = layer.attn + if attn.qkv_lora is not None: + q_a = _require_tensor(state, _attn_key(layer_idx, "q_proj", "lora_A")).to( + device=attn.qkv_lora.lora_a.device, dtype=attn.qkv_lora.lora_a.dtype + ) + k_a = _require_tensor(state, _attn_key(layer_idx, "k_proj", "lora_A")).to(q_a) + v_a = _require_tensor(state, _attn_key(layer_idx, "v_proj", "lora_A")).to(q_a) + if strict and (not torch.equal(q_a, k_a) or not torch.equal(q_a, v_a)): + raise ValueError( + "Megatron Lite fused qkv_lora requires q/k/v lora_A tensors to match." + ) + q_a_local = ( + _slice_lora_rank_partition(q_a, ps) + if _is_rank_partitioned_lora_a(attn.qkv_lora, ps) + else q_a.contiguous() + ) + q_b = _slice_tp_output( + _require_tensor(state, _attn_key(layer_idx, "q_proj", "lora_B")).to( + device=attn.qkv_lora.lora_b.device, dtype=attn.qkv_lora.lora_b.dtype + ), + q_width_local, + ps, + ) + k_b = _slice_tp_output( + _require_tensor(state, _attn_key(layer_idx, "k_proj", "lora_B")).to( + device=attn.qkv_lora.lora_b.device, dtype=attn.qkv_lora.lora_b.dtype + ), + kv_width_local, + ps, + ) + v_b = _slice_tp_output( + _require_tensor(state, _attn_key(layer_idx, "v_proj", "lora_B")).to( + device=attn.qkv_lora.lora_b.device, dtype=attn.qkv_lora.lora_b.dtype + ), + kv_width_local, + ps, + ) + attn.qkv_lora.lora_a.data.copy_(q_a_local) + attn.qkv_lora.lora_b.data.copy_( + _pack_local_mcore_qkv_b( + q_b, + k_b, + v_b, + num_heads_local=q_heads_local, + num_kv_heads_local=kv_heads_local, + head_dim=model_cfg.head_dim, + ) + ) + loaded += 2 + + if attn.proj_lora is not None: + proj_a = _slice_tp_input( + _require_tensor(state, _attn_key(layer_idx, "o_proj", "lora_A")).to( + device=attn.proj_lora.lora_a.device, dtype=attn.proj_lora.lora_a.dtype + ), + attn_in_width_local, + ps, + ) + proj_b = _require_tensor(state, _attn_key(layer_idx, "o_proj", "lora_B")).to( + device=attn.proj_lora.lora_b.device, dtype=attn.proj_lora.lora_b.dtype + ) + if _is_output_partitioned_lora_b(attn.proj_lora, ps): + proj_b = _slice_tp_output(proj_b, attn.proj_lora.lora_b.shape[0], ps) + attn.proj_lora.lora_a.data.copy_(proj_a) + attn.proj_lora.lora_b.data.copy_(proj_b) + loaded += 2 + + experts = layer.moe.experts + expert_start = ps.ep_rank * experts.num_local_experts + expert_stop = expert_start + experts.num_local_experts + if experts.fc1_lora is not None: + if _expert_lora_is_shared(experts.fc1_lora): + local_gate_a = [] + local_gate_b = [] + local_up_b = [] + for expert_idx in range(expert_start, expert_stop): + gate_a = _require_tensor( + state, _expert_key(layer_idx, expert_idx, "gate_proj", "lora_A") + ).to( + device=experts.fc1_lora.lora_a.device, + dtype=experts.fc1_lora.lora_a.dtype, + ) + up_a = _require_tensor( + state, _expert_key(layer_idx, expert_idx, "up_proj", "lora_A") + ).to(gate_a) + if strict and not torch.equal(gate_a, up_a): + raise ValueError( + "Megatron Lite fused fc1_lora requires gate/up lora_A tensors to match." + ) + local_gate_a.append(gate_a) + local_gate_b.append( + _require_tensor( + state, _expert_key(layer_idx, expert_idx, "gate_proj", "lora_B") + ).to( + device=experts.fc1_lora.lora_b.device, + dtype=experts.fc1_lora.lora_b.dtype, + ) + ) + local_up_b.append( + _require_tensor( + state, _expert_key(layer_idx, expert_idx, "up_proj", "lora_B") + ).to( + device=experts.fc1_lora.lora_b.device, + dtype=experts.fc1_lora.lora_b.dtype, + ) + ) + if strict: + if any( + not torch.equal(local_gate_a[0], value) for value in local_gate_a[1:] + ): + raise ValueError( + "Megatron Lite shared expert fc1_lora can only import PEFT adapters " + "whose local expert gate lora_A tensors are identical." + ) + if any( + not torch.equal(local_gate_b[0], value) for value in local_gate_b[1:] + ): + raise ValueError( + "Megatron Lite shared expert fc1_lora can only import PEFT adapters " + "whose local expert gate lora_B tensors are identical." + ) + if any(not torch.equal(local_up_b[0], value) for value in local_up_b[1:]): + raise ValueError( + "Megatron Lite shared expert fc1_lora can only import PEFT adapters " + "whose local expert up lora_B tensors are identical." + ) + experts.fc1_lora.lora_a.data.copy_(local_gate_a[0]) + experts.fc1_lora.lora_b.data.copy_( + torch.cat([local_gate_b[0], local_up_b[0]], dim=0) + ) + loaded += 2 + else: + for local_idx, expert_idx in enumerate(range(expert_start, expert_stop)): + gate_a = _require_tensor( + state, _expert_key(layer_idx, expert_idx, "gate_proj", "lora_A") + ).to( + device=experts.fc1_lora.lora_a.device, + dtype=experts.fc1_lora.lora_a.dtype, + ) + up_a = _require_tensor( + state, _expert_key(layer_idx, expert_idx, "up_proj", "lora_A") + ).to(gate_a) + if strict and not torch.equal(gate_a, up_a): + raise ValueError( + "Megatron Lite fused fc1_lora requires gate/up lora_A tensors to match." + ) + gate_b = _require_tensor( + state, _expert_key(layer_idx, expert_idx, "gate_proj", "lora_B") + ).to( + device=experts.fc1_lora.lora_b.device, + dtype=experts.fc1_lora.lora_b.dtype, + ) + up_b = _require_tensor( + state, _expert_key(layer_idx, expert_idx, "up_proj", "lora_B") + ).to( + device=experts.fc1_lora.lora_b.device, + dtype=experts.fc1_lora.lora_b.dtype, + ) + experts.fc1_lora.lora_a.data[local_idx].copy_(gate_a) + experts.fc1_lora.lora_b.data[local_idx].copy_( + torch.cat([gate_b, up_b], dim=0) + ) + loaded += 2 + + if experts.fc2_lora is not None: + if _expert_lora_is_shared(experts.fc2_lora): + local_a = [] + local_b = [] + for expert_idx in range(expert_start, expert_stop): + local_a.append( + _require_tensor( + state, _expert_key(layer_idx, expert_idx, "down_proj", "lora_A") + ).to( + device=experts.fc2_lora.lora_a.device, + dtype=experts.fc2_lora.lora_a.dtype, + ) + ) + local_b.append( + _require_tensor( + state, _expert_key(layer_idx, expert_idx, "down_proj", "lora_B") + ).to( + device=experts.fc2_lora.lora_b.device, + dtype=experts.fc2_lora.lora_b.dtype, + ) + ) + if strict: + if any(not torch.equal(local_a[0], value) for value in local_a[1:]): + raise ValueError( + "Megatron Lite shared expert fc2_lora can only import PEFT adapters " + "whose local expert down lora_A tensors are identical." + ) + if any(not torch.equal(local_b[0], value) for value in local_b[1:]): + raise ValueError( + "Megatron Lite shared expert fc2_lora can only import PEFT adapters " + "whose local expert down lora_B tensors are identical." + ) + experts.fc2_lora.lora_a.data.copy_(local_a[0]) + experts.fc2_lora.lora_b.data.copy_(local_b[0]) + loaded += 2 + else: + for local_idx, expert_idx in enumerate(range(expert_start, expert_stop)): + experts.fc2_lora.lora_a.data[local_idx].copy_( + _require_tensor( + state, _expert_key(layer_idx, expert_idx, "down_proj", "lora_A") + ).to( + device=experts.fc2_lora.lora_a.device, + dtype=experts.fc2_lora.lora_a.dtype, + ) + ) + experts.fc2_lora.lora_b.data[local_idx].copy_( + _require_tensor( + state, _expert_key(layer_idx, expert_idx, "down_proj", "lora_B") + ).to( + device=experts.fc2_lora.lora_b.device, + dtype=experts.fc2_lora.lora_b.dtype, + ) + ) + loaded += 2 + + return {"loaded_tensors": loaded} + + +def load_lora_adapter( + chunks: list[nn.Module] | tuple[nn.Module, ...], + adapter_dir: str | Path, + model_cfg: Qwen3MoEConfig, + ps: ParallelState, + *, + strict: bool = True, + lora_config: LoraConfig | dict[str, Any] | None = None, +) -> dict[str, Any]: + from safetensors.torch import load_file + + path = Path(adapter_dir) / "adapter_model.safetensors" + state = load_file(str(path), device="cpu") + config_path = Path(adapter_dir) / "adapter_config.json" + if config_path.exists(): + adapter_config = json.loads(config_path.read_text()) + _validate_adapter_config(chunks, state, adapter_config, lora_config=lora_config) + elif lora_config is not None: + expected = normalize_lora_config(lora_config) + state_rank = _infer_state_rank(state) + if state_rank is not None and state_rank != expected.rank: + raise ValueError( + f"Adapter tensor rank {state_rank} does not match expected rank {expected.rank}." + ) + return load_lora_adapter_state(chunks, state, model_cfg, ps, strict=strict) + + +__all__ = [ + "export_lora_adapter_state", + "load_lora_adapter", + "load_lora_adapter_state", + "save_lora_adapter", +] diff --git a/experimental/lite/megatron/lite/model/qwen3_moe/lite/model.py b/experimental/lite/megatron/lite/model/qwen3_moe/lite/model.py new file mode 100644 index 00000000000..cc5ee916a14 --- /dev/null +++ b/experimental/lite/megatron/lite/model/qwen3_moe/lite/model.py @@ -0,0 +1,621 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Native Qwen3MoE: TransformerLayer + Qwen3MoEModel. + +Attention and MoE come from primitive/modules; this file only +defines the model-specific composition (Layer stacking, PP layout, +loss computation). +""" + +from __future__ import annotations + +from contextlib import nullcontext + +import torch +import torch.nn as nn +import transformer_engine.pytorch as te + +from megatron.lite.model.qwen3_moe.config import Qwen3MoEConfig +from megatron.lite.primitive.modules.dispatcher import TokenDispatcher +from megatron.lite.primitive.modules.experts import Experts +from megatron.lite.primitive.modules.gqa import GQAttention +from megatron.lite.primitive.modules.lora import LoraConfig +from megatron.lite.primitive.modules.router import TopKRouter +from megatron.lite.primitive.ops.cross_entropy import vocab_parallel_cross_entropy +from megatron.lite.primitive.ops.linear_cross_entropy import linear_cross_entropy +from megatron.lite.primitive.ops.logprob import vocab_parallel_entropy +from megatron.lite.primitive.parallel import ( + ParallelState, + VanillaColumnParallelLinear, + VocabParallelEmbedding, + VocabParallelOutput, + build_pipeline_chunk_layout, + gather_from_sequence_parallel, + roll_packed_thd_left, + scatter_to_sequence_parallel, +) +from megatron.lite.primitive.utils import build_fp8_recipe + +# --------------------------------------------------------------------------- +# MoE Layer (thin assembly over megatron.lite.primitive.modules) +# --------------------------------------------------------------------------- + + +class MoELayer(nn.Module): + def __init__( + self, + config: Qwen3MoEConfig, + ps: ParallelState, + *, + use_deepep: bool = True, + router_bias_rate: float = 0.0, + fp8: bool = False, + moe_act_recompute: bool = False, + lora_config: LoraConfig | dict | None = None, + ): + super().__init__() + # Match Qwen3-MoE's `load_balancing_type="none"` setting: no aux loss. + self.router = TopKRouter( + config, ps, router_bias_rate=router_bias_rate, compute_aux_loss=False + ) + self.experts = Experts( + config, ps, fp8=fp8, moe_act_recompute=moe_act_recompute, lora_config=lora_config + ) + self.dispatcher = TokenDispatcher( + config.num_experts, config.hidden_size, ps, use_deepep=use_deepep + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + input_shape = x.shape + if x.dim() == 3: + x_2d = x.view(-1, x.size(-1)) + else: + x_2d = x + + scores, indices = self.router(x_2d) + dispatched, tpe, permuted_probs = self.dispatcher.dispatch(x_2d, scores, indices) + del scores, indices + self.dispatcher.wait_dispatch_event() + expert_out = self.experts( + dispatched, + tpe, + permuted_probs, + tokens_per_expert_list=getattr(self.dispatcher, "_local_tpe_list", None), + ) + del dispatched, tpe, permuted_probs + combined = self.dispatcher.combine(expert_out) + del expert_out + + return combined.view(input_shape).to(x.dtype) + + +# --------------------------------------------------------------------------- +# Transformer Layer + Model +# --------------------------------------------------------------------------- + +_SP_GRAD_SUFFIXES: tuple[str, ...] = ( + ".attn.qkv.linear.layer_norm_weight", + ".mlp_norm.weight", + ".q_norm.weight", + ".k_norm.weight", + ".moe.router.gate.weight", + ".enorm.weight", + ".hnorm.weight", + ".final_layernorm.weight", +) + + +def _collect_sp_grad_params(model: nn.Module) -> list[nn.Parameter]: + """Collect non-TP-sharded params needing coalesced all_reduce after backward.""" + params = [] + for name, p in model.named_parameters(): + if any(name.endswith(s) for s in _SP_GRAD_SUFFIXES) or name == "norm.weight": + params.append(p) + return params + + +class TransformerLayer(nn.Module): + def __init__( + self, + config: Qwen3MoEConfig, + ps: ParallelState, + layer_idx: int, + *, + use_deepep: bool = True, + router_bias_rate: float = 0.0, + fp8: bool = False, + moe_act_recompute: bool = False, + use_thd: bool = False, + lora_config: LoraConfig | dict | None = None, + ): + super().__init__() + self.layer_idx = layer_idx + + # Declaration order follows MC's TransformerLayer (self_attention → + # pre_mlp_layernorm → mlp). `named_parameters()` iterates in + # declaration order, and MC's `DistributedDataParallel` lays out + # gradient buckets by that order; mismatching it changes fp32 master + # shard layouts and breaks bitwise alignment from step 1 onwards. + self.attn = GQAttention( + hidden_size=config.hidden_size, + num_attention_heads=config.num_attention_heads, + num_key_value_heads=config.num_key_value_heads, + head_dim=config.head_dim, + ps=ps, + rms_norm_eps=config.rms_norm_eps, + rope_theta=config.rope_theta, + use_thd=use_thd, + qkv_layout="mcore", + lora_config=lora_config, + ) + self.mlp_norm = te.RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.moe = MoELayer( + config, + ps, + use_deepep=use_deepep, + router_bias_rate=router_bias_rate, + fp8=fp8, + moe_act_recompute=moe_act_recompute, + lora_config=lora_config, + ) + + def forward( + self, x: torch.Tensor, position_ids: torch.Tensor | None = None, packed_seq_params=None + ) -> torch.Tensor: + residual = x + h = self.attn(x, position_ids=position_ids, packed_seq_params=packed_seq_params) + x = residual + h + + residual = x + h = self.mlp_norm(x) + moe_out = self.moe(h) + x = residual + moe_out + + return x + + +class MTPLossAutoScaler(torch.autograd.Function): + """Attach MTP loss gradients to the main LM hidden state.""" + + main_loss_backward_scale: float = 1.0 + + @staticmethod + def forward(ctx, output: torch.Tensor, mtp_loss: torch.Tensor): + ctx.save_for_backward(mtp_loss) + return output + + @staticmethod + def backward(ctx, grad_output: torch.Tensor): + (mtp_loss,) = ctx.saved_tensors + scaled_mtp_grad = torch.ones_like(mtp_loss) * MTPLossAutoScaler.main_loss_backward_scale + return grad_output, scaled_mtp_grad + + @staticmethod + def set_loss_scale(scale: torch.Tensor | float) -> None: + if isinstance(scale, torch.Tensor): + scale = float(scale.detach().float().item()) + MTPLossAutoScaler.main_loss_backward_scale = float(scale) + + +class MultiTokenPredictionLayer(nn.Module): + """MCore-style MTP layer for the THD SFT lite path.""" + + def __init__( + self, + config: Qwen3MoEConfig, + ps: ParallelState, + layer_idx: int, + *, + embedding: VocabParallelEmbedding, + use_deepep: bool, + router_bias_rate: float, + fp8: bool, + moe_act_recompute: bool, + use_thd: bool, + detach_encoder: bool, + lora_config: LoraConfig | dict | None, + ): + super().__init__() + self.ps = ps + self.embedding = embedding + self.detach_encoder = detach_encoder + self.enorm = te.RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.hnorm = te.RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.eh_proj = VanillaColumnParallelLinear( + config.hidden_size * 2, config.hidden_size, ps, sp=ps.tp_size > 1, gather_output=True + ) + self.transformer_layer = TransformerLayer( + config, + ps, + config.num_hidden_layers + layer_idx, + use_deepep=use_deepep, + router_bias_rate=router_bias_rate, + fp8=fp8, + moe_act_recompute=moe_act_recompute, + use_thd=use_thd, + lora_config=lora_config, + ) + self.final_layernorm = te.RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + def forward( + self, + *, + input_ids: torch.Tensor, + position_ids: torch.Tensor | None, + hidden_states: torch.Tensor, + rotary_position_ids: torch.Tensor | None = None, + packed_seq_params=None, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]: + attention_position_ids = ( + rotary_position_ids if rotary_position_ids is not None else position_ids + ) + input_ids, _ = roll_packed_thd_left(input_ids, packed_seq_params=packed_seq_params, dims=-1) + if position_ids is not None: + position_ids, _ = roll_packed_thd_left( + position_ids, packed_seq_params=packed_seq_params, dims=-1 + ) + decoder_input = self.embedding(input_ids) + decoder_input = scatter_to_sequence_parallel(decoder_input, self.ps) + + if self.detach_encoder: + decoder_input = decoder_input.detach() + hidden_states = hidden_states.detach() + + decoder_input = self.enorm(decoder_input) + hidden_states = self.hnorm(hidden_states) + hidden_states = torch.cat((decoder_input, hidden_states), dim=-1) + hidden_states = self.eh_proj(hidden_states) + hidden_states = scatter_to_sequence_parallel(hidden_states, self.ps) + hidden_states = self.transformer_layer( + hidden_states, position_ids=attention_position_ids, packed_seq_params=packed_seq_params + ) + hidden_states = self.final_layernorm(hidden_states) + return hidden_states, input_ids, position_ids + + +class MultiTokenPredictionBlock(nn.Module): + def __init__( + self, + config: Qwen3MoEConfig, + ps: ParallelState, + *, + embedding: VocabParallelEmbedding, + use_deepep: bool, + router_bias_rate: float, + fp8: bool, + moe_act_recompute: bool, + use_thd: bool, + detach_encoder: bool, + repeated_layer: bool, + lora_config: LoraConfig | dict | None, + ): + super().__init__() + self.num_layers = config.num_nextn_predict_layers + self.repeated_layer = repeated_layer + layers_to_build = 1 if repeated_layer else self.num_layers + self.layers = nn.ModuleList( + [ + MultiTokenPredictionLayer( + config, + ps, + idx, + embedding=embedding, + use_deepep=use_deepep, + router_bias_rate=router_bias_rate, + fp8=fp8, + moe_act_recompute=moe_act_recompute, + use_thd=use_thd, + detach_encoder=detach_encoder, + lora_config=lora_config, + ) + for idx in range(layers_to_build) + ] + ) + + def forward( + self, + *, + input_ids: torch.Tensor, + position_ids: torch.Tensor | None, + hidden_states: torch.Tensor, + packed_seq_params=None, + ) -> list[torch.Tensor]: + outputs: list[torch.Tensor] = [] + rotary_position_ids = position_ids + for depth in range(self.num_layers): + layer = self.layers[0] if self.repeated_layer else self.layers[depth] + hidden_states, input_ids, position_ids = layer( + input_ids=input_ids, + position_ids=position_ids, + hidden_states=hidden_states, + rotary_position_ids=rotary_position_ids, + packed_seq_params=packed_seq_params, + ) + outputs.append(hidden_states) + return outputs + + +def _temperature_to_float(temperature: float | torch.Tensor) -> float: + if isinstance(temperature, torch.Tensor): + if temperature.numel() != 1: + raise ValueError( + "Megatron Lite fused/MTP SFT currently supports scalar temperature only." + ) + return float(temperature.detach().float().item()) + return float(temperature) + + +class Qwen3MoEModel(nn.Module): + def __init__( + self, + config: Qwen3MoEConfig, + ps: ParallelState, + vpp: int | None = None, + vpp_chunk_id: int | None = None, + *, + use_deepep: bool = False, + fp8: bool = False, + recompute_modules: list[str] | None = None, + router_bias_rate: float = 0.0, + use_thd: bool = False, + mtp_enable: bool = False, + mtp_enable_train: bool = False, + mtp_detach_encoder: bool = False, + lora_config: LoraConfig | dict | None = None, + ): + super().__init__() + self.config = config + self.ps = ps + self.fp8 = fp8 + self.mtp_enable_train = bool(mtp_enable and mtp_enable_train) + self.mtp_loss_scaling_factor = config.mtp_loss_scaling_factor + self._input_tensor: torch.Tensor | None = None + layout = build_pipeline_chunk_layout(config.num_hidden_layers, ps, vpp, vpp_chunk_id) + self.layer_indices = layout.layer_indices + has_embed = layout.has_embed + has_head = layout.has_head + self.pre_process = has_embed + self.post_process = has_head + self.share_embeddings_and_output_weights = False + + self.embed: VocabParallelEmbedding | None = None + if has_embed: + self.embed = VocabParallelEmbedding(config.vocab_size, config.hidden_size, ps) + + _recompute = recompute_modules or [] + moe_act_recompute = "moe_act" in _recompute and "moe" not in _recompute + self.layers = nn.ModuleList( + [ + TransformerLayer( + config, + ps, + idx, + use_deepep=use_deepep, + router_bias_rate=router_bias_rate, + fp8=fp8, + moe_act_recompute=moe_act_recompute, + use_thd=use_thd, + lora_config=lora_config, + ) + for idx in self.layer_indices + ] + ) + + self.norm: nn.Module | None = None + self.head: VocabParallelOutput | None = None + if has_head: + self.norm = te.RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.head = VocabParallelOutput(config.vocab_size, config.hidden_size, ps) + + self.mtp_embed: VocabParallelEmbedding | None = None + self.mtp: MultiTokenPredictionBlock | None = None + if mtp_enable and config.num_nextn_predict_layers > 0 and self.head is not None: + mtp_embedding = self.embed + if mtp_embedding is None: + mtp_embedding = VocabParallelEmbedding(config.vocab_size, config.hidden_size, ps) + self.mtp_embed = mtp_embedding + self.mtp = MultiTokenPredictionBlock( + config, + ps, + embedding=mtp_embedding, + use_deepep=use_deepep, + router_bias_rate=router_bias_rate, + fp8=fp8, + moe_act_recompute=moe_act_recompute, + use_thd=use_thd, + detach_encoder=mtp_detach_encoder, + repeated_layer=config.mtp_use_repeated_layer, + lora_config=lora_config, + ) + + self.sp_params: list[nn.Parameter] = [] + if ps.tp_size > 1: + self.sp_params = _collect_sp_grad_params(self) + + def set_input_tensor(self, input_tensor): + if isinstance(input_tensor, list): + if len(input_tensor) > 1: + raise ValueError("Qwen3MoEModel expects a single pipeline input tensor.") + input_tensor = input_tensor[0] if input_tensor else None + self._input_tensor = input_tensor + + def forward( + self, + input_ids: torch.Tensor | None = None, + hidden_states: torch.Tensor | None = None, + position_ids: torch.Tensor | None = None, + packed_seq_params=None, + labels: torch.Tensor | None = None, + loss_mask: torch.Tensor | None = None, + temperature: float | torch.Tensor = 1.0, + use_fused_kernels: bool = False, + calculate_entropy: bool = False, + return_log_probs: bool = True, + ) -> dict: + if self.embed is not None: + assert input_ids is not None + h = self.embed(input_ids) + else: + if hidden_states is None: + hidden_states = self._input_tensor + assert hidden_states is not None + h = hidden_states + + fp8_ctx = ( + te.fp8_autocast(enabled=True, fp8_recipe=build_fp8_recipe()) + if self.fp8 + else nullcontext() + ) + + with fp8_ctx: + if self.embed is not None: + h = scatter_to_sequence_parallel(h, self.ps) + for layer in self.layers: + h = layer(h, position_ids=position_ids, packed_seq_params=packed_seq_params) + # Head path is SP-aware: norm runs on SP-sharded [S/tp, B, H] and + # head's internal all-gather happens inside VocabParallelOutput. + # Mirrors MC GPTModel's final_layernorm → output_layer(sp=True). + + output = {"hidden_states": h} + + if self.head is not None: + hidden_for_head = self.norm(h) + + if labels is not None: + temperature_value = _temperature_to_float(temperature) + mtp_result = self._apply_mtp_loss( + hidden_for_head, + input_ids=input_ids, + position_ids=position_ids, + labels=labels, + loss_mask=loss_mask, + packed_seq_params=packed_seq_params, + temperature=temperature_value, + use_fused_kernels=use_fused_kernels, + ) + if mtp_result is not None: + hidden_for_head, mtp_loss = mtp_result + output["mtp_loss"] = mtp_loss + labels_sb = labels.transpose(0, 1).contiguous() + if use_fused_kernels: + hidden_full = gather_from_sequence_parallel(hidden_for_head, self.ps) + log_probs, entropy = linear_cross_entropy( + hidden_full, + self._head_weight_for_fused_ce(hidden_full), + labels_sb, + temperature_value, + self.ps.tp_group, + ) + token_loss = -log_probs + output["loss"] = token_loss.mean() + if return_log_probs: + output["log_probs"] = log_probs.transpose(0, 1).contiguous() + if calculate_entropy: + output["entropy"] = entropy.transpose(0, 1).contiguous() + else: + logits = self.head(hidden_for_head) + if temperature_value != 1.0: + logits = logits / temperature_value + token_loss = vocab_parallel_cross_entropy(logits, labels_sb, self.ps.tp_group) + output["loss"] = token_loss.mean() + if return_log_probs: + output["log_probs"] = (-token_loss).transpose(0, 1).contiguous() + if calculate_entropy: + entropy = vocab_parallel_entropy(logits, self.ps.tp_group) + output["entropy"] = entropy.transpose(0, 1).contiguous() + + if labels is None: + logits = self.head(hidden_for_head) + output["logits"] = self.head.gather(logits) + + return output + + def _apply_mtp_loss( + self, + hidden_states: torch.Tensor, + *, + input_ids: torch.Tensor | None, + position_ids: torch.Tensor | None, + labels: torch.Tensor, + loss_mask: torch.Tensor | None, + packed_seq_params, + temperature: float, + use_fused_kernels: bool, + ) -> tuple[torch.Tensor, torch.Tensor] | None: + if self.mtp is None: + return None + if not self.mtp_enable_train: + return None + if input_ids is None: + raise ValueError("MTP training requires input_ids.") + if loss_mask is None: + loss_mask = torch.ones_like(labels, dtype=torch.float32) + else: + loss_mask = loss_mask.to(dtype=torch.float32) + + mtp_hidden_states = self.mtp( + input_ids=input_ids, + position_ids=position_ids, + hidden_states=hidden_states, + packed_seq_params=packed_seq_params, + ) + + mtp_labels = labels.clone() + mtp_loss_mask = loss_mask.clone() + mtp_loss_values = [] + for mtp_hidden in mtp_hidden_states: + mtp_labels, _ = roll_packed_thd_left( + mtp_labels, packed_seq_params=packed_seq_params, dims=-1 + ) + mtp_loss_mask, num_tokens = roll_packed_thd_left( + mtp_loss_mask, packed_seq_params=packed_seq_params, dims=-1 + ) + labels_sb = mtp_labels.transpose(0, 1).contiguous() + mask_sb = mtp_loss_mask.transpose(0, 1).contiguous() + + if use_fused_kernels: + mtp_hidden_full = gather_from_sequence_parallel(mtp_hidden, self.ps) + log_probs, _entropy = linear_cross_entropy( + mtp_hidden_full, + self._head_weight_for_fused_ce(mtp_hidden_full), + labels_sb, + temperature, + self.ps.tp_group, + ) + token_loss = -log_probs + else: + logits = self.head(mtp_hidden) + if temperature != 1.0: + logits = logits / temperature + token_loss = vocab_parallel_cross_entropy(logits, labels_sb, self.ps.tp_group) + token_loss = token_loss * mask_sb.to(dtype=token_loss.dtype) + num_tokens = num_tokens.to(dtype=token_loss.dtype).clamp_min(1.0) + mtp_loss_values.append(token_loss.sum() / num_tokens) + + mtp_loss_scale = self.mtp_loss_scaling_factor / max(len(mtp_hidden_states), 1) + hidden_states = MTPLossAutoScaler.apply( + hidden_states, mtp_loss_scale * token_loss / num_tokens + ) + + if not mtp_loss_values: + return None + return ( + hidden_states, + torch.stack([loss.detach().float() for loss in mtp_loss_values]).mean(), + ) + + def _head_weight_for_fused_ce(self, hidden_states: torch.Tensor) -> torch.Tensor: + assert self.head is not None + weight = self.head.col.linear.weight + if weight.dtype == hidden_states.dtype: + return weight + return weight.to(dtype=hidden_states.dtype) + + +__all__ = [ + "MoELayer", + "MTPLossAutoScaler", + "MultiTokenPredictionBlock", + "MultiTokenPredictionLayer", + "Qwen3MoEModel", + "TransformerLayer", +] diff --git a/experimental/lite/megatron/lite/model/qwen3_moe/lite/protocol.py b/experimental/lite/megatron/lite/model/qwen3_moe/lite/protocol.py new file mode 100644 index 00000000000..ab02ccab832 --- /dev/null +++ b/experimental/lite/megatron/lite/model/qwen3_moe/lite/protocol.py @@ -0,0 +1,330 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Qwen3MoE lite impl — model protocol for Megatron Lite runtime. + +This file is the reference implementation of the Megatron Lite model protocol. +New model authors: copy this file and adapt. + +Protocol convention (what runtime calls): + Required: + ImplConfig — @dataclass, per-impl knobs + build_model_config(source, **overrides) → ModelConfig + build_model(model_cfg, *, impl_cfg) → ModelBundle + Optional (in ModelBundle.extras or module-level): + load_hf_weights(chunk, hf_path, model_cfg, ps) — HF weight loading + export_hf_weights(chunks, model_cfg, ps) — HF weight export + vocab_size(model_cfg) -> int — benchmark metadata + Escape hatch: + create_runtime(hf_path, cfg) -> Runtime — fully override runtime +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import torch +import torch.nn as nn +from megatron.lite.model.protocol_utils import ( + add_cross_entropy_fusion, + add_loss_context_kwargs, + pack_thd_forward_kwargs, + set_cross_entropy_fusion, + unpack_thd_forward_output, +) +from megatron.lite.model.qwen3_moe.common import is_expert_param +from megatron.lite.model.qwen3_moe.config import Qwen3MoEConfig +from megatron.lite.model.qwen3_moe.lite.checkpoint import EXPERT_CLASSIFIER, PLACEMENT_FN +from megatron.lite.model.qwen3_moe.lite.checkpoint import load_hf_weights as _load_hf_weights_impl +from megatron.lite.model.qwen3_moe.lite.model import MTPLossAutoScaler, Qwen3MoEModel +from megatron.lite.primitive.bundle import ModelBundle +from megatron.lite.primitive.modules.lora import ( + LoraConfig, + freeze_non_lora_params, + normalize_lora_config, + trainable_param_stats, +) +from megatron.lite.primitive.parallel import ParallelState, init_parallel +from megatron.lite.primitive.recompute import apply_recompute, parse_recompute_spec +from megatron.lite.runtime.contracts import OptimizerConfig, ParallelConfig +from megatron.lite.runtime.contracts.data import PackedBatch + +__all__ = [ + "EXPERT_CLASSIFIER", + "ImplConfig", + "PLACEMENT_FN", + "build_model", + "build_model_config", + "export_hf_weights", + "load_hf_weights", + "vocab_size", +] + +# --------------------------------------------------------------------------- +# ImplConfig +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class ImplConfig: + """Lite impl knobs. Constructed by runtime from user config.""" + + parallel: ParallelConfig = field(default_factory=ParallelConfig) + optimizer: str | None = "dist_opt" # None = no optimizer (inference) + recompute: list[str] = field(default_factory=list) + offload: list[str] = field(default_factory=list) + use_deepep: bool = False + use_thd: bool = False + cross_entropy_fusion: bool = False + router_aux_loss_coef: float | None = None + router_bias_rate: float = 0.0 + # User-level OptimizerConfig threaded through the runtime. + optimizer_config: OptimizerConfig | None = None + mtp_enable: bool = False + mtp_enable_train: bool = False + mtp_detach_encoder: bool = False + mtp_loss_scaling_factor: float = 0.1 + mtp_use_repeated_layer: bool | None = None + deterministic: bool = True + lora: LoraConfig | dict | None = None + + +# --------------------------------------------------------------------------- +# Module map for recompute/offload +# --------------------------------------------------------------------------- + +MODULE_MAP = { + "core_attn": lambda layer: layer.attn.core_attn, + "experts": lambda layer: layer.moe.experts, + "moe": lambda layer: layer.moe, + "router": lambda layer: layer.moe.router, + "mlp_norm": lambda layer: layer.mlp_norm, + "attn_proj": lambda layer: layer.attn.proj, +} + + +# --------------------------------------------------------------------------- +# Required: build_model_config +# --------------------------------------------------------------------------- + + +def build_model_config(source: str | Path | dict, **overrides) -> Qwen3MoEConfig: + """Build Qwen3MoE architecture config from HF source.""" + if isinstance(source, dict): + cfg = Qwen3MoEConfig._from_hf_dict(source) + else: + cfg = Qwen3MoEConfig.from_hf(str(source)) + for k, v in overrides.items(): + if hasattr(cfg, k): + setattr(cfg, k, v) + return cfg + + +# --------------------------------------------------------------------------- +# Required: build_model +# --------------------------------------------------------------------------- + + +def _forward_step(model: nn.Module, batch: PackedBatch) -> dict: + kwargs = pack_thd_forward_kwargs(model, batch) + add_loss_context_kwargs(kwargs, include_return_log_probs=True) + add_cross_entropy_fusion(kwargs, model) + return model(**kwargs) + + +def _forward_step_bshd(model: nn.Module, batch: PackedBatch) -> dict: + labels = batch.labels.reshape(1, -1) if batch.labels is not None else None + return model(input_ids=batch.input_ids.reshape(1, -1), labels=labels, packed_seq_params=None) + + +def unpack_forward_output(model: nn.Module, batch: PackedBatch, output) -> Any: + return unpack_thd_forward_output(model, batch, output) + + +def build_model(model_cfg: Qwen3MoEConfig, *, impl_cfg: ImplConfig) -> ModelBundle: + """Build lite Qwen3MoE: model, parallel state, optimizer — everything. + + Model owns all construction. Runtime just consumes the ModelBundle. + """ + p = impl_cfg.parallel + lora_config = normalize_lora_config(impl_cfg.lora) + + # ── validation ── + if impl_cfg.use_deepep and (p.etp is not None and p.etp > 1): + raise ValueError("use_deepep and etp>1 are mutually exclusive") + + # ── override model config from impl_cfg ── + if impl_cfg.router_aux_loss_coef is not None: + model_cfg.router_aux_loss_coef = impl_cfg.router_aux_loss_coef + mtp_enable = bool(impl_cfg.mtp_enable) + mtp_enable_train = mtp_enable and bool(impl_cfg.mtp_enable_train) + if mtp_enable: + if model_cfg.num_nextn_predict_layers <= 0: + raise ValueError("mtp_enable=True but HF config has no num_nextn_predict_layers.") + model_cfg.mtp_loss_scaling_factor = impl_cfg.mtp_loss_scaling_factor + if impl_cfg.mtp_use_repeated_layer is not None: + model_cfg.mtp_use_repeated_layer = impl_cfg.mtp_use_repeated_layer + else: + model_cfg.num_nextn_predict_layers = 0 + + # ── parallel state (model creates its own) ── + ps = init_parallel(p) + deterministic = impl_cfg.deterministic + + # ── build chunks ── + recompute_spec = parse_recompute_spec(impl_cfg.recompute) + model_kwargs: dict[str, Any] = dict( + use_deepep=impl_cfg.use_deepep, + fp8=False, + recompute_modules=recompute_spec, + router_bias_rate=impl_cfg.router_bias_rate, + use_thd=impl_cfg.use_thd, + mtp_enable=mtp_enable, + mtp_enable_train=mtp_enable_train, + mtp_detach_encoder=impl_cfg.mtp_detach_encoder, + lora_config=lora_config, + ) + + vpp = None if p.vpp == 1 else p.vpp + if vpp is None: + chunks = [Qwen3MoEModel(model_cfg, ps, **model_kwargs).to(torch.bfloat16).cuda()] + else: + chunks = [] + for i in range(vpp): + chunks.append( + Qwen3MoEModel(model_cfg, ps, vpp=vpp, vpp_chunk_id=i, **model_kwargs) + .to(torch.bfloat16) + .cuda() + ) + + set_cross_entropy_fusion(chunks, impl_cfg.cross_entropy_fusion) + + # ── recompute ── + if recompute_spec: + for chunk in chunks: + apply_recompute(chunk.layers, recompute_spec, MODULE_MAP) + + # ── offload ── + if impl_cfg.offload: + from megatron.lite.primitive.recompute import apply_offload + + for chunk in chunks: + apply_offload(chunk.layers, impl_cfg.offload, MODULE_MAP) + + lora_stats = None + if lora_config.enabled: + lora_stats = {"chunks": []} + for chunk in chunks: + freeze_stats = freeze_non_lora_params(chunk) + trainable_stats = trainable_param_stats(chunk) + lora_stats["chunks"].append({**freeze_stats, **trainable_stats}) + + # ── optimizer (model chooses which primitive) ── + optimizer = None + finalize_grads = None + post_model_load_hook = None + if impl_cfg.optimizer == "dist_opt": + from megatron.lite.primitive.optimizers.megatron_wrap import ( + build_dist_opt_training_optimizer, + ) + + optimizer, finalize_grads = build_dist_opt_training_optimizer( + chunks, + model_cfg=model_cfg, + impl_cfg=impl_cfg, + ps=ps, + model_name="qwen3_moe", + is_expert=is_expert_param, + deterministic=deterministic, + ) + from megatron.lite.primitive.ckpt import attach_model_sharded_state_dict + + attach_model_sharded_state_dict( + chunks, ps, get_placements=PLACEMENT_FN, is_expert=is_expert_param + ) + optimizer_backend = "dist_opt" + elif impl_cfg.optimizer == "fsdp2": + optimizer_backend = "fsdp2" + + def _post_model_load_hook(): + from megatron.lite.model.qwen3_moe.lite.model import TransformerLayer + from megatron.lite.primitive.optimizers.fsdp2 import build_fsdp2_training_optimizer + + return { + "optimizer": build_fsdp2_training_optimizer( + chunks, + impl_cfg.optimizer_config, + ps, + unit_modules=(TransformerLayer,), + expert_classifier=is_expert_param, + deterministic=deterministic, + vpp=impl_cfg.parallel.vpp, + # Non-layer params stay under the root FSDP2 unit. The fused + # CE path reads head.col.linear.weight directly, and the + # embedding path is also driven from model.forward(). + leaf_module_names=(), + ) + } + + post_model_load_hook = _post_model_load_hook + elif impl_cfg.optimizer is None: + optimizer_backend = "none" + else: + raise ValueError(f"Unknown qwen3_moe lite optimizer: {impl_cfg.optimizer!r}.") + + from megatron.lite.primitive.modules.moe import MoEAuxLossAutoScaler + + def _pre_forward_hook(loss_scale): + MoEAuxLossAutoScaler.set_loss_scale(loss_scale) + MTPLossAutoScaler.set_loss_scale(loss_scale) + + return ModelBundle( + chunks=chunks, + parallel_state=ps, + optimizer=optimizer, + finalize_grads=finalize_grads, + forward_step=_forward_step if impl_cfg.use_thd else _forward_step_bshd, + extras={ + "model_cfg": model_cfg, + # Lite's router uses megatron.lite's MoEAuxLossAutoScaler; hand the + # classmethod directly as the per-microbatch hook. + "pre_forward_hook": _pre_forward_hook, + "optimizer_backend": optimizer_backend, + "post_model_load_hook": post_model_load_hook, + "lora_config": lora_config, + "lora_stats": lora_stats, + }, + ) + + +# --------------------------------------------------------------------------- +# Optional: load_hf_weights +# --------------------------------------------------------------------------- + + +def load_hf_weights( + chunk: nn.Module, hf_path: str, model_cfg: Qwen3MoEConfig, ps: ParallelState +) -> None: + """Load HF pretrained weights into model chunk.""" + if not hf_path: + return + _load_hf_weights_impl(chunk, hf_path, model_cfg, ps) + + +def export_hf_weights( + chunks: list[nn.Module], model_cfg: Qwen3MoEConfig, ps: ParallelState, **kwargs +): + """Export HF weights from model chunks.""" + from megatron.lite.model.qwen3_moe.lite.checkpoint import export_hf_weights as _export + + for chunk in chunks: + yield from _export(chunk, model_cfg, ps, **kwargs) + + +# --------------------------------------------------------------------------- +# Tooling metadata (benchmark / debug) +# --------------------------------------------------------------------------- + + +def vocab_size(model_cfg: Qwen3MoEConfig) -> int | None: + return getattr(model_cfg, "vocab_size", None) diff --git a/experimental/lite/megatron/lite/model/qwen3_moe/stats.py b/experimental/lite/megatron/lite/model/qwen3_moe/stats.py new file mode 100644 index 00000000000..77c3fd6215d --- /dev/null +++ b/experimental/lite/megatron/lite/model/qwen3_moe/stats.py @@ -0,0 +1,26 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Model-level Qwen3MoE benchmark statistics.""" + +from __future__ import annotations + +from megatron.lite.model.qwen3_moe.config import Qwen3MoEConfig + + +def activated_params(model_cfg: Qwen3MoEConfig) -> int | None: + try: + h = model_cfg.hidden_size + n_layers = model_cfg.num_hidden_layers + n_q = model_cfg.num_attention_heads + n_kv = model_cfg.num_key_value_heads + d = model_cfg.head_dim + attn = h * (n_q + n_kv + n_kv) * d + (n_q * d) * h + router = h * model_cfg.num_experts + inter = model_cfg.moe_intermediate_size + expert = h * (inter * 2) + inter * h + experts_active = model_cfg.num_experts_per_tok * expert + return int((attn + router + experts_active) * n_layers) + except (AttributeError, TypeError, ValueError): + return None + + +__all__ = ["activated_params"] diff --git a/experimental/lite/megatron/lite/model/registry.py b/experimental/lite/megatron/lite/model/registry.py new file mode 100644 index 00000000000..9ca75846ba6 --- /dev/null +++ b/experimental/lite/megatron/lite/model/registry.py @@ -0,0 +1,186 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Model and protocol registry.""" + +from __future__ import annotations + +import importlib +import json +from pathlib import Path + +# --------------------------------------------------------------------------- +# Registry tables (populated by register_model) +# --------------------------------------------------------------------------- + +# model_name → model package module path +MODEL_PACKAGES: dict[str, str] = {} + +# HF model_type string → Megatron Lite model_name +_HF_MODEL_TYPE_MAP: dict[str, str] = {} + +# (model_name, impl) → runtime_model_name +_IMPL_TO_RUNTIME_MODEL: dict[tuple[str, str], str] = {} + +# runtime_model_name → protocol module path +TRAIN_RUNTIME_MODULES: dict[str, str] = {} + +# --------------------------------------------------------------------------- +# Registration API +# --------------------------------------------------------------------------- + + +def register_model( + model_name: str, + *, + package: str, + hf_model_types: list[str] | None = None, + impls: dict[str, str] | None = None, +) -> None: + """Register a model and all its implementations in one call. + + Args: + model_name: Megatron Lite model name (e.g. ``"qwen3"``). + package: Model package module path (e.g. ``"megatron.lite.model.qwen3_moe"``). + hf_model_types: HF ``model_type`` strings that map to this model. + impls: ``{impl_name: protocol_module_path}``. + The first impl is also registered as the bare ``model_name`` runtime key. + + Example:: + + register_model( + "qwen3", + package="megatron.lite.model.qwen3_moe", + hf_model_types=["qwen3_moe", "qwen2_moe"], + impls={ + "lite": "megatron.lite.model.qwen3_moe.lite.protocol", + }, + ) + """ + MODEL_PACKAGES[model_name] = package + + if hf_model_types: + for hf_type in hf_model_types: + _HF_MODEL_TYPE_MAP[hf_type] = model_name + + if impls: + for i, (impl_name, proto_module) in enumerate(impls.items()): + runtime_key = model_name if i == 0 else f"{model_name}_{impl_name}" + _IMPL_TO_RUNTIME_MODEL[(model_name, impl_name)] = runtime_key + TRAIN_RUNTIME_MODULES[runtime_key] = proto_module + + +# --------------------------------------------------------------------------- +# Built-in models +# --------------------------------------------------------------------------- + +_QWEN3_MOE_LITE = "megatron.lite.model.qwen3_moe.lite.protocol" + +register_model( + "qwen3", + package="megatron.lite.model.qwen3_moe", + hf_model_types=["qwen3_moe", "qwen2_moe"], + impls={"lite": _QWEN3_MOE_LITE}, +) + +register_model( + "qwen3_moe", package="megatron.lite.model.qwen3_moe", impls={"lite": _QWEN3_MOE_LITE} +) + +register_model( + "qwen3_5", + package="megatron.lite.model.qwen3_5", + hf_model_types=["qwen3_5_moe"], + impls={"lite": "megatron.lite.model.qwen3_5.lite.protocol"}, +) + +register_model( + "kimi_k2", + package="megatron.lite.model.kimi_k2", + hf_model_types=["kimi_k2", "deepseek_v3"], + impls={"lite": "megatron.lite.model.kimi_k2.lite.protocol"}, +) + +register_model( + "glm5", + package="megatron.lite.model.glm5", + hf_model_types=["glm_moe_dsa"], + impls={"lite": "megatron.lite.model.glm5.lite.protocol"}, +) + +register_model( + "deepseek_v4", + package="megatron.lite.model.deepseek_v4", + hf_model_types=["deepseek_v4"], + impls={"lite": "megatron.lite.model.deepseek_v4.lite.protocol"}, +) + + +# --------------------------------------------------------------------------- +# Lookup functions +# --------------------------------------------------------------------------- + + +def get_model_package(model_name: str): + if model_name not in MODEL_PACKAGES: + raise ValueError(f"Unknown model: {model_name!r}. Available: {list(MODEL_PACKAGES)}") + return importlib.import_module(MODEL_PACKAGES[model_name]) + + +def get_train_runtime_module(model_name: str): + if model_name in TRAIN_RUNTIME_MODULES: + return importlib.import_module(TRAIN_RUNTIME_MODULES[model_name]) + raise ValueError(f"No protocol module for: {model_name!r}") + + +def resolve_runtime_model_name(model_name: str, impl: str) -> str: + key = (model_name, impl) + if key not in _IMPL_TO_RUNTIME_MODEL: + raise ValueError( + f"No runtime for ({model_name!r}, {impl!r}). " f"Known: {list(_IMPL_TO_RUNTIME_MODEL)}" + ) + return _IMPL_TO_RUNTIME_MODEL[key] + + +def resolve_model_type_from_hf(source: str | Path | dict) -> str: + """Resolve Megatron Lite model_name from an HF source. + + Args: + source: One of: + - Directory path (str/Path) containing ``config.json`` + - Path to a ``config.json`` file directly + - A dict / HF config object with a ``model_type`` key + """ + if isinstance(source, dict): + hf_config = source + elif hasattr(source, "model_type"): + # HF PretrainedConfig object + hf_config = {"model_type": source.model_type} + else: + p = Path(source) + if p.is_file(): + config_path = p + elif p.is_dir(): + config_path = p / "config.json" + else: + raise FileNotFoundError(f"Not a file or directory: {source}") + if not config_path.exists(): + raise FileNotFoundError(f"No config.json found at {source}") + with open(config_path) as f: + hf_config = json.load(f) + + hf_model_type = hf_config.get("model_type", "") + native_name = _HF_MODEL_TYPE_MAP.get(hf_model_type) + if native_name is not None: + return native_name + raise ValueError( + f"Cannot resolve model_type={hf_model_type!r}. " + f"Known: {list(_HF_MODEL_TYPE_MAP)}. Set model_name explicitly." + ) + + +__all__ = [ + "get_model_package", + "get_train_runtime_module", + "register_model", + "resolve_model_type_from_hf", + "resolve_runtime_model_name", +] diff --git a/experimental/lite/megatron/lite/primitive/__init__.py b/experimental/lite/megatron/lite/primitive/__init__.py new file mode 100644 index 00000000000..17c992fc88d --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Public primitive-layer entrypoints.""" diff --git a/experimental/lite/megatron/lite/primitive/bundle.py b/experimental/lite/megatron/lite/primitive/bundle.py new file mode 100644 index 00000000000..439968c1c6f --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/bundle.py @@ -0,0 +1,29 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""ModelBundle — return type of protocol.build_model().""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import Any + +import torch.nn as nn + +from megatron.lite.primitive.parallel.state import ParallelState + + +@dataclass +class ModelBundle: + """Everything runtime needs to run a training loop. + + Returned by protocol.build_model(). Model owns the construction + of all fields — runtime just consumes them. + """ + + chunks: list[nn.Module] + parallel_state: ParallelState + optimizer: Any | None = None + finalize_grads: Callable[[], None] | None = None + forward_step: Callable[..., dict] | None = None + # extra metadata (expert_classifier, model_cfg, etc.) + extras: dict[str, Any] = field(default_factory=dict) diff --git a/experimental/lite/megatron/lite/primitive/ckpt/__init__.py b/experimental/lite/megatron/lite/primitive/ckpt/__init__.py new file mode 100644 index 00000000000..4c2842a1220 --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/ckpt/__init__.py @@ -0,0 +1,20 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Checkpoint helpers.""" + +from megatron.lite.primitive.ckpt.dcp import load_training_checkpoint, save_training_checkpoint +from megatron.lite.primitive.ckpt.hf_weights import HFWeights + +__all__ = [ + "HFWeights", + "attach_model_sharded_state_dict", + "load_training_checkpoint", + "save_training_checkpoint", +] + + +def __getattr__(name: str): + if name == "attach_model_sharded_state_dict": + from megatron.lite.primitive.ckpt.distckpt import attach_model_sharded_state_dict + + return attach_model_sharded_state_dict + raise AttributeError(name) diff --git a/experimental/lite/megatron/lite/primitive/ckpt/dcp.py b/experimental/lite/megatron/lite/primitive/ckpt/dcp.py new file mode 100644 index 00000000000..f53fe705877 --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/ckpt/dcp.py @@ -0,0 +1,638 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +""" +DCP (Distributed Checkpoint) framework for training checkpoints. + +Model-agnostic: takes a placement function to describe how each parameter is sharded. +HF weight loading/saving is model-specific and lives in models//checkpoint.py. +""" + +from __future__ import annotations + +import os +import random +from collections.abc import Iterable +from pathlib import Path +from typing import Any + +import numpy as np +import torch # pyright: ignore[reportMissingImports] +import torch.distributed as dist # pyright: ignore[reportMissingImports] +import torch.distributed.checkpoint as dcp # pyright: ignore[reportMissingImports] +import torch.nn as nn # pyright: ignore[reportMissingImports] +from torch.distributed.device_mesh import DeviceMesh # pyright: ignore[reportMissingImports] +from torch.distributed.tensor import DTensor # pyright: ignore[reportMissingImports] + +from megatron.lite.primitive.parallel import ParallelState +from megatron.lite.primitive.protocols import ( + ExpertClassifierFn, + PlacementFn, + default_expert_classifier, + default_placement_fn, +) + + +def save_training_checkpoint( + model: nn.Module | Iterable[nn.Module], + optimizer, + step: int | str, + path: str | None = None, + config=None, + ps: ParallelState | None = None, + get_placements: PlacementFn = default_placement_fn, + is_expert: ExpertClassifierFn = default_expert_classifier, + *, + use_dcp: bool | None = True, + save_rng: bool = True, + save_model: bool = True, + save_optimizer: bool = True, +) -> None: + """Save training checkpoint using DTensor + DCP for automatic resharding.""" + if path is None and isinstance(step, str): + path = step + step = 0 + if path is None: + raise ValueError("checkpoint path is required") + step = int(step) + if use_dcp is None: + use_dcp = True + if not use_dcp: + _save_local_training_checkpoint(model, optimizer, step, path, save_rng=save_rng) + return + if _supports_dist_opt_distckpt(model, optimizer): + ckpt_path = os.path.join(path, f"step_{step}") + os.makedirs(ckpt_path, exist_ok=True) + _save_dist_opt_checkpoint( + model, optimizer, step, ckpt_path, save_model=save_model, save_optimizer=save_optimizer + ) + if save_rng: + _save_rng_sidecar(ckpt_path) + log_rank0(f"Saved dist_opt checkpoint at step {step} to {ckpt_path}") + return + if config is None or ps is None: + raise ValueError("DCP checkpointing requires config and ParallelState.") + if not isinstance(model, nn.Module): + raise TypeError("DCP checkpointing currently expects a single nn.Module.") + dense_mesh, expert_mesh = _build_meshes(config) + state_dict: dict = {"step": step} + # Pipeline stages own DIFFERENT parameters but their local layers re-index + # to 0..N, so without a per-stage prefix the DCP FQNs collide across pp ranks + # (stage0 layer0 and stage1 layer1 both -> "model.0.layers.0..."), corrupting + # the round-trip. Mirror distckpt's pp-aware keying: disjoint keyspace per stage. + model_prefix = f"model_pp{ps.pp_rank}" if ps.pp_size > 1 else "model" + + if save_model: + for name, param in model.named_parameters(): + placements = get_placements(name) + mesh = expert_mesh if is_expert(name) else dense_mesh + state_dict[f"{model_prefix}.{name}"] = _dcp_tensor_from_param(param, mesh, placements) + + ckpt_path = os.path.join(path, f"step_{step}") + os.makedirs(ckpt_path, exist_ok=True) + dcp.save(state_dict, checkpoint_id=ckpt_path) + if save_optimizer: + _save_optimizer_checkpoint(optimizer, ckpt_path) + if save_rng: + _save_rng_sidecar(ckpt_path) + log_rank0(f"Saved training checkpoint at step {step} to {ckpt_path}") + + +def load_training_checkpoint( + model: nn.Module | Iterable[nn.Module], + optimizer, + path: str, + config=None, + ps: ParallelState | None = None, + get_placements: PlacementFn = default_placement_fn, + is_expert: ExpertClassifierFn = default_expert_classifier, + *, + use_dcp: bool | None = True, + load_rng: bool = True, + load_parameter_state_update_legacy_format: bool = False, + load_model: bool = True, + load_optimizer: bool = True, +) -> int: + """Load training checkpoint with automatic resharding across different parallel configs.""" + if use_dcp is None: + use_dcp = True + if not use_dcp: + return _load_local_training_checkpoint( + model, + optimizer, + path, + load_rng=load_rng, + load_parameter_state_update_legacy_format=load_parameter_state_update_legacy_format, + ) + ckpt_path = _resolve_step_checkpoint_path(path) + if _supports_dist_opt_distckpt(model, optimizer): + step = _load_dist_opt_checkpoint( + model, optimizer, ckpt_path, load_model=load_model, load_optimizer=load_optimizer + ) + if load_rng: + _load_rng_sidecar(ckpt_path) + log_rank0(f"Loaded dist_opt checkpoint from {path} at step {step}") + return step + if config is None or ps is None: + raise ValueError("DCP checkpointing requires config and ParallelState.") + if not isinstance(model, nn.Module): + raise TypeError("DCP checkpointing currently expects a single nn.Module.") + dense_mesh, expert_mesh = _build_meshes(config) + + state_dict: dict = {"step": 0} + # Same pp-aware keying as save (see save_training_checkpoint): per-stage + # disjoint keyspace so pp ranks don't read each other's colliding FQNs. + model_prefix = f"model_pp{ps.pp_rank}" if ps.pp_size > 1 else "model" + + if load_model: + for name, param in model.named_parameters(): + placements = get_placements(name) + mesh = expert_mesh if is_expert(name) else dense_mesh + state_dict[f"{model_prefix}.{name}"] = _empty_dcp_tensor_like_param( + param, mesh, placements + ) + + dcp.load(state_dict, checkpoint_id=ckpt_path) + + if load_model: + for name, param in model.named_parameters(): + key = f"{model_prefix}.{name}" + if key in state_dict: + t = state_dict[key] + with torch.no_grad(): + _copy_tensor_(param, t) + + if load_optimizer: + _load_optimizer_checkpoint(optimizer, ckpt_path) + + step = state_dict.get("step", 0) + if load_rng: + _load_rng_sidecar(ckpt_path) + log_rank0(f"Loaded training checkpoint from {path} at step {step}") + return step + + +def _resolve_step_checkpoint_path(path: str) -> str: + if os.path.basename(path).startswith("step_"): + return path + + step_dirs = sorted( + [d for d in os.listdir(path) if d.startswith("step_")], key=lambda d: int(d.split("_")[1]) + ) + if step_dirs: + return os.path.join(path, step_dirs[-1]) + return path + + +def _supports_dist_opt_distckpt(model: nn.Module | Iterable[nn.Module], optimizer) -> bool: + try: + from megatron.lite.primitive.ckpt.distckpt import supports_dist_opt_distckpt + except ModuleNotFoundError as exc: + if exc.name != "megatron.core": + raise + return False + + return supports_dist_opt_distckpt(model, optimizer) + + +def _save_dist_opt_checkpoint( + model: nn.Module | Iterable[nn.Module], + optimizer, + step: int, + path: str, + *, + save_model: bool, + save_optimizer: bool, +) -> None: + from megatron.lite.primitive.ckpt.distckpt import save_dist_opt_checkpoint + + save_dist_opt_checkpoint( + model, optimizer, step, path, save_model=save_model, save_optimizer=save_optimizer + ) + + +def _load_dist_opt_checkpoint( + model: nn.Module | Iterable[nn.Module], + optimizer, + path: str, + *, + load_model: bool, + load_optimizer: bool, +) -> int: + from megatron.lite.primitive.ckpt.distckpt import load_dist_opt_checkpoint + + return load_dist_opt_checkpoint( + model, optimizer, path, load_model=load_model, load_optimizer=load_optimizer + ) + + +def _optimizer_checkpoint_path(path: str) -> str: + rank = dist.get_rank() if dist.is_initialized() else 0 + return os.path.join(path, f"optimizer_rank_{rank}.pt") + + +def _save_optimizer_checkpoint(optimizer, path: str) -> None: + if optimizer is None: + log_rank0("Skipping optimizer checkpoint save because optimizer is None") + return + state_dict_fn = getattr(optimizer, "state_dict", None) + if not callable(state_dict_fn): + raise TypeError(f"Optimizer {type(optimizer).__name__} does not provide state_dict().") + torch.save(state_dict_fn(), _optimizer_checkpoint_path(path)) + + +def _load_optimizer_checkpoint(optimizer, path: str) -> None: + if optimizer is None: + log_rank0("Skipping optimizer checkpoint load because optimizer is None") + return + ckpt_path = _optimizer_checkpoint_path(path) + if not os.path.exists(ckpt_path): + log_rank0(f"No optimizer checkpoint found at {ckpt_path}; loading model state only") + return + load_state_dict_fn = getattr(optimizer, "load_state_dict", None) + if not callable(load_state_dict_fn): + raise TypeError(f"Optimizer {type(optimizer).__name__} does not provide load_state_dict().") + state = torch.load(ckpt_path, map_location="cpu", weights_only=False) + load_state_dict_fn(state) + + +def _model_chunks(model: nn.Module | Iterable[nn.Module]) -> list[nn.Module]: + if isinstance(model, nn.Module): + return [model] + chunks = list(model) + if not all(isinstance(chunk, nn.Module) for chunk in chunks): + raise TypeError("checkpoint model chunks must be nn.Module instances.") + return chunks + + +def _to_local_tensor(tensor: Any) -> torch.Tensor: + local_tensor = getattr(tensor, "_local_tensor", None) + if isinstance(local_tensor, torch.Tensor): + return local_tensor + to_local = getattr(tensor, "to_local", None) + if callable(to_local): + return to_local() + return tensor + + +def _is_dtensor_like(tensor: Any) -> bool: + return ( + callable(getattr(tensor, "to_local", None)) + and hasattr(tensor, "device_mesh") + and hasattr(tensor, "placements") + ) + + +def _dcp_tensor_from_param(param: torch.Tensor, mesh: DeviceMesh, placements: list) -> DTensor: + if _is_dtensor_like(param): + return _dtensor_from_dtensor_like_param(param, _to_local_tensor(param).detach()) + return DTensor.from_local(_to_local_tensor(param).detach(), mesh, placements) + + +def _empty_dcp_tensor_like_param( + param: torch.Tensor, mesh: DeviceMesh, placements: list +) -> DTensor: + if _is_dtensor_like(param): + return _dtensor_from_dtensor_like_param(param, torch.empty_like(_to_local_tensor(param))) + return DTensor.from_local(torch.empty_like(_to_local_tensor(param)), mesh, placements) + + +def _dtensor_from_dtensor_like_param(param: torch.Tensor, local_tensor: torch.Tensor) -> DTensor: + return DTensor.from_local( + local_tensor, + param.device_mesh, + param.placements, + shape=tuple(param.shape), + stride=tuple(param.stride()), + ) + + +def _copy_tensor_(target: torch.Tensor, src: torch.Tensor) -> None: + local_target = _to_local_tensor(target) + local_src = _to_local_tensor(src).to(device=local_target.device, dtype=local_target.dtype) + if isinstance(local_target, torch.Tensor) and local_target is not target: + local_target.copy_(local_src) + else: + target.copy_(local_src) + + +def _chunk_tensor_state(module: nn.Module) -> dict[str, torch.Tensor]: + state: dict[str, torch.Tensor] = {} + for name, param in module.named_parameters(): + state[f"param.{name}"] = _to_local_tensor(param.detach()).cpu().clone() + for name, buffer in module.named_buffers(): + state[f"buffer.{name}"] = _to_local_tensor(buffer.detach()).cpu().clone() + return state + + +def _load_chunk_tensor_state(module: nn.Module, state: dict[str, torch.Tensor]) -> None: + params = dict(module.named_parameters()) + buffers = dict(module.named_buffers()) + missing: list[str] = [] + for key, src in state.items(): + kind, name = key.split(".", 1) + if kind == "param" and name in params: + with torch.no_grad(): + _copy_tensor_(params[name], src) + elif kind == "buffer" and name in buffers: + with torch.no_grad(): + _copy_tensor_(buffers[name], src) + else: + missing.append(key) + if missing: + raise RuntimeError(f"checkpoint contains unknown tensor keys: {missing}") + + +def _local_checkpoint_file(path: str | os.PathLike[str]) -> Path: + ckpt_path = Path(path) + if ckpt_path.is_dir() or ckpt_path.suffix == "": + if _is_distributed_checkpoint_ranked(): + return ckpt_path / f"training_state_{_rank_suffix()}.pt" + return ckpt_path / "training_state.pt" + return ckpt_path + + +def _local_optimizer_parameter_state_file(ckpt_file: Path) -> Path: + return ckpt_file.with_name(f"{ckpt_file.stem}.optimizer_parameter_state{ckpt_file.suffix}") + + +def _rank_suffix() -> str: + if dist.is_available() and dist.is_initialized(): + return f"rank_{dist.get_rank():05d}" + return "rank_00000" + + +def _is_distributed_checkpoint_ranked() -> bool: + return dist.is_available() and dist.is_initialized() + + +def _rng_sidecar_file(path: str | os.PathLike[str]) -> Path: + return Path(path) / f"rng_state_{_rank_suffix()}.pt" + + +def _cpu_clone(tensor: torch.Tensor | None) -> torch.Tensor | None: + if tensor is None: + return None + return tensor.detach().cpu().clone() + + +def _get_cuda_rng_state() -> torch.Tensor | None: + if not torch.cuda.is_initialized(): + return None + return _cpu_clone(torch.cuda.get_rng_state()) + + +def _get_cuda_rng_tracker_states() -> dict[str, torch.Tensor]: + if not torch.cuda.is_initialized(): + return {} + + from megatron.core import tensor_parallel + + states = tensor_parallel.get_cuda_rng_tracker().get_states() + return {name: _cpu_clone(state) for name, state in states.items() if state is not None} + + +def _get_rng_state() -> dict[str, Any]: + return { + "random_rng_state": random.getstate(), + "np_rng_state": np.random.get_state(), + "torch_rng_state": _cpu_clone(torch.get_rng_state()), + "cuda_rng_state": _get_cuda_rng_state(), + "rng_tracker_states": _get_cuda_rng_tracker_states(), + } + + +def _restore_cuda_rng_tracker_states(states: dict[str, torch.Tensor]) -> None: + if not states or not torch.cuda.is_initialized(): + return + try: + from megatron.core import tensor_parallel + + tracker = tensor_parallel.get_cuda_rng_tracker() + graph_safe = tensor_parallel.is_graph_safe_cuda_rng_tracker(tracker) + restored = { + name: tensor_parallel.convert_cuda_rng_state(state, to_graphable=graph_safe) + for name, state in states.items() + } + tracker.set_states(restored) + except Exception as exc: + raise RuntimeError("Failed to restore Megatron tensor-parallel RNG tracker state.") from exc + + +def _restore_rng_state(state: dict[str, Any] | None) -> None: + if not state: + return + random.setstate(state["random_rng_state"]) + np.random.set_state(state["np_rng_state"]) + torch.set_rng_state(state["torch_rng_state"]) + cuda_rng_state = state.get("cuda_rng_state") + if cuda_rng_state is not None and torch.cuda.is_initialized(): + torch.cuda.set_rng_state(cuda_rng_state) + _restore_cuda_rng_tracker_states(state.get("rng_tracker_states", {})) + + +def _save_rng_sidecar(path: str | os.PathLike[str]) -> None: + rng_file = _rng_sidecar_file(path) + rng_file.parent.mkdir(parents=True, exist_ok=True) + torch.save(_get_rng_state(), rng_file) + + +def _load_rng_sidecar(path: str | os.PathLike[str]) -> None: + rng_file = _rng_sidecar_file(path) + if not rng_file.exists(): + log_rank0(f"RNG sidecar not found at {rng_file}; skipping RNG restore.") + return + _restore_rng_state(torch.load(rng_file, map_location="cpu", weights_only=False)) + + +def _save_local_training_checkpoint( + model: nn.Module | Iterable[nn.Module], + optimizer, + step: int, + path: str, + *, + save_rng: bool = True, +) -> None: + chunks = _model_chunks(model) + ckpt_file = _local_checkpoint_file(path) + ckpt_file.parent.mkdir(parents=True, exist_ok=True) + save_parameter_state = getattr(optimizer, "save_parameter_state", None) + optimizer_parameter_state_file = ( + _local_optimizer_parameter_state_file(ckpt_file) if callable(save_parameter_state) else None + ) + state = { + "format": "megatron_lite.local_training.v1", + "step": int(step), + "model": [_chunk_tensor_state(chunk) for chunk in chunks], + "optimizer": optimizer.state_dict() if optimizer is not None else None, + "optimizer_parameter_state": ( + optimizer_parameter_state_file.name + if optimizer_parameter_state_file is not None + else None + ), + "rng_state": _get_rng_state() if save_rng else None, + } + torch.save(state, ckpt_file) + if optimizer_parameter_state_file is not None: + save_parameter_state(str(optimizer_parameter_state_file)) + log_rank0(f"Saved local training checkpoint at step {step} to {ckpt_file}") + + +def _load_local_training_checkpoint( + model: nn.Module | Iterable[nn.Module], + optimizer, + path: str, + *, + load_rng: bool = True, + load_parameter_state_update_legacy_format: bool = False, +) -> int: + ckpt_file = _local_checkpoint_file(path) + state = torch.load(ckpt_file, map_location="cpu", weights_only=False) + if state.get("format") != "megatron_lite.local_training.v1": + raise RuntimeError(f"Unsupported local checkpoint format in {ckpt_file}") + chunks = _model_chunks(model) + chunk_states = state.get("model") + if not isinstance(chunk_states, list) or len(chunk_states) != len(chunks): + raise RuntimeError("Checkpoint model chunk count does not match target model.") + for chunk, chunk_state in zip(chunks, chunk_states, strict=True): + _load_chunk_tensor_state(chunk, chunk_state) + if optimizer is not None and state.get("optimizer") is not None: + optimizer.load_state_dict(state["optimizer"]) + parameter_state_name = state.get("optimizer_parameter_state") + load_parameter_state = getattr(optimizer, "load_parameter_state", None) + if parameter_state_name is not None and callable(load_parameter_state): + load_parameter_state( + str(ckpt_file.with_name(parameter_state_name)), + update_legacy_format=load_parameter_state_update_legacy_format, + ) + else: + reload_model_params = getattr(optimizer, "reload_model_params", None) + if callable(reload_model_params): + reload_model_params() + if load_rng: + _restore_rng_state(state.get("rng_state")) + step = int(state.get("step", 0)) + log_rank0(f"Loaded local training checkpoint from {ckpt_file} at step {step}") + return step + + +def _build_meshes(config): + """Build separate meshes for dense and expert parameters. + + Dense mesh [PP, DP, CP, TP] — matches init_parallel dense decomposition. + Expert mesh [PP, EDP, EP, ETP] — matches init_parallel expert decomposition. + + Both meshes use C-order layout so the innermost (rightmost) dimension + corresponds to the fastest-changing rank index, consistent with + init_parallel's rank = (...) * inner_size + inner_rank formula. + """ + ws = dist.get_world_size() + tp = int(config.tp or 1) + ep = int(config.ep or 1) + etp = max(int(config.etp or 1), 1) + cp = max(int(config.cp or 1), 1) + pp = max(int(config.pp or 1), 1) + + dense_dp = ws // (tp * cp * pp) + expert_dp = ws // (etp * ep * pp) + + ranks = torch.arange(ws) + dense_mesh = DeviceMesh("cuda", ranks.reshape(pp, dense_dp, cp, tp)) + expert_mesh = DeviceMesh("cuda", ranks.reshape(pp, expert_dp, ep, etp)) + return dense_mesh, expert_mesh + + +def log_rank0(msg: str) -> None: + if not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0: + print(f"[megatron.lite] {msg}", flush=True) + + +# ====================================================================== +# QKV / FC1 canonicalize for DCP (interleaved-TP ↔ canonical layout) +# ====================================================================== + + +def _ag(data, size, group, dim=0): + from megatron.lite.primitive.ckpt.hf_weights import allgather_concat + + return allgather_concat(data, size, group, dim) + + +def canonicalize_qkv_for_dcp(model, num_attention_heads, num_key_value_heads, head_dim, ps): + """Rearrange fused QKV from interleaved-TP to canonical (Q|K|V) for DCP save.""" + if ps.tp_size <= 1: + return + from megatron.lite.primitive.utils import ensure_divisible + + nq = ensure_divisible(num_attention_heads, ps.tp_size) * head_dim + nkv = ensure_divisible(num_key_value_heads, ps.tp_size) * head_dim + for name, param in model.named_parameters(): + if "qkv" not in name or "layer_norm" in name: + continue + full = _ag(param.data, ps.tp_size, ps.tp_group) + cs = param.data.shape[0] + q, k, v = [], [], [] + for r in range(ps.tp_size): + s = full[r * cs : (r + 1) * cs] + q.append(s[:nq]) + k.append(s[nq : nq + nkv]) + v.append(s[nq + nkv :]) + canon = torch.cat([torch.cat(q), torch.cat(k), torch.cat(v)], dim=0) + param.data.copy_(canon.chunk(ps.tp_size, dim=0)[ps.tp_rank]) + + +def decanon_qkv_after_dcp(model, num_attention_heads, num_key_value_heads, head_dim, ps): + """Reverse of canonicalize_qkv_for_dcp.""" + if ps.tp_size <= 1: + return + qs = num_attention_heads * head_dim + kvs = num_key_value_heads * head_dim + for name, param in model.named_parameters(): + if "qkv" not in name or "layer_norm" in name: + continue + full = _ag(param.data, ps.tp_size, ps.tp_group) + ql = full[:qs].chunk(ps.tp_size)[ps.tp_rank] + kl = full[qs : qs + kvs].chunk(ps.tp_size)[ps.tp_rank] + vl = full[qs + kvs :].chunk(ps.tp_size)[ps.tp_rank] + param.data.copy_(torch.cat([ql, kl, vl], dim=0)) + + +def canonicalize_fc1_for_dcp(model, ps): + """Rearrange fused gate-up FC1 from interleaved-ETP to canonical for DCP save.""" + if ps.etp_size <= 1: + return + for name, param in model.named_parameters(): + if "experts" not in name or "fc1" not in name: + continue + full = _ag(param.data, ps.etp_size, ps.etp_group) + cs = param.data.shape[0] + ffn = cs // 2 + g, u = [], [] + for r in range(ps.etp_size): + s = full[r * cs : (r + 1) * cs] + g.append(s[:ffn]) + u.append(s[ffn:]) + canon = torch.cat([torch.cat(g), torch.cat(u)], dim=0) + param.data.copy_(canon.chunk(ps.etp_size, dim=0)[ps.etp_rank]) + + +def decanon_fc1_after_dcp(model, ps): + """Reverse of canonicalize_fc1_for_dcp.""" + if ps.etp_size <= 1: + return + for name, param in model.named_parameters(): + if "experts" not in name or "fc1" not in name: + continue + full = _ag(param.data, ps.etp_size, ps.etp_group) + ffn = full.shape[0] // 2 + gl = full[:ffn].chunk(ps.etp_size)[ps.etp_rank] + ul = full[ffn:].chunk(ps.etp_size)[ps.etp_rank] + param.data.copy_(torch.cat([gl, ul], dim=0)) + + +__all__ = [ + "canonicalize_fc1_for_dcp", + "canonicalize_qkv_for_dcp", + "decanon_fc1_after_dcp", + "decanon_qkv_after_dcp", + "load_training_checkpoint", + "save_training_checkpoint", +] diff --git a/experimental/lite/megatron/lite/primitive/ckpt/distckpt.py b/experimental/lite/megatron/lite/primitive/ckpt/distckpt.py new file mode 100644 index 00000000000..8c466b4b596 --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/ckpt/distckpt.py @@ -0,0 +1,567 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Megatron Core distributed checkpoint bridge for MLite dist_opt.""" + +from __future__ import annotations + +import os +from collections.abc import Callable, Iterable, MutableMapping +from dataclasses import replace +from types import MethodType +from typing import Any + +import torch +import torch.nn as nn + +from megatron.core import dist_checkpointing +from megatron.core.dist_checkpointing.mapping import ShardedTensor +from megatron.lite.primitive.parallel import ParallelState +from megatron.lite.primitive.protocols import ( + ExpertClassifierFn, + PlacementFn, + default_expert_classifier, + default_placement_fn, +) + +_DISTOPT_METADATA = { + "distrib_optim_sharding_type": "fully_reshardable", + "distrib_optim_fully_reshardable_mem_efficient": False, + "chained_optim_avoid_prefix": True, +} + + +def attach_model_sharded_state_dict( + model_chunks: Iterable[nn.Module], + ps: ParallelState, + *, + get_placements: PlacementFn = default_placement_fn, + is_expert: ExpertClassifierFn = default_expert_classifier, +) -> None: + """Attach an MLite-local mcore sharded_state_dict method to dist_opt chunks.""" + + for chunk in model_chunks: + chunk.sharded_state_dict = MethodType( # type: ignore[method-assign] + _build_bound_sharded_state_dict(ps, get_placements, is_expert), chunk + ) + chunk._mlite_dist_opt_sharded_state_dict = True # type: ignore[attr-defined] + chunk._mlite_dist_opt_parallel_state = ps # type: ignore[attr-defined] + + +def supports_dist_opt_distckpt(model: nn.Module | Iterable[nn.Module], optimizer: Any) -> bool: + """Return whether this model/optimizer pair can use mcore dist_checkpointing.""" + + if optimizer is not None and not callable(getattr(optimizer, "sharded_state_dict", None)): + return False + return all( + bool(getattr(chunk, "_mlite_dist_opt_sharded_state_dict", False)) + and callable(getattr(chunk, "sharded_state_dict", None)) + for chunk in _model_chunks(model) + ) + + +def save_dist_opt_checkpoint( + model: nn.Module | Iterable[nn.Module], + optimizer: Any, + step: int, + checkpoint_dir: str, + *, + save_model: bool = True, + save_optimizer: bool = True, +) -> None: + """Save model and DistributedOptimizer state through mcore dist_checkpointing.""" + + os.makedirs(checkpoint_dir, exist_ok=True) + metadata = _dist_opt_checkpoint_metadata(optimizer) + model_sd = _model_sharded_state_dict(model) if save_model or save_optimizer else {} + state_dict: dict[str, Any] = {"step": int(step)} + if save_model: + state_dict.update(model_sd) + if save_optimizer and optimizer is not None: + _synchronize_native_optimizer_steps(optimizer) + patches = _patch_empty_native_optimizer_state_dicts(optimizer, fallback_step=step) + try: + state_dict["optimizer"] = optimizer.sharded_state_dict( + _single_or_all_model_state(model_sd), metadata=metadata + ) + finally: + _restore_state_dict_patches(patches) + dist_checkpointing.save( + state_dict, + checkpoint_dir, + validate_access_integrity=False, + content_metadata=metadata, + ) + + +def load_dist_opt_checkpoint( + model: nn.Module | Iterable[nn.Module], + optimizer: Any, + checkpoint_dir: str, + *, + load_model: bool = True, + load_optimizer: bool = True, +) -> int: + """Load a mcore dist_checkpointing checkpoint into model and DistributedOptimizer.""" + + metadata = _dist_opt_checkpoint_metadata(optimizer) + model_sd = _model_sharded_state_dict(model) if load_model or load_optimizer else {} + load_sd: dict[str, Any] = {"step": 0} + if load_model: + load_sd.update(model_sd) + if load_optimizer and optimizer is not None: + patches = _patch_empty_native_optimizer_state_dicts(optimizer, fallback_step=0) + try: + load_sd["optimizer"] = optimizer.sharded_state_dict( + _single_or_all_model_state(model_sd), is_loading=True, metadata=metadata + ) + finally: + _restore_state_dict_patches(patches) + # torch>=2.6 flips torch.load's weights_only default to True, which rejects the trusted dist_opt + # common state (mcore's load_common torch.loads optimizer/scheduler classes like AdamW). We are + # loading our OWN checkpoint -> force weights_only=False for the duration of the load. + _orig_torch_load = torch.load + + def _trusted_torch_load(*args, **kwargs): + kwargs.setdefault("weights_only", False) + return _orig_torch_load(*args, **kwargs) + + torch.load = _trusted_torch_load + try: + state_dict = dist_checkpointing.load( + load_sd, checkpoint_dir, validate_access_integrity=False + ) + finally: + torch.load = _orig_torch_load + if load_model: + _load_model_state_dict(model, state_dict) + if load_optimizer and optimizer is not None and "optimizer" in state_dict: + load_patches = _patch_native_optimizer_step_load(optimizer) + try: + optimizer.load_state_dict(state_dict["optimizer"]) + finally: + _restore_set_state_patches(load_patches) + _synchronize_native_optimizer_steps(optimizer) + elif load_model and optimizer is not None: + reload_model_params = getattr(optimizer, "reload_model_params", None) + if callable(reload_model_params): + reload_model_params() + return int(state_dict.get("step", 0)) + + +def _synchronize_native_optimizer_steps(optimizer: Any) -> None: + """Align torch optimizer per-parameter steps before mcore fallback checkpointing.""" + + seen: set[int] = set() + + def visit(obj: Any) -> None: + obj_id = id(obj) + if obj_id in seen: + return + seen.add(obj_id) + + for child in _iter_optimizer_children(obj): + visit(child) + + state = getattr(obj, "state", None) + if isinstance(state, MutableMapping): + _synchronize_step_mapping(state) + + visit(optimizer) + + +def _patch_empty_native_optimizer_state_dicts( + optimizer: Any, *, fallback_step: int +) -> list[tuple[Any, Any]]: + patches: list[tuple[Any, Any]] = [] + for dist_opt in _iter_distributed_optimizers(optimizer): + inner = getattr(dist_opt, "optimizer", None) + state = getattr(inner, "state", None) + if not isinstance(state, MutableMapping) or state: + continue + original_state_dict = dist_opt.state_dict + + def patched_state_dict( + original_state_dict=original_state_dict, dist_opt=dist_opt, fallback_step=fallback_step + ): + try: + return original_state_dict() + except AssertionError: + return _empty_native_optimizer_state_dict(dist_opt, fallback_step) + + dist_opt.state_dict = patched_state_dict # type: ignore[method-assign] + patches.append((dist_opt, original_state_dict)) + return patches + + +def _restore_state_dict_patches(patches: list[tuple[Any, Any]]) -> None: + for dist_opt, original_state_dict in patches: + dist_opt.state_dict = original_state_dict # type: ignore[method-assign] + + +def _patch_native_optimizer_step_load(optimizer: Any) -> list[tuple[Any, Any]]: + patches: list[tuple[Any, Any]] = [] + for dist_opt in _iter_distributed_optimizers(optimizer): + original_set_state = dist_opt._set_main_param_and_optimizer_states + + def patched_set_state( + model_param, tensors, dist_opt=dist_opt, original_set_state=original_set_state + ): + removed_step = _pop_optimizer_step_for_model_param(dist_opt, model_param, tensors) + try: + return original_set_state(model_param, tensors) + finally: + if removed_step is not None: + state, step = removed_step + state["step"] = step + + dist_opt._set_main_param_and_optimizer_states = patched_set_state # type: ignore[method-assign] + patches.append((dist_opt, original_set_state)) + return patches + + +def _restore_set_state_patches(patches: list[tuple[Any, Any]]) -> None: + for dist_opt, original_set_state in patches: + dist_opt._set_main_param_and_optimizer_states = original_set_state # type: ignore[method-assign] + + +def _pop_optimizer_step_for_model_param( + dist_opt: Any, model_param, tensors: dict[str, Any] +) -> tuple[MutableMapping, Any] | None: + if "step" in tensors: + return None + try: + group_index, group_order = dist_opt.model_param_group_index_map[model_param] + main_param = dist_opt.optimizer.param_groups[group_index]["params"][group_order] + state = dist_opt.optimizer.state[main_param] + except (KeyError, IndexError, TypeError): + return None + if not isinstance(state, MutableMapping) or "step" not in state: + return None + return state, state.pop("step") + + +def _iter_distributed_optimizers(optimizer: Any) -> Iterable[Any]: + seen: set[int] = set() + + def visit(obj: Any): + obj_id = id(obj) + if obj_id in seen: + return + seen.add(obj_id) + + inner = _safe_inner_optimizer(obj) + if ( + callable(getattr(obj, "sharded_state_dict", None)) + and hasattr(obj, "gbuf_ranges") + and hasattr(obj, "buffers") + and inner is not None + ): + yield obj + + for child in _iter_optimizer_children(obj, known_inner=inner): + yield from visit(child) + + yield from visit(optimizer) + + +def _dist_opt_checkpoint_metadata(optimizer: Any) -> dict[str, Any]: + dist_opts = tuple(_iter_distributed_optimizers(optimizer)) + return { + **_DISTOPT_METADATA, + "distrib_optim_fully_reshardable_mem_efficient": bool(dist_opts) and all(getattr(opt, "data_parallel_group_gloo", None) is not None for opt in dist_opts), + } + + +def _iter_optimizer_children(obj: Any, *, known_inner: Any | None = None) -> Iterable[Any]: + chained = getattr(obj, "chained_optimizers", None) + if isinstance(chained, Iterable): + yield from chained + + sub_optimizers = getattr(obj, "sub_optimizers", None) + if isinstance(sub_optimizers, Iterable): + yield from sub_optimizers + + inner = _safe_inner_optimizer(obj) if known_inner is None else known_inner + if inner is not None and inner is not obj: + yield inner + + +def _safe_inner_optimizer(obj: Any) -> Any | None: + if isinstance(getattr(obj, "chained_optimizers", None), Iterable): + # Megatron-Core ChainedOptimizer exposes `.optimizer` only for the + # single-optimizer compatibility case; multi-optimizer PP/EP chains + # assert on access. The children above are the real traversal targets. + return None + return getattr(obj, "optimizer", None) + + +def _empty_native_optimizer_state_dict(dist_opt: Any, fallback_step: int) -> dict[str, Any]: + inner_state_dict = dist_opt.optimizer.state_dict() + optimizer_state = { + key: ([group.copy() for group in value] if key == "param_groups" else value) + for key, value in inner_state_dict.items() + if key != "state" + } + for param_group in optimizer_state["param_groups"]: + param_group.pop("params", None) + param_group["step"] = int(fallback_step) + state_dict: dict[str, Any] = {"optimizer": optimizer_state} + grad_scaler = getattr(dist_opt, "grad_scaler", None) + if grad_scaler: + state_dict["grad_scaler"] = grad_scaler.state_dict() + return state_dict + + +def _synchronize_step_mapping(state: MutableMapping) -> None: + steps: list[Any] = [] + for param_state in state.values(): + if isinstance(param_state, MutableMapping) and "step" in param_state: + steps.append(param_state["step"]) + if not steps: + return + target = max(_step_as_int(step) for step in steps) + for param_state in state.values(): + if isinstance(param_state, MutableMapping) and "step" in param_state: + param_state["step"] = _step_like(param_state["step"], target) + + +def _step_as_int(step: Any) -> int: + if isinstance(step, torch.Tensor): + return int(step.detach().cpu().item()) + return int(step) + + +def _step_like(reference: Any, value: int) -> Any: + if isinstance(reference, torch.Tensor): + return torch.full_like(reference, value) + return value + + +def _build_bound_sharded_state_dict( + ps: ParallelState, get_placements: PlacementFn, is_expert: ExpertClassifierFn +) -> Callable: + def sharded_state_dict( + self, + prefix: str = "", + sharded_offsets: tuple[tuple[int, int, int], ...] = (), + metadata: dict | None = None, + ) -> dict[str, ShardedTensor]: + del metadata + return _module_sharded_state_dict( + _wrapped_module(self), + ps, + get_placements=get_placements, + is_expert=is_expert, + prefix=prefix, + sharded_offsets=sharded_offsets, + ) + + return sharded_state_dict + + +def _module_sharded_state_dict( + module: nn.Module, + ps: ParallelState, + *, + get_placements: PlacementFn, + is_expert: ExpertClassifierFn, + prefix: str = "", + sharded_offsets: tuple[tuple[int, int, int], ...] = (), +) -> dict[str, ShardedTensor]: + state: dict[str, ShardedTensor] = {} + for name, param in module.named_parameters(): + state[f"{prefix}{name}"] = _make_sharded_tensor( + f"{prefix}{name}", + param, + ps, + placements=get_placements(name), + expert=is_expert(name), + sharded_offsets=sharded_offsets, + ) + for name, buffer in module.named_buffers(): + state[f"{prefix}{name}"] = _make_sharded_tensor( + f"{prefix}{name}", + buffer, + ps, + placements=get_placements(name), + expert=is_expert(name), + sharded_offsets=sharded_offsets, + ) + return state + + +def _make_sharded_tensor( + key: str, + tensor: torch.Tensor, + ps: ParallelState, + *, + placements: list, + expert: bool, + sharded_offsets: tuple[tuple[int, int, int], ...] = (), +) -> ShardedTensor: + rank_offsets, replica_id = _rank_offsets_and_replica_id(placements, ps, expert=expert) + return ShardedTensor.from_rank_offsets( + key, tensor, *sharded_offsets, *rank_offsets, replica_id=replica_id + ) + + +def _rank_offsets_and_replica_id( + placements: list, ps: ParallelState, *, expert: bool +) -> tuple[tuple[tuple[int, int, int], ...], tuple[int, ...]]: + ranks, sizes = _mesh_ranks_and_sizes(ps, expert=expert) + axis_fragments: dict[int, tuple[int, int]] = {} + for placement, rank, size in zip(placements, ranks, sizes, strict=True): + if _is_shard_placement(placement): + dim = _shard_dim(placement) + if dim is None: + raise ValueError(f"Unsupported Shard placement without dim: {placement!r}.") + prev_rank, prev_size = axis_fragments.get(dim, (0, 1)) + axis_fragments[dim] = (prev_rank * size + rank, prev_size * size) + rank_offsets = tuple((dim, rank, size) for dim, (rank, size) in axis_fragments.items()) + return rank_offsets, _replica_id(placements, ps, expert=expert) + + +def _replica_id(placements: list, ps: ParallelState, *, expert: bool) -> tuple[int, int, int]: + # PP stages own different parameters. They are not replicas of one + # another, so PP rank must not make a shard non-main. + if expert: + return ( + 0, + _replica_axis_rank(placements, 2, ps.ep_rank), + _replica_axis_rank(placements, 1, ps.expert_dp_rank), + ) + dp_cp_rank = ( + 0 + if _placement_is_sharded(placements, 1) or _placement_is_sharded(placements, 2) + else ps.dp_cp_rank + ) + return (0, _replica_axis_rank(placements, 3, ps.tp_rank), int(dp_cp_rank)) + + +def _replica_axis_rank(placements: list, axis: int, rank: int) -> int: + return 0 if _placement_is_sharded(placements, axis) else int(rank) + + +def _placement_is_sharded(placements: list, axis: int) -> bool: + return axis < len(placements) and _is_shard_placement(placements[axis]) + + +def _mesh_ranks_and_sizes(ps: ParallelState, *, expert: bool) -> tuple[list[int], list[int]]: + if expert: + return ( + [ps.pp_rank, ps.expert_dp_rank, ps.ep_rank, ps.etp_rank], + [ps.pp_size, ps.expert_dp_size, ps.ep_size, ps.etp_size], + ) + return ( + [ps.pp_rank, ps.dp_rank, ps.cp_rank, ps.tp_rank], + [ps.pp_size, ps.dp_size, ps.cp_size, ps.tp_size], + ) + + +def _is_shard_placement(placement: Any) -> bool: + return type(placement).__name__ == "Shard" + + +def _shard_dim(placement: Any) -> int | None: + dim = getattr(placement, "dim", None) + if dim is None: + dim = getattr(placement, "_dim", None) + return None if dim is None else int(dim) + + +def _model_sharded_state_dict(model: nn.Module | Iterable[nn.Module]) -> dict[str, Any]: + chunks = _model_chunks(model) + ps = _chunk_parallel_state(chunks[0]) if chunks else None + return { + _model_chunk_key(ps, idx, len(chunks)): _chunk_sharded_state_dict( + chunk, _model_chunk_sharded_key_prefix(ps, idx, len(chunks)) + ) + for idx, chunk in enumerate(chunks) + } + + +def _model_chunk_key(ps: ParallelState | None, idx: int, num_chunks: int) -> str: + if ps is not None and ps.pp_size > 1: + key = f"model_pp{ps.pp_rank}" + if num_chunks > 1: + key = f"{key}_vpp{idx}" + return key + if num_chunks == 1: + return "model" + return f"model{idx}" + + +def _model_chunk_sharded_key_prefix(ps: ParallelState | None, idx: int, num_chunks: int) -> str: + if ps is None and num_chunks == 1: + return "" + if ps is not None and ps.pp_size <= 1 and num_chunks == 1: + return "" + return f"{_model_chunk_key(ps, idx, num_chunks)}." + + +def _chunk_sharded_state_dict(chunk: nn.Module, sharded_key_prefix: str) -> dict[str, Any]: + chunk_sd = chunk.sharded_state_dict() # type: ignore[attr-defined] + if not sharded_key_prefix: + return chunk_sd + return { + key: ( + replace(value, key=f"{sharded_key_prefix}{value.key}") + if isinstance(value, ShardedTensor) + else value + ) + for key, value in chunk_sd.items() + } + + +def _single_or_all_model_state(model_sd: dict[str, Any]) -> dict[str, Any]: + if "model" in model_sd: + return model_sd["model"] + return model_sd + + +def _load_model_state_dict( + model: nn.Module | Iterable[nn.Module], state_dict: dict[str, Any] +) -> None: + chunks = _model_chunks(model) + if len(chunks) == 1 and "model" in state_dict: + _wrapped_module(chunks[0]).load_state_dict(state_dict["model"], strict=False) + return + ps = _chunk_parallel_state(chunks[0]) if chunks else None + if len(chunks) == 1 and ps is not None and ps.pp_size > 1: + key = _model_chunk_key(ps, 0, 1) + if key in state_dict: + _wrapped_module(chunks[0]).load_state_dict(state_dict[key], strict=False) + return + for idx, chunk in enumerate(chunks): + key = _model_chunk_key(ps, idx, len(chunks)) + if key in state_dict: + _wrapped_module(chunk).load_state_dict(state_dict[key], strict=False) + + +def _chunk_parallel_state(chunk: nn.Module) -> ParallelState | None: + ps = getattr(chunk, "_mlite_dist_opt_parallel_state", None) + if ps is not None: + return ps + wrapped = _wrapped_module(chunk) + return getattr(wrapped, "_mlite_dist_opt_parallel_state", None) + + +def _model_chunks(model: nn.Module | Iterable[nn.Module]) -> list[nn.Module]: + if isinstance(model, nn.Module): + return list(model) if isinstance(model, nn.ModuleList) else [model] + chunks = list(model) + if not all(isinstance(chunk, nn.Module) for chunk in chunks): + raise TypeError("distckpt model chunks must be nn.Module instances.") + return chunks + + +def _wrapped_module(model: nn.Module) -> nn.Module: + module = getattr(model, "module", None) + return module if isinstance(module, nn.Module) else model + + +__all__ = [ + "attach_model_sharded_state_dict", + "load_dist_opt_checkpoint", + "save_dist_opt_checkpoint", + "supports_dist_opt_distckpt", +] diff --git a/experimental/lite/megatron/lite/primitive/ckpt/hf_weights.py b/experimental/lite/megatron/lite/primitive/ckpt/hf_weights.py new file mode 100644 index 00000000000..71879b5ba16 --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/ckpt/hf_weights.py @@ -0,0 +1,622 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""HF ↔ Megatron Lite weight conversion. + +Everything needed for HuggingFace safetensors ↔ Megatron Lite model conversion: +- HFWeights protocol (model implements this) +- SafeTensorReader / save_safetensors (file I/O) +- Tensor utilities (split_dim, allgather_concat, remap_layer_index, ...) +- Generic load_hf_weights / export_hf_weights / save_hf_weights (orchestration) +""" + +from __future__ import annotations + +import json +import os +import re +from collections.abc import Generator +from pathlib import Path +from typing import Protocol, runtime_checkable + +import torch +import torch.distributed as dist +import torch.nn as nn +from safetensors import safe_open +from safetensors.torch import save_file as _safe_save + +try: + from torch.distributed.tensor import DTensor +except Exception: # pragma: no cover - older torch without DTensor + DTensor = None # type: ignore[assignment] + + +def _materialize_dtensor(tensor: torch.Tensor) -> torch.Tensor: + """Reconstruct a plain local tensor from an FSDP2 ``DTensor`` parameter. + + FSDP2 (``fully_shard``) stores parameters as ``DTensor`` sharded over the + data-parallel mesh, while manual TP/EP sharding leaves each rank holding its + own local shard as a regular tensor. The HF export gather (``_gather_dense`` / + ``_gather_expert``) and the downstream rollout weight sender both assume plain + ``torch.Tensor`` inputs; handing them a ``DTensor`` raises + ``aten.copy_.default: got mixed torch.Tensor and DTensor``. ``full_tensor()`` + gathers the FSDP shards back into the full (TP/EP-local) tensor; non-DTensor + params (dist_opt backend) pass through untouched. + """ + if DTensor is not None and isinstance(tensor, DTensor): + return tensor.full_tensor() + return tensor + + +@runtime_checkable +class HFWeights(Protocol): + """Protocol for HF ↔ Megatron Lite weight conversion. + + Model-specific implementations only do tensor math, never distributed comm. + """ + + def weight_map(self) -> dict[str, list[str]]: + """Megatron Lite param name → [HF param names]. Multiple = concat (QKV, gate+up).""" + ... + + def hf_to_native(self, native_name: str, hf_tensors: list[torch.Tensor]) -> torch.Tensor: + """Convert HF tensors → single Megatron Lite tensor (e.g. merge QKV).""" + ... + + def native_to_hf( + self, native_name: str, tensor: torch.Tensor + ) -> list[tuple[str, torch.Tensor]]: + """Convert Megatron Lite tensor → [(hf_name, hf_tensor)] (e.g. split QKV back).""" + ... + + def tp_spec(self, native_name: str) -> tuple[int, int] | None: + """TP sharding: ``(split_dim, 0=TP|1=ETP)``, or None if replicated.""" + ... + + def qkv_spec(self, native_name: str) -> tuple[int, int, int] | None: + """If native_name is a fused QKV weight, return (num_q_heads, num_kv_heads, head_dim). + + Needed for correct GQA TP sharding — Q/K/V must be split independently. + Return None for non-QKV parameters. + """ + return None + + @property + def num_experts(self) -> int: + """Total number of experts (needed for EP gather index math).""" + ... + + def is_expert(self, native_name: str) -> bool: + """Whether this param belongs to an expert (for EP sharding).""" + ... + + def expert_global_id(self, native_name: str) -> int | None: + """Global expert ID from synthetic name. None if not expert.""" + ... + + def expert_local_name(self, native_name: str, local_idx: int) -> str: + """Synthetic expert name → actual model param name.""" + ... + + +# ====================================================================== +# SafeTensors I/O +# ====================================================================== + + +class SafeTensorReader: + """Read individual tensors from an HF safetensors directory.""" + + def __init__(self, path: str): + self.path = Path(path) + self.index = self._load_index() + + def _load_index(self) -> dict[str, str]: + idx_file = self.path / "model.safetensors.index.json" + if idx_file.exists(): + with open(idx_file) as f: + return json.load(f)["weight_map"] + return {} + + def get_tensor(self, name: str) -> torch.Tensor: + if self.index: + filepath = self.path / self.index[name] + else: + filepath = self.path / "model.safetensors" + with safe_open(str(filepath), framework="pt", device="cpu") as f: + return f.get_tensor(name) + + +def unwrap_model(model: nn.Module) -> nn.Module: + """Strip nested wrapper modules like DDP -> model.""" + base_model = model + seen: set[int] = set() + while hasattr(base_model, "module"): + ident = id(base_model) + if ident in seen: + break + seen.add(ident) + next_model = base_model.module + if not isinstance(next_model, nn.Module) or next_model is base_model: + break + base_model = next_model + return base_model + + +def save_safetensors( + tensors: dict[str, torch.Tensor], path: str, filename: str = "model.safetensors" +) -> None: + os.makedirs(path, exist_ok=True) + _safe_save(tensors, os.path.join(path, filename)) + + +def _resolve_export_dtype(export_dtype: str | torch.dtype | None) -> torch.dtype | None: + if export_dtype is None: + return None + if isinstance(export_dtype, torch.dtype): + return export_dtype + normalized = str(export_dtype).lower() + aliases = { + "bf16": torch.bfloat16, + "bfloat16": torch.bfloat16, + "fp16": torch.float16, + "float16": torch.float16, + "half": torch.float16, + "fp32": torch.float32, + "float32": torch.float32, + "float": torch.float32, + } + if normalized not in aliases: + raise ValueError(f"Unsupported export_dtype={export_dtype!r}") + return aliases[normalized] + + +def _cast_export_tensor(tensor: torch.Tensor, export_dtype: torch.dtype | None) -> torch.Tensor: + if export_dtype is None or not tensor.is_floating_point(): + return tensor + return tensor.to(dtype=export_dtype) + + +# ====================================================================== +# Tensor utilities +# ====================================================================== + + +def split_dim(tensor: torch.Tensor, rank: int, world: int, dim: int = 0) -> torch.Tensor: + if world <= 1: + return tensor + return tensor.chunk(world, dim=dim)[rank].contiguous() + + +def split_qkv( + tensor: torch.Tensor, rank: int, world: int, num_q_heads: int, num_kv_heads: int, head_dim: int +) -> torch.Tensor: + """TP-shard a fused [Q, K, V] weight, splitting Q/K/V heads independently. + + Naive ``split_dim`` would slice across the Q/K/V boundary incorrectly + when num_q_heads != num_kv_heads (GQA). + """ + if world <= 1: + return tensor + q_size = num_q_heads * head_dim + kv_size = num_kv_heads * head_dim + q = tensor[:q_size] + k = tensor[q_size : q_size + kv_size] + v = tensor[q_size + kv_size :] + q_shard = q.chunk(world, dim=0)[rank] + k_shard = k.chunk(world, dim=0)[rank] + v_shard = v.chunk(world, dim=0)[rank] + return torch.cat([q_shard, k_shard, v_shard], dim=0).contiguous() + + +def split_gate_up(tensor: torch.Tensor, rank: int, world: int) -> torch.Tensor: + if world <= 1: + return tensor + ffn = tensor.shape[0] // 2 + gate = tensor[:ffn].chunk(world, dim=0)[rank] + up = tensor[ffn:].chunk(world, dim=0)[rank] + return torch.cat([gate, up], dim=0).contiguous() + + +def allgather_concat( + tensor: torch.Tensor, world_size: int, group: dist.ProcessGroup | None, dim: int +) -> torch.Tensor: + gathered = [torch.empty_like(tensor) for _ in range(world_size)] + dist.all_gather(gathered, tensor.contiguous(), group=group) + return torch.cat(gathered, dim=dim) + + +def remap_layer_index(name: str, global_to_local: dict[int, int]) -> str | None: + if not global_to_local: + return name + m = re.match(r"(layers\.)(\d+)(\..*)", name) + if not m: + return name + gidx = int(m.group(2)) + if gidx not in global_to_local: + return None + return f"{m.group(1)}{global_to_local[gidx]}{m.group(3)}" + + +def extract_layer_idx(name: str) -> int: + m = re.search(r"layers\.(\d+)\.", name) + return int(m.group(1)) if m else 0 + + +def parse_expert_idx(name: str) -> int: + m = re.search(r"weight(\d+)$", name) + return int(m.group(1)) if m else 0 + + +def set_expert_idx(name: str, idx: int) -> str: + return re.sub(r"weight\d+$", f"weight{idx}", name) + + +def to_global_layer_name(name: str, layer_map: dict[int, int]) -> str: + if not layer_map: + return name + + def _replace(m: re.Match) -> str: + return f"layers.{layer_map.get(int(m.group(1)), int(m.group(1)))}." + + return re.sub(r"layers\.(\d+)\.", _replace, name) + + +def gather_gate_up(tensor: torch.Tensor, world_size: int, group: dist.ProcessGroup) -> torch.Tensor: + ffn_local = tensor.shape[0] // 2 + gate_full = allgather_concat(tensor[:ffn_local], world_size, group, dim=0) + up_full = allgather_concat(tensor[ffn_local:], world_size, group, dim=0) + return torch.cat([gate_full, up_full], dim=0) + + +# ====================================================================== +# Generic load / export / save using HFWeights +# ====================================================================== + + +def load_hf_weights( + model: nn.Module, hf_path: str, spec: HFWeights, ps, *, vocab_size: int | None = None +) -> None: + """Load HF safetensors into a Megatron Lite model using HFWeights. + + Handles PP layer filtering, TP split, EP shard assignment. + ``ps`` is a ParallelState (lazy import to avoid GPU dep at module level). + """ + from megatron.lite.primitive.parallel import pad_vocab_for_tp + from megatron.lite.primitive.utils import log_rank0 + + base_model = unwrap_model(model) + reader = SafeTensorReader(hf_path) + wmap = spec.weight_map() + + global_to_local: dict[int, int] = ( + {gi: li for li, gi in enumerate(base_model.layer_indices)} + if hasattr(base_model, "layer_indices") + else {} + ) + + state = base_model.state_dict() + loaded: dict[str, torch.Tensor] = {} + num_experts_total = getattr(spec, "num_experts", None) + expert_shard = None + if num_experts_total is None: + expert_ids = [spec.expert_global_id(name) for name in wmap] + expert_ids = [expert_id for expert_id in expert_ids if expert_id is not None] + if expert_ids: + num_experts_total = max(expert_ids) + 1 + if num_experts_total: + from megatron.lite.primitive.utils import ensure_divisible + + experts_per_rank = ensure_divisible(num_experts_total, ps.ep_size) + local_start = ps.ep_rank * experts_per_rank + expert_shard = (experts_per_rank, local_start) + + for native_name, hf_names in wmap.items(): + mapped = remap_layer_index(native_name, global_to_local) + if mapped is None: + continue + + expert_gid = spec.expert_global_id(mapped) + if expert_gid is not None: + _load_expert_weight( + mapped, hf_names, reader, spec, ps, loaded, expert_gid, expert_shard + ) + continue + + hf_tensors = [reader.get_tensor(n) for n in hf_names] + tensor = spec.hf_to_native(mapped, hf_tensors) + + tp_info = spec.tp_spec(mapped) + if tp_info is not None: + split_d, tp_or_etp = tp_info + if tp_or_etp == 0: + if vocab_size is not None and ("embed" in mapped or "head" in mapped): + padded = pad_vocab_for_tp(vocab_size, ps.tp_size) + if tensor.size(0) < padded: + pad = torch.zeros( + padded - tensor.size(0), *tensor.shape[1:], dtype=tensor.dtype + ) + tensor = torch.cat([tensor, pad], dim=0) + qkv = spec.qkv_spec(mapped) if hasattr(spec, "qkv_spec") else None + if qkv is not None: + tensor = split_qkv(tensor, ps.tp_rank, ps.tp_size, *qkv) + else: + tensor = split_dim(tensor, ps.tp_rank, ps.tp_size, dim=split_d) + else: + tensor = split_dim(tensor, ps.etp_rank, ps.etp_size, dim=split_d) + + actual = _resolve_param_name(mapped, state) + if actual: + loaded[actual] = tensor.to(dtype=torch.bfloat16) + + for name, param in base_model.named_parameters(): + if name in loaded: + param.data.copy_(loaded[name]) + elif "lora" in name.lower() or "adapter" in name.lower(): + continue + else: + log_rank0(f"WARNING: {name} not loaded from checkpoint") + + +def _load_expert_weight(native_name, hf_names, reader, spec, ps, loaded, expert_gid, expert_shard): + if expert_shard is None: + raise RuntimeError("Expert weight encountered but expert shard metadata is unavailable.") + experts_per_rank, local_start = expert_shard + if expert_gid < local_start or expert_gid >= local_start + experts_per_rank: + return + + hf_tensors = [reader.get_tensor(n) for n in hf_names] + tensor = spec.hf_to_native(native_name, hf_tensors) + + if ps.etp_size > 1: + tp_info = spec.tp_spec(native_name) + if tp_info is not None: + split_d, _ = tp_info + if "fc1" in native_name: + tensor = split_gate_up(tensor, ps.etp_rank, ps.etp_size) + else: + tensor = split_dim(tensor, ps.etp_rank, ps.etp_size, dim=split_d) + + loaded[spec.expert_local_name(native_name, expert_gid - local_start)] = tensor.to( + dtype=torch.bfloat16 + ) + + +def _resolve_param_name(name: str, state_dict: dict) -> str | None: + if name in state_dict: + return name + for key in state_dict: + if name in key: + return key + return None + + +def export_hf_weights( + model: nn.Module | list[nn.Module], + spec: HFWeights, + ps, + *, + vocab_size: int | None = None, + limit: int | None = None, + rank0_only: bool = False, + export_dtype: str | torch.dtype | None = None, +) -> Generator[tuple[str, torch.Tensor], None, None]: + """Export model weights as HF-format (name, tensor) pairs. + + Gathers across TP/ETP/EP/PP so the output is the full unsharded HF state on + every participating rank. RL weight sync needs every colocated rollout rank + to receive weights; save paths can pass ``rank0_only=True`` to avoid + materializing duplicate writers. + """ + if isinstance(model, nn.ModuleList): + chunks: list[nn.Module] = list(model) + elif isinstance(model, list): + chunks = model + else: + chunks = [model] + + rank = dist.get_rank() if dist.is_initialized() else 0 + resolved_export_dtype = _resolve_export_dtype(export_dtype) + + if ps.pp_size <= 1: + exported_params = 0 + expert_groups: dict[str, list[tuple[int, str, torch.Tensor]]] = {} + for chunk in chunks: + base_chunk = unwrap_model(chunk) + layer_map = ( + {i: base_chunk.layer_indices[i] for i in range(len(base_chunk.layer_indices))} + if hasattr(base_chunk, "layer_indices") + else {} + ) + for name, param in base_chunk.named_parameters(): + gname = to_global_layer_name(name, layer_map) + tensor = _materialize_dtensor(param.data.detach()) + + gathered_one: dict[str, torch.Tensor] = {} + if spec.is_expert(gname): + if limit is None: + expert_groups.setdefault(_expert_group_key(gname), []).append( + (parse_expert_idx(gname), gname, tensor) + ) + exported_params += 1 + continue + _gather_expert(gname, tensor, spec, ps, gathered_one) + else: + gathered_one[gname] = _gather_dense(gname, tensor, spec, ps) + + exported_params += 1 + if not rank0_only or rank == 0: + for native_name, gathered_tensor in gathered_one.items(): + if vocab_size is not None and ( + "embed" in native_name or "head" in native_name + ): + gathered_tensor = gathered_tensor[:vocab_size] + for hf_name, hf_tensor in spec.native_to_hf(native_name, gathered_tensor): + yield hf_name, _cast_export_tensor(hf_tensor, resolved_export_dtype) + + if limit is not None and exported_params >= limit: + return + + for group_key in sorted(expert_groups): + gathered_group: dict[str, torch.Tensor] = {} + _gather_expert_group(expert_groups[group_key], spec, ps, gathered_group) + if not rank0_only or rank == 0: + for native_name in sorted(gathered_group, key=parse_expert_idx): + gathered_tensor = gathered_group[native_name] + for hf_name, hf_tensor in spec.native_to_hf(native_name, gathered_tensor): + yield hf_name, _cast_export_tensor(hf_tensor, resolved_export_dtype) + return + + gathered: dict[str, torch.Tensor] = {} + for chunk in chunks: + base_chunk = unwrap_model(chunk) + # Map local layer indices to global for PP + layer_map = ( + {i: base_chunk.layer_indices[i] for i in range(len(base_chunk.layer_indices))} + if hasattr(base_chunk, "layer_indices") + else {} + ) + for name, param in base_chunk.named_parameters(): + gname = to_global_layer_name(name, layer_map) + t = _materialize_dtensor(param.data.detach()) + + if spec.is_expert(gname): + _gather_expert(gname, t, spec, ps, gathered) + else: + gathered[gname] = _gather_dense(gname, t, spec, ps) + + # PP gather + if ps.pp_size > 1: + all_states: list[dict | None] = [None] * ps.pp_size + dist.all_gather_object(all_states, gathered, group=ps.pp_group) + gathered = {} + for s in all_states: + if s is not None: + gathered.update(s) + + rank = dist.get_rank() if dist.is_initialized() else 0 + if rank0_only and rank != 0: + return + + # Vocab trim + if vocab_size is not None: + for key in list(gathered.keys()): + if "embed" in key or "head" in key: + gathered[key] = gathered[key][:vocab_size] + + # Convert Megatron Lite names → HF names via spec + for native_name, tensor in gathered.items(): + for hf_name, hf_tensor in spec.native_to_hf(native_name, tensor): + yield hf_name, _cast_export_tensor(hf_tensor, resolved_export_dtype) + + +def _gather_dense(name: str, tensor: torch.Tensor, spec: HFWeights, ps) -> torch.Tensor: + """Gather a dense (non-expert) param across TP.""" + custom_gather = getattr(spec, "gather_dense", None) + if callable(custom_gather): + gathered = custom_gather(name, tensor, ps) + if gathered is not None: + return gathered.cpu() + + tp_info = spec.tp_spec(name) + if tp_info is not None and ps.tp_size > 1: + split_d, tp_or_etp = tp_info + if tp_or_etp == 0: + tensor = allgather_concat(tensor, ps.tp_size, ps.tp_group, dim=split_d) + return tensor.cpu() + + +def _gather_expert( + name: str, tensor: torch.Tensor, spec: HFWeights, ps, out: dict[str, torch.Tensor] +) -> None: + """Gather an expert param across ETP + EP.""" + tensor = _gather_expert_etp(name, tensor, spec, ps) + + # EP gather: global_id = ep_rank * n_local + local_id. + local_idx = parse_expert_idx(name) + if ps.ep_size > 1 and ps.ep_group is not None: + n_local = spec.num_experts // ps.ep_size + ep_gathered = [torch.empty_like(tensor) for _ in range(ps.ep_size)] + dist.all_gather(ep_gathered, tensor.contiguous(), group=ps.ep_group) + for ep_rank, ep_tensor in enumerate(ep_gathered): + global_idx = ep_rank * n_local + local_idx + out[set_expert_idx(name, global_idx)] = ep_tensor.cpu() + else: + out[name] = tensor.cpu() + + +def _gather_expert_etp(name: str, tensor: torch.Tensor, spec: HFWeights, ps) -> torch.Tensor: + # ETP gather + if ps.etp_size > 1 and ps.etp_group is not None: + tp_info = spec.tp_spec(name) + if tp_info is not None: + split_d, _ = tp_info + if "fc1" in name: + return gather_gate_up(tensor, ps.etp_size, ps.etp_group) + return allgather_concat(tensor, ps.etp_size, ps.etp_group, dim=split_d) + return tensor + + +def _expert_group_key(name: str) -> str: + return re.sub(r"weight\d+$", "weight", name) + + +def _gather_expert_group( + entries: list[tuple[int, str, torch.Tensor]], spec: HFWeights, ps, out: dict[str, torch.Tensor] +) -> None: + """Gather local experts in one EP collective per layer/kind.""" + prepared = [ + (local_idx, name, _gather_expert_etp(name, tensor, spec, ps)) + for local_idx, name, tensor in sorted(entries) + ] + packed_group_name = getattr(spec, "packed_expert_group_name", None) + if callable(packed_group_name): + packed_name = packed_group_name(prepared[0][1]) + if packed_name is not None: + if ps.ep_size <= 1 or ps.ep_group is None: + out[packed_name] = torch.stack( + [tensor.contiguous() for _, _, tensor in prepared], dim=0 + ).cpu() + return + + sample = prepared[0][2] + packed = torch.empty((spec.num_experts, *sample.shape), dtype=sample.dtype, device="cpu") + for batch_start in range(0, len(prepared), 4): + batch = prepared[batch_start : batch_start + 4] + stacked = torch.stack([tensor.contiguous() for _, _, tensor in batch], dim=0) + ep_gathered = [torch.empty_like(stacked) for _ in range(ps.ep_size)] + dist.all_gather(ep_gathered, stacked, group=ps.ep_group) + for ep_rank, ep_tensor in enumerate(ep_gathered): + for batch_idx, (local_idx, _, _) in enumerate(batch): + packed[ep_rank * (spec.num_experts // ps.ep_size) + local_idx].copy_(ep_tensor[batch_idx]) + out[packed_name] = packed + return + + if ps.ep_size <= 1 or ps.ep_group is None: + for _, name, tensor in prepared: + out[name] = tensor.cpu() + return + + n_local = spec.num_experts // ps.ep_size + stacked = torch.stack([tensor.contiguous() for _, _, tensor in prepared], dim=0) + ep_gathered = [torch.empty_like(stacked) for _ in range(ps.ep_size)] + dist.all_gather(ep_gathered, stacked, group=ps.ep_group) + for ep_rank, ep_tensor in enumerate(ep_gathered): + for slot, (local_idx, name, _) in enumerate(prepared): + global_idx = ep_rank * n_local + local_idx + out[set_expert_idx(name, global_idx)] = ep_tensor[slot].cpu() + + +def save_hf_weights( + model: nn.Module | list[nn.Module], + hf_path: str, + spec: HFWeights, + ps, + *, + vocab_size: int | None = None, +) -> None: + """Export + write to safetensors.""" + rank = dist.get_rank() if dist.is_initialized() else 0 + out = dict(export_hf_weights(model, spec, ps, vocab_size=vocab_size, rank0_only=True)) + if rank == 0 and out: + save_safetensors(out, hf_path) + if dist.is_initialized(): + dist.barrier() diff --git a/experimental/lite/megatron/lite/primitive/config.py b/experimental/lite/megatron/lite/primitive/config.py new file mode 100644 index 00000000000..c13bbf4fb86 --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/config.py @@ -0,0 +1,54 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Configuration utilities. + +Model-specific configs live under `megatron.lite/models/*/config.py` +as standalone dataclasses. +""" + +from __future__ import annotations + +import json +from dataclasses import fields as dc_fields +from pathlib import Path +from typing import Any + + +def to_dataclass(oc_cfg, cls): + """Convert an OmegaConf DictConfig back to a typed dataclass instance.""" + from omegaconf import OmegaConf + + init_names = {f.name for f in dc_fields(cls) if f.init} + d = OmegaConf.to_container(oc_cfg, resolve=True) + return cls(**{k: v for k, v in d.items() if k in init_names}) + + +def load_hf_config_dict(path_or_name: str) -> dict[str, Any]: + """Load HF config dict from local path or Hub.""" + p = Path(path_or_name) + + if p.is_file() and p.name == "config.json": + with open(p) as f: + return json.load(f) + + if p.is_dir(): + cfg_file = p / "config.json" + if cfg_file.exists(): + with open(cfg_file) as f: + return json.load(f) + raise FileNotFoundError(f"No config.json in {p}") + + try: + from transformers import AutoConfig + + hf_config = AutoConfig.from_pretrained(path_or_name, trust_remote_code=True) + return hf_config.to_dict() + except ImportError as err: + raise ImportError( + f"'{path_or_name}' is not a local path. " + "Install transformers to load from HuggingFace Hub: pip install transformers" + ) from err + except Exception as e: + raise ValueError(f"Failed to load config from '{path_or_name}': {e}") from e + + +__all__ = ["load_hf_config_dict", "to_dataclass"] diff --git a/experimental/lite/megatron/lite/primitive/data.py b/experimental/lite/megatron/lite/primitive/data.py new file mode 100644 index 00000000000..2f854dfceb2 --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/data.py @@ -0,0 +1,196 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Batch generation utilities for Megatron Lite runtime.""" + +from __future__ import annotations + +import os + +import torch # pyright: ignore[reportMissingImports] + +from megatron.lite.primitive.utils.packed_seq import PackedSeqParams + +_TRUE_ENV_VALUES = {"1", "true", "yes", "on"} +_FALSE_ENV_VALUES = {"0", "false", "no", "off"} + + +def _read_bool_env(name: str) -> bool | None: + value = os.environ.get(name) + if value is None or value.strip() == "": + return None + normalized = value.strip().lower() + if normalized in _TRUE_ENV_VALUES: + return True + if normalized in _FALSE_ENV_VALUES: + return False + raise ValueError(f"{name} must be a boolean value, got {value!r}") + + +def _resolve_thd_padding(seq_len: int, cp_size: int) -> tuple[int, int, bool]: + """Return (padded_seq_len, alignment_multiple, pad_enabled) for THD input.""" + if seq_len <= 0: + raise ValueError(f"seq_len must be positive, got {seq_len}") + if cp_size < 1: + raise ValueError(f"cp_size must be >= 1, got {cp_size}") + + pad_to_alignment = _read_bool_env("MEGATRON_LITE_THD_PAD_TO_ALIGNMENT") + if pad_to_alignment is None: + pad_to_alignment = cp_size > 1 + + pad_multiple_env = os.environ.get("MEGATRON_LITE_THD_PAD_MULTIPLE", "auto").strip().lower() + if pad_multiple_env in ("", "auto", "cp", "min_cp"): + align_size = cp_size * 2 if cp_size > 1 else 1 + else: + try: + align_size = int(pad_multiple_env) + except ValueError as exc: + raise ValueError( + "MEGATRON_LITE_THD_PAD_MULTIPLE must be 'auto' or a positive integer, " + f"got {pad_multiple_env!r}" + ) from exc + if align_size <= 0: + raise ValueError( + "MEGATRON_LITE_THD_PAD_MULTIPLE must be 'auto' or a positive integer, " + f"got {pad_multiple_env!r}" + ) + + if not pad_to_alignment: + return seq_len, align_size, False + + seq_len_padded = ((seq_len + align_size - 1) // align_size) * align_size + return seq_len_padded, align_size, True + + +def fixed_batches( + vocab_size: int, + seq_len: int, + num_steps: int, + batch_size: int = 1, + device: str = "cuda", + seed: int = 42, +) -> list[tuple[torch.Tensor, torch.Tensor]]: + """Deterministic (input_ids, labels) pairs, identical across all ranks.""" + g = torch.Generator().manual_seed(seed) + batches = [] + for _ in range(num_steps): + ids = torch.randint(0, vocab_size, (batch_size, seq_len), generator=g).to(device) + labels = torch.randint(0, vocab_size, (batch_size, seq_len), generator=g).to(device) + batches.append((ids, labels)) + return batches + + +def infinite_batches( + vocab_size: int, seq_len: int, batch_size: int = 1, device: str = "cuda", seed: int = 42 +): + """Infinite deterministic batch generator (for throughput benchmarks).""" + g = torch.Generator(device=device).manual_seed(seed) + while True: + yield { + "input_ids": torch.randint( + 0, vocab_size, (batch_size, seq_len), device=device, generator=g + ), + "labels": torch.randint( + 0, vocab_size, (batch_size, seq_len), device=device, generator=g + ), + } + + +def infinite_batches_thd( + vocab_size: int, + seq_len: int, + *, + cp_size: int = 1, + cp_rank: int = 0, + device: str = "cuda", + seed: int = 42, +): + """Infinite deterministic batch generator in THD (packed variable-length) format. + + Produces plain dicts consumed directly by native lite model forwards + (input_ids 2-D batch=1, mrope position_ids (3,1,T), pre-built PackedSeqParams). + + cp_size/cp_rank should be supplied explicitly by the caller (for example + from handle._parallel_state). The defaults keep older single-CP call sites + working, but CP runs must pass the real CP rank and world size. + + When context parallelism is active (cp_size > 1), each rank's packed tokens are + split via zigzag striping. position_ids stay FULL because lite's RoPE + (is_thd_format=False) auto-slices emb internally, same as BSH path. + """ + if cp_size < 1: + raise ValueError(f"cp_size must be >= 1, got {cp_size}") + if cp_rank < 0 or cp_rank >= cp_size: + raise ValueError(f"cp_rank must be in [0, {cp_size}), got {cp_rank}") + + g = torch.Generator(device=device).manual_seed(seed) + seq_len_padded, _align_size, _pad_to_alignment = _resolve_thd_padding(seq_len, cp_size) + + # position_ids always stay full length; RoPE auto-slices for CP. + position_ids = ( + torch.arange(seq_len_padded, device=device) + .view(1, 1, -1) + .expand(3, 1, seq_len_padded) + .contiguous() + ) + cu_seqlens = torch.tensor([0, seq_len], dtype=torch.int32, device=device) + cu_seqlens_padded = torch.tensor([0, seq_len_padded], dtype=torch.int32, device=device) + + if cp_size > 1: + from megatron.lite.primitive.parallel.cp import zigzag_split_for_cp # noqa: I001 + + # cu_seqlens stays full; MC GDN internally divides by cp_size. + packed_seq_params = PackedSeqParams( + qkv_format="thd", + cu_seqlens_q=cu_seqlens, + cu_seqlens_kv=cu_seqlens, + max_seqlen_q=seq_len_padded, + max_seqlen_kv=seq_len_padded, + cu_seqlens_q_padded=cu_seqlens_padded, + cu_seqlens_kv_padded=cu_seqlens_padded, + ) + while True: + ids_full = torch.randint(0, vocab_size, (seq_len,), device=device, generator=g) + lbl_full = torch.randint(0, vocab_size, (seq_len,), device=device, generator=g) + if seq_len_padded != seq_len: + ids_padded = torch.zeros(seq_len_padded, dtype=ids_full.dtype, device=device) + lbl_padded = torch.zeros(seq_len_padded, dtype=lbl_full.dtype, device=device) + ids_padded[:seq_len] = ids_full + lbl_padded[:seq_len] = lbl_full + else: + ids_padded = ids_full + lbl_padded = lbl_full + ids_cp = zigzag_split_for_cp(ids_padded, cp_rank, cp_size, seq_dim=0) + lbl_cp = zigzag_split_for_cp(lbl_padded, cp_rank, cp_size, seq_dim=0) + yield { + "input_ids": ids_cp.unsqueeze(0), # (1, T/cp) + "labels": lbl_cp.unsqueeze(0), # (1, T/cp) + "position_ids": position_ids, # FULL (3, 1, T) + "packed_seq_params": packed_seq_params, + } + else: + packed_seq_params = PackedSeqParams( + qkv_format="thd", + cu_seqlens_q=cu_seqlens, + cu_seqlens_kv=cu_seqlens, + max_seqlen_q=seq_len_padded, + max_seqlen_kv=seq_len_padded, + cu_seqlens_q_padded=cu_seqlens_padded, + cu_seqlens_kv_padded=cu_seqlens_padded, + ) + while True: + input_ids = torch.zeros((1, seq_len_padded), dtype=torch.long, device=device) + labels = torch.zeros((1, seq_len_padded), dtype=torch.long, device=device) + input_ids[:, :seq_len] = torch.randint( + 0, vocab_size, (1, seq_len), device=device, generator=g + ) + labels[:, :seq_len] = torch.randint( + 0, vocab_size, (1, seq_len), device=device, generator=g + ) + yield { + "input_ids": input_ids, + "labels": labels, + "position_ids": position_ids, + "packed_seq_params": packed_seq_params, + } + + +__all__ = ["fixed_batches", "infinite_batches", "infinite_batches_thd"] diff --git a/experimental/lite/megatron/lite/primitive/deterministic.py b/experimental/lite/megatron/lite/primitive/deterministic.py new file mode 100644 index 00000000000..f1a736bf690 --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/deterministic.py @@ -0,0 +1,35 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Deterministic computation utilities for bitwise reproducible experiments.""" + +from __future__ import annotations + +import os + +import torch + +_is_deterministic: bool = False +_TRUE_VALUES = {"1", "true", "yes", "on"} +_DETERMINISTIC_ENV = "MEGATRON_LITE_DETERMINISTIC" + + +def deterministic_requested() -> bool: + """Return whether bitwise-validation deterministic mode was requested.""" + return os.environ.get(_DETERMINISTIC_ENV, "").strip().lower() in _TRUE_VALUES + + +def set_deterministic(seed: int = 42) -> None: + """Enable all deterministic flags. Must call before first CUDA op in process.""" + global _is_deterministic + os.environ.setdefault("CUBLAS_WORKSPACE_CONFIG", ":4096:8") + os.environ["NVTE_ALLOW_NONDETERMINISTIC_ALGO"] = "0" + torch.use_deterministic_algorithms(True, warn_only=False) + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + torch.manual_seed(seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(seed) + _is_deterministic = True + + +def is_deterministic() -> bool: + return _is_deterministic diff --git a/experimental/lite/megatron/lite/primitive/kernels/__init__.py b/experimental/lite/megatron/lite/primitive/kernels/__init__.py new file mode 100644 index 00000000000..39af219f47e --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/kernels/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Optional kernel shims used by MLite primitives.""" + +from __future__ import annotations + +__all__: list[str] = [] diff --git a/experimental/lite/megatron/lite/primitive/kernels/dsa_kernels.py b/experimental/lite/megatron/lite/primitive/kernels/dsa_kernels.py new file mode 100644 index 00000000000..7e56ec30388 --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/kernels/dsa_kernels.py @@ -0,0 +1,1146 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + +""" +DSA kernel wrappers for Megatron's DSv4 sparse attention. + +Mirrors the three integration paths of the old standalone ``dsa_kernels`` +package, but built on top of + +* :mod:`cudnn.deepseek_sparse_attention` (a.k.a. ``DSA``) — CuTe-DSL backward + + indexer score kernels + TRT-LLM radix top-K, shipped as part of + cuDNN Frontend. +* :mod:`flash_mla` — production sparse-attention forward kernel, expected to + be available as a separate PyPI package. + +Public API (same shape as the old ``dsa_kernels`` package): + +* ``build_flat_topk_idxs`` / ``local_to_global_flat`` — index helpers. +* ``dsa_sparse_attn`` — Path A / Path C step 2, differentiable sparse attention. +* ``indexer_topk`` — Path C inference indexer scoring + top-K. +* ``fused_indexer_sparse_attn`` — Path B training, fused indexer loss + + sparse attention with shared backward. +""" + +from __future__ import annotations + +from importlib import import_module +from typing import Callable, Optional, Tuple + +import torch +from torch import Tensor + +# --------------------------------------------------------------------------- +# Lazy kernel imports +# --------------------------------------------------------------------------- + + +_flash_mla_sparse_fwd = None +_DSA = None +_indexer_fwd_sm90: Optional[Callable] = None +_indexer_fwd_sm100: Optional[Callable] = None + + +def _ensure_flash_mla(): + """Lazily import the FlashMLA sparse-forward kernel. + + FlashMLA ships ``flash_mla_sparse_fwd`` with a multi-head-KV signature; + :func:`_dsa_fwd_flash_mla` below is a thin adapter that unbatches the + DSA-shape inputs and pads ``TopK`` to the alignment expected by + FlashMLA's SM90 / SM100 kernels. + """ + global _flash_mla_sparse_fwd + if _flash_mla_sparse_fwd is not None: + return + + try: + from flash_mla import flash_mla_sparse_fwd as _fwd + except ImportError as e: + raise ImportError( + "FlashMLA is required for DSA sparse attention forward. " + "Install from https://github.com/deepseek-ai/FlashMLA/tree/nv_dev " + "so that `from flash_mla import flash_mla_sparse_fwd` succeeds." + ) from e + _flash_mla_sparse_fwd = _fwd + + +def _get_topk_alignment() -> int: + """Minimum ``TopK`` alignment required by the current GPU architecture. + + * SM90 : dual-warpgroup loop steps by 2 blocks → ``2 * B_TOPK = 128`` + * SM100: single-pipeline loop steps by 1 block → ``B_TOPK`` (64 for + head64, 128 for head128). DSA uses ``D = 512`` which maps to the + head64 kernel path → 64. + """ + sm = torch.cuda.get_device_capability() + if sm[0] >= 10: + return 64 + return 128 + + +def _dsa_fwd_flash_mla( + q: Tensor, + kv: Tensor, + topk_idxs: Tensor, + softmax_scale: float, + d_v: int = 512, + attn_sink: Optional[Tensor] = None, + topk_length: Optional[Tensor] = None, + indexer_topk: int = 0, +) -> Tuple[Tensor, Tensor, Optional[Tensor]]: + """DSA-shaped adapter around :func:`flash_mla.flash_mla_sparse_fwd`. + + Accepts flat (unbatched) tensors with global indices; pads ``TopK`` to + the GPU-specific alignment; returns ``(out, lse, lse_indexer)``. + """ + assert not ( + indexer_topk > 0 and topk_length is not None + ), "indexer_topk > 0 requires non-compact mode (topk_length must be None)" + _ensure_flash_mla() + + _total_S_q, _H, _D = q.shape + TopK = topk_idxs.shape[-1] + topk_align = _get_topk_alignment() + TopK_padded = (TopK + topk_align - 1) // topk_align * topk_align + if TopK_padded != TopK: + pad_width = TopK_padded - TopK + topk_idxs = torch.nn.functional.pad(topk_idxs, (0, pad_width), value=-1) + + kv_3d = kv.unsqueeze(1) # (total_S_kv, 1, D) h_kv=1 + indices = topk_idxs.unsqueeze(1) # (total_S_q, 1, TopK_padded) h_kv=1 + + with torch.cuda.nvtx.range("flash_mla_sparse_fwd"): + res = _flash_mla_sparse_fwd( + q, + kv_3d, + indices, + softmax_scale, + d_v=d_v, + attn_sink=attn_sink, + topk_length=topk_length, + indexer_topk=indexer_topk, + ) + if indexer_topk > 0: + out, _max_logits, lse, lse_indexer = res + else: + out, _max_logits, lse = res + lse_indexer = None + + if indexer_topk > 0: + # When indexer_topk == total TopK, lse_indexer should equal lse but + # the kernel may not snapshot correctly; fall back to lse. + if indexer_topk >= TopK: + return out, lse, lse.clone() + return out, lse, lse_indexer + return out, lse, None + + +def _ensure_dsa_namespace(): + """Lazily import the cudnn-frontend DSA namespace.""" + global _DSA + if _DSA is not None: + return + try: + from cudnn import DSA as _ns + except ImportError as e: + try: + from cudnn.deepseek_sparse_attention import DSA as _ns + except ImportError: + raise ImportError( + "cudnn-frontend DSA namespace not available. Install with " + "`pip install nvidia-cudnn-frontend[cutedsl]`; newer " + "versions expose it as `cudnn.deepseek_sparse_attention.DSA`." + ) from e + _DSA = _ns + + +def _load_indexer_fwd_sm90(): + """Load the H100 SM90 indexer forward entry only when it is selected.""" + global _indexer_fwd_sm90 + if _indexer_fwd_sm90 is None: + try: + module = import_module( + "cudnn.deepseek_sparse_attention.indexer_forward._interface_sm90" + ) + _indexer_fwd_sm90 = module.indexer_fwd + except (AttributeError, ImportError) as exc: + raise ImportError( + "H100 DSA indexer forward requires the SM90 cudnn route " + "`cudnn.deepseek_sparse_attention.indexer_forward._interface_sm90.indexer_fwd`." + ) from exc + return _indexer_fwd_sm90 + + +def _load_indexer_fwd_sm100(): + """Load the Blackwell SM100 indexer forward entry only when it is selected.""" + global _indexer_fwd_sm100 + if _indexer_fwd_sm100 is None: + try: + module = import_module("cudnn.deepseek_sparse_attention.indexer_forward._interface") + _indexer_fwd_sm100 = module.indexer_fwd + except (AttributeError, ImportError) as exc: + raise ImportError( + "Blackwell DSA indexer forward requires the SM100 cudnn route " + "`cudnn.deepseek_sparse_attention.indexer_forward._interface.indexer_fwd` " + "(exported by cudnn-frontend as indexer_fwd_sm100)." + ) from exc + return _indexer_fwd_sm100 + + +def _select_indexer_forward(device): + major, _minor = torch.cuda.get_device_capability(device) + if major == 9: + return _load_indexer_fwd_sm90() + if major >= 10: + return _load_indexer_fwd_sm100() + return None + + +def _dsa_indexer_forward_wrapper( + q: Tensor, + k: Tensor, + w: Tensor, + *, + ratio: int = 4, + qhead_per_kv_head: Optional[int] = None, + sm_scale: float = 1.0, + cu_seqlens_q: Optional[Tensor] = None, + cu_seqlens_k: Optional[Tensor] = None, + max_seqlen_q: Optional[int] = None, + max_seqlen_k: Optional[int] = None, +): + """Route indexer forward to architecture-specific CuTe-DSL backends.""" + if q.is_cuda: + indexer_fwd = _select_indexer_forward(q.device) + if indexer_fwd is not None: + return { + "scores": indexer_fwd( + q, + k, + w, + ratio=ratio, + qhead_per_kv_head=qhead_per_kv_head, + sm_scale=sm_scale, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k=cu_seqlens_k, + max_seqlen_q=max_seqlen_q, + max_seqlen_k=max_seqlen_k, + ) + } + _ensure_dsa_namespace() + return _DSA.indexer_forward_wrapper( + q, + k, + w, + ratio=ratio, + qhead_per_kv_head=qhead_per_kv_head, + sm_scale=sm_scale, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k=cu_seqlens_k, + max_seqlen_q=max_seqlen_q, + max_seqlen_k=max_seqlen_k, + ) + + +# --------------------------------------------------------------------------- +# Index helpers +# --------------------------------------------------------------------------- + + +def local_to_global_flat(local_idxs: Tensor, batch_size: int, seqlen_kv: int) -> Tensor: + """Convert local per-batch indices to global flat indices. + + Follows the convention used by FlashMLA / SparseAttentionBackward: + flat row order is SBHD ``row[s * B + b]``; global index is + ``local * B + b`` for valid entries and ``-1`` otherwise. + + Args: + local_idxs: ``(b, sq, topk)`` int, values in ``[0, seqlen_kv)`` or -1. + batch_size: ``B``. + seqlen_kv: KV sequence length per batch (used for shape assertions + only; callers compute the values). + + Returns: + ``(sq*b, topk)`` int32. + """ + b, sq, topk = local_idxs.shape + assert b == batch_size + + idxs_sb = local_idxs.permute(1, 0, 2).reshape(sq * b, topk) + valid = idxs_sb >= 0 + batch_ids = torch.arange(sq * b, device=local_idxs.device) % b + batch_ids_exp = batch_ids.unsqueeze(1).expand_as(idxs_sb) + idxs_sb = torch.where(valid, idxs_sb * b + batch_ids_exp, idxs_sb) + return idxs_sb.int() + + +def build_flat_topk_idxs( + *idx_groups: Tensor, batch_size: int, seqlen_kv: int, compact: bool = False +) -> Tuple[Tensor, Optional[Tensor]]: + """Combine local per-batch index groups and convert to flat global form. + + Each *idx_group* is ``(b, sq, topk_i)`` with local per-batch KV indices + (already in ``kv_full`` index space, i.e. with any compressed-position + offset applied). ``-1`` marks invalid positions. + + Args: + *idx_groups: one or more ``(b, sq, topk_i)`` int tensors. + batch_size: ``B``. + seqlen_kv: total KV sequence length per batch. + compact: if True, pack valid entries to the front of each row and + additionally return ``topk_length``; if False, leave as-is and + return ``None``. + + Returns: + ``(topk_idxs, topk_length)`` where + ``topk_idxs`` is ``(sq*b, total_topk)`` int32 (flat global) and + ``topk_length`` is ``(sq*b,)`` int32 when ``compact``, else ``None``. + """ + combined = torch.cat(idx_groups, dim=-1) # (b, sq, total_topk) + b, sq, total_topk = combined.shape + + # Globalize first, compact second. Both ops are element-wise + (-1)-preserving, + # so swapping the order is a no-op for correctness; the win is that the + # global indices come out already in (sq*b, total_topk) flat layout, which is + # exactly the row order the cuDNN compactify kernel returns its per-row + # ``length`` in — no extra permute on the length tensor. + global_idxs = local_to_global_flat(combined, b, seqlen_kv) + + topk_length_flat = None + if compact: + if global_idxs.is_cuda: + # Fast path: single warp-per-row CuTe DSL kernel from cuDNN's DSA + # namespace. Replaces a stable argsort + gather + sum + permute + # chain with one global-load + global-store per element. + _ensure_dsa_namespace() + res = _DSA.compactify_wrapper(global_idxs) + global_idxs, topk_length_flat = res["indices"], res["topk_length"] + else: + # CPU fallback so the unit tests that exercise this helper without + # CUDA still work. Production callers always go through the CUDA + # path above. + valid_mask = global_idxs >= 0 + sorted_indices = valid_mask.int().argsort(dim=-1, descending=True, stable=True) + global_idxs = global_idxs.gather(-1, sorted_indices) + topk_length_flat = valid_mask.sum(dim=-1).int() + + return global_idxs, topk_length_flat + + +# --------------------------------------------------------------------------- +# Path A + Path C step 2: differentiable sparse attention +# --------------------------------------------------------------------------- + + +class SparseAttnFunc(torch.autograd.Function): + """SM100 sparse attention fwd + bwd on flat tensors. + + Forward uses :mod:`flash_mla`; backward uses cuDNN Frontend's + :attr:`cudnn.DSA.sparse_attention_backward_wrapper`. + """ + + @staticmethod + def forward( + ctx, + q: Tensor, # (total_sq, H, D) bf16 + kv: Tensor, # (total_skv, D) bf16 + attn_sink: Tensor, # (H,) f32 + topk_idxs: Tensor, # (total_sq, TopK) int32 global + topk_length: Optional[Tensor], # (total_sq,) int32 or None + softmax_scale: float, + indexer_topk: int, + value_dim: Optional[int], + ) -> Tuple[Tensor, Tensor, Optional[Tensor]]: + """Run FlashMLA sparse-attention forward and save tensors for backward.""" + out, lse, lse_indexer = _dsa_fwd_flash_mla( + q, + kv, + topk_idxs, + softmax_scale, + attn_sink=attn_sink, + topk_length=topk_length, + indexer_topk=indexer_topk, + d_v=512 if value_dim is None else value_dim, + ) + + ctx.save_for_backward(q, kv, attn_sink, topk_idxs, out, lse) + ctx.softmax_scale = softmax_scale + ctx.topk_length = topk_length + return out, lse, lse_indexer + + @staticmethod + def backward(ctx, dO, d_lse, d_lse_indexer): + """Compute sparse-attention backward via cuDNN DSA wrapper.""" + _ensure_dsa_namespace() + + q, kv, attn_sink, topk_idxs, out, lse = ctx.saved_tensors + + result = _DSA.sparse_attention_backward_wrapper( + q, + kv, + out, + dO, + lse, + attn_sink, + topk_idxs, + softmax_scale=ctx.softmax_scale, + topk_length=ctx.topk_length, + ) + dq, dkv, d_sink = result["dq"], result["dkv"], result["d_sink"] + return dq, dkv, d_sink, None, None, None, None, None + + +def dsa_sparse_attn( + query: Tensor, + kv: Tensor, + attn_sink: Tensor, + topk_idxs: Tensor, + softmax_scale: float, + topk_length: Optional[Tensor] = None, + indexer_topk: int = 0, + value_dim: Optional[int] = None, +) -> Tensor: + """Sparse attention (Path A / Path C step 2). + + Args: + query: ``(sq, b, np, d)`` bf16 SBHD. + kv: ``(skv, b, d)`` bf16 SBD (K=V). + attn_sink: ``(np,)`` f32. + topk_idxs: ``(sq*b, topk)`` int32 — **flat global** indices produced + by :func:`build_flat_topk_idxs`. + softmax_scale: scalar float. + topk_length: ``(sq*b,)`` int32 — optional compact fast-path. Must be + ``None`` when ``indexer_topk > 0`` (FlashMLA constraint). + indexer_topk: int; ``0`` for Paths A/C, positive for Path B to enable + FlashMLA's ``lse_indexer`` output. + value_dim: FlashMLA value dimension. Defaults to ``512`` to preserve + the existing DSA wrapper behavior. + + Returns: + ``(sq, b, np * d_v)`` bf16 output. + """ + sq, b, np_, d = query.shape + skv = kv.shape[0] + + q_flat = query.reshape(sq * b, np_, d) + kv_flat = kv.reshape(skv * b, d) + + out_flat, _lse, _lse_indexer = SparseAttnFunc.apply( + q_flat, kv_flat, attn_sink, topk_idxs, topk_length, softmax_scale, indexer_topk, value_dim + ) + + d_v = out_flat.shape[-1] + return out_flat.reshape(sq, b, np_, d_v).reshape(sq, b, np_ * d_v) + + +# --------------------------------------------------------------------------- +# Path C inference: indexer scoring + top-K +# --------------------------------------------------------------------------- + + +def _indexer_topk_bshd( + q_bshd: Tensor, k_bsd: Tensor, w_bsh: Tensor, topk: int, ratio: int = 4 +) -> Tuple[Tensor, Tensor, Tensor]: + """BSHD-layout core for :func:`indexer_topk`. + + Internal entry point used by both the public SBHD wrapper and Path B's + ``FusedIndexerSparseAttnFunc.forward`` so the SBHD→BSHD permute can be + performed once at the call site and reused across both the indexer + forward and the score-backward kernels (predict / target). + + Args: + q_bshd: ``(b, sq, idx_nh, idx_hd)`` bf16, C-contiguous. + k_bsd: ``(b, sk, idx_hd)`` bf16, C-contiguous. + w_bsh: ``(b, sq, idx_nh)`` bf16, C-contiguous, **already + ``indexer_softmax_scale``-scaled** by the caller. + topk: number of top-K indices to return per query. + ratio: compression ratio for the kernel's causal mask. + + Returns: + ``(topk_indices, topk_length, scores)`` where: + + * ``topk_indices``: ``(b, sq, topk)`` int32, invalid slots ``-1``. + * ``topk_length``: ``(b, sq)`` int32, per-row valid count. + * ``scores``: ``(b, sq, sk)`` fp32, raw scores from + :attr:`cudnn.DSA.indexer_forward_wrapper` with ``-inf`` on + causally-masked positions. + """ + _ensure_dsa_namespace() + + b, sq, _idx_nh, _idx_hd = q_bshd.shape + sk = k_bsd.shape[1] + device = q_bshd.device + + k_bshd = k_bsd.unsqueeze(2) # (b, sk, 1, idx_hd) + + scores = _dsa_indexer_forward_wrapper(q_bshd, k_bshd, w_bsh, ratio=ratio)[ + "scores" + ] # (b, sq, sk) fp32, -inf on masked positions + + # Top-K selection via the TRT-LLM CuTe-DSL radix kernel. + n_rows = b * sq + scores_flat = scores.reshape(n_rows, sk).contiguous() + q_idx = torch.arange(sq, device=device) + valid_per_q = ((q_idx + 1) // ratio).clamp(max=sk).to(torch.int32) # (sq,) + seq_lens = valid_per_q.repeat(b) # (b*sq,), row-major over (b, sq) + + topk_k = min(topk, sk) + tk_result = _DSA.indexer_top_k_wrapper( + scores_flat, seq_lens, top_k=topk_k, next_n=1, return_val=False + ) + topk_indices = tk_result["indices"].view(b, sq, topk_k) + + if topk_k < topk: + pad = torch.full((b, sq, topk - topk_k), -1, dtype=torch.int32, device=device) + topk_indices = torch.cat([topk_indices, pad], dim=-1) + + topk_length = (topk_indices >= 0).sum(dim=-1).int() # (b, sq) + return topk_indices.int(), topk_length, scores + + +def _sbhd_to_bshd_indexer_inputs( + q_indexer: Tensor, k_indexer: Tensor, weights: Tensor, indexer_softmax_scale: float +) -> Tuple[Tensor, Tensor, Tensor, Tensor]: + """Permute the indexer inputs SBHD→BSHD once, returning both the raw + BSHD weights and (when needed) a separate scaled copy. + + The ``relu(c·x) = c·relu(x)`` trick lets us push the indexer softmax + scale onto ``W`` (``(B, S_q, H)``, small) instead of the score tensor + (``(B, S_q, S_k)``, big). The raw ``w_bsh`` is preserved for the + backward GEMM path, which takes ``sm_scale`` directly. When + ``indexer_softmax_scale == 1.0`` the two views alias each other. + + Returns ``(q_bshd, k_bsd, w_bsh, w_bsh_scaled)``. + """ + q_bshd = q_indexer.permute(1, 0, 2, 3).contiguous() + k_bsd = k_indexer.permute(1, 0, 2).contiguous() + w_bsh = weights.permute(1, 0, 2).contiguous() + + if indexer_softmax_scale != 1.0: + w_bsh_scaled = (w_bsh.float() * indexer_softmax_scale).to(w_bsh.dtype) + else: + w_bsh_scaled = w_bsh + + return q_bshd, k_bsd, w_bsh, w_bsh_scaled + + +def indexer_topk( + q_indexer: Tensor, + k_indexer: Tensor, + weights: Tensor, + topk: int, + ratio: int = 4, + indexer_softmax_scale: float = 1.0, +) -> Tuple[Tensor, Tensor]: + """Score + top-K selection for inference (no KL loss, no backward). + + Built on cuDNN Frontend's CuTe-DSL indexer forward kernel followed by + TRT-LLM's radix top-K kernel. + + Args: + q_indexer: ``(sq, b, idx_nh, idx_hd)`` bf16 SBHD. + k_indexer: ``(sk, b, idx_hd)`` bf16 SBD. + weights: ``(sq, b, idx_nh)`` bf16 SBH — raw (unscaled) weights. + topk: number of top-K indices to select. + ratio: compression ratio for the causal mask. + indexer_softmax_scale: scale applied to the indexer ``Q @ K^T`` + scores (typically ``idx_hd ** -0.5``). Applied internally via + the weights-scaling trick (``relu(c·x) = c·relu(x)`` for + ``c > 0``) so the caller passes raw weights. Default ``1.0`` + means weights are treated as already-scaled. + + Returns: + topk_indices: ``(b, sq, topk)`` int32 — local per-batch indices into + ``k_indexer``; invalid positions are ``-1``. + topk_length: ``(b, sq)`` int32 — per-query valid count. + """ + q_bshd, k_bsd, _w_bsh_raw, w_bsh_scaled = _sbhd_to_bshd_indexer_inputs( + q_indexer, k_indexer, weights, indexer_softmax_scale + ) + topk_indices, topk_length, _ = _indexer_topk_bshd(q_bshd, k_bsd, w_bsh_scaled, topk, ratio) + return topk_indices, topk_length + + +# --------------------------------------------------------------------------- +# Path B: fused indexer + sparse attention (training) +# --------------------------------------------------------------------------- + + +_CLIP_PROB_MIN = torch.finfo(torch.float32).tiny # kept compatible w/ cudnn kernel + + +def _compute_indexer_predict( + q_indexer_bshd: Tensor, + k_indexer_bsd: Tensor, + weights_bsh: Tensor, + topk_indices: Tensor, + qhead_per_kv_head: int, +) -> Tensor: + """Compute ``predict`` distribution (softmax over top-K of indexer scores). + + Wraps :attr:`cudnn.DSA.sparse_indexer_score_recompute_wrapper`. + + Args: + q_indexer_bshd: ``(B, S_q, H_q, D)`` bf16. + k_indexer_bsd: ``(B, S_k, D)`` bf16. + weights_bsh: ``(B, S_q, H_q)`` bf16. + topk_indices: ``(B, S_q, topk)`` int32. + qhead_per_kv_head: ``H_q`` (MQA). + + Returns: + predict: ``(B, S_q, topk)`` fp32, softmax over the top-K axis. + """ + _ensure_dsa_namespace() + result = _DSA.sparse_indexer_score_recompute_wrapper( + q_indexer_bshd, + k_indexer_bsd, + weights_bsh, + topk_indices, + qhead_per_kv_head=qhead_per_kv_head, + ) + return result["predict"] + + +def _compute_attn_target( + q_attn_bshd: Tensor, + k_attn_bsd: Tensor, + lse: Tensor, + topk_indices: Tensor, + softmax_scale: float, + qhead_per_kv_head: int, +) -> Tensor: + """Compute ``target`` distribution (L1-normalised head-sum softmax). + + Wraps :attr:`cudnn.DSA.sparse_attn_score_recompute_wrapper`. + + Shapes match :func:`_compute_indexer_predict`; ``lse`` is + ``(B, S_q, H_q)`` FP32 (comes from the attention forward pass). + """ + _ensure_dsa_namespace() + result = _DSA.sparse_attn_score_recompute_wrapper( + q_attn_bshd, + k_attn_bsd, + lse, + topk_indices, + softmax_scale, + qhead_per_kv_head=qhead_per_kv_head, + ) + return result["target"] + + +def _kl_loss_from_target_predict( + target: Tensor, + predict: Tensor, + topk_indices: Tensor, + loss_coeff: float, + calculate_per_token_loss: bool = False, +) -> Tensor: + """KL(target || predict) reduced over ``(B, S_q)`` and scaled by loss_coeff. + + Rows with no valid top-K positions (early query rows with ratio causal + masking) contribute 0 to the loss — the sparse score kernels produce + garbage for those rows, mirroring ``compute_dsa_indexer_loss``'s + ``row_valid`` handling. The default mean is taken over all ``(B, S_q)`` + positions. Per-token-loss mode returns a raw local sum so finalize can + apply the global token divisor. + """ + eps = _CLIP_PROB_MIN + t = target.clamp(min=eps) + p = predict.clamp(min=eps) + kl_per_row = (t * (torch.log(t) - torch.log(p))).sum(dim=-1) # (B, S_q) + + row_valid = (topk_indices >= 0).any(dim=-1) # (B, S_q) + kl_per_row = torch.where(row_valid, kl_per_row, torch.zeros_like(kl_per_row)) + loss = kl_per_row.sum() if calculate_per_token_loss else kl_per_row.mean() + return loss_coeff * loss + + +# --------------------------------------------------------------------------- +# Dense path (``sparse_loss=False``) — full-KV indexer loss +# --------------------------------------------------------------------------- + + +def _compute_dense_indexer_score( + q_indexer_bshd: Tensor, + k_indexer_bshd: Tensor, + weights_bsh: Tensor, + qhead_per_kv_head: int, + indexer_softmax_scale: float, + ratio: int, +) -> Tuple[Tensor, Tensor]: + """Dense indexer score forward over the full ``S_k`` axis. + + Wraps :attr:`cudnn.DSA.dense_indexer_score_recompute_wrapper`. Returns + ``(out, denom)`` where + + * ``out`` : ``(B, S_q, S_k)`` fp32, the raw head-reduced score + ``S[b,q,k] = indexer_softmax_scale * sum_h ReLU(Q_h · K_k^T) · W_{b,q,h}`` + with the kernel's ``ratio``-causal mask applied to invalid columns. + * ``denom`` : ``(B, S_q)`` fp32, the LSE denom of ``out`` along + ``S_k`` — i.e. ``predict = exp(out - denom[..., None])`` is the + indexer softmax distribution over the full KV. + + Both outputs are forwarded into :func:`_kl_loss_from_dense_scores` + *and* saved for the dense-path backward, where the dense indexer-grad + kernel consumes them directly. + """ + _ensure_dsa_namespace() + result = _DSA.dense_indexer_score_recompute_wrapper( + q_indexer_bshd, + k_indexer_bshd, + weights_bsh, + qhead_per_kv_head=qhead_per_kv_head, + sm_scale=indexer_softmax_scale, + ratio=ratio, + ) + return result["out"], result["denom"] + + +def _compute_dense_attn_score( + q_attn_bshd: Tensor, + k_attn_bshd: Tensor, + lse: Tensor, + qhead_per_kv_head: int, + softmax_scale: float, + ratio: int, +) -> Tuple[Tensor, Tensor]: + """Dense attention score forward over the full ``S_k`` axis. + + Wraps :attr:`cudnn.DSA.dense_attn_score_recompute_wrapper`. Returns + ``(out, denom)`` where + + * ``out`` : ``(B, S_q, S_k)`` fp32, the head-summed unnormalized + attention probability ``S[b,q,k] = sum_h exp(Q_h · K_k^T · scale - LSE[b,q,h])`` + with ``ratio`` causal mask applied. + * ``denom`` : ``(B, S_q)`` fp32, the L1-norm denom ``sum_k S[b,q,:]``. + ``target = out / denom[..., None]`` is the L1-normalized + head-summed attention distribution. + """ + _ensure_dsa_namespace() + result = _DSA.dense_attn_score_recompute_wrapper( + q_attn_bshd, + k_attn_bshd, + lse, + softmax_scale, + qhead_per_kv_head=qhead_per_kv_head, + ratio=ratio, + ) + return result["out"], result["denom"] + + +def _kl_loss_from_dense_scores( + attn_score: Tensor, + attn_l1norm: Tensor, + index_score: Tensor, + index_lse: Tensor, + loss_coeff: float, + calculate_per_token_loss: bool = False, +) -> Tensor: + """KL(target || predict) over the **full** KV axis, averaged over ``(B, S_q)``. + + Derives ``target = attn_score / attn_l1norm`` (L1-normalised, matches + ``compute_dsa_indexer_loss``'s ``attention_scores / sum`` step) and + ``log_predict = index_score - index_lse`` (LSE-normalised log-softmax), + then computes ``KL = sum_k target * (log target - log predict)`` and + scales by ``loss_coeff``. + + Rows where the kernel's ``ratio`` causal mask leaves no valid KV + position have ``attn_l1norm <= 0`` (L1) or ``index_lse == -inf`` + (LSE); those rows contribute 0 to the loss — the same ``row_valid`` + semantics as the reference ``compute_dsa_indexer_loss``. + """ + eps = _CLIP_PROB_MIN + # row_valid: rows with at least one un-masked KV position. + row_valid = (attn_l1norm > eps) & torch.isfinite(index_lse) + + # Safe denoms: replace invalid rows with a finite value so target / + # log-predict don't produce NaN; the row mask zeroes their KL below. + safe_l1 = attn_l1norm.clamp(min=eps) + safe_lse = torch.where(row_valid, index_lse, torch.zeros_like(index_lse)) + + target = attn_score / safe_l1.unsqueeze(-1) + target_clamped = target.clamp(min=eps) + # Per-position validity: the indexer-score kernel emits -inf at + # ratio-masked positions; those contribute 0 to KL by the + # ``0 · log(0/p) = 0`` convention. Without this gate, the eps-clamp + # on target makes the term ``eps · (log eps - (-inf)) = +inf``. + position_valid = torch.isfinite(index_score) + safe_index_score = torch.where(position_valid, index_score, torch.zeros_like(index_score)) + log_predict = safe_index_score - safe_lse.unsqueeze(-1) + + kl_terms = target_clamped * (torch.log(target_clamped) - log_predict) + kl_terms = torch.where(position_valid, kl_terms, torch.zeros_like(kl_terms)) + kl_per_row = kl_terms.sum(dim=-1) # (B, S_q) + kl_per_row = torch.where(row_valid, kl_per_row, torch.zeros_like(kl_per_row)) + loss = kl_per_row.sum() if calculate_per_token_loss else kl_per_row.mean() + return loss_coeff * loss + + +class FusedIndexerSparseAttnFunc(torch.autograd.Function): + """Path B: fused indexer (+KL loss) + sparse attention in one autograd. + + Differentiable w.r.t. ``query``, ``kv_full``, ``attn_sink``, + ``q_indexer``, ``k_indexer``, ``weights``. + + Two indexer-loss variants, selected by the ``sparse_loss`` argument + (matches ``compute_dsa_indexer_loss`` in the reference ``dsa.py``): + + * **Sparse loss** (``sparse_loss=True``) — KL is computed only over + the top-K KV positions the indexer has selected. + * **Dense loss** (``sparse_loss=False``, the default) — KL is + computed over *all* causally valid KV positions. + + Both variants share the FlashMLA sparse-attention forward + the + cuDNN sparse-attn backward; only the indexer-loss path branches. + """ + + @staticmethod + def forward( + ctx, + # Sparse attn inputs (differentiable) + query: Tensor, # (sq, b, np, d) bf16 + kv_full: Tensor, # (skv, b, d) bf16 + attn_sink: Tensor, # (np,) f32 + # Window indices (not differentiable) + window_idxs: Tensor, # (b, sq, win_topk) int32 + # Indexer inputs (differentiable) + q_indexer: Tensor, # (sq, b, idx_nh, idx_hd) bf16 + k_indexer: Tensor, # (n_comp, b, idx_hd) bf16 + weights: Tensor, # (sq, b, idx_nh) bf16 — raw (unscaled) + # Scalars + indexer_topk: int, + ratio: int, + softmax_scale: float, + indexer_softmax_scale: float, + loss_coeff: float, + sparse_loss: bool, + kv_offset: int, + calculate_per_token_loss: bool, + value_dim: Optional[int], + ) -> Tuple[Tensor, Tensor]: + """Fused forward: indexer scoring, sparse attention, KL loss, and indexer backward.""" + _ensure_dsa_namespace() + + sq, b, np_, d = query.shape + skv = kv_full.shape[0] + n_comp = k_indexer.shape[0] + idx_nh, idx_hd = q_indexer.shape[2], q_indexer.shape[3] + + requested_topk = indexer_topk + effective_topk = min(requested_topk, n_comp) + + # ---- 1. Permute indexer inputs SBHD->BSHD ONCE. ------------------- + q_idx_bshd, k_idx_bsd, w_bsh, w_bsh_scaled = _sbhd_to_bshd_indexer_inputs( + q_indexer, k_indexer, weights, indexer_softmax_scale + ) + + # ---- 2. Indexer scoring + top-K (with scores retained). ------------- + topk_indices_cmp, _, indexer_scores = _indexer_topk_bshd( + q_idx_bshd, k_idx_bsd, w_bsh_scaled, effective_topk, ratio + ) # topk_indices_cmp: (b, sq, effective_topk) int32; indexer_scores: (b, sq, n_comp) fp32 + + # ---- 3. Combine indices (indexer first, then window). -------------- + compress_topk_idxs = torch.where(topk_indices_cmp >= 0, topk_indices_cmp + kv_offset, -1) + if requested_topk > effective_topk: + pad = torch.full( + (b, sq, requested_topk - effective_topk), + -1, + device=compress_topk_idxs.device, + dtype=compress_topk_idxs.dtype, + ) + compress_topk_idxs = torch.cat([compress_topk_idxs, pad], dim=-1) + combined_local = torch.cat([compress_topk_idxs, window_idxs], dim=-1) + global_idxs = local_to_global_flat(combined_local, b, skv) + + # ---- 4. FlashMLA forward (non-compact, indexer_topk > 0). --------- + q_flat = query.reshape(sq * b, np_, d) + kv_flat = kv_full.reshape(skv * b, d) + out_flat, lse, lse_indexer = _dsa_fwd_flash_mla( + q_flat, + kv_flat, + global_idxs, + softmax_scale, + attn_sink=attn_sink, + topk_length=None, + indexer_topk=requested_topk, + d_v=512 if value_dim is None else value_dim, + ) + + # ---- 5. Derive predict from indexer_scores, compute target. -------- + # Attention-path tensors (detached — loss is not differentiable through them). + q_attn_bshd = query.detach().permute(1, 0, 2, 3).contiguous() + k_attn_compressed_bsd = kv_full[kv_offset:].detach().permute(1, 0, 2).contiguous() + lse_indexer_bsqh = lse_indexer.reshape(sq, b, np_).permute(1, 0, 2) + + if sparse_loss: + # Derive predict: gather topk scores from indexer_scores → softmax. + safe_indices = topk_indices_cmp.clamp(min=0).long() + gathered_scores = torch.gather(indexer_scores, dim=2, index=safe_indices) + gathered_scores = torch.where( + topk_indices_cmp >= 0, gathered_scores, torch.finfo(torch.float32).min + ) + predict = torch.softmax(gathered_scores, dim=-1) # (b, sq, topk) fp32 + + target = _compute_attn_target( + q_attn_bshd, + k_attn_compressed_bsd, + lse_indexer_bsqh, + topk_indices_cmp, + softmax_scale, + qhead_per_kv_head=np_, + ) + + if loss_coeff > 0: + indexer_loss = _kl_loss_from_target_predict( + target, predict, topk_indices_cmp, loss_coeff, calculate_per_token_loss + ) + else: + indexer_loss = torch.zeros((), device=query.device, dtype=torch.float32) + else: + # Dense: use full indexer_scores directly + logsumexp. + index_score = indexer_scores # (b, sq, n_comp) fp32 + index_lse = torch.logsumexp(indexer_scores, dim=-1) # (b, sq) fp32 + + attn_score, attn_l1norm = _compute_dense_attn_score( + q_attn_bshd, + k_attn_compressed_bsd.unsqueeze(2), + lse_indexer_bsqh, + qhead_per_kv_head=np_, + softmax_scale=softmax_scale, + ratio=ratio, + ) + + if loss_coeff > 0: + indexer_loss = _kl_loss_from_dense_scores( + attn_score, + attn_l1norm, + index_score, + index_lse, + loss_coeff, + calculate_per_token_loss, + ) + else: + indexer_loss = torch.zeros((), device=query.device, dtype=torch.float32) + + # ---- 6. Eagerly compute indexer backward (grad_loss=1). ------------ + # The actual grad_loss scaling is deferred to backward (when + # DSAIndexerLossAutoScaler provides the correct scale). + indexer_loss_coeff = loss_coeff + if calculate_per_token_loss: + indexer_loss_coeff = loss_coeff * (b * sq) + + unit_grad_loss = torch.ones((), device=query.device, dtype=torch.float32) + + if loss_coeff > 0: + if sparse_loss: + attn_score_for_bwd = target.clone() + index_score_for_bwd = predict.clone() + ig = _DSA.indexer_backward_wrapper( + q_idx_bshd, + w_bsh, + k_idx_bsd, + attn_score_for_bwd, + index_score_for_bwd, + topk_indices_cmp, + sm_scale=indexer_softmax_scale, + loss_coeff=indexer_loss_coeff, + grad_loss=unit_grad_loss, + block_I=128, + ) + else: + attn_score_for_bwd = attn_score.clone() + index_score_for_bwd = index_score.clone() + ig = _DSA.dense_indexer_backward_wrapper( + q_idx_bshd, + w_bsh, + k_idx_bsd, + attn_score_for_bwd, + attn_l1norm, + index_score_for_bwd, + index_lse, + sm_scale=indexer_softmax_scale, + loss_coeff=indexer_loss_coeff, + grad_loss=unit_grad_loss, + ratio=ratio, + block_I=128, + ) + # BSHD -> SBHD (match input layout). + precomputed_grad_q_indexer = ig["d_index_q"].permute(1, 0, 2, 3).contiguous() + precomputed_grad_k_indexer = ig["d_index_k"].permute(1, 0, 2).contiguous() + precomputed_grad_weights = ig["d_weights"].permute(1, 0, 2).contiguous() + else: + precomputed_grad_q_indexer = torch.zeros_like(q_indexer) + precomputed_grad_k_indexer = torch.zeros_like(k_indexer) + precomputed_grad_weights = torch.zeros_like(weights) + + # ---- 7. Save context (only sparse-attn bwd tensors + indexer grads). + ctx.save_for_backward( + q_flat, + kv_flat, + attn_sink, + global_idxs, + out_flat, + lse, + precomputed_grad_q_indexer, + precomputed_grad_k_indexer, + precomputed_grad_weights, + ) + ctx.softmax_scale = softmax_scale + ctx.sq = sq + ctx.b = b + ctx.np_ = np_ + ctx.d = d + ctx.skv = skv + + # ---- 8. Return. --------------------------------------------------- + d_v = out_flat.shape[-1] + output = out_flat.reshape(sq, b, np_, d_v).reshape(sq, b, np_ * d_v) + return output, indexer_loss + + @staticmethod + def backward(ctx, grad_output, grad_loss): + """Backward: sparse attention bwd + scale pre-computed indexer grads.""" + ( + q_flat, + kv_flat, + attn_sink, + global_idxs, + out_flat, + lse, + precomputed_grad_q_indexer, + precomputed_grad_k_indexer, + precomputed_grad_weights, + ) = ctx.saved_tensors + + sq, b, np_, d = ctx.sq, ctx.b, ctx.np_, ctx.d + skv = ctx.skv + + # ---- 1. Sparse attn backward. ------------------------------------- + d_v = out_flat.shape[-1] + dO_flat = grad_output.reshape(sq * b, np_, d_v) + + attn_bwd = _DSA.sparse_attention_backward_wrapper( + q_flat, + kv_flat, + out_flat, + dO_flat, + lse, + attn_sink, + global_idxs, + softmax_scale=ctx.softmax_scale, + topk_length=None, + ) + grad_query = attn_bwd["dq"].reshape(sq, b, np_, d) + grad_kv_full = attn_bwd["dkv"].reshape(skv, b, d) + d_sink = attn_bwd["d_sink"] + + # ---- 2. Scale pre-computed indexer grads by grad_loss. ------------- + grad_q_indexer = precomputed_grad_q_indexer * grad_loss + grad_k_indexer = precomputed_grad_k_indexer * grad_loss + grad_weights = precomputed_grad_weights * grad_loss + + # Grads: query, kv_full, attn_sink, window_idxs, q_indexer, k_indexer, + # weights, indexer_topk, ratio, softmax_scale, indexer_softmax_scale, + # loss_coeff, sparse_loss, kv_offset, calculate_per_token_loss, value_dim + return ( + grad_query, + grad_kv_full, + d_sink, + None, + grad_q_indexer, + grad_k_indexer, + grad_weights, + None, + None, + None, + None, + None, + None, + None, + None, + None, + ) + + +def fused_indexer_sparse_attn( + query: Tensor, + kv_full: Tensor, + attn_sink: Tensor, + window_idxs: Tensor, + q_indexer: Tensor, + k_indexer: Tensor, + weights: Tensor, + indexer_topk: int, + ratio: int, + softmax_scale: float, + indexer_softmax_scale: float = 1.0, + loss_coeff: float = 0.0, + sparse_loss: bool = False, + kv_offset: int = 0, + calculate_per_token_loss: bool = False, + value_dim: Optional[int] = None, +) -> Tuple[Tensor, Tensor]: + """Path B (training): fused indexer (+KL loss) + sparse attention. + + See :class:`FusedIndexerSparseAttnFunc` for the detailed data flow. + + Args: + query: ``(sq, b, np, d)`` bf16 SBHD — attention query. + kv_full: ``(skv, b, d)`` bf16 SBD — original + compressed KV. + attn_sink: ``(np,)`` f32 — learnable sink per head. + window_idxs: ``(b, sq, win_topk)`` int32 — local window indices. + q_indexer: ``(sq, b, idx_nh, idx_hd)`` bf16 — indexer query. + k_indexer: ``(n_comp, b, idx_hd)`` bf16 — indexer key (compressed). + weights: ``(sq, b, idx_nh)`` bf16 — raw indexer weights. + indexer_topk: number of top-K compressed positions to select. + ratio: compression ratio used for the causal mask. + softmax_scale: attention ``Q @ K^T`` scale, typically + ``1/sqrt(v_head_dim)``. + indexer_softmax_scale: indexer ``Q @ K^T`` scale, typically + ``1/sqrt(idx_hd)``. Applied internally — caller passes raw + (unscaled) ``weights``. + loss_coeff: coefficient scaling the KL divergence loss. + sparse_loss: if ``True``, KL is computed only over the top-K + positions (cheap, less informative); if ``False`` (the + default, matches ``transformer_config.dsa_indexer_use_sparse_loss``), + KL is computed over the full causally-valid KV (more + informative, matches the DeepSeek-V3.2 paper, larger + intermediate-tensor footprint). See + :class:`FusedIndexerSparseAttnFunc` for the full data flow + of each variant. + kv_offset: start of compressed region within ``kv_full``. + calculate_per_token_loss: if True, report raw local KL sum and + compensate the cuDNN backward wrappers' local averaging. + value_dim: FlashMLA value dimension. Defaults to ``512`` to preserve + the existing DSA wrapper behavior. + + Returns: + ``(output, indexer_loss)`` where ``output`` is ``(sq, b, np * d_v)`` + bf16 and ``indexer_loss`` is a scalar f32. + """ + return FusedIndexerSparseAttnFunc.apply( + query, + kv_full, + attn_sink, + window_idxs, + q_indexer, + k_indexer, + weights, + indexer_topk, + ratio, + softmax_scale, + indexer_softmax_scale, + loss_coeff, + sparse_loss, + kv_offset, + calculate_per_token_loss, + value_dim, + ) + + +__all__ = [ + "build_flat_topk_idxs", + "local_to_global_flat", + "dsa_sparse_attn", + "indexer_topk", + "fused_indexer_sparse_attn", +] diff --git a/experimental/lite/megatron/lite/primitive/kernels/grouped_gemm.py b/experimental/lite/megatron/lite/primitive/kernels/grouped_gemm.py new file mode 100644 index 00000000000..957eac77fdb --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/kernels/grouped_gemm.py @@ -0,0 +1,26 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Optional grouped GEMM kernel accessors.""" + +from __future__ import annotations + +try: + import grouped_gemm # pyright: ignore[reportMissingImports] +except Exception: # pragma: no cover - optional fused kernel or missing shared libraries. + grouped_gemm = None # type: ignore[assignment] + + +def grouped_gemm_is_available() -> bool: + return grouped_gemm is not None + + +def assert_grouped_gemm_is_available() -> None: + if grouped_gemm is None: + raise AssertionError( + "Grouped GEMM is not available. Please install the grouped_gemm package." + ) + + +ops = grouped_gemm.ops if grouped_gemm_is_available() else None + + +__all__ = ["assert_grouped_gemm_is_available", "grouped_gemm_is_available", "ops"] diff --git a/experimental/lite/megatron/lite/primitive/kernels/jit.py b/experimental/lite/megatron/lite/primitive/kernels/jit.py new file mode 100644 index 00000000000..63e630b7a92 --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/kernels/jit.py @@ -0,0 +1,27 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Local JIT/compile decorator shim for small fused Python kernels.""" + +from __future__ import annotations + +import os + +import torch + + +def noop_decorator(func): + return func + + +def _build_jit_fuser(): + if os.environ.get("MEGATRON_LITE_DISABLE_JIT_FUSER", "0") == "1": + return noop_decorator + compile_fn = getattr(torch, "compile", None) + if compile_fn is not None: + return compile_fn + return torch.jit.script + + +jit_fuser = _build_jit_fuser() + + +__all__ = ["jit_fuser", "noop_decorator"] diff --git a/experimental/lite/megatron/lite/primitive/kernels/swiglu.py b/experimental/lite/megatron/lite/primitive/kernels/swiglu.py new file mode 100644 index 00000000000..ec805a57c7f --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/kernels/swiglu.py @@ -0,0 +1,141 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""SwiGLU fused autograd helpers for MLite.""" + +from __future__ import annotations + +import torch +import torch.nn.functional as F + +from megatron.lite.primitive.kernels.jit import jit_fuser + + +@jit_fuser +def swiglu(y): + y_1, y_2 = torch.chunk(y, 2, -1) + return F.silu(y_1) * y_2 + + +@jit_fuser +def bias_swiglu(y, bias): + y = y + bias + return swiglu(y) + + +@jit_fuser +def weighted_swiglu(y, weights): + dtype = y.dtype + result = swiglu(y) * weights + return result.to(dtype) + + +@jit_fuser +def swiglu_back(g, y): + y_1, y_2 = torch.chunk(y, 2, -1) + return torch.cat( + (g * torch.sigmoid(y_1) * (1 + y_1 * (1 - torch.sigmoid(y_1))) * y_2, g * F.silu(y_1)), -1 + ) + + +@jit_fuser +def bias_swiglu_back(g, y, bias): + y = y + bias + return swiglu_back(g, y) + + +@jit_fuser +def weighted_swiglu_back(g, y, weights): + input_dtype = y.dtype + weight_dtype = weights.dtype + input_grad = swiglu_back(g * weights, y) + weights_grad = swiglu(y) * g.to(weight_dtype) + weights_grad = torch.sum(weights_grad, dim=-1, keepdim=True) + return input_grad.to(input_dtype), weights_grad.to(weight_dtype) + + +class BiasSwiGLUFunction(torch.autograd.Function): + @staticmethod + def forward(ctx, input, bias, fp8_input_store=False, cpu_offload_input=False): + input_for_backward = input.to(torch.float8_e4m3fn) if fp8_input_store else input + if cpu_offload_input: + input_for_backward.activation_offloading = True + if bias is not None: + bias.activation_offloading = True + ctx.save_for_backward(input_for_backward, bias) + ctx.ori_input_dtype = input.dtype + ctx.fp8_input_store = fp8_input_store + return bias_swiglu(input, bias) + + @staticmethod + def backward(ctx, grad_output): + input, bias = ctx.saved_tensors + input = input.to(ctx.ori_input_dtype) if ctx.fp8_input_store else input + grad = bias_swiglu_back(grad_output, input, bias) + return grad, grad, None, None + + +class SwiGLUFunction(torch.autograd.Function): + @staticmethod + def forward(ctx, input, fp8_input_store=False, cpu_offload_input=False): + input_for_backward = input.to(torch.float8_e4m3fn) if fp8_input_store else input + if cpu_offload_input: + input_for_backward.activation_offloading = True + ctx.save_for_backward(input_for_backward) + ctx.ori_input_dtype = input.dtype + ctx.fp8_input_store = fp8_input_store + return swiglu(input) + + @staticmethod + def backward(ctx, grad_output): + (input,) = ctx.saved_tensors + input = input.to(ctx.ori_input_dtype) if ctx.fp8_input_store else input + grad = swiglu_back(grad_output, input) + return grad, None, None + + +class WeightedSwiGLUFunction(torch.autograd.Function): + @staticmethod + def forward(ctx, input, weights, fp8_input_store=False): + input_for_backward = input.to(torch.float8_e4m3fn) if fp8_input_store else input + ctx.save_for_backward(input_for_backward, weights) + ctx.ori_input_dtype = input.dtype + ctx.fp8_input_store = fp8_input_store + return weighted_swiglu(input, weights) + + @staticmethod + def backward(ctx, grad_output): + input, weights = ctx.saved_tensors + input = input.to(ctx.ori_input_dtype) if ctx.fp8_input_store else input + input_grad, weights_grad = weighted_swiglu_back(grad_output, input, weights) + return input_grad, weights_grad, None + + +def bias_swiglu_impl(input, bias, fp8_input_store=False, cpu_offload_input=False): + original_shape = input.shape + assert len(original_shape) in [2, 3] + input = input.view(-1, original_shape[-1]) + if bias is not None: + output = BiasSwiGLUFunction.apply(input, bias, fp8_input_store, cpu_offload_input) + else: + output = SwiGLUFunction.apply(input, fp8_input_store, cpu_offload_input) + return ( + output + if len(original_shape) == 2 + else output.view(original_shape[0], original_shape[1], -1) + ) + + +def weighted_bias_swiglu_impl(input, bias, weights, fp8_input_store=False): + original_shape = input.shape + assert len(original_shape) in [2, 3] + input = input.view(-1, original_shape[-1]) + if bias is not None: + raise NotImplementedError("Bias is not supported for weighted swiglu fusion") + output = WeightedSwiGLUFunction.apply(input, weights, fp8_input_store) + return ( + output + if len(original_shape) == 2 + else output.view(original_shape[0], original_shape[1], -1) + ) + + +__all__ = ["bias_swiglu_impl", "swiglu", "weighted_bias_swiglu_impl"] diff --git a/experimental/lite/megatron/lite/primitive/modules/__init__.py b/experimental/lite/megatron/lite/primitive/modules/__init__.py new file mode 100644 index 00000000000..dab0fa137fd --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/modules/__init__.py @@ -0,0 +1,55 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Shared model modules owned by Megatron Lite.""" + +from __future__ import annotations + +_EXPORTS = { + "Experts": ("megatron.lite.primitive.modules.experts", "Experts"), + "GatedDeltaNet": ("megatron.lite.primitive.modules.gated_delta_net", "GatedDeltaNet"), + "GQAttention": ("megatron.lite.primitive.modules.gqa", "GQAttention"), + "MTPBlock": ("megatron.lite.primitive.modules.mtp", "MTPBlock"), + "MTPDecoderLayer": ("megatron.lite.primitive.modules.mtp", "MTPDecoderLayer"), + "MTPLossAutoScaler": ("megatron.lite.primitive.modules.mtp", "MTPLossAutoScaler"), + "MoEAuxLossAutoScaler": ("megatron.lite.primitive.modules.moe", "MoEAuxLossAutoScaler"), + "MultimodalRotaryEmbedding": ( + "megatron.lite.primitive.modules.mrope", + "MultimodalRotaryEmbedding", + ), + "SigmoidTopKRouter": ("megatron.lite.primitive.modules.router", "SigmoidTopKRouter"), + "SwiGLUMLP": ("megatron.lite.primitive.modules.mlp", "SwiGLUMLP"), + "TokenDispatcher": ("megatron.lite.primitive.modules.dispatcher", "TokenDispatcher"), + "TopKRouter": ("megatron.lite.primitive.modules.router", "TopKRouter"), + "_AllToAll": ("megatron.lite.primitive.modules.moe", "_AllToAll"), + "split_grouped_qkvg": ("megatron.lite.primitive.modules.gqa_utils", "split_grouped_qkvg"), +} + + +def __getattr__(name: str): + if name not in _EXPORTS: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + import importlib + + module_name, attr_name = _EXPORTS[name] + module = importlib.import_module(module_name) + value = getattr(module, attr_name) + globals()[name] = value + return value + + +__all__ = [ + "Experts", + "GatedDeltaNet", + "GQAttention", + "MTPBlock", + "MTPDecoderLayer", + "MTPLossAutoScaler", + "MoEAuxLossAutoScaler", + "MultimodalRotaryEmbedding", + "SigmoidTopKRouter", + "SwiGLUMLP", + "split_grouped_qkvg", + "TokenDispatcher", + "TopKRouter", + "_AllToAll", +] diff --git a/experimental/lite/megatron/lite/primitive/modules/attention/__init__.py b/experimental/lite/megatron/lite/primitive/modules/attention/__init__.py new file mode 100644 index 00000000000..f13e7670125 --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/modules/attention/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +from megatron.lite.primitive.modules.attention.dsa import ( + DynamicSparseAttention, + RMSNorm, + build_rope_cache, + build_rotary_embeddings, +) +from megatron.lite.primitive.modules.attention.mla import MultiLatentAttention + +__all__ = [ + "DynamicSparseAttention", + "MultiLatentAttention", + "RMSNorm", + "build_rope_cache", + "build_rotary_embeddings", +] diff --git a/experimental/lite/megatron/lite/primitive/modules/attention/cp.py b/experimental/lite/megatron/lite/primitive/modules/attention/cp.py new file mode 100644 index 00000000000..957e2e60d9a --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/modules/attention/cp.py @@ -0,0 +1,93 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +from typing import Any + +import torch +import torch.distributed as dist +from torch.distributed.nn.functional import all_gather + + +def _all_gather_cp(tensor: torch.Tensor, group: dist.ProcessGroup) -> list[torch.Tensor]: + return list(all_gather(tensor.contiguous(), group=group)) + + +def iter_cp_sources(tensor, position_ids, *, cp_rank, cp_size, cp_group): + if cp_size <= 1: + yield cp_rank, tensor, position_ids + return + if cp_group is None: + raise RuntimeError("CP source iteration requires a context-parallel process group.") + tensor_parts = _all_gather_cp(tensor, cp_group) + position_parts = _all_gather_cp(position_ids.to(dtype=torch.long), cp_group) + for rank, (source_tensor, source_positions) in enumerate(zip(tensor_parts, position_parts)): + yield rank, source_tensor, source_positions + + +def _gather_contiguous_tail(tensor, *, tail_len, cp_size, cp_group, seq_dim): + if cp_size <= 1 or tail_len <= 0: + return None + if cp_group is None: + raise RuntimeError("CP chunk-tail gather requires a context-parallel process group.") + if tensor.size(seq_dim) < tail_len: + raise ValueError(f"CP chunk tail needs len >= {tail_len}, got {tensor.size(seq_dim)}.") + tail = tensor.narrow(seq_dim, tensor.size(seq_dim) - tail_len, tail_len) + return _all_gather_cp(tail.contiguous(), cp_group) + + +def compress_contiguous_chunks_for_cp( + compressor, + tensor, + *, + position_ids, + cp_rank, + cp_size, + cp_group, + compress_kwargs: dict[str, Any] | None = None, + seq_dim=1, + compressed_seq_dim=2, +): + kwargs = compress_kwargs or {} + compress_ratio = int(compressor.compress_ratio) + if cp_size <= 1: + compressed = compressor(tensor, position_ids=position_ids, **kwargs) + if compressed is None: + return None + cutoff = (tensor.size(seq_dim) // compress_ratio) * compress_ratio + comp_pos = position_ids[:, :cutoff:compress_ratio] + return compressed, comp_pos + + drop_prefix = 0 + tail_parts = None + if compressor.overlap: + tail_parts = _gather_contiguous_tail( + tensor, + tail_len=compress_ratio, + cp_size=cp_size, + cp_group=cp_group, + seq_dim=seq_dim, + ) + zero_tail = tensor.new_zeros(()) + for tail in tail_parts: + zero_tail = zero_tail + tail.to(dtype=tensor.dtype).sum() * 0.0 + tensor = tensor + zero_tail + if tail_parts is not None and cp_rank > 0: + prefix = tail_parts[cp_rank - 1].to(device=tensor.device, dtype=tensor.dtype) + prefix_pos = position_ids[:, :compress_ratio] - compress_ratio + tensor = torch.cat([prefix, tensor], dim=seq_dim) + position_ids = torch.cat([prefix_pos, position_ids], dim=1) + drop_prefix = 1 + + compressed = compressor(tensor, position_ids=position_ids, **kwargs) + if compressed is None: + return None + cutoff = (tensor.size(seq_dim) // compress_ratio) * compress_ratio + comp_pos = position_ids[:, :cutoff:compress_ratio] + if drop_prefix: + compressed = compressed.narrow( + compressed_seq_dim, + drop_prefix, + compressed.size(compressed_seq_dim) - drop_prefix, + ) + comp_pos = comp_pos[:, drop_prefix:] + if compressed.size(compressed_seq_dim) == 0: + return None + return compressed, comp_pos diff --git a/experimental/lite/megatron/lite/primitive/modules/attention/csa.py b/experimental/lite/megatron/lite/primitive/modules/attention/csa.py new file mode 100644 index 00000000000..4cee8ee7528 --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/modules/attention/csa.py @@ -0,0 +1,714 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +import math +from typing import Any + +import torch +import torch.nn as nn +import transformer_engine.pytorch as te +from megatron.lite.primitive.modules.attention.dsa import rotate_activation +from megatron.lite.primitive.modules.attention.cp import ( + compress_contiguous_chunks_for_cp, + iter_cp_sources, +) +from megatron.lite.primitive.parallel.state import ParallelState +from megatron.lite.primitive.utils.rotary import ( + _yarn_find_correction_range, + _yarn_linear_ramp_mask, +) + + +class GroupedLinear(nn.Module): + def __init__(self, in_features_per_group: int, out_features: int, n_groups: int): + super().__init__() + self.in_features_per_group = in_features_per_group + self.out_features = out_features + self.n_groups = n_groups + self.weight = nn.Parameter(torch.empty(out_features, in_features_per_group)) + nn.init.kaiming_uniform_(self.weight, a=math.sqrt(5)) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + out_per_group = self.out_features // self.n_groups + weight = self.weight.view(self.n_groups, out_per_group, self.in_features_per_group) + return torch.einsum("...gd,god->...go", x, weight) + + +def build_rope_cos_sin( + position_ids: torch.Tensor, + rope_head_dim: int, + rope_theta: float, + *, + device: torch.device, + dtype: torch.dtype, +) -> tuple[torch.Tensor, torch.Tensor]: + inv_freq = 1.0 / ( + rope_theta + ** (torch.arange(0, rope_head_dim, 2, device=device, dtype=torch.float32) / rope_head_dim) + ) + freqs = torch.einsum("bs,d->bsd", position_ids.to(torch.float32), inv_freq) + emb = torch.cat([freqs, freqs], dim=-1) + return emb.cos().to(dtype=dtype), emb.sin().to(dtype=dtype) + + +def build_yarn_rope_cos_sin( + position_ids: torch.Tensor, + rope_head_dim: int, + rope_theta: float, + *, + config: Any, + device: torch.device, + dtype: torch.dtype, +) -> tuple[torch.Tensor, torch.Tensor]: + dim = rope_head_dim + inv_freq_extra = 1.0 / ( + rope_theta ** (torch.arange(0, dim, 2, device=device, dtype=torch.float32) / dim) + ) + inv_freq_inter = 1.0 / ( + config.rotary_scaling_factor + * rope_theta ** (torch.arange(0, dim, 2, device=device, dtype=torch.float32) / dim) + ) + low, high = _yarn_find_correction_range( + config.beta_fast, + config.beta_slow, + dim, + rope_theta, + config.original_max_position_embeddings, + ) + inv_freq_mask = 1.0 - _yarn_linear_ramp_mask(low, high, dim // 2, device) + inv_freq = inv_freq_inter * (1 - inv_freq_mask) + inv_freq_extra * inv_freq_mask + freqs = torch.einsum("bs,d->bsd", position_ids.to(torch.float32), inv_freq) + emb = torch.cat([freqs, freqs], dim=-1) + return emb.cos().to(dtype=dtype), emb.sin().to(dtype=dtype) + + +def build_compressed_rope_cos_sin( + position_ids: torch.Tensor, + rope_head_dim: int, + rope_theta: float, + *, + config: Any, + use_yarn: bool, + device: torch.device, + dtype: torch.dtype, +) -> tuple[torch.Tensor, torch.Tensor]: + if use_yarn: + return build_yarn_rope_cos_sin( + position_ids, + rope_head_dim, + rope_theta, + config=config, + device=device, + dtype=dtype, + ) + return build_rope_cos_sin(position_ids, rope_head_dim, rope_theta, device=device, dtype=dtype) + + +def apply_partial_rope( + x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor, rope_head_dim: int +) -> torch.Tensor: + if rope_head_dim == 0: + return x + rope = x[..., -rope_head_dim:] + tail = x[..., :-rope_head_dim] + rope_pairs = rope.unflatten(-1, (-1, 2)) + a, b = rope_pairs[..., 0], rope_pairs[..., 1] + c = cos[..., : rope_head_dim // 2] + s = sin[..., : rope_head_dim // 2] + while c.ndim < a.ndim: + c = c.unsqueeze(1) + s = s.unsqueeze(1) + out_a = a * c - b * s + out_b = a * s + b * c + rope_out = torch.stack([out_a, out_b], dim=-1).flatten(-2) + return torch.cat([tail, rope_out], dim=-1) + + +class CompressedSequenceCompressor(nn.Module): + def __init__(self, config: Any, compress_ratio: int, head_dim: int, *, rotate: bool = False): + super().__init__() + self.config = config + self.compress_ratio = compress_ratio + self.head_dim = head_dim + self.rope_head_dim = min(config.qk_rope_head_dim, head_dim) + self.overlap = compress_ratio == 4 + self.coff = 2 if self.overlap else 1 + self.rotate = rotate + self.wkv = nn.Linear(config.hidden_size, self.coff * head_dim, bias=False) + self.wgate = nn.Linear(config.hidden_size, self.coff * head_dim, bias=False) + self.ape = nn.Parameter( + torch.empty(compress_ratio, self.coff * head_dim, dtype=torch.float32) + ) + self.norm = te.RMSNorm(head_dim, eps=config.rms_norm_eps) + nn.init.normal_(self.ape, mean=0.0, std=config.initializer_range) + + def _overlap_transform(self, tensor: torch.Tensor, fill_value: float) -> torch.Tensor: + bsz, n_blocks, ratio, _, head_dim = tensor.shape + out = tensor.new_full((bsz, n_blocks, 2 * ratio, head_dim), fill_value) + out[:, :, ratio:] = tensor[:, :, :, 1] + out[:, 1:, :ratio] = tensor[:, :-1, :, 0] + return out + + def forward( + self, x: torch.Tensor, *, position_ids: torch.Tensor, rope_theta: float + ) -> torch.Tensor | None: + bsz, seq_len, _ = x.shape + ratio = self.compress_ratio + n_blocks = seq_len // ratio + if n_blocks == 0: + return None + cutoff = n_blocks * ratio + content = self.wkv(x[:, :cutoff]) + gate = self.wgate(x[:, :cutoff]) + content = content.view(bsz, n_blocks, ratio, self.coff, self.head_dim) + gate = gate.view_as(content) + gate = gate + self.ape.view(1, 1, ratio, self.coff, self.head_dim).to(gate.device) + if self.overlap: + content = self._overlap_transform(content, 0.0) + gate = self._overlap_transform(gate, float("-inf")) + else: + content = content.squeeze(3) + gate = gate.squeeze(3) + weights = torch.softmax(gate.float(), dim=2).to(dtype=content.dtype) + compressed = self.norm((content * weights).sum(dim=2)).unsqueeze(1) + compressed_positions = position_ids[:, :cutoff:ratio] + cos, sin = build_compressed_rope_cos_sin( + compressed_positions, + self.rope_head_dim, + rope_theta, + config=self.config, + use_yarn=self.compress_ratio > 1, + device=x.device, + dtype=compressed.dtype, + ) + compressed = apply_partial_rope(compressed, cos, sin, self.rope_head_dim) + return rotate_activation(compressed) if self.rotate else compressed + + +def _source_scores_mask( + q_positions: torch.Tensor, k_positions: torch.Tensor, *, sliding_window: int +) -> torch.Tensor: + q_pos = q_positions.unsqueeze(-1) + k_pos = k_positions.unsqueeze(1) + return (k_pos <= q_pos) & (k_pos >= q_pos - sliding_window + 1) + + +def _compressed_scores_mask( + q_positions: torch.Tensor, comp_positions: torch.Tensor, *, ratio: int +) -> torch.Tensor: + visible = (q_positions + 1) // ratio + comp_ids = comp_positions // ratio + return comp_ids.unsqueeze(1) < visible.unsqueeze(-1) + + +def _window_topk_indices( + batch: int, seq_len: int, window: int, *, device: torch.device +) -> torch.Tensor: + topk = max(1, min(int(window), seq_len)) + query_pos = torch.arange(seq_len, device=device).view(seq_len, 1) + offsets = torch.arange(topk, device=device) + indices = (query_pos - topk + 1).clamp(min=0) + offsets + indices = torch.where(indices > query_pos, -1, indices) + return indices.unsqueeze(0).expand(batch, -1, -1).to(torch.int32) + + +def _load_dsa_kernels(): + from megatron.lite.primitive.kernels import dsa_kernels + + return dsa_kernels + + +class CompressedSparseAttentionIndexer(nn.Module): + def __init__(self, config, compress_ratio: int): + super().__init__() + self.config = config + self.index_n_heads = config.index_n_heads + self.index_head_dim = config.index_head_dim + self.index_topk = config.index_topk + self.rope_head_dim = min(config.qk_rope_head_dim, config.index_head_dim) + self.softmax_scale = self.index_head_dim**-0.5 + self.wq_b = nn.Linear( + config.q_lora_rank, config.index_n_heads * config.index_head_dim, bias=False + ) + self.weights_proj = nn.Linear(config.hidden_size, config.index_n_heads, bias=False) + self.compressor = CompressedSequenceCompressor( + config, compress_ratio, config.index_head_dim, rotate=True + ) + + +class CompressedSparseAttention(nn.Module): + def __init__( + self, + config, + *, + layer_idx: int, + ps: ParallelState, + ): + super().__init__() + self.config = config + self.ps = ps + self.attention_backend = "torch" + self.num_heads = config.num_attention_heads + self.head_dim = config.head_dim + self.rope_head_dim = config.qk_rope_head_dim + self.num_heads_per_group = config.num_attention_heads // config.o_groups + # MTP layers use layer_idx == num_hidden_layers (+i), which is past the + # per-decoder-layer compress_ratios list (length num_hidden_layers); fall + # back to the last real layer's ratio so the MTP CSA still builds. + if config.compress_ratios: + _cr_idx = min(layer_idx, len(config.compress_ratios) - 1) + self.compress_ratio = config.compress_ratios[_cr_idx] + else: + self.compress_ratio = 0 + self.wq_a = nn.Linear(config.hidden_size, config.q_lora_rank, bias=False) + self.q_norm = te.RMSNorm(config.q_lora_rank, eps=config.rms_norm_eps) + self.wq_b = nn.Linear(config.q_lora_rank, self.num_heads * self.head_dim, bias=False) + self.wkv = nn.Linear(config.hidden_size, self.head_dim, bias=False) + self.kv_norm = te.RMSNorm(config.head_dim, eps=config.rms_norm_eps) + self.wo_a = GroupedLinear( + self.num_heads_per_group * self.head_dim, + config.o_groups * config.o_lora_rank, + config.o_groups, + ) + self.wo_b = nn.Linear(config.o_groups * config.o_lora_rank, config.hidden_size, bias=False) + self.sinks = nn.Parameter(torch.zeros(self.num_heads)) + self.compressor = ( + CompressedSequenceCompressor(config, self.compress_ratio, self.head_dim) + if self.compress_ratio > 1 + else None + ) + self.indexer = ( + CompressedSparseAttentionIndexer(config, self.compress_ratio) + if self.compress_ratio == 4 + else None + ) + + def forward( + self, + x: torch.Tensor, + *, + position_ids: torch.Tensor, + attention_mask: torch.Tensor | None = None, + ) -> torch.Tensor: + if self.ps.cp_size > 1 and attention_mask is not None: + raise ValueError("CP expects attention_mask=None; masks are derived from position_ids.") + batch, seq_len, _ = x.shape + attention_rope_theta = ( + self.config.compress_rope_theta if self.compress_ratio > 1 else self.config.rope_theta + ) + cos, sin = build_compressed_rope_cos_sin( + position_ids, + self.rope_head_dim, + attention_rope_theta, + config=self.config, + use_yarn=self.compress_ratio > 1, + device=x.device, + dtype=x.dtype, + ) + q_low = self.q_norm(self.wq_a(x)) + q = self.wq_b(q_low).view(batch, seq_len, self.num_heads, self.head_dim).transpose(1, 2) + q = q * torch.rsqrt( + q.float().pow(2).mean(dim=-1, keepdim=True) + self.config.rms_norm_eps + ).to(dtype=q.dtype) + kv = self.kv_norm(self.wkv(x)).view(batch, seq_len, 1, self.head_dim).transpose(1, 2) + q = apply_partial_rope(q, cos, sin, self.rope_head_dim) + kv = apply_partial_rope(kv, cos, sin, self.rope_head_dim) + use_sparse_backend = self.attention_backend not in {"local", "eager", "torch"} + if ( + use_sparse_backend + and self.compress_ratio == 4 + and self.ps.cp_size == 1 + and self.compressor is not None + and self.indexer is not None + ): + return self._forward_fused_dsa_cp1( + x, + q, + q_low, + kv, + position_ids=position_ids, + cos=cos, + sin=sin, + attention_mask=attention_mask, + ) + if use_sparse_backend and self.ps.cp_size == 1 and attention_mask is None: + return self._forward_fused_sparse_no_indexer_cp1( + x, + q, + kv, + position_ids=position_ids, + cos=cos, + sin=sin, + ) + + dense_score_parts = [] + dense_value_parts = [] + for _source_rank, source_kv, source_pos in iter_cp_sources( + kv, + position_ids, + cp_rank=self.ps.cp_rank, + cp_size=self.ps.cp_size, + cp_group=self.ps.cp_group, + ): + source_heads = source_kv.expand(-1, self.num_heads, -1, -1) + scores = torch.matmul(q.float(), source_heads.float().transpose(-1, -2)) / ( + self.head_dim**0.5 + ) + source_mask = _source_scores_mask( + position_ids, + source_pos, + sliding_window=self.config.sliding_window, + ).unsqueeze(1) + scores = scores.masked_fill(~source_mask, -float("inf")) + dense_score_parts.append(scores) + dense_value_parts.append(source_heads) + dense_scores = torch.cat(dense_score_parts, dim=-1) + if attention_mask is not None: + dense_scores = dense_scores + attention_mask.to(dtype=dense_scores.dtype) + score_parts = [dense_scores] + value_parts = [torch.cat(dense_value_parts, dim=2)] + + if self.compressor is not None: + compressed_pack = compress_contiguous_chunks_for_cp( + self.compressor, + x, + position_ids=position_ids, + cp_rank=self.ps.cp_rank, + cp_size=self.ps.cp_size, + cp_group=self.ps.cp_group, + compress_kwargs={"rope_theta": self.config.compress_rope_theta}, + ) + else: + compressed_pack = None + if compressed_pack is not None: + compressed, compressed_pos = compressed_pack + compressed, compressed_pos = self._gather_cp_sources( + compressed, compressed_pos, seq_dim=2 + ) + compressed_values = compressed.expand(-1, self.num_heads, -1, -1) + compressed_scores = torch.matmul( + q.float(), compressed_values.float().transpose(-1, -2) + ) / (self.head_dim**0.5) + compressed_valid = _compressed_scores_mask( + position_ids, + compressed_pos, + ratio=self.compress_ratio, + ).unsqueeze(1) + compressed_scores = compressed_scores.masked_fill(~compressed_valid, -float("inf")) + + if self.indexer is not None: + index_comp_pack = compress_contiguous_chunks_for_cp( + self.indexer.compressor, + x, + position_ids=position_ids, + cp_rank=self.ps.cp_rank, + cp_size=self.ps.cp_size, + cp_group=self.ps.cp_group, + compress_kwargs={"rope_theta": self.config.compress_rope_theta}, + ) + if index_comp_pack is not None: + index_comp, index_pos = index_comp_pack + index_comp, index_pos = self._gather_cp_sources( + index_comp, index_pos, seq_dim=2 + ) + idx_cos, idx_sin = build_compressed_rope_cos_sin( + position_ids, + self.indexer.rope_head_dim, + self.config.compress_rope_theta, + config=self.config, + use_yarn=self.compress_ratio > 1, + device=x.device, + dtype=x.dtype, + ) + q_idx = ( + self.indexer.wq_b(q_low) + .view( + batch, seq_len, self.indexer.index_n_heads, self.indexer.index_head_dim + ) + .transpose(1, 2) + ) + q_idx = apply_partial_rope(q_idx, idx_cos, idx_sin, self.indexer.rope_head_dim) + index_weights = ( + self.indexer.weights_proj(x).float() + * (self.indexer.index_n_heads**-0.5) + * self.indexer.softmax_scale + ) + k_idx = index_comp.squeeze(1) + index_scores = torch.einsum( + "bhsd,btd->bsht", q_idx.float(), k_idx.float() + ).relu() + index_scores = (index_scores * index_weights.unsqueeze(-1)).sum(dim=2) + index_valid = _compressed_scores_mask( + position_ids, + index_pos, + ratio=self.compress_ratio, + ) + index_scores = index_scores.masked_fill(~index_valid, -float("inf")) + topk = min(self.indexer.index_topk, index_scores.size(-1)) + topk_indices = index_scores.topk(topk, dim=-1).indices + topk_mask = torch.zeros_like(index_scores, dtype=torch.bool) + topk_mask.scatter_(-1, topk_indices, True) + compressed_scores = compressed_scores + index_scores.unsqueeze(1) + compressed_scores = compressed_scores.masked_fill( + ~topk_mask.unsqueeze(1), -float("inf") + ) + score_parts.append(compressed_scores) + value_parts.append(compressed_values) + + scores = torch.cat(score_parts, dim=-1) + sink = ( + self.sinks.view(1, self.num_heads, 1, 1) + .expand(batch, -1, seq_len, -1) + .to(dtype=scores.dtype) + ) + combined_scores = torch.cat([scores, sink], dim=-1) + combined_scores = combined_scores - combined_scores.max(dim=-1, keepdim=True).values + probs = torch.softmax(combined_scores, dim=-1, dtype=torch.float32).to(dtype=q.dtype) + + context = q.new_zeros(batch, self.num_heads, seq_len, self.head_dim) + offset = 0 + for values in value_parts: + next_offset = offset + values.size(2) + partial = torch.matmul(probs[..., offset:next_offset], values) + context = context + partial + offset = next_offset + + return self._project_context(context, cos, sin) + + def _project_context( + self, context: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor + ) -> torch.Tensor: + context = apply_partial_rope(context, cos, -sin, self.rope_head_dim).transpose(1, 2) + batch, seq_len = context.shape[:2] + grouped = context.reshape( + batch, seq_len, self.config.o_groups, self.num_heads_per_group * self.head_dim + ) + return self.wo_b(self.wo_a(grouped).flatten(2)) + + def _gather_cp_sources( + self, tensor: torch.Tensor, position_ids: torch.Tensor, *, seq_dim: int + ) -> tuple[torch.Tensor, torch.Tensor]: + parts, pos_parts = [], [] + for _rank, source, source_pos in iter_cp_sources( + tensor, + position_ids, + cp_rank=self.ps.cp_rank, + cp_size=self.ps.cp_size, + cp_group=self.ps.cp_group, + ): + parts.append(source) + pos_parts.append(source_pos) + pos = torch.cat(pos_parts, dim=1) + return torch.cat(parts, dim=seq_dim), pos + + def _forward_fused_sparse_no_indexer_cp1( + self, + x: torch.Tensor, + q: torch.Tensor, + kv: torch.Tensor, + *, + position_ids: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, + ) -> torch.Tensor: + dsa_kernels = _load_dsa_kernels() + batch, seq_len, _ = x.shape + query = q.transpose(1, 2).transpose(0, 1).contiguous() + kv_full = kv.squeeze(1) + kv_full = kv_full.transpose(0, 1).contiguous() + window_idxs = _window_topk_indices( + batch, + seq_len, + self.config.sliding_window, + device=x.device, + ) + + compressed = None + if self.compressor is not None and self.compress_ratio > 1: + compressed = self.compressor( + x, + position_ids=position_ids, + rope_theta=self.config.compress_rope_theta, + ) + if compressed is not None: + compressed_kv = compressed.squeeze(1) + kv_full = torch.cat([kv_full, compressed_kv.transpose(0, 1).contiguous()], dim=0) + + if compressed is not None: + n_compressed = compressed.size(2) + comp_idx = torch.arange(n_compressed, device=x.device).view(1, n_compressed) + valid_per_pos = ( + torch.arange(1, seq_len + 1, device=x.device) // self.compress_ratio + ).view(seq_len, 1) + compress_topk_idxs = torch.where( + comp_idx < valid_per_pos, + comp_idx + seq_len, + torch.full_like(comp_idx, -1), + ) + compress_topk_idxs = ( + compress_topk_idxs.unsqueeze(0).expand(batch, -1, -1).to(torch.int32) + ) + flat_idxs, _flat_tlen = dsa_kernels.build_flat_topk_idxs( + window_idxs, + compress_topk_idxs, + batch_size=batch, + seqlen_kv=kv_full.size(0), + ) + else: + flat_idxs, _flat_tlen = dsa_kernels.build_flat_topk_idxs( + window_idxs, + batch_size=batch, + seqlen_kv=kv_full.size(0), + ) + + out = dsa_kernels.dsa_sparse_attn( + query, + kv_full, + self.sinks.float(), + flat_idxs, + self.head_dim**-0.5, + ) + context = ( + out.view(seq_len, batch, self.num_heads, self.head_dim).permute(1, 2, 0, 3).contiguous() + ) + return self._project_context(context, cos, sin) + + def _forward_fused_dsa_cp1( + self, + x: torch.Tensor, + q: torch.Tensor, + q_low: torch.Tensor, + kv: torch.Tensor, + *, + position_ids: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, + attention_mask: torch.Tensor | None, + ) -> torch.Tensor: + if self.ps.cp_size != 1: + raise NotImplementedError("DeepSeek V4 fused DSA path currently supports CP=1 only.") + if attention_mask is not None: + raise NotImplementedError( + "DeepSeek V4 fused DSA path currently supports causal masking only." + ) + dsa_kernels = _load_dsa_kernels() + # The cuDNN SM90 indexer requires seqlen_q <= seqlen_k * ratio, but the + # compressor floors to seq_len // ratio blocks, so a seq_len that is not a + # multiple of ratio leaves the last query token(s) without a compressed key + # block. Right-pad to a multiple of ratio (the causal tail attends only real + # tokens and is sliced off the output) so seqlen_k * ratio == seqlen_q. + orig_seq_len = x.shape[1] + ratio = self.compress_ratio + pad = (-orig_seq_len) % ratio + if pad: + pos_tail = position_ids[:, -1:] + torch.arange( + 1, pad + 1, device=position_ids.device, dtype=position_ids.dtype + ) + position_ids = torch.cat([position_ids, pos_tail], dim=1) + x = torch.nn.functional.pad(x, (0, 0, 0, pad)) + q = torch.nn.functional.pad(q, (0, 0, 0, pad)) + q_low = torch.nn.functional.pad(q_low, (0, 0, 0, pad)) + kv = torch.nn.functional.pad(kv, (0, 0, 0, pad)) + batch, seq_len, _ = x.shape + compressed = self.compressor( + x, + position_ids=position_ids, + rope_theta=self.config.compress_rope_theta, + ) + index_comp = self.indexer.compressor( + x, + position_ids=position_ids, + rope_theta=self.config.compress_rope_theta, + ) + if compressed is None or index_comp is None: + raise RuntimeError("DeepSeek V4 fused DSA requires at least one compressed KV entry.") + compressed_kv = compressed.squeeze(1) + index_k = index_comp.squeeze(1).transpose(0, 1).contiguous() + kv_full = torch.cat([kv.squeeze(1), compressed_kv], dim=1) + kv_full = kv_full.transpose(0, 1).contiguous() + + idx_cos, idx_sin = build_compressed_rope_cos_sin( + position_ids, + self.indexer.rope_head_dim, + self.config.compress_rope_theta, + config=self.config, + use_yarn=self.compress_ratio > 1, + device=x.device, + dtype=x.dtype, + ) + q_indexer = self.indexer.wq_b(q_low).view( + batch, seq_len, self.indexer.index_n_heads, self.indexer.index_head_dim + ) + q_indexer = q_indexer.transpose(1, 2) + q_indexer = apply_partial_rope(q_indexer, idx_cos, idx_sin, self.indexer.rope_head_dim) + q_indexer = rotate_activation(q_indexer) + q_indexer = q_indexer.transpose(1, 2).transpose(0, 1).contiguous() + weights_indexer = ( + (self.indexer.weights_proj(x).to(dtype=x.dtype) * (self.indexer.index_n_heads**-0.5)) + .transpose(0, 1) + .contiguous() + ) + indexer_topk = int(self.indexer.index_topk) + if indexer_topk <= 0: + raise RuntimeError("DeepSeek V4 fused DSA requires positive indexer_topk.") + window_idxs = _window_topk_indices( + batch, + seq_len, + self.config.sliding_window, + device=x.device, + ) + query = q.transpose(1, 2).transpose(0, 1).contiguous() + sink = self.sinks.float() + + if self.training and torch.is_grad_enabled(): + out, _indexer_loss = dsa_kernels.fused_indexer_sparse_attn( + query, + kv_full, + sink, + window_idxs, + q_indexer, + index_k, + weights_indexer, + indexer_topk, + self.compress_ratio, + self.head_dim**-0.5, + self.indexer.softmax_scale, + 0.0, + sparse_loss=False, + kv_offset=seq_len, + calculate_per_token_loss=False, + ) + else: + topk_indices, _topk_length = dsa_kernels.indexer_topk( + q_indexer, + index_k, + weights_indexer, + indexer_topk, + self.compress_ratio, + indexer_softmax_scale=self.indexer.softmax_scale, + ) + topk_indices = torch.where( + topk_indices >= 0, + topk_indices + seq_len, + topk_indices, + ).to(torch.int32) + flat_idxs, flat_tlen = dsa_kernels.build_flat_topk_idxs( + window_idxs, + topk_indices, + batch_size=batch, + seqlen_kv=kv_full.size(0), + compact=True, + ) + out = dsa_kernels.dsa_sparse_attn( + query, + kv_full, + sink, + flat_idxs, + self.head_dim**-0.5, + topk_length=flat_tlen, + ) + + context = ( + out.view(seq_len, batch, self.num_heads, self.head_dim).permute(1, 2, 0, 3).contiguous() + ) + if pad: + context = context[:, :, :orig_seq_len, :].contiguous() + return self._project_context(context, cos, sin) diff --git a/experimental/lite/megatron/lite/primitive/modules/attention/dsa.py b/experimental/lite/megatron/lite/primitive/modules/attention/dsa.py new file mode 100644 index 00000000000..8956d4a9f01 --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/modules/attention/dsa.py @@ -0,0 +1,697 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Dynamic Sparse Attention. + +The module is model-agnostic: callers pass architecture dimensions directly and +keep model config classes out of the primitive layer. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import torch +import torch.nn as nn +import transformer_engine.pytorch as te + +from megatron.lite.primitive.parallel.cp import ( + zigzag_reconstruct_from_cp_parts, + zigzag_slice_for_cp, +) +from megatron.lite.primitive.parallel.thd import ( + reconstruct_packed_from_cp_parts, + split_packed_to_cp_local, +) + +from megatron.lite.primitive.kernels import dsa_kernels as _dsa_kernels + +if TYPE_CHECKING: + from megatron.lite.primitive.modules.attention.mla import MultiLatentAttention + + +def _fused_indexer_sparse_attn(*args, value_dim: int | None = None, **kwargs): + try: + return _dsa_kernels.fused_indexer_sparse_attn(*args, value_dim=value_dim, **kwargs) + except TypeError as exc: + if "value_dim" not in str(exc): + raise + return _dsa_kernels.fused_indexer_sparse_attn(*args, **kwargs) + + +class DSAIndexerLossAutoScaler(torch.autograd.Function): + """Attach the DSA indexer loss to the output without changing forward values.""" + + main_loss_backward_scale: torch.Tensor | None = None + + @staticmethod + def forward(ctx, output: torch.Tensor, indexer_loss: torch.Tensor) -> torch.Tensor: + ctx.save_for_backward(indexer_loss) + return output + + @staticmethod + def backward(ctx, grad_output: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + (indexer_loss,) = ctx.saved_tensors + if DSAIndexerLossAutoScaler.main_loss_backward_scale is None: + DSAIndexerLossAutoScaler.main_loss_backward_scale = torch.tensor( + 1.0, device=indexer_loss.device + ) + indexer_loss_backward_scale = DSAIndexerLossAutoScaler.main_loss_backward_scale + scaled_indexer_loss_grad = torch.ones_like(indexer_loss) * indexer_loss_backward_scale + return grad_output, scaled_indexer_loss_grad + + @staticmethod + def set_loss_scale(scale: torch.Tensor) -> None: + if DSAIndexerLossAutoScaler.main_loss_backward_scale is None: + DSAIndexerLossAutoScaler.main_loss_backward_scale = scale + else: + DSAIndexerLossAutoScaler.main_loss_backward_scale.copy_(scale) + + +RMSNorm = te.RMSNorm + + +def _hadamard_transform_torch(x: torch.Tensor, scale: float) -> torch.Tensor: + n = x.shape[-1] + if n <= 0 or n & (n - 1): + raise ValueError(f"Hadamard rotation requires power-of-two dim, got {n}") + original_shape = x.shape + y = x.reshape(-1, n) + h = 1 + while h < n: + y = y.reshape(-1, n // (h * 2), h * 2) + left = y[..., :h] + right = y[..., h:] + y = torch.cat([left + right, left - right], dim=-1) + h *= 2 + return y.reshape(original_shape) * scale + + +try: + from fast_hadamard_transform import hadamard_transform as _fast_hadamard_transform +except Exception: # pragma: no cover - optional CUDA extension + _fast_hadamard_transform = None + + +def rotate_activation(x: torch.Tensor) -> torch.Tensor: + x = x.to(torch.bfloat16) if x.dtype != torch.bfloat16 else x + scale = x.shape[-1] ** -0.5 + if _fast_hadamard_transform is not None and x.is_cuda: + return _fast_hadamard_transform(x, scale=scale) + return _hadamard_transform_torch(x, scale=scale) + + +def build_rope_cache( + *, dim: int, max_position_embeddings: int, rope_theta: float, device: torch.device | None = None +) -> tuple[torch.Tensor, torch.Tensor]: + inv_freq = 1.0 / ( + rope_theta ** (torch.arange(0, dim, 2, dtype=torch.float32, device=device) / dim) + ) + positions = torch.arange(max_position_embeddings, dtype=torch.float32, device=device) + freqs = torch.outer(positions, inv_freq) + return freqs.cos(), freqs.sin() + + +def build_rotary_embeddings( + *, position_ids: torch.Tensor, dim: int, rope_theta: float, dtype: torch.dtype +) -> tuple[torch.Tensor, torch.Tensor]: + device = position_ids.device + inv_freq = 1.0 / ( + rope_theta + ** (torch.arange(0, dim, 2, dtype=torch.int64, device=device).to(torch.float32) / dim) + ) + inv_freq_expanded = inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1) + position_ids_expanded = position_ids[:, None, :].float() + device_type = device.type if isinstance(device.type, str) and device.type != "mps" else "cpu" + with torch.autocast(device_type=device_type, enabled=False): + freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2) + emb = torch.cat((freqs, freqs), dim=-1) + cos = emb.cos() + sin = emb.sin() + return cos.to(dtype=dtype), sin.to(dtype=dtype) + + +def rotate_half(x: torch.Tensor) -> torch.Tensor: + half = x.shape[-1] // 2 + x1 = x[..., :half] + x2 = x[..., half:] + return torch.cat((-x2, x1), dim=-1) + + +def apply_rotary_pos_emb( + x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor, *, unsqueeze_dim: int +) -> torch.Tensor: + cos = cos.unsqueeze(unsqueeze_dim) + sin = sin.unsqueeze(unsqueeze_dim) + return (x * cos) + (rotate_half(x) * sin) + + +def apply_rotary_emb( + x: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, + position_ids: torch.Tensor, + *, + interleaved: bool = True, +) -> torch.Tensor: + if position_ids.dim() == 3: + position_ids = position_ids[0] + if position_ids.dim() == 1: + position_ids = position_ids.unsqueeze(0) + + input_dtype = x.dtype + x = x.float() + cos = cos.to(device=x.device)[position_ids].float().unsqueeze(2) + sin = sin.to(device=x.device)[position_ids].float().unsqueeze(2) + if interleaved: + x_even = x[..., 0::2] + x_odd = x[..., 1::2] + out = torch.empty_like(x) + out[..., 0::2] = x_even * cos - x_odd * sin + out[..., 1::2] = x_even * sin + x_odd * cos + return out.to(input_dtype) + + half = x.shape[-1] // 2 + x1, x2 = x[..., :half], x[..., half:] + return torch.cat([x1 * cos - x2 * sin, x1 * sin + x2 * cos], dim=-1).to(input_dtype) + + +def _rotary_embeddings_from_cache( + cos: torch.Tensor, + sin: torch.Tensor, + position_ids: torch.Tensor, + *, + device: torch.device, + dtype: torch.dtype, + dim: int, +) -> tuple[torch.Tensor, torch.Tensor]: + if cos.dim() == 3 and sin.dim() == 3: + return cos.to(device=device, dtype=dtype), sin.to(device=device, dtype=dtype) + + if position_ids.dim() == 3: + position_ids = position_ids[0] + if position_ids.dim() == 1: + position_ids = position_ids.unsqueeze(0) + cos = cos.to(device=device)[position_ids].float() + sin = sin.to(device=device)[position_ids].float() + if cos.shape[-1] * 2 == dim: + cos = torch.cat((cos, cos), dim=-1) + sin = torch.cat((sin, sin), dim=-1) + return cos.to(dtype=dtype), sin.to(dtype=dtype) + + +def _all_gather_cp(tensor: torch.Tensor, *, cp_size: int, cp_group) -> list[torch.Tensor]: + if cp_size <= 1: + return [tensor] + if cp_group is None: + raise RuntimeError("CP>1 requires a context-parallel process group.") + from torch.distributed.nn.functional import all_gather + + return list(all_gather(tensor.contiguous(), group=cp_group)) + + +class DSAIndexer(nn.Module): + """Compute per-token top-k key indices for Dynamic Sparse Attention.""" + + def __init__( + self, + *, + hidden_size: int, + q_lora_rank: int, + qk_rope_head_dim: int, + index_n_heads: int, + index_head_dim: int, + index_topk: int, + rope_interleaved: bool = True, + layer_norm_eps: float = 1e-5, + rope_first: bool = False, + use_hadamard: bool = True, + ): + super().__init__() + if index_head_dim < qk_rope_head_dim: + raise ValueError("index_head_dim must be >= qk_rope_head_dim") + self.num_heads = index_n_heads + self.head_dim = index_head_dim + self.qk_rope_head_dim = qk_rope_head_dim + self.qk_nope_head_dim = index_head_dim - qk_rope_head_dim + self.index_topk = index_topk + self.rope_interleaved = rope_interleaved + self.rope_first = rope_first + self.use_hadamard = use_hadamard + + self.wq_b = nn.Linear(q_lora_rank, index_n_heads * index_head_dim, bias=False) + self.wk = nn.Linear(hidden_size, index_head_dim, bias=False) + self.k_norm = nn.LayerNorm(index_head_dim, eps=layer_norm_eps) + self.weights_proj = nn.Linear(hidden_size, index_n_heads, bias=False) + self.softmax_scale = index_head_dim**-0.5 + + def forward_before_topk( + self, + x: torch.Tensor, + q_resid: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, + position_ids: torch.Tensor, + attention_mask: torch.Tensor | None = None, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Project GLM5 indexer inputs for Megatron's fused DSA kernels.""" + if attention_mask is not None: + raise NotImplementedError( + "GLM5 fused DSA indexer only supports causal masking; custom " + "attention_mask is not supported." + ) + batch, seq_len, _ = x.shape + cos, sin = _rotary_embeddings_from_cache( + cos, sin, position_ids, device=x.device, dtype=x.dtype, dim=self.qk_rope_head_dim + ) + + q = self.wq_b(q_resid).view(batch, seq_len, self.num_heads, self.head_dim) + + k = self.k_norm(self.wk(x)) + if self.rope_first: + q_pe, q_nope = torch.split(q, [self.qk_rope_head_dim, self.qk_nope_head_dim], dim=-1) + k_pe, k_nope = torch.split(k, [self.qk_rope_head_dim, self.qk_nope_head_dim], dim=-1) + else: + q_nope, q_pe = torch.split(q, [self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1) + k_nope, k_pe = torch.split(k, [self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1) + q_pe = apply_rotary_pos_emb(q_pe, cos, sin, unsqueeze_dim=2) + k_pe = apply_rotary_pos_emb(k_pe.unsqueeze(2), cos, sin, unsqueeze_dim=2) + k_pe = k_pe.squeeze(2) + + if self.rope_first: + q = torch.cat([q_pe, q_nope], dim=-1) + k = torch.cat([k_pe, k_nope], dim=-1) + else: + q = torch.cat([q_nope, q_pe], dim=-1) + k = torch.cat([k_nope, k_pe], dim=-1) + if self.use_hadamard: + q = rotate_activation(q) + k = rotate_activation(k) + + weights = self.weights_proj(x) * (self.num_heads**-0.5) + return ( + q.transpose(0, 1).contiguous(), + k.transpose(0, 1).contiguous(), + weights.transpose(0, 1).contiguous(), + ) + + def forward( + self, + x: torch.Tensor, + q_resid: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, + position_ids: torch.Tensor, + attention_mask: torch.Tensor | None = None, + ) -> torch.Tensor: + q, k, weights = self.forward_before_topk( + x, q_resid, cos, sin, position_ids, attention_mask=attention_mask + ) + topk_indices, _ = _dsa_kernels.indexer_topk( + q, + k, + weights, + min(self.index_topk, k.shape[0]), + 1, + indexer_softmax_scale=self.softmax_scale, + ) + return topk_indices + + +class DynamicSparseAttention(nn.Module): + """Correctness-first DSA attention path.""" + + @staticmethod + def dense_attention_cls() -> type[MultiLatentAttention]: + from megatron.lite.primitive.modules.attention.mla import MultiLatentAttention + + return MultiLatentAttention + + def __init__( + self, + *, + hidden_size: int, + num_attention_heads: int, + q_lora_rank: int, + kv_lora_rank: int, + qk_nope_head_dim: int, + qk_rope_head_dim: int, + v_head_dim: int, + index_n_heads: int, + index_head_dim: int, + index_topk: int, + rms_norm_eps: float, + rope_interleaved: bool = True, + latent_rms_norm_eps: float | None = None, + indexer_layer_norm_eps: float = 1e-5, + indexer_rope_interleaved: bool | None = None, + indexer_rope_first: bool = False, + indexer_use_hadamard: bool = True, + indexer_loss_coeff: float = 0.0, + indexer_use_sparse_loss: bool = False, + calculate_per_token_loss: bool = False, + cp_size: int = 1, + cp_rank: int = 0, + cp_group=None, + ): + super().__init__() + if cp_size < 1: + raise ValueError(f"cp_size must be >= 1, got {cp_size}") + if not 0 <= cp_rank < cp_size: + raise ValueError(f"cp_rank must be in [0, {cp_size}), got {cp_rank}") + self.num_heads = num_attention_heads + self.q_lora_rank = q_lora_rank + self.kv_lora_rank = kv_lora_rank + self.qk_nope_head_dim = qk_nope_head_dim + self.qk_rope_head_dim = qk_rope_head_dim + self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim + self.v_head_dim = v_head_dim + self.rope_interleaved = rope_interleaved + self.softmax_scale = self.qk_head_dim**-0.5 + self.indexer_loss_coeff = indexer_loss_coeff + self.indexer_use_sparse_loss = indexer_use_sparse_loss + self.calculate_per_token_loss = calculate_per_token_loss + self.cp_size = cp_size + self.cp_rank = cp_rank + self.cp_group = cp_group + latent_rms_norm_eps = rms_norm_eps if latent_rms_norm_eps is None else latent_rms_norm_eps + indexer_rope_interleaved = ( + rope_interleaved if indexer_rope_interleaved is None else indexer_rope_interleaved + ) + + self.q_a_proj = nn.Linear(hidden_size, q_lora_rank, bias=False) + self.q_a_layernorm = RMSNorm(q_lora_rank, eps=latent_rms_norm_eps) + self.q_b_proj = nn.Linear(q_lora_rank, num_attention_heads * self.qk_head_dim, bias=False) + self.kv_a_proj_with_mqa = nn.Linear( + hidden_size, kv_lora_rank + qk_rope_head_dim, bias=False + ) + self.kv_a_layernorm = RMSNorm(kv_lora_rank, eps=latent_rms_norm_eps) + self.kv_b_proj = nn.Linear( + kv_lora_rank, num_attention_heads * (qk_nope_head_dim + v_head_dim), bias=False + ) + self.o_proj = nn.Linear(num_attention_heads * v_head_dim, hidden_size, bias=False) + self.indexer = DSAIndexer( + hidden_size=hidden_size, + q_lora_rank=q_lora_rank, + qk_rope_head_dim=qk_rope_head_dim, + index_n_heads=index_n_heads, + index_head_dim=index_head_dim, + index_topk=index_topk, + rope_interleaved=indexer_rope_interleaved, + layer_norm_eps=indexer_layer_norm_eps, + rope_first=indexer_rope_first, + use_hadamard=indexer_use_hadamard, + ) + self.register_buffer( + "attn_sink", + torch.full((num_attention_heads,), -1.0e20, dtype=torch.float32), + persistent=False, + ) + + def forward( + self, + x: torch.Tensor, + *, + cos: torch.Tensor, + sin: torch.Tensor, + position_ids: torch.Tensor, + attention_mask: torch.Tensor | None = None, + packed_seq_params=None, + ) -> torch.Tensor: + if attention_mask is not None: + raise NotImplementedError( + "GLM5 fused DSA only supports causal masking; custom attention_mask " + "is not supported." + ) + if packed_seq_params is not None: + if self.cp_size > 1: + x, position_ids = self._gather_packed_cp_inputs(x, position_ids, packed_seq_params) + cos, sin = self._gather_packed_cp_rotary(cos, sin, packed_seq_params, x.device) + out = self._forward_packed_full(x, cos, sin, position_ids, packed_seq_params) + if self.cp_size > 1: + out = split_packed_to_cp_local( + out, + cu_seqlens_padded=self._packed_cu_seqlens(packed_seq_params, x.device), + cp_size=self.cp_size, + cp_rank=self.cp_rank, + dim=1, + ) + return out + + cp_restore = self.cp_size > 1 + if cp_restore: + x, position_ids, attention_mask = self._gather_cp_inputs( + x, position_ids, attention_mask + ) + + out = self._forward_dense_full(x, cos, sin, position_ids) + if cp_restore: + out = zigzag_slice_for_cp(out, self.cp_rank, self.cp_size, seq_dim=1) + return out + + def _forward_packed_full( + self, + x: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, + position_ids: torch.Tensor, + packed_seq_params, + ) -> torch.Tensor: + cu_seqlens = self._packed_cu_seqlens(packed_seq_params, x.device) + if position_ids.dim() == 1: + position_ids = position_ids.unsqueeze(0) + if position_ids.shape[-1] != x.shape[1]: + raise ValueError( + "GLM5 packed DynamicSparseAttention position_ids must cover the reconstructed packed tokens, " + f"got {tuple(position_ids.shape)} for packed length {x.shape[1]}." + ) + pieces = [] + for idx in range(int(cu_seqlens.numel()) - 1): + start = int(cu_seqlens[idx].item()) + end = int(cu_seqlens[idx + 1].item()) + if end <= start: + continue + seg_cos, seg_sin = self._slice_rotary_cache(cos, sin, start, end) + pieces.append( + self._forward_dense_full( + x[:, start:end, :], + seg_cos, + seg_sin, + position_ids[:, start:end], + ) + ) + if pieces: + return torch.cat(pieces, dim=1) + return x.new_empty(x.shape[0], 0, self.o_proj.out_features) + + def _forward_dense_full( + self, + x: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, + position_ids: torch.Tensor, + ) -> torch.Tensor: + + batch, seq_len, _ = x.shape + q_resid = self.q_a_layernorm(self.q_a_proj(x)) + q = self.q_b_proj(q_resid).view(batch, seq_len, self.num_heads, self.qk_head_dim) + q_nope, q_pe = torch.split(q, [self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1) + cos, sin = _rotary_embeddings_from_cache( + cos, sin, position_ids, device=x.device, dtype=x.dtype, dim=self.qk_rope_head_dim + ) + + q_pe = apply_rotary_pos_emb(q_pe, cos, sin, unsqueeze_dim=2) + k_up_weight, v_up_weight = self._split_kv_b_weights() + q_nope = torch.einsum("bshd,hdr->bshr", q_nope, k_up_weight) + query_states = torch.cat([q_nope, q_pe], dim=-1).transpose(0, 1).contiguous() + + kv_latent, k_pe = torch.split( + self.kv_a_proj_with_mqa(x), [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1 + ) + kv_latent = self.kv_a_layernorm(kv_latent) + k_pe = apply_rotary_pos_emb(k_pe.unsqueeze(2), cos, sin, unsqueeze_dim=2).squeeze(2) + kv_full = torch.cat([kv_latent, k_pe], dim=-1).transpose(0, 1).contiguous() + + q_indexer, k_indexer, weights_indexer = self.indexer.forward_before_topk( + x.detach(), q_resid.detach(), cos, sin, position_ids + ) + effective_indexer_topk = min(self.indexer.index_topk, seq_len) + + if self.training and torch.is_grad_enabled(): + window_idxs = torch.empty(batch, seq_len, 0, device=x.device, dtype=torch.int32) + out, indexer_loss = _fused_indexer_sparse_attn( + query_states, + kv_full, + self.attn_sink.float(), + window_idxs, + q_indexer, + k_indexer, + weights_indexer, + self.indexer.index_topk, + 1, + self.softmax_scale, + self.indexer.softmax_scale, + self.indexer_loss_coeff, + sparse_loss=self.indexer_use_sparse_loss, + kv_offset=0, + calculate_per_token_loss=self.calculate_per_token_loss, + value_dim=self.kv_lora_rank, + ) + if self.indexer_loss_coeff > 0: + out = DSAIndexerLossAutoScaler.apply(out, indexer_loss) + else: + topk_indices, _ = _dsa_kernels.indexer_topk( + q_indexer, + k_indexer, + weights_indexer, + effective_indexer_topk, + 1, + indexer_softmax_scale=self.indexer.softmax_scale, + ) + flat_idxs, flat_tlen = _dsa_kernels.build_flat_topk_idxs( + topk_indices, batch_size=batch, seqlen_kv=seq_len, compact=True + ) + out = _dsa_kernels.dsa_sparse_attn( + query_states, + kv_full, + self.attn_sink.float(), + flat_idxs, + self.softmax_scale, + topk_length=flat_tlen, + value_dim=self.kv_lora_rank, + ) + + out = out.view(seq_len, batch, self.num_heads, self.kv_lora_rank) + out = out.permute(1, 0, 2, 3).contiguous() + out = torch.einsum("bshr,hvr->bshv", out, v_up_weight) + out = out.reshape(batch, seq_len, self.num_heads * self.v_head_dim) + return self.o_proj(out) + + def _split_kv_b_weights(self) -> tuple[torch.Tensor, torch.Tensor]: + kv_b = self.kv_b_proj.weight.view( + self.num_heads, self.qk_nope_head_dim + self.v_head_dim, self.kv_lora_rank + ) + return (kv_b[:, : self.qk_nope_head_dim, :], kv_b[:, self.qk_nope_head_dim :, :]) + + def _gather_cp_inputs( + self, x: torch.Tensor, position_ids: torch.Tensor, attention_mask: torch.Tensor | None + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]: + local_batch, local_seq = x.shape[:2] + x_parts = _all_gather_cp(x, cp_size=self.cp_size, cp_group=self.cp_group) + full_x = zigzag_reconstruct_from_cp_parts(x_parts, seq_dim=1) + full_seq = full_x.shape[1] + + full_position_ids = self._full_cp_position_ids( + position_ids, batch=local_batch, local_seq=local_seq, full_seq=full_seq, device=x.device + ) + if attention_mask is not None: + expected = (full_seq, full_seq) + if tuple(attention_mask.shape[-2:]) != expected: + raise NotImplementedError( + "GLM5 DynamicSparseAttention CP attention_mask must already cover the reconstructed " + f"full sequence {expected}, got {tuple(attention_mask.shape)}." + ) + return full_x, full_position_ids, attention_mask + + def _full_cp_position_ids( + self, + position_ids: torch.Tensor, + *, + batch: int, + local_seq: int, + full_seq: int, + device: torch.device, + ) -> torch.Tensor: + if position_ids.dim() == 3: + position_ids = position_ids[0] + if position_ids.dim() == 1: + position_ids = position_ids.unsqueeze(0).expand(batch, -1) + if position_ids.shape[-1] == full_seq: + return position_ids.to(device=device, dtype=torch.long) + if position_ids.shape[-1] != local_seq: + raise ValueError( + "GLM5 DynamicSparseAttention CP position_ids must be either local or full sequence length, " + f"got {tuple(position_ids.shape)} for local_seq={local_seq}, full_seq={full_seq}." + ) + + pos_parts = _all_gather_cp( + position_ids.to(device=device, dtype=torch.long), + cp_size=self.cp_size, + cp_group=self.cp_group, + ) + return zigzag_reconstruct_from_cp_parts(pos_parts, seq_dim=1) + + def _gather_packed_cp_inputs( + self, x: torch.Tensor, position_ids: torch.Tensor, packed_seq_params + ) -> tuple[torch.Tensor, torch.Tensor]: + local_seq = x.shape[1] + cu_seqlens = self._packed_cu_seqlens(packed_seq_params, x.device) + full_seq = int(cu_seqlens[-1].item()) + x_parts = _all_gather_cp(x, cp_size=self.cp_size, cp_group=self.cp_group) + full_x = reconstruct_packed_from_cp_parts( + x_parts, cu_seqlens_padded=cu_seqlens, cp_size=self.cp_size, dim=1 + ) + + if position_ids.dim() == 1: + position_ids = position_ids.unsqueeze(0) + position_ids = position_ids.to(device=x.device, dtype=torch.long) + if position_ids.shape[-1] == full_seq: + return full_x, position_ids + if position_ids.shape[-1] != local_seq: + raise ValueError( + "GLM5 packed DynamicSparseAttention CP position_ids must be either local or full packed length, " + f"got {tuple(position_ids.shape)} for local_seq={local_seq}, full_seq={full_seq}." + ) + pos_parts = _all_gather_cp(position_ids, cp_size=self.cp_size, cp_group=self.cp_group) + full_position_ids = reconstruct_packed_from_cp_parts( + pos_parts, cu_seqlens_padded=cu_seqlens, cp_size=self.cp_size, dim=1 + ) + return full_x, full_position_ids + + def _gather_packed_cp_rotary( + self, cos: torch.Tensor, sin: torch.Tensor, packed_seq_params, device: torch.device + ) -> tuple[torch.Tensor, torch.Tensor]: + if cos.dim() != 3 or sin.dim() != 3: + return cos, sin + cu_seqlens = self._packed_cu_seqlens(packed_seq_params, device) + full_seq = int(cu_seqlens[-1].item()) + if cos.shape[1] == full_seq and sin.shape[1] == full_seq: + return cos, sin + cos_parts = _all_gather_cp(cos, cp_size=self.cp_size, cp_group=self.cp_group) + sin_parts = _all_gather_cp(sin, cp_size=self.cp_size, cp_group=self.cp_group) + full_cos = reconstruct_packed_from_cp_parts( + cos_parts, cu_seqlens_padded=cu_seqlens, cp_size=self.cp_size, dim=1 + ) + full_sin = reconstruct_packed_from_cp_parts( + sin_parts, cu_seqlens_padded=cu_seqlens, cp_size=self.cp_size, dim=1 + ) + return full_cos, full_sin + + @staticmethod + def _packed_cu_seqlens(packed_seq_params, device: torch.device) -> torch.Tensor: + cu_seqlens = getattr(packed_seq_params, "cu_seqlens_q_padded", None) + if cu_seqlens is None: + cu_seqlens = getattr(packed_seq_params, "cu_seqlens_q", None) + if cu_seqlens is None: + raise ValueError("GLM5 packed DynamicSparseAttention requires packed cu_seqlens.") + return cu_seqlens.to(device=device, dtype=torch.int32) + + @staticmethod + def _slice_rotary_cache( + cos: torch.Tensor, sin: torch.Tensor, start: int, end: int + ) -> tuple[torch.Tensor, torch.Tensor]: + if cos.dim() == 3 and sin.dim() == 3: + return cos[:, start:end, :], sin[:, start:end, :] + return cos, sin + + +__all__ = [ + "DSAIndexer", + "DSAIndexerLossAutoScaler", + "DynamicSparseAttention", + "RMSNorm", + "apply_rotary_emb", + "apply_rotary_pos_emb", + "build_rope_cache", + "build_rotary_embeddings", + "rotate_activation", + "rotate_half", +] diff --git a/experimental/lite/megatron/lite/primitive/modules/attention/hca.py b/experimental/lite/megatron/lite/primitive/modules/attention/hca.py new file mode 100644 index 00000000000..7593ee46505 --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/modules/attention/hca.py @@ -0,0 +1,66 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +def split_sinkhorn( + mixes: torch.Tensor, + hc_scale: torch.Tensor, + hc_base: torch.Tensor, + hc_mult: int, + iters: int, + eps: float, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + split_sizes = [hc_mult, hc_mult, hc_mult * hc_mult] + pre_mix, post_mix, comb_mix = mixes.split(split_sizes, dim=-1) + base_pre, base_post, base_comb = hc_base.to(dtype=mixes.dtype, device=mixes.device).split( + split_sizes, dim=-1 + ) + scale = hc_scale.to(dtype=mixes.dtype, device=mixes.device) + pre = torch.sigmoid(pre_mix * scale[0] + base_pre) + post = 2 * torch.sigmoid(post_mix * scale[1] + base_post) + comb_logits = (comb_mix * scale[2] + base_comb).view(*comb_mix.shape[:-1], hc_mult, hc_mult) + comb = torch.exp(comb_logits - comb_logits.max(dim=-1, keepdim=True).values) + for _ in range(iters): + comb = comb / comb.sum(dim=-1, keepdim=True).clamp(min=eps) + comb = comb / comb.sum(dim=-2, keepdim=True).clamp(min=eps) + return pre, post, comb + + +class HyperConnection(nn.Module): + def __init__(self, hidden_size: int, hc_mult: int, sinkhorn_iters: int, eps: float): + super().__init__() + mix = (2 + hc_mult) * hc_mult + self.hidden_size = hidden_size + self.hc_mult = hc_mult + self.sinkhorn_iters = sinkhorn_iters + self.eps = eps + self.fn = nn.Parameter(torch.empty(mix, hc_mult * hidden_size, dtype=torch.float32)) + self.base = nn.Parameter(torch.zeros(mix, dtype=torch.float32)) + self.scale = nn.Parameter(torch.ones(3, dtype=torch.float32)) + nn.init.xavier_uniform_(self.fn) + + def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + if x.dim() == 3: + x = x.unsqueeze(2).expand(*x.shape[:2], self.hc_mult, x.size(-1)) + shape, dtype = x.shape, x.dtype + xf = x.flatten(2) + rms_inv = 1.0 / (xf.norm(dim=-1, keepdim=True) / math.sqrt(xf.shape[-1]) + self.eps) + mixes = F.linear(xf, self.fn.to(device=x.device, dtype=dtype)) * rms_inv + pre, post, comb = split_sinkhorn( + mixes, self.scale, self.base, self.hc_mult, self.sinkhorn_iters, self.eps + ) + y = torch.sum(pre.unsqueeze(-1) * xf.view(shape), dim=2) + return y.to(dtype), post, comb + + @staticmethod + def post( + x: torch.Tensor, residual: torch.Tensor, post: torch.Tensor, comb: torch.Tensor + ) -> torch.Tensor: + dtype = x.dtype + placed = post.to(dtype).unsqueeze(-1) * x.unsqueeze(-2) + mixed = torch.matmul(comb.to(dtype), residual.to(dtype)) + return placed + mixed diff --git a/experimental/lite/megatron/lite/primitive/modules/attention/mhc.py b/experimental/lite/megatron/lite/primitive/modules/attention/mhc.py new file mode 100644 index 00000000000..dba32bd4d8e --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/modules/attention/mhc.py @@ -0,0 +1,27 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class MultiHeadHyperConnectionHead(nn.Module): + def __init__(self, hidden_size: int, hc_mult: int, eps: float): + super().__init__() + self.hidden_size = hidden_size + self.hc_mult = hc_mult + self.eps = eps + self.hc_fn = nn.Parameter(torch.empty(hc_mult, hc_mult * hidden_size, dtype=torch.float32)) + self.hc_base = nn.Parameter(torch.zeros(hc_mult, dtype=torch.float32)) + self.hc_scale = nn.Parameter(torch.ones(1, dtype=torch.float32)) + nn.init.xavier_uniform_(self.hc_fn) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + if x.dim() == 3: + return x + shape, dtype = x.shape, x.dtype + xf = x.flatten(2).float() + rsqrt = torch.rsqrt(xf.square().mean(-1, keepdim=True) + self.eps) + mixes = F.linear(xf, self.hc_fn.float()) * rsqrt + pre = torch.sigmoid(mixes * self.hc_scale.float() + self.hc_base.float()) + self.eps + y = torch.sum(pre.unsqueeze(-1) * xf.view(shape), dim=2) + return y.to(dtype) diff --git a/experimental/lite/megatron/lite/primitive/modules/attention/mla.py b/experimental/lite/megatron/lite/primitive/modules/attention/mla.py new file mode 100644 index 00000000000..792c27471a3 --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/modules/attention/mla.py @@ -0,0 +1,418 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Shared Multi-Latent Attention primitive.""" + +from __future__ import annotations + +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F +import transformer_engine.pytorch as te +from megatron.lite.primitive.utils.rope import ( + _apply_rotary_pos_emb_bshd, + _apply_rotary_pos_emb_thd, +) +from megatron.lite.primitive.utils.rotary import ( + RotaryEmbedding, + YarnRotaryEmbedding, + _yarn_get_mscale, +) + +from megatron.lite.primitive.parallel import ( + ColumnParallelLinear, + ParallelState, + RowParallelLinear, + gather_from_sequence_parallel, +) +from megatron.lite.primitive.parallel.cp import ( + zigzag_reconstruct_from_cp_parts, + zigzag_slice_for_cp, +) +from megatron.lite.primitive.parallel.thd import ( + reconstruct_packed_from_cp_parts, + split_packed_to_cp_local, +) + +_KEPT_PSP_FIELDS = ( + "qkv_format", + "cu_seqlens_q", + "cu_seqlens_kv", + "cu_seqlens_q_padded", + "cu_seqlens_kv_padded", + "max_seqlen_q", + "max_seqlen_kv", +) + + +def _apply_mla_rope_bshd(t: torch.Tensor, freqs: torch.Tensor, *, mscale: float) -> torch.Tensor: + return _apply_rotary_pos_emb_bshd( + t, freqs, rotary_interleaved=False, mscale=mscale, mla_rotary_interleaved=True + ) + + +def _apply_mla_rope_thd( + t: torch.Tensor, + cu_seqlens: torch.Tensor, + freqs: torch.Tensor, + *, + mscale: float, + cp_group, +) -> torch.Tensor: + return _apply_rotary_pos_emb_thd( + t, + cu_seqlens, + freqs, + rotary_interleaved=False, + mscale=mscale, + cp_group=cp_group, + mla_rotary_interleaved=True, + ) + + +class MultiLatentAttention(nn.Module): + """Native MLA composition using lite parallel linears and TE core attention.""" + + _cp_stream: torch.cuda.Stream | None = None + + def __init__( + self, + *, + hidden_size: int, + num_attention_heads: int, + q_lora_rank: int, + kv_lora_rank: int, + qk_nope_head_dim: int, + qk_rope_head_dim: int, + v_head_dim: int, + ps: ParallelState, + rms_norm_eps: float = 1e-6, + rope_theta: float = 10_000.0, + rope_scaling: dict | None = None, + use_thd: bool = False, + ): + super().__init__() + if num_attention_heads % ps.tp_size != 0: + raise ValueError("num_attention_heads must be divisible by tensor parallel size") + self.ps = ps + self.num_heads_local = num_attention_heads // ps.tp_size + self.q_lora_rank = q_lora_rank + self.kv_lora_rank = kv_lora_rank + self.qk_nope_head_dim = qk_nope_head_dim + self.qk_rope_head_dim = qk_rope_head_dim + self.v_head_dim = v_head_dim + self.q_head_dim = qk_nope_head_dim + qk_rope_head_dim + + self.linear_proj = RowParallelLinear( + num_attention_heads * v_head_dim, + hidden_size, + ps, + bias=False, + ) + self.linear_q_down_proj = nn.Linear(hidden_size, q_lora_rank, bias=False) + self.linear_q_up_proj = ColumnParallelLinear( + q_lora_rank, + num_attention_heads * self.q_head_dim, + ps, + bias=False, + normalization="RMSNorm", + eps=rms_norm_eps, + ) + self.linear_kv_down_proj = nn.Linear( + hidden_size, + kv_lora_rank + qk_rope_head_dim, + bias=False, + ) + self.linear_kv_up_proj = ColumnParallelLinear( + kv_lora_rank, + num_attention_heads * (qk_nope_head_dim + v_head_dim), + ps, + bias=False, + normalization="RMSNorm", + eps=rms_norm_eps, + ) + + rope_scaling = dict(rope_scaling or {}) + rope_type = rope_scaling.get("type", "rope") + if rope_type == "yarn": + factor = float(rope_scaling.get("factor", 1.0)) + self.rotary = YarnRotaryEmbedding( + qk_rope_head_dim, + rotary_base=rope_theta, + scaling_factor=factor, + original_max_position_embeddings=int( + rope_scaling.get("original_max_position_embeddings", 4096) + ), + beta_fast=float(rope_scaling.get("beta_fast", 32.0)), + beta_slow=float(rope_scaling.get("beta_slow", 1.0)), + mscale=float(rope_scaling.get("mscale", 1.0)), + mscale_all_dim=float(rope_scaling.get("mscale_all_dim", 1.0)), + cp_group=ps.cp_group if ps.cp_size > 1 else None, + ) + attn_mscale = _yarn_get_mscale(factor, float(rope_scaling.get("mscale_all_dim", 1.0))) + elif rope_type == "rope": + self.rotary = RotaryEmbedding( + kv_channels=qk_rope_head_dim, + rotary_base=rope_theta, + use_cpu_initialization=False, + cp_group=ps.cp_group if ps.cp_size > 1 else None, + ) + attn_mscale = 1.0 + else: + raise ValueError(f"Unsupported MLA rope type: {rope_type!r}") + self._softmax_scale = attn_mscale * attn_mscale / math.sqrt(self.q_head_dim) + self._query_scale = 1.0 + + cp_kwargs = {} + if ps.cp_size > 1: + if MultiLatentAttention._cp_stream is None: + MultiLatentAttention._cp_stream = torch.cuda.Stream() + cp_kwargs = dict( + cp_group=ps.cp_group, + cp_global_ranks=ps.cp_global_ranks, + cp_stream=MultiLatentAttention._cp_stream, + ) + self._use_torch_core = ps.cp_size > 1 and v_head_dim != self.q_head_dim + self.core_attn = None + if not self._use_torch_core: + dpa_kwargs = dict(cp_kwargs, softmax_scale=self._softmax_scale) + kv_channels = ( + (self.q_head_dim, v_head_dim) if v_head_dim != self.q_head_dim else self.q_head_dim + ) + self.core_attn = te.DotProductAttention( + num_attention_heads=self.num_heads_local, + kv_channels=kv_channels, + attention_dropout=0.0, + attn_mask_type="causal", + qkv_format="thd" if use_thd else "sbhd", + **dpa_kwargs, + ) + + def forward(self, x: torch.Tensor, packed_seq_params=None) -> torch.Tensor: + q_compressed = self.linear_q_down_proj(x) + kv_combined = self.linear_kv_down_proj(x) + kv_compressed, k_pos_emb = kv_combined.split( + [self.kv_lora_rank, self.qk_rope_head_dim], + dim=-1, + ) + if self.ps.tp_size > 1: + k_pos_emb = gather_from_sequence_parallel(k_pos_emb, self.ps) + + q_proj = self.linear_q_up_proj(q_compressed) + q = q_proj.view( + *q_proj.shape[:-1], + self.num_heads_local, + self.q_head_dim, + ) + kv_proj = self.linear_kv_up_proj(kv_compressed) + kv = kv_proj.view( + *kv_proj.shape[:-1], + self.num_heads_local, + self.qk_nope_head_dim + self.v_head_dim, + ) + q_nope, q_pos = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1) + k_nope, value = kv.split([self.qk_nope_head_dim, self.v_head_dim], dim=-1) + k_pos = k_pos_emb.unsqueeze(-2) + + is_thd = packed_seq_params is not None + if is_thd: + q_nope = q_nope.squeeze(1) + q_pos = q_pos.squeeze(1) + k_nope = k_nope.squeeze(1) + value = value.squeeze(1) + k_pos = k_pos.squeeze(1) + + q_pos, k_pos = self._apply_rope(q_pos, k_pos, packed_seq_params) + if k_pos.dim() == q_nope.dim(): + k_pos = k_pos.expand(*q_nope.shape[:-1], self.qk_rope_head_dim) + else: + k_pos = k_pos.expand(-1, -1, self.num_heads_local, -1) + query = torch.cat([q_nope, q_pos], dim=-1).contiguous() + key = torch.cat([k_nope, k_pos], dim=-1).contiguous() + value = value.contiguous() + if self._query_scale != 1.0: + query = query * self._query_scale + + if is_thd: + if self._use_torch_core: + out = self._torch_core_attention_thd( + query, + key, + value, + packed_seq_params=packed_seq_params, + ).reshape(query.size(0), 1, -1) + else: + psp_kwargs = { + k: getattr(packed_seq_params, k) + for k in _KEPT_PSP_FIELDS + if getattr(packed_seq_params, k, None) is not None + } + assert self.core_attn is not None + out = self.core_attn( + query, + key, + value, + core_attention_bias_type="no_bias", + attn_mask_type="padding_causal", + **psp_kwargs, + ).reshape(query.size(0), 1, -1) + else: + if self._use_torch_core: + out = self._torch_core_attention(query, key, value) + else: + assert self.core_attn is not None + out = self.core_attn(query, key, value, core_attention_bias_type="no_bias") + if out.dim() > x.dim(): + out = out.reshape(*out.shape[:-2], self.num_heads_local * self.v_head_dim) + return self.linear_proj(out) + + def _torch_core_attention( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + ) -> torch.Tensor: + local_seq = query.size(0) + if self.ps.cp_size > 1: + from torch.distributed.nn.functional import all_gather + + query_parts = all_gather(query.contiguous(), group=self.ps.cp_group) + key_parts = all_gather(key.contiguous(), group=self.ps.cp_group) + value_parts = all_gather(value.contiguous(), group=self.ps.cp_group) + query = zigzag_reconstruct_from_cp_parts(query_parts, seq_dim=0) + key = zigzag_reconstruct_from_cp_parts(key_parts, seq_dim=0) + value = zigzag_reconstruct_from_cp_parts(value_parts, seq_dim=0) + + q = query.permute(1, 2, 0, 3) + k = key.permute(1, 2, 0, 3) + v = value.permute(1, 2, 0, 3) + scale = self._softmax_scale if self._query_scale == 1.0 else None + out = F.scaled_dot_product_attention( + q, + k, + v, + dropout_p=0.0, + is_causal=True, + scale=scale, + ) + out = out.permute(2, 0, 1, 3).contiguous() + if self.ps.cp_size > 1: + out = zigzag_slice_for_cp(out, self.ps.cp_rank, self.ps.cp_size, seq_dim=0) + if out.size(0) != local_seq: + raise RuntimeError("CP MLA output shard has unexpected sequence length.") + return out + + def _torch_core_attention_thd( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + *, + packed_seq_params, + ) -> torch.Tensor: + cu_seqlens = getattr(packed_seq_params, "cu_seqlens_q_padded", None) + if cu_seqlens is None: + cu_seqlens = getattr(packed_seq_params, "cu_seqlens_q", None) + if cu_seqlens is None: + raise ValueError("Packed THD MLA fallback requires cu_seqlens.") + + local_tokens = query.size(0) + if self.ps.cp_size > 1: + from torch.distributed.nn.functional import all_gather + + query = reconstruct_packed_from_cp_parts( + list(all_gather(query.contiguous(), group=self.ps.cp_group)), + cu_seqlens_padded=cu_seqlens, + cp_size=self.ps.cp_size, + dim=0, + ) + key = reconstruct_packed_from_cp_parts( + list(all_gather(key.contiguous(), group=self.ps.cp_group)), + cu_seqlens_padded=cu_seqlens, + cp_size=self.ps.cp_size, + dim=0, + ) + value = reconstruct_packed_from_cp_parts( + list(all_gather(value.contiguous(), group=self.ps.cp_group)), + cu_seqlens_padded=cu_seqlens, + cp_size=self.ps.cp_size, + dim=0, + ) + + outputs = [] + scale = self._softmax_scale if self._query_scale == 1.0 else None + for idx in range(int(cu_seqlens.numel()) - 1): + start = int(cu_seqlens[idx].item()) + end = int(cu_seqlens[idx + 1].item()) + if end <= start: + continue + q = query[start:end].permute(1, 0, 2).unsqueeze(0) + k = key[start:end].permute(1, 0, 2).unsqueeze(0) + v = value[start:end].permute(1, 0, 2).unsqueeze(0) + out = F.scaled_dot_product_attention( + q, + k, + v, + dropout_p=0.0, + is_causal=True, + scale=scale, + ) + outputs.append(out.squeeze(0).permute(1, 0, 2).contiguous()) + full_out = torch.cat(outputs, dim=0) if outputs else value.new_empty(value.shape) + if self.ps.cp_size <= 1: + return full_out + local_out = split_packed_to_cp_local( + full_out, + cu_seqlens_padded=cu_seqlens, + cp_size=self.ps.cp_size, + cp_rank=self.ps.cp_rank, + dim=0, + ) + if local_out.size(0) != local_tokens: + raise RuntimeError("CP THD MLA output shard has unexpected token count.") + return local_out + + def _apply_rope(self, q_pos: torch.Tensor, k_pos: torch.Tensor, packed_seq_params): + is_thd = packed_seq_params is not None + if is_thd: + max_q = getattr(packed_seq_params, "max_seqlen_q", None) + max_kv = getattr(packed_seq_params, "max_seqlen_kv", None) + seq_len = ( + int(max(max_q, max_kv)) + if max_q is not None and max_kv is not None + else int(packed_seq_params.cu_seqlens_q[-1]) + ) + freqs = self.rotary(seq_len, packed_seq=True) + if isinstance(freqs, tuple): + freqs, mscale = freqs + else: + mscale = 1.0 + q_pos = _apply_mla_rope_thd( + q_pos, + packed_seq_params.cu_seqlens_q, + freqs, + mscale=mscale, + cp_group=self.ps.cp_group, + ) + k_pos = _apply_mla_rope_thd( + k_pos, + packed_seq_params.cu_seqlens_kv, + freqs, + mscale=mscale, + cp_group=self.ps.cp_group, + ) + return q_pos, k_pos + + seq_len = q_pos.size(0) * self.ps.cp_size + freqs = self.rotary(seq_len) + if isinstance(freqs, tuple): + freqs, mscale = freqs + else: + mscale = 1.0 + return ( + _apply_mla_rope_bshd(q_pos, freqs, mscale=mscale), + _apply_mla_rope_bshd(k_pos, freqs, mscale=mscale), + ) + + +__all__ = ["MultiLatentAttention"] diff --git a/experimental/lite/megatron/lite/primitive/modules/dispatcher.py b/experimental/lite/megatron/lite/primitive/modules/dispatcher.py new file mode 100644 index 00000000000..59554b32ca1 --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/modules/dispatcher.py @@ -0,0 +1,580 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Token dispatcher: AllToAll and DeepEP dispatch/combine.""" + +from __future__ import annotations + +import os + +import torch # pyright: ignore[reportMissingImports] +import torch.distributed as dist # pyright: ignore[reportMissingImports] + +from megatron.lite.primitive.modules.moe import _AllToAll +from megatron.lite.primitive.parallel import ParallelState +from megatron.lite.primitive.utils import ensure_divisible +from megatron.lite.primitive.utils.moe import permute, unpermute + +try: + import deep_ep # pyright: ignore[reportMissingImports] + from deep_ep.utils import EventHandle, EventOverlap # pyright: ignore[reportMissingImports] +except ImportError: + deep_ep = None # type: ignore + EventHandle = None # type: ignore + EventOverlap = None # type: ignore + + +def _hidden_bytes(hidden_size: int) -> int: + return hidden_size * 2 + + +def _build_deepep_buffer(group: dist.ProcessGroup, hidden_size: int): + if deep_ep is None: + raise RuntimeError("DeepEP buffer requested but deep_ep is not installed.") + + group_size = dist.get_world_size(group=group) + hidden_bytes = _hidden_bytes(hidden_size) + num_nvl_bytes = 0 + num_rdma_bytes = 0 + + for config in ( + deep_ep.Buffer.get_dispatch_config(group_size), + deep_ep.Buffer.get_combine_config(group_size), + ): + num_nvl_bytes = max( + config.get_nvl_buffer_size_hint(hidden_bytes, group_size), num_nvl_bytes + ) + num_rdma_bytes = max( + config.get_rdma_buffer_size_hint(hidden_bytes, group_size), num_rdma_bytes + ) + + return deep_ep.Buffer(group=group, num_nvl_bytes=num_nvl_bytes, num_rdma_bytes=num_rdma_bytes) + + +def _use_moe_permute_fusion() -> bool: + return os.environ.get("MEGATRON_LITE_MOE_PERMUTE_FUSION", "0") == "1" + + +def _tensor_hidden_bytes(x: torch.Tensor) -> int: + return x.size(1) * max(x.element_size(), 2) + + +class _DeepEPDispatch(torch.autograd.Function): + @staticmethod + def forward( + ctx, + buffer, + hidden_states: torch.Tensor, + topk_indices: torch.Tensor, + topk_scores: torch.Tensor, + num_experts: int, + async_finish: bool, + allocate_on_comm_stream: bool, + ): + previous_event = ( + EventOverlap(EventHandle()) + if async_finish and EventHandle is not None and EventOverlap is not None + else None + ) + ( + num_tokens_per_rank, + num_tokens_per_rdma_rank, + num_tokens_per_expert, + is_token_in_rank, + event, + ) = buffer.get_dispatch_layout( + topk_indices, + num_experts=num_experts, + previous_event=previous_event, + async_finish=async_finish, + allocate_on_comm_stream=allocate_on_comm_stream, + ) + (recv_hidden, recv_indices, recv_probs, recv_per_expert, handle, after_event) = ( + buffer.dispatch( + hidden_states.contiguous(), + topk_idx=topk_indices, + topk_weights=topk_scores.float(), + num_tokens_per_rank=num_tokens_per_rank, + num_tokens_per_rdma_rank=num_tokens_per_rdma_rank, + is_token_in_rank=is_token_in_rank, + num_tokens_per_expert=num_tokens_per_expert, + previous_event=event, + async_finish=async_finish, + allocate_on_comm_stream=allocate_on_comm_stream, + ) + ) + if async_finish: + after_event.current_stream_wait() + + ctx.buffer = buffer + ctx.handle = handle + ctx.async_finish = async_finish + ctx.allocate_on_comm_stream = allocate_on_comm_stream + recv_per_expert_tensor = torch.tensor( + recv_per_expert, dtype=torch.int64, device=recv_hidden.device + ) + return recv_hidden, recv_indices, recv_probs, recv_per_expert_tensor, handle + + @staticmethod + def backward( + ctx, grad_recv_hidden, grad_recv_indices, grad_recv_probs, grad_recv_per_expert, grad_handle + ): + del grad_recv_indices, grad_recv_per_expert, grad_handle + previous_event = ( + EventOverlap(EventHandle()) + if ctx.async_finish and EventHandle is not None and EventOverlap is not None + else None + ) + grad_scores = None if grad_recv_probs is None else grad_recv_probs.float() + grad_hidden, grad_topk_scores, after_event = ctx.buffer.combine( + grad_recv_hidden.contiguous(), + ctx.handle, + topk_weights=grad_scores, + previous_event=previous_event, + async_finish=ctx.async_finish, + allocate_on_comm_stream=ctx.allocate_on_comm_stream, + ) + if ctx.async_finish: + after_event.current_stream_wait() + return None, grad_hidden, None, grad_topk_scores, None, None, None + + +class _DeepEPCombine(torch.autograd.Function): + @staticmethod + def forward( + ctx, + buffer, + rank_grouped: torch.Tensor, + handle, + async_finish: bool, + allocate_on_comm_stream: bool, + ): + previous_event = ( + EventOverlap(EventHandle()) + if async_finish and EventHandle is not None and EventOverlap is not None + else None + ) + combined, _, after_event = buffer.combine( + rank_grouped, + handle, + previous_event=previous_event, + async_finish=async_finish, + allocate_on_comm_stream=allocate_on_comm_stream, + ) + if async_finish: + after_event.current_stream_wait() + ctx.buffer = buffer + ctx.handle = handle + ctx.async_finish = async_finish + ctx.allocate_on_comm_stream = allocate_on_comm_stream + return combined + + @staticmethod + def backward(ctx, grad_output): + previous_event = ( + EventOverlap(EventHandle()) + if ctx.async_finish and EventHandle is not None and EventOverlap is not None + else None + ) + grad_rank_grouped, _, _, _, _, after_event = ctx.buffer.dispatch( + grad_output.contiguous(), + handle=ctx.handle, + previous_event=previous_event, + async_finish=ctx.async_finish, + allocate_on_comm_stream=ctx.allocate_on_comm_stream, + ) + if ctx.async_finish: + after_event.current_stream_wait() + return None, grad_rank_grouped, None, None, None + + +class TokenDispatcher: + + def __init__( + self, num_experts: int, hidden_size: int, ps: ParallelState, *, use_deepep: bool = True + ): + self.ps = ps + self.num_experts = num_experts + self.ep_size = ps.ep_size + self.num_local_experts = ensure_divisible(num_experts, ps.ep_size) + + self.use_deepep = use_deepep and deep_ep is not None and ps.ep_size > 1 + if self.use_deepep: + assert ps.tp_ep_group is not None + self.buffer = _build_deepep_buffer(ps.tp_ep_group, hidden_size) + + self._row_id_map: torch.Tensor | None = None + self._restore_shape: tuple | None = None + self._input_splits: list[int] | None = None + self._output_splits: list[int] | None = None + self._handle = None + self._deepep_event = None + + if self.ep_size > 1 and self.num_local_experts > 1: + chunk_idxs = torch.arange(self.ep_size * self.num_local_experts) + self._sort_by_experts = ( + chunk_idxs.reshape(self.ep_size, self.num_local_experts).T.ravel().tolist() + ) + self._restore_by_ranks = ( + chunk_idxs.reshape(self.num_local_experts, self.ep_size).T.ravel().tolist() + ) + + def dispatch( + self, hidden_states: torch.Tensor, topk_scores: torch.Tensor, topk_indices: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]: + if self.ep_size <= 1: + return self._dispatch_local(hidden_states, topk_scores, topk_indices) + if self.use_deepep: + return self._dispatch_deepep(hidden_states, topk_scores, topk_indices) + dispatched, tpe, sorted_scores = self._dispatch_alltoall( + hidden_states, topk_scores, topk_indices + ) + return dispatched, tpe, sorted_scores + + def combine(self, expert_output: torch.Tensor) -> torch.Tensor: + if self.ep_size <= 1: + return self._combine_local(expert_output) + if self.use_deepep: + return self._combine_deepep(expert_output) + return self._combine_alltoall(expert_output) + + def submit_deepep_combine( + self, expert_output: torch.Tensor, *, allocate_on_comm_stream: bool = False + ): + if not self.use_deepep: + raise RuntimeError("submit_deepep_combine requires DeepEP combine.") + rank_grouped = unpermute( + expert_output, + self._row_id_map, + restore_shape=self._restore_shape, + fused=_use_moe_permute_fusion(), + ) + previous_event = ( + EventOverlap(EventHandle()) + if EventHandle is not None and EventOverlap is not None + else None + ) + combined = self.buffer.combine( + rank_grouped, + self._handle, + previous_event=previous_event, + async_finish=True, + allocate_on_comm_stream=allocate_on_comm_stream, + ) + event = None + if isinstance(combined, tuple): + if len(combined) >= 3: + event = combined[2] + combined = combined[0] + return {"combined": combined, "event": event} + + def finish_deepep_combine(self, state): + if not self.use_deepep: + raise RuntimeError("finish_deepep_combine requires DeepEP combine.") + event = state.get("event") + if event is not None: + event.current_stream_wait() + self._row_id_map = None + self._restore_shape = None + self._handle = None + self._local_tpe_list = None + return state["combined"] + + def _dispatch_local(self, hidden_states, topk_scores, topk_indices): + t, h = hidden_states.shape + e = self.num_experts + + routing_map = torch.zeros(t, e, dtype=torch.bool, device=hidden_states.device) + routing_map.scatter_(1, topk_indices, True) + num_out = int(routing_map.sum().item()) + + probs_2d = torch.zeros(t, e, dtype=topk_scores.dtype, device=hidden_states.device) + probs_2d.scatter_(1, topk_indices, topk_scores) + + permuted, permuted_probs, sorted_indices = permute( + hidden_states, + routing_map, + probs=probs_2d, + num_out_tokens=num_out, + fused=_use_moe_permute_fusion(), + )[:3] + + self._row_id_map = sorted_indices + self._restore_shape = hidden_states.shape + + tokens_per_expert = routing_map.sum(dim=0).to(torch.int64) + return permuted, tokens_per_expert, permuted_probs + + def _combine_local(self, expert_output): + result = unpermute( + expert_output, + self._row_id_map, + restore_shape=self._restore_shape, + fused=_use_moe_permute_fusion(), + ) + self._row_id_map = None + self._restore_shape = None + return result + + def _dispatch_alltoall(self, hidden_states, topk_scores, topk_indices): + t, h = hidden_states.shape + e = self.num_experts + + routing_map = torch.zeros(t, e, dtype=torch.bool, device=hidden_states.device) + routing_map.scatter_(1, topk_indices, True) + # Use the actual number of routed (token, expert) pairs from routing_map + # rather than t * topk: hash routing (ds4) can map a token's topk slots to + # DUPLICATE experts, which scatter_ dedups, so t*topk would overcount and + # leave permuted.size(0) != sum(input_splits) (all-to-all split mismatch). + # Unique-topk routers (every other model) have routing_map.sum() == t*topk, + # so this is a no-op for them. + num_out = int(routing_map.sum().item()) + + probs_2d = torch.zeros(t, e, dtype=topk_scores.dtype, device=hidden_states.device) + probs_2d.scatter_(1, topk_indices, topk_scores) + + permuted, permuted_probs, sorted_indices = permute( + hidden_states, + routing_map, + probs=probs_2d, + num_out_tokens=num_out, + fused=_use_moe_permute_fusion(), + )[:3] + self._row_id_map = sorted_indices + self._restore_shape = hidden_states.shape + + tokens_per_expert = routing_map.sum(dim=0).to(torch.int64) + tpe_by_rank = tokens_per_expert.view(self.ep_size, self.num_local_experts).sum(dim=1) + self._input_splits = tpe_by_rank.tolist() + + global_tpe_flat = tokens_per_expert.new_empty(self.ep_size * e) + dist.all_gather_into_tensor(global_tpe_flat, tokens_per_expert, group=self.ps.ep_group) + global_tpe_2d = global_tpe_flat.view(self.ep_size, e) + ep_rank = dist.get_rank(group=self.ps.ep_group) + my_start = ep_rank * self.num_local_experts + recv_tpe_2d = global_tpe_2d[:, my_start : my_start + self.num_local_experts].contiguous() + self._output_splits = recv_tpe_2d.sum(dim=1).tolist() + + recv_flat = _AllToAll.apply( + permuted, self._input_splits, self._output_splits, self.ps.ep_group + ) + recv_scores = _AllToAll.apply( + permuted_probs.unsqueeze(-1), self._input_splits, self._output_splits, self.ps.ep_group + ) + + if self.num_local_experts > 1: + chunk_sizes = recv_tpe_2d.ravel().tolist() + chunks = torch.split(recv_flat, chunk_sizes, dim=0) + score_chunks = torch.split(recv_scores, chunk_sizes, dim=0) + sort_idxs = self._sort_by_experts + restore_idxs = self._restore_by_ranks + dispatched = torch.cat([chunks[i] for i in sort_idxs], dim=0) + permuted_probs_out = torch.cat([score_chunks[i] for i in sort_idxs], dim=0) + self._combine_chunk_sizes = [chunk_sizes[i] for i in sort_idxs] + self._combine_restore_idxs = restore_idxs + else: + dispatched = recv_flat + permuted_probs_out = recv_scores + self._combine_chunk_sizes = None + self._combine_restore_idxs = None + + recv_tpe = recv_tpe_2d.sum(dim=0) + return dispatched, recv_tpe, permuted_probs_out.squeeze(-1) + + def _combine_alltoall(self, expert_output): + if self._combine_chunk_sizes is not None: + chunks = torch.split(expert_output, self._combine_chunk_sizes, dim=0) + restore_idxs = ( + self._combine_restore_idxs + if self._combine_restore_idxs is not None + else self._restore_by_ranks + ) + rank_grouped = torch.cat([chunks[i] for i in restore_idxs], dim=0) + else: + rank_grouped = expert_output + + combined = _AllToAll.apply( + rank_grouped, self._output_splits, self._input_splits, self.ps.ep_group + ) + result = unpermute( + combined, + self._row_id_map, + restore_shape=self._restore_shape, + fused=_use_moe_permute_fusion(), + ) + self._row_id_map = None + self._restore_shape = None + self._input_splits = None + self._output_splits = None + self._combine_chunk_sizes = None + self._combine_restore_idxs = None + self._local_tpe_list = None + return result + + def submit_deepep_dispatch( + self, hidden_states, topk_scores, topk_indices, *, allocate_on_comm_stream: bool = False + ): + if not self.use_deepep: + raise RuntimeError("submit_deepep_dispatch requires DeepEP dispatch.") + previous_event = ( + EventOverlap(EventHandle()) + if EventHandle is not None and EventOverlap is not None + else None + ) + ( + num_tokens_per_rank, + num_tokens_per_rdma_rank, + num_tokens_per_expert, + is_token_in_rank, + event, + ) = self.buffer.get_dispatch_layout( + topk_indices, + num_experts=self.num_experts, + previous_event=previous_event, + async_finish=True, + allocate_on_comm_stream=allocate_on_comm_stream, + ) + + topk_scores = topk_scores.float() + recv_hidden, recv_indices, recv_probs, recv_per_expert, handle, event = ( + self.buffer.dispatch( + hidden_states, + topk_idx=topk_indices, + topk_weights=topk_scores, + num_tokens_per_rank=num_tokens_per_rank, + num_tokens_per_rdma_rank=num_tokens_per_rdma_rank, + is_token_in_rank=is_token_in_rank, + num_tokens_per_expert=num_tokens_per_expert, + previous_event=event, + async_finish=True, + allocate_on_comm_stream=allocate_on_comm_stream, + ) + ) + return { + "recv_hidden": recv_hidden, + "recv_indices": recv_indices, + "recv_probs": recv_probs, + "recv_per_expert": recv_per_expert, + "handle": handle, + "event": event, + } + + def finish_deepep_dispatch(self, state): + if not self.use_deepep: + raise RuntimeError("finish_deepep_dispatch requires DeepEP dispatch.") + self._handle = state["handle"] + self._deepep_event = state["event"] + self.wait_dispatch_event() + return self._finish_deepep_dispatch( + state["recv_hidden"], + state["recv_indices"], + state["recv_probs"], + state["recv_per_expert"], + ) + + def _finish_deepep_dispatch( + self, + recv_hidden: torch.Tensor, + recv_indices: torch.Tensor, + recv_probs: torch.Tensor, + recv_per_expert, + ): + if isinstance(recv_per_expert, torch.Tensor): + recv_per_expert = [int(x) for x in recv_per_expert.detach().cpu().tolist()] + local_tpe = torch.tensor( + recv_per_expert[: self.num_local_experts], dtype=torch.int64, device=recv_hidden.device + ) + self._local_tpe_list = [int(x) for x in recv_per_expert[: self.num_local_experts]] + rows = recv_hidden.size(0) + recv_indices = recv_indices.to(torch.long) + routing_map = torch.zeros( + rows, self.num_local_experts, dtype=torch.bool, device=recv_hidden.device + ) + probs_2d = torch.zeros( + rows, self.num_local_experts, dtype=recv_probs.dtype, device=recv_hidden.device + ) + valid = recv_indices >= 0 + row_ids = torch.arange(rows, device=recv_hidden.device).unsqueeze(1) + row_ids = row_ids.expand_as(recv_indices)[valid] + expert_ids = recv_indices[valid] + routing_map[row_ids, expert_ids] = True + probs_2d[row_ids, expert_ids] = recv_probs[valid] + num_out = sum(int(x) for x in recv_per_expert) + dispatched, permuted_probs, sorted_indices = permute( + recv_hidden, + routing_map, + probs=probs_2d, + num_out_tokens=num_out, + fused=_use_moe_permute_fusion(), + )[:3] + self._row_id_map = sorted_indices + self._restore_shape = recv_hidden.shape + if os.environ.get("MEGATRON_LITE_DEEPEP_DEBUG_METADATA") == "1": + ep_rank = dist.get_rank(group=self.ps.ep_group) + print( + "[DEEPEP_METADATA] " + f"ep_rank={ep_rank} recv_rows={int(recv_hidden.shape[0])} " + f"expert_rows={int(dispatched.shape[0])} " + f"recv_indices_shape={tuple(recv_indices.shape)} " + f"recv_per_expert_len={len(recv_per_expert)} " + f"recv_per_expert_sum={sum(int(x) for x in recv_per_expert)} " + f"recv_per_expert_head={recv_per_expert[: self.num_local_experts]} " + f"local_tpe_sum={int(local_tpe.sum().item())}", + flush=True, + ) + if os.environ.get("MEGATRON_LITE_DEEPEP_SKIP_DISPATCH_METADATA_CHECK") != "1" and int( + local_tpe.sum().item() + ) != int(dispatched.shape[0]): + ep_rank = dist.get_rank(group=self.ps.ep_group) + raise RuntimeError( + "DeepEP dispatch metadata mismatch: " + f"ep_rank={ep_rank} dispatched_tokens={int(dispatched.shape[0])} " + f"local_tpe={local_tpe.tolist()} recv_per_expert_len={len(recv_per_expert)}" + ) + return dispatched, local_tpe, permuted_probs + + def _dispatch_deepep(self, hidden_states, topk_scores, topk_indices): + if torch.is_grad_enabled(): + recv_hidden, recv_indices, recv_probs, recv_per_expert, handle = _DeepEPDispatch.apply( + self.buffer, + hidden_states, + topk_indices, + topk_scores.float(), + self.num_experts, + False, + False, + ) + self._handle = handle + self._deepep_event = None + return self._finish_deepep_dispatch( + recv_hidden, recv_indices, recv_probs, recv_per_expert + ) + state = self.submit_deepep_dispatch( + hidden_states, topk_scores, topk_indices, allocate_on_comm_stream=False + ) + return self.finish_deepep_dispatch(state) + + def wait_dispatch_event(self): + if self._deepep_event is not None: + self._deepep_event.current_stream_wait() + self._deepep_event = None + + def _combine_deepep(self, expert_output): + rank_grouped = unpermute( + expert_output, + self._row_id_map, + restore_shape=self._restore_shape, + fused=_use_moe_permute_fusion(), + ) + if torch.is_grad_enabled(): + combined = _DeepEPCombine.apply(self.buffer, rank_grouped, self._handle, False, False) + else: + combined = self.buffer.combine(rank_grouped, self._handle) + if isinstance(combined, tuple): + combined = combined[0] + self._row_id_map = None + self._restore_shape = None + self._handle = None + self._local_tpe_list = None + return combined + + +__all__ = ["TokenDispatcher"] diff --git a/experimental/lite/megatron/lite/primitive/modules/experts.py b/experimental/lite/megatron/lite/primitive/modules/experts.py new file mode 100644 index 00000000000..e3ca4b51ed7 --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/modules/experts.py @@ -0,0 +1,230 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""MoE expert compute: SwiGLU fusions, _AllReduceETP, and Experts.""" + +from __future__ import annotations + +import os +from contextlib import contextmanager +from typing import Any + +import torch # pyright: ignore[reportMissingImports] +import torch.distributed as dist # pyright: ignore[reportMissingImports] +import torch.nn as nn # pyright: ignore[reportMissingImports] +import transformer_engine.pytorch as te # pyright: ignore[reportMissingImports] + +from megatron.lite.primitive.kernels.swiglu import bias_swiglu_impl, weighted_bias_swiglu_impl +from megatron.lite.primitive.modules.lora import ( + LoraConfig, + SharedGroupedLinearLoRA, + normalize_lora_config, +) +from megatron.lite.primitive.parallel import ParallelState +from megatron.lite.primitive.recompute import CheckpointWithoutOutput +from megatron.lite.primitive.utils import ensure_divisible + +__all__ = ["Experts", "_AllReduceETP"] + + +@contextmanager +def _expert_nvtx_range(name: str): + if os.environ.get("MEGATRON_LITE_EP_EXPERT_NVTX") != "1" or not torch.cuda.is_available(): + yield + return + torch.cuda.nvtx.range_push(name) + try: + yield + finally: + torch.cuda.nvtx.range_pop() + + +def swiglu_with_probs( + y: torch.Tensor, probs: torch.Tensor | None, swiglu_limit: float = 0.0 +) -> torch.Tensor: + """SwiGLU with optional expert probability scaling.""" + if swiglu_limit > 0: + gate, up = y.chunk(2, dim=-1) + up = torch.clamp(up.float(), min=-swiglu_limit, max=swiglu_limit) + gate = torch.clamp(gate.float(), max=swiglu_limit) + out = torch.nn.functional.silu(gate) * up + if probs is not None: + out = out * probs + return out.to(dtype=y.dtype) + if probs is not None: + return weighted_bias_swiglu_impl(y, bias=None, weights=probs) + return bias_swiglu_impl(y, bias=None) + + +class _AllReduceETP(torch.autograd.Function): + """AllReduce with proper autograd: grad(AllReduce) = AllReduce.""" + + @staticmethod + def forward(ctx, x, group): + ctx.group = group + dist.all_reduce(x, group=group) + return x + + @staticmethod + def backward(ctx, grad): + return grad, None + + +class Experts(nn.Module): + + def __init__( + self, + config: Any, + ps: ParallelState, + *, + fp8: bool = False, + moe_act_recompute: bool = False, + lora_config: LoraConfig | dict | None = None, + ): + super().__init__() + self.num_local_experts = ensure_divisible(config.num_experts, ps.ep_size) + self.fp8 = fp8 + self.moe_act_recompute = moe_act_recompute + self.etp_group = ps.etp_group if ps.etp_size > 1 else None + self.swiglu_limit = float(getattr(config, "swiglu_limit", 0.0) or 0.0) + + self.fc1 = te.GroupedLinear( + self.num_local_experts, + config.hidden_size, + config.moe_intermediate_size * 2 // ps.etp_size, + bias=False, + params_dtype=torch.bfloat16, + ) + self.fc2 = te.GroupedLinear( + self.num_local_experts, + config.moe_intermediate_size // ps.etp_size, + config.hidden_size, + bias=False, + params_dtype=torch.bfloat16, + ) + lora = normalize_lora_config(lora_config) + self.fc1_lora: SharedGroupedLinearLoRA | None = None + self.fc2_lora: SharedGroupedLinearLoRA | None = None + if lora.enabled and lora.targets_module("linear_fc1"): + self.fc1_lora = SharedGroupedLinearLoRA( + self.num_local_experts, + config.hidden_size, + config.moe_intermediate_size * 2 // ps.etp_size, + lora.rank, + alpha=lora.alpha, + dropout=lora.dropout, + ) + if lora.enabled and lora.targets_module("linear_fc2"): + self.fc2_lora = SharedGroupedLinearLoRA( + self.num_local_experts, + config.moe_intermediate_size // ps.etp_size, + config.hidden_size, + lora.rank, + alpha=lora.alpha, + dropout=lora.dropout, + ) + if ps.tp_size > 1 and ps.ep_size == 1 and ps.etp_size == 1: + tp_group = ps.tp_group + for module in (self.fc1, self.fc2, self.fc1_lora, self.fc2_lora): + if module is None: + continue + for param in module.parameters(): + + def _ar(grad, g=tp_group): + dist.all_reduce(grad, op=dist.ReduceOp.SUM, group=g) + return grad + + param.register_hook(_ar) + + def forward( + self, + x: torch.Tensor, + tokens_per_expert: torch.Tensor, + permuted_probs: torch.Tensor | None = None, + tokens_per_expert_list: list[int] | None = None, + ) -> torch.Tensor: + m_splits = ( + tokens_per_expert.tolist() + if tokens_per_expert_list is None + else list(tokens_per_expert_list) + ) + pad_mask = None + if self.fp8: + x, permuted_probs, m_splits, pad_mask = self._fp8_pad(x, permuted_probs, m_splits) + + etp_real_len = x.shape[0] + if self.etp_group is not None: + max_len = torch.tensor([etp_real_len], device=x.device, dtype=torch.int64) + dist.all_reduce(max_len, op=dist.ReduceOp.MAX, group=self.etp_group) + max_len = int(max_len.item()) + if etp_real_len < max_len: + x = torch.cat( + [ + x, + torch.zeros( + max_len - etp_real_len, x.shape[1], dtype=x.dtype, device=x.device + ), + ], + dim=0, + ) + if permuted_probs is not None: + permuted_probs = torch.cat( + [ + permuted_probs, + torch.zeros( + max_len - etp_real_len, dtype=permuted_probs.dtype, device=x.device + ), + ], + dim=0, + ) + m_splits = list(m_splits) + m_splits[-1] += max_len - etp_real_len + + probs = permuted_probs.unsqueeze(-1) if permuted_probs is not None else None + with _expert_nvtx_range("ep_experts.forward"): + if self.moe_act_recompute and probs is not None: + act_ckpt = CheckpointWithoutOutput(preserve_rng_state=True) + fc1_out = self.fc1(x, m_splits) + if self.fc1_lora is not None: + fc1_out = fc1_out + self.fc1_lora(x, m_splits) + h = act_ckpt.checkpoint(swiglu_with_probs, fc1_out, probs, self.swiglu_limit) + out = self.fc2(h, m_splits) + if self.fc2_lora is not None: + out = out + self.fc2_lora(h, m_splits) + act_ckpt.discard_output_and_register_recompute(out) + else: + fc1_out = self.fc1(x, m_splits) + if self.fc1_lora is not None: + fc1_out = fc1_out + self.fc1_lora(x, m_splits) + h = swiglu_with_probs(fc1_out, probs, self.swiglu_limit) + out = self.fc2(h, m_splits) + if self.fc2_lora is not None: + out = out + self.fc2_lora(h, m_splits) + + if self.etp_group is not None: + out = _AllReduceETP.apply(out, self.etp_group) + out = out[:etp_real_len] + + if pad_mask is not None: + out = out[pad_mask] + return out + + @staticmethod + def _fp8_pad(x, permuted_probs, m_splits): + padded = [(s + 15) // 16 * 16 for s in m_splits] + if padded == m_splits: + return x, permuted_probs, m_splits, None + device, dtype = x.device, x.dtype + total_padded = sum(padded) + x_pad = torch.zeros(total_padded, x.size(1), device=device, dtype=dtype) + mask = torch.zeros(total_padded, dtype=torch.bool, device=device) + probs_pad = None + if permuted_probs is not None: + probs_pad = torch.zeros(total_padded, device=device, dtype=permuted_probs.dtype) + src_off, dst_off = 0, 0 + for real, pad in zip(m_splits, padded, strict=True): + x_pad[dst_off : dst_off + real] = x[src_off : src_off + real] + mask[dst_off : dst_off + real] = True + if probs_pad is not None: + probs_pad[dst_off : dst_off + real] = permuted_probs[src_off : src_off + real] + src_off += real + dst_off += pad + return x_pad, probs_pad, padded, mask diff --git a/experimental/lite/megatron/lite/primitive/modules/gated_delta_net.py b/experimental/lite/megatron/lite/primitive/modules/gated_delta_net.py new file mode 100644 index 00000000000..a0c18f2b8ba --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/modules/gated_delta_net.py @@ -0,0 +1,476 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Qwen-style Gated DeltaNet primitive.""" + +from __future__ import annotations + +import torch +import torch.nn as nn +import torch.nn.functional as F +import transformer_engine.pytorch as te + +from megatron.lite.primitive.kernels.jit import jit_fuser +from megatron.lite.primitive.ops.gated_delta_rule import ( + l2norm, + torch_chunk_gated_delta_rule, +) +from megatron.lite.primitive.parallel import ( + ColumnParallelLinear, + ParallelState, + RowParallelLinear, +) +from megatron.lite.primitive.parallel.cp import ( + contiguous_to_zigzag_chunks, + zigzag_reconstruct_from_cp_parts, + zigzag_slice_for_cp, + zigzag_to_contiguous_chunks, +) +from megatron.lite.primitive.parallel.thd import ( + reconstruct_packed_from_cp_parts, + split_packed_to_cp_local, +) +from megatron.lite.primitive.utils import ensure_divisible + + +try: + from fla.modules.convolution import ( + causal_conv1d as _fla_causal_conv1d, # pyright: ignore[reportMissingImports] + ) + from fla.ops.gated_delta_rule import ( + chunk_gated_delta_rule as _fla_chunk_gated_delta_rule, # pyright: ignore[reportMissingImports] + ) + + _HAS_FLA = True +except ImportError: + _HAS_FLA = False + +try: + from fla.ops.cp import build_cp_context as _fla_build_cp_context # pyright: ignore[reportMissingImports] +except ImportError: + _fla_build_cp_context = None + +_CONV_PAD_ALIGNMENT = 4096 + + +class GatedDeltaNet(nn.Module): + """Native Gated DeltaNet with dense/packed all-gather CP support.""" + + def __init__( + self, + *, + hidden_size: int, + linear_num_key_heads: int, + linear_key_head_dim: int, + linear_num_value_heads: int, + linear_value_head_dim: int, + linear_conv_kernel_dim: int, + rms_norm_eps: float, + ps: ParallelState, + deterministic: bool = False, + cp_mode: str = "replicated", + ): + super().__init__() + if cp_mode not in {"sharded", "replicated"}: + raise ValueError(f"Unsupported GatedDeltaNet CP mode: {cp_mode!r}.") + self.ps = ps + self.deterministic = bool(deterministic) + self.cp_mode = cp_mode + self.num_k_heads = linear_num_key_heads + self.num_v_heads = linear_num_value_heads + self.dk = linear_key_head_dim + self.dv = linear_value_head_dim + self.v_heads_per_k_head = ensure_divisible(self.num_v_heads, self.num_k_heads) + self.num_k_heads_local = ensure_divisible(self.num_k_heads, ps.tp_size) + self.num_v_heads_local = ensure_divisible(self.num_v_heads, ps.tp_size) + self.qk_dim = self.num_k_heads * self.dk + self.v_dim = self.num_v_heads * self.dv + self.qk_dim_local = self.num_k_heads_local * self.dk + self.v_dim_local = self.num_v_heads_local * self.dv + self.in_proj_dim = self.qk_dim * 2 + self.v_dim * 2 + self.num_v_heads * 2 + + self.in_proj = ColumnParallelLinear( + hidden_size, + self.in_proj_dim, + ps, + bias=False, + normalization="RMSNorm", + eps=rms_norm_eps, + zero_centered_gamma=True, + ) + conv_dim_local = self.qk_dim_local * 2 + self.v_dim_local + self.conv1d = nn.Conv1d( + in_channels=conv_dim_local, + out_channels=conv_dim_local, + kernel_size=linear_conv_kernel_dim, + groups=conv_dim_local, + bias=False, + padding=linear_conv_kernel_dim - 1, + ) + self.dt_bias = nn.Parameter( + torch.ones(self.num_v_heads_local, dtype=torch.float32) + ) + self.A_log = nn.Parameter( + torch.zeros(self.num_v_heads_local, dtype=torch.float32) + ) + self.norm = te.RMSNorm(self.dv, eps=rms_norm_eps, zero_centered_gamma=True) + self.o_proj = RowParallelLinear(self.v_dim, hidden_size, ps, bias=False) + self._cp_context_cache: dict[ + tuple[int, int, torch.device], tuple[torch.Tensor, object] + ] = {} + + def forward( + self, x: torch.Tensor, position_ids: torch.Tensor | None = None, packed_seq_params=None + ) -> torch.Tensor: + del position_ids + is_packed = packed_seq_params is not None + qkvzba = self.in_proj(x).transpose(0, 1).contiguous() + cu_seqlens = self._packed_cu_seqlens(packed_seq_params) if is_packed else None + cp_context = None + replicated = False + if self.ps.cp_size > 1: + if self.ps.cp_group is None: + raise RuntimeError("CP>1 requires ParallelState.cp_group.") + if self.cp_mode == "replicated": + qkvzba, cu_seqlens = self._replicate_cp_qkvzba(qkvzba, cu_seqlens) + replicated = True + else: + if not _HAS_FLA or _fla_build_cp_context is None: + raise NotImplementedError( + "GatedDeltaNet all-gather CP requires FLA kernels." + ) + if not is_packed and qkvzba.shape[0] > 1: + raise ValueError( + "GatedDeltaNet all-gather CP with SBHD inputs currently requires " + "micro_batch_size == 1. Use packed THD input or micro_batch_size=1." + ) + qkvzba = self._cp_swap_qkvzba( + qkvzba, + cu_seqlens if is_packed else None, + to_contiguous=True, + ) + cu_seqlens, cp_context = self._build_cp_context(qkvzba, cu_seqlens) + batch, seq_len = qkvzba.shape[:2] + query, key, value, gate, beta, alpha = self._split_proj(qkvzba) + qkv = torch.cat( + [ + query.reshape(batch, seq_len, -1), + key.reshape(batch, seq_len, -1), + value.reshape(batch, seq_len, -1), + ], + dim=-1, + ) + + qkv = self._causal_conv1d( + qkv, seq_len, cu_seqlens=cu_seqlens, cp_context=cp_context + ) + query, key, value, gate, beta, alpha = self._prepare_qkv( + qkv, gate, beta, alpha, batch, seq_len + ) + g, beta = self._compute_g_and_beta(self.A_log, self.dt_bias, alpha, beta) + out, _ = self._gated_delta_rule( + query, + key, + value, + g, + beta, + initial_state=None, + output_final_state=False, + cu_seqlens=cu_seqlens, + cp_context=cp_context, + ) + + out = self._apply_gated_norm(out, gate) + out = out.reshape(batch, seq_len, self.v_dim_local) + if self.ps.cp_size > 1: + if replicated: + out = self._slice_replicated_output(out, cu_seqlens) + else: + out = self._cp_swap_qkvzba( + out, + cu_seqlens if is_packed else None, + to_contiguous=False, + ) + batch, seq_len = out.shape[:2] + out = out.transpose(0, 1).contiguous() + return self.o_proj(out) + + def _all_gather_cp_tensor(self, tensor: torch.Tensor) -> list[torch.Tensor]: + if self.ps.cp_size <= 1: + return [tensor] + try: + from torch.distributed.nn.functional import all_gather + + return list(all_gather(tensor, group=self.ps.cp_group)) + except Exception: + parts = [torch.empty_like(tensor) for _ in range(self.ps.cp_size)] + torch.distributed.all_gather(parts, tensor, group=self.ps.cp_group) + return parts + + def _replicate_cp_qkvzba( + self, + qkvzba: torch.Tensor, + cu_seqlens: torch.Tensor | None, + ) -> tuple[torch.Tensor, torch.Tensor | None]: + if cu_seqlens is None: + parts = self._all_gather_cp_tensor(qkvzba) + return zigzag_reconstruct_from_cp_parts(parts, seq_dim=1), None + if qkvzba.shape[0] != 1: + raise ValueError("Packed THD GatedDeltaNet expects a single packed batch row.") + parts = self._all_gather_cp_tensor(qkvzba[0].contiguous()) + full = reconstruct_packed_from_cp_parts( + parts, + cu_seqlens_padded=cu_seqlens, + cp_size=self.ps.cp_size, + dim=0, + ) + return full.unsqueeze(0).contiguous(), cu_seqlens + + def _slice_replicated_output( + self, + out: torch.Tensor, + cu_seqlens: torch.Tensor | None, + ) -> torch.Tensor: + if cu_seqlens is None: + return zigzag_slice_for_cp(out, self.ps.cp_rank, self.ps.cp_size, seq_dim=1) + if out.shape[0] != 1: + raise ValueError("Packed THD GatedDeltaNet expects a single packed batch row.") + local = split_packed_to_cp_local( + out[0].contiguous(), + cu_seqlens_padded=cu_seqlens, + cp_size=self.ps.cp_size, + cp_rank=self.ps.cp_rank, + dim=0, + ) + return local.unsqueeze(0).contiguous() + + def _cp_swap_qkvzba( + self, + tensor: torch.Tensor, + cu_seqlens: torch.Tensor | None, + *, + to_contiguous: bool, + ) -> torch.Tensor: + if cu_seqlens is None: + swap = ( + zigzag_to_contiguous_chunks + if to_contiguous + else contiguous_to_zigzag_chunks + ) + return swap(tensor, self.ps.cp_group, seq_dim=1) + if tensor.shape[0] != 1: + raise ValueError( + "Packed THD GatedDeltaNet expects a single packed batch row." + ) + local_cu_seqlens = cu_seqlens // self.ps.cp_size + pieces = [] + swap = ( + zigzag_to_contiguous_chunks + if to_contiguous + else contiguous_to_zigzag_chunks + ) + for idx in range(int(local_cu_seqlens.numel()) - 1): + start = int(local_cu_seqlens[idx].item()) + end = int(local_cu_seqlens[idx + 1].item()) + if end <= start: + continue + pieces.append(swap(tensor[:, start:end, :], self.ps.cp_group, seq_dim=1)) + if not pieces: + return tensor + return torch.cat(pieces, dim=1).contiguous() + + def _build_cp_context( + self, + qkvzba: torch.Tensor, + cu_seqlens: torch.Tensor | None, + ) -> tuple[torch.Tensor, object]: + if _fla_build_cp_context is None: + raise NotImplementedError( + "GatedDeltaNet all-gather CP requires FLA cp context." + ) + if cu_seqlens is not None: + return ( + cu_seqlens, + _fla_build_cp_context( + cu_seqlens=cu_seqlens, + group=self.ps.cp_group, + conv1d_kernel_size=self.conv1d.kernel_size[0], + ), + ) + + batch, local_seq_len = qkvzba.shape[:2] + global_seq_len = local_seq_len * self.ps.cp_size + cache_key = (global_seq_len, batch, qkvzba.device) + cached = self._cp_context_cache.get(cache_key) + if cached is None: + dense_cu_seqlens = ( + torch.arange(batch + 1, device=qkvzba.device, dtype=torch.long) + * global_seq_len + ) + cached = ( + dense_cu_seqlens, + _fla_build_cp_context( + cu_seqlens=dense_cu_seqlens, + group=self.ps.cp_group, + conv1d_kernel_size=self.conv1d.kernel_size[0], + ), + ) + self._cp_context_cache[cache_key] = cached + return cached + + def _causal_conv1d( + self, + qkv: torch.Tensor, + seq_len: int, + *, + cu_seqlens: torch.Tensor | None, + cp_context, + ) -> torch.Tensor: + if _HAS_FLA and (cp_context is not None or cu_seqlens is not None or not self.deterministic): + orig_seq_len = qkv.shape[1] + pad_n = 0 if cp_context is not None else (-orig_seq_len % _CONV_PAD_ALIGNMENT) + conv_input = qkv + conv_cu_seqlens = cu_seqlens + if pad_n > 0: + conv_input = F.pad(qkv, (0, 0, 0, pad_n)) + if conv_cu_seqlens is not None: + conv_cu_seqlens = conv_cu_seqlens.clone() + conv_cu_seqlens[-1] += pad_n + kwargs = {} + if cp_context is not None: + kwargs["cp_context"] = cp_context + qkv, _ = _fla_causal_conv1d( + x=conv_input, + weight=self.conv1d.weight.squeeze(1), + bias=None, + activation="silu", + cu_seqlens=conv_cu_seqlens, + **kwargs, + ) + if pad_n > 0: + qkv = qkv[:, :orig_seq_len, :] + return qkv + if cu_seqlens is None and cp_context is None: + return F.silu( + self.conv1d(qkv.transpose(1, 2))[:, :, :seq_len].transpose(1, 2) + ) + raise NotImplementedError("GatedDeltaNet packed THD requires FLA causal conv.") + + def _gated_delta_rule( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + *, + initial_state: torch.Tensor | None, + output_final_state: bool, + cu_seqlens: torch.Tensor | None, + cp_context, + ) -> tuple[torch.Tensor, torch.Tensor | None]: + if _HAS_FLA and (cp_context is not None or not self.deterministic): + kwargs = {} + if cp_context is not None: + kwargs["cp_context"] = cp_context + return _fla_chunk_gated_delta_rule( + query, + key, + value, + g=g, + beta=beta, + initial_state=initial_state, + output_final_state=output_final_state, + use_qk_l2norm_in_kernel=False, + cu_seqlens=cu_seqlens, + **kwargs, + ) + if cp_context is not None: + raise NotImplementedError( + "GatedDeltaNet all-gather CP requires FLA gated delta rule." + ) + return torch_chunk_gated_delta_rule( + query, + key, + value, + g=g, + beta=beta, + initial_state=initial_state, + output_final_state=output_final_state, + ) + + @staticmethod + def _packed_cu_seqlens(packed_seq_params) -> torch.Tensor: + cu_seqlens = ( + packed_seq_params.cu_seqlens_q_padded + if getattr(packed_seq_params, "cu_seqlens_q_padded", None) is not None + else packed_seq_params.cu_seqlens_q + ) + if cu_seqlens is None: + raise ValueError( + "packed_seq_params must carry cu_seqlens_q for CP GatedDeltaNet." + ) + return cu_seqlens + + def _split_proj(self, qkvzba: torch.Tensor): + q, k, v, z, b, a = qkvzba.split( + [ + self.qk_dim_local, + self.qk_dim_local, + self.v_dim_local, + self.v_dim_local, + self.num_v_heads_local, + self.num_v_heads_local, + ], + dim=-1, + ) + batch, seq_len = qkvzba.shape[:2] + return ( + q.reshape(batch, seq_len, self.num_k_heads_local, self.dk), + k.reshape(batch, seq_len, self.num_k_heads_local, self.dk), + v.reshape(batch, seq_len, self.num_v_heads_local, self.dv), + z.reshape(batch, seq_len, self.num_v_heads_local, self.dv), + b.reshape(batch, seq_len, self.num_v_heads_local), + a.reshape(batch, seq_len, self.num_v_heads_local), + ) + + def _prepare_qkv( + self, qkv: torch.Tensor, gate, beta, alpha, batch: int, seq_len: int + ): + query_key, value = qkv.split([2 * self.qk_dim_local, self.v_dim_local], dim=-1) + query_key = query_key.reshape( + batch, seq_len, 2 * self.num_k_heads_local, self.dk + ) + value = value.reshape(batch, seq_len, self.num_v_heads_local, self.dv) + query, key = query_key.split(self.num_k_heads_local, dim=2) + query = self._l2norm(query.contiguous()) + key = self._l2norm(key.contiguous()) + if self.v_heads_per_k_head > 1: + query = query.repeat_interleave(self.v_heads_per_k_head, dim=2) + key = key.repeat_interleave(self.v_heads_per_k_head, dim=2) + return ( + query.contiguous(), + key.contiguous(), + value.contiguous(), + gate.contiguous(), + beta.contiguous(), + alpha.contiguous(), + ) + + def _l2norm(self, x: torch.Tensor) -> torch.Tensor: + return l2norm(x) + + @staticmethod + def _compute_g_and_beta(A_log, dt_bias, alpha, beta): + g = -A_log.exp() * F.softplus(alpha.float() + dt_bias) + return g, beta.sigmoid() + + @jit_fuser + def _apply_gated_norm(self, x: torch.Tensor, gate: torch.Tensor) -> torch.Tensor: + x_dtype = x.dtype + x = x.reshape(-1, x.shape[-1]) + y = self.norm(x) + gate = gate.reshape(-1, gate.shape[-1]) + y = y * F.silu(gate.float()) + return y.to(x_dtype) + + +__all__ = ["GatedDeltaNet"] diff --git a/experimental/lite/megatron/lite/primitive/modules/gqa.py b/experimental/lite/megatron/lite/primitive/modules/gqa.py new file mode 100644 index 00000000000..7bb10d36001 --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/modules/gqa.py @@ -0,0 +1,310 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Grouped Query Attention + Rotary Embedding. + +Model-agnostic: takes explicit params instead of model-specific config. +Supports sequence parallel, context parallel, and THD (packed sequences). +""" + +from __future__ import annotations + +import torch +import torch.nn as nn +import transformer_engine.pytorch as te + +from megatron.lite.primitive.modules.gqa_utils import split_grouped_qkvg +from megatron.lite.primitive.modules.lora import LinearLoRA, LoraConfig, normalize_lora_config +from megatron.lite.primitive.modules.mrope import MultimodalRotaryEmbedding +from megatron.lite.primitive.parallel import ColumnParallelLinear, ParallelState, RowParallelLinear +from megatron.lite.primitive.utils import ensure_divisible +from megatron.lite.primitive.utils.rope import _apply_rotary_pos_emb_bshd, _apply_rotary_pos_emb_thd +from megatron.lite.primitive.utils.rotary import RotaryEmbedding + +# Whitelist of MC PackedSeqParams fields accepted by TE DotProductAttention.forward(). +# MC-only fields (local_cp_size, cp_group, total_tokens, seq_idx) are excluded. +# Mirror the TE DotProductAttention packed-sequence argument whitelist. +_KEPT_PSP_FIELDS = ( + "qkv_format", + "cu_seqlens_q", + "cu_seqlens_kv", + "cu_seqlens_q_padded", + "cu_seqlens_kv_padded", + "max_seqlen_q", + "max_seqlen_kv", +) + + +class GQAttention(nn.Module): + """Grouped Query Attention with TE DotProductAttention. + + Model-agnostic: all architecture params passed explicitly. + """ + + _cp_stream: torch.cuda.Stream | None = None + + def __init__( + self, + hidden_size: int, + num_attention_heads: int, + num_key_value_heads: int, + head_dim: int, + ps: ParallelState, + *, + rms_norm_eps: float = 1e-6, + rope_theta: float = 1_000_000.0, + rotary_percent: float = 1.0, + use_thd: bool = False, + output_gate: bool = False, + use_fp32_rope: bool = False, + zero_centered_gamma: bool = False, + qkv_layout: str = "flat", + lora_config: LoraConfig | dict | None = None, + mrope_section: list[int] | None = None, + ): + super().__init__() + self.num_heads_local = ensure_divisible(num_attention_heads, ps.tp_size) + self.num_kv_heads_local = ensure_divisible(num_key_value_heads, ps.tp_size) + self.head_dim = head_dim + self.ps = ps + self._output_gate = output_gate + self._use_fp32_rope = use_fp32_rope + self._qkv_eps = rms_norm_eps + self._qkv_zero_centered_gamma = zero_centered_gamma + if qkv_layout not in {"flat", "mcore"}: + raise ValueError(f"Unsupported qkv_layout={qkv_layout!r}") + self._qkv_layout = qkv_layout + self._mrope_section = list(mrope_section) if mrope_section is not None else None + + # Declaration order follows MC's `SelfAttention` submodule order + # (linear_proj → linear_qkv → q_layernorm → k_layernorm). `named_ + # parameters()` iterates in registration order, and MC's + # `DistributedDataParallel` partitions gradient buckets by that order. + # Mismatched order would put bucket boundaries in different places, + # producing different per-rank fp32 master shard layouts and + # non-bitwise step-1 divergence. + self.proj = RowParallelLinear(num_attention_heads * head_dim, hidden_size, ps, bias=False) + q_cols = num_attention_heads * (2 if output_gate else 1) + qkv_size = (q_cols + 2 * num_key_value_heads) * head_dim + self.qkv = ColumnParallelLinear( + hidden_size, + qkv_size, + ps, + bias=False, + normalization="RMSNorm", + eps=rms_norm_eps, + zero_centered_gamma=zero_centered_gamma, + ) + self.q_norm = te.RMSNorm( + head_dim, eps=rms_norm_eps, zero_centered_gamma=zero_centered_gamma + ) + self.k_norm = te.RMSNorm( + head_dim, eps=rms_norm_eps, zero_centered_gamma=zero_centered_gamma + ) + + lora = normalize_lora_config(lora_config) + self.qkv_lora: LinearLoRA | None = None + self.proj_lora: LinearLoRA | None = None + if lora.enabled and lora.targets_module("linear_qkv"): + self.qkv_lora = LinearLoRA( + hidden_size, + self.qkv.local_out, + lora.rank, + alpha=lora.alpha, + dropout=lora.dropout, + sequence_parallel_input=self.qkv.use_sp, + tp_group=ps.tp_group, + rank_partition_size=ps.tp_size, + rank_partitioned_a=ps.tp_size > 1, + a_tensor_model_parallel=ps.tp_size > 1, + b_tensor_model_parallel=ps.tp_size > 1, + ) + if lora.enabled and lora.targets_module("linear_proj"): + self.proj_lora = LinearLoRA( + self.proj.local_in, + hidden_size, + lora.rank, + alpha=lora.alpha, + dropout=lora.dropout, + tp_group=ps.tp_group, + tp_rank=ps.tp_rank, + sequence_parallel_scatter_output=self.proj.use_sp, + input_parallel_reduce=ps.tp_size > 1, + output_partition_size=ps.tp_size, + output_partitioned_b=ps.tp_size > 1, + a_tensor_model_parallel=ps.tp_size > 1, + b_tensor_model_parallel=ps.tp_size > 1, + ) + + if self._mrope_section is None: + self.rotary = RotaryEmbedding( + kv_channels=head_dim, + rotary_percent=rotary_percent, + rotary_interleaved=False, + rotary_base=int(rope_theta), + use_cpu_initialization=False, + cp_group=ps.cp_group if ps.cp_size > 1 else None, + ) + else: + self.rotary = MultimodalRotaryEmbedding( + kv_channels=head_dim, + rotary_percent=rotary_percent, + rotary_base=rope_theta, + cp_group=ps.cp_group if ps.cp_size > 1 else None, + ) + cp_kwargs = {} + if ps.cp_size > 1: + if GQAttention._cp_stream is None: + GQAttention._cp_stream = torch.cuda.Stream() + cp_kwargs = dict( + cp_group=ps.cp_group, + cp_global_ranks=ps.cp_global_ranks, + cp_stream=GQAttention._cp_stream, + ) + + self.core_attn = te.DotProductAttention( + num_attention_heads=self.num_heads_local, + kv_channels=head_dim, + num_gqa_groups=self.num_kv_heads_local, + attention_dropout=0.0, + attn_mask_type="causal", + qkv_format="thd" if use_thd else "sbhd", + **cp_kwargs, + ) + + def forward( + self, x: torch.Tensor, position_ids: torch.Tensor | None = None, packed_seq_params=None + ) -> torch.Tensor: + qkv = self.qkv(x) + if self.qkv_lora is not None: + qkv = qkv + self.qkv_lora(self._qkv_lora_input(x)) + q, gate, k, v = self._split_qkv(qkv) + + is_thd = packed_seq_params is not None + if is_thd: + q, k, v = q.squeeze(1), k.squeeze(1), v.squeeze(1) + + q = self.q_norm(q) + k = self.k_norm(k) + + # RoPE uses the local unfused rotate-half helpers. + if self._use_fp32_rope: + orig_dtype = q.dtype + q, k = q.float(), k.float() + if self._mrope_section is not None: + if position_ids is None: + raise ValueError("MRoPE attention requires position_ids.") + # Packed THD position_ids are already CP-local after protocol.forward + # prepares the VERL batch; avoid slicing MRoPE frequencies twice. + freqs = self.rotary(position_ids, self._mrope_section, packed_seq=is_thd) + if is_thd: + q = _apply_rotary_pos_emb_bshd(q[:, None], freqs).squeeze(1) + k = _apply_rotary_pos_emb_bshd(k[:, None], freqs).squeeze(1) + else: + q = _apply_rotary_pos_emb_bshd(q, freqs) + k = _apply_rotary_pos_emb_bshd(k, freqs) + elif is_thd: + max_q = getattr(packed_seq_params, "max_seqlen_q", None) + max_kv = getattr(packed_seq_params, "max_seqlen_kv", None) + if max_q is None or max_kv is None: + seq_len_for_rope = int(packed_seq_params.cu_seqlens_q[-1]) + else: + seq_len_for_rope = int(max(max_q, max_kv)) + # Packed THD uses max per-sequence padded length, not total packed tokens. + # The THD apply helper handles CP-zigzag frequency slicing per sequence. + freqs = self.rotary(seq_len_for_rope, packed_seq=True) + q = _apply_rotary_pos_emb_thd( + q, + packed_seq_params.cu_seqlens_q, + freqs, + rotary_interleaved=False, + mscale=1.0, + cp_group=self.ps.cp_group, + ) + k = _apply_rotary_pos_emb_thd( + k, + packed_seq_params.cu_seqlens_kv, + freqs, + rotary_interleaved=False, + mscale=1.0, + cp_group=self.ps.cp_group, + ) + else: + # q is CP-zigzag pre-sliced; rotary needs FULL seq len, + # its internal get_pos_emb_on_this_cp_rank re-slices to local len. + local_seq_len = q.size(0) + seq_len_for_rope = local_seq_len * self.ps.cp_size + freqs = self.rotary(seq_len_for_rope) + q = _apply_rotary_pos_emb_bshd(q, freqs, rotary_interleaved=False, mscale=1.0) + k = _apply_rotary_pos_emb_bshd(k, freqs, rotary_interleaved=False, mscale=1.0) + if self._use_fp32_rope: + q, k = q.to(orig_dtype), k.to(orig_dtype) + + if is_thd: + psp_kwargs = { + k: getattr(packed_seq_params, k) + for k in _KEPT_PSP_FIELDS + if getattr(packed_seq_params, k, None) is not None + } + attn_out = self.core_attn( + q, + k, + v, + core_attention_bias_type="no_bias", + attn_mask_type="padding_causal", + **psp_kwargs, + ) + attn_out = attn_out.reshape(attn_out.size(0), 1, -1) + else: + attn_out = self.core_attn(q, k, v, core_attention_bias_type="no_bias") + if attn_out.dim() > x.dim(): + shape = attn_out.shape + attn_out = attn_out.reshape(*shape[:-2], self.num_heads_local * self.head_dim) + + if gate is not None: + gate_fp32 = gate.reshape(attn_out.shape).float().sigmoid() + attn_out = (attn_out.float() * gate_fp32).to(attn_out.dtype) + output = self.proj(attn_out) + if self.proj_lora is not None: + output = output + self.proj_lora(attn_out) + return output + + def _qkv_lora_input(self, x: torch.Tensor) -> torch.Tensor: + linear = self.qkv.linear + if not hasattr(linear, "layer_norm_weight"): + return x + weight = linear.layer_norm_weight + if self._qkv_zero_centered_gamma: + weight = weight + 1 + variance = x.float().pow(2).mean(dim=-1, keepdim=True) + x_norm = x.float() * torch.rsqrt(variance + self._qkv_eps) + return (x_norm * weight.float()).to(x.dtype) + + def _split_qkv(self, qkv: torch.Tensor): + nq, nkv, hd = self.num_heads_local, self.num_kv_heads_local, self.head_dim + lead = qkv.shape[:-1] + if self._qkv_layout == "mcore": + q_per_group = ensure_divisible(nq, nkv) + if self._output_gate: + return split_grouped_qkvg(qkv, num_heads=nq, num_kv_heads=nkv, head_dim=hd) + + qkv = qkv.view(*lead, nkv, (q_per_group + 2) * hd) + q = qkv[..., : q_per_group * hd].reshape(*lead, nq, hd) + k = qkv[..., q_per_group * hd : (q_per_group + 1) * hd] + v = qkv[..., (q_per_group + 1) * hd : (q_per_group + 2) * hd] + return q, None, k, v + + if self._output_gate: + q_block = qkv[..., : nq * 2 * hd].reshape(*lead, nq, 2 * hd) + kv_block = qkv[..., nq * 2 * hd :].reshape(*lead, 2 * nkv, hd) + q = q_block[..., :hd] + gate = q_block[..., hd:] + k = kv_block[..., :nkv, :] + v = kv_block[..., nkv:, :] + return q, gate, k, v + qkv = qkv.view(*lead, nq + 2 * nkv, hd) + # Match MCore SelfAttention's split path: keep q/k/v as views instead + # of inserting copy nodes, since those copy nodes alter backward + # accumulation at the qkv boundary under TP/CP. + q = qkv[..., :nq, :] + k = qkv[..., nq : nq + nkv, :] + v = qkv[..., nq + nkv :, :] + return q, None, k, v diff --git a/experimental/lite/megatron/lite/primitive/modules/gqa_utils.py b/experimental/lite/megatron/lite/primitive/modules/gqa_utils.py new file mode 100644 index 00000000000..2ef329b2c20 --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/modules/gqa_utils.py @@ -0,0 +1,29 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Pure grouped-query attention helpers.""" + +from __future__ import annotations + +import torch + +from megatron.lite.primitive.utils import ensure_divisible + + +def split_grouped_qkvg( + qkv: torch.Tensor, *, num_heads: int, num_kv_heads: int, head_dim: int +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + lead = qkv.shape[:-1] + q_heads_per_group = ensure_divisible(num_heads, num_kv_heads) + group_width = (2 * q_heads_per_group + 2) * head_dim + grouped = qkv.reshape(*lead, num_kv_heads, group_width) + query, gate, key, value = grouped.split( + [q_heads_per_group * head_dim, q_heads_per_group * head_dim, head_dim, head_dim], dim=-1 + ) + return ( + query.reshape(*lead, num_heads, head_dim), + gate.reshape(*lead, num_heads, head_dim), + key.reshape(*lead, num_kv_heads, head_dim), + value.reshape(*lead, num_kv_heads, head_dim), + ) + + +__all__ = ["split_grouped_qkvg"] diff --git a/experimental/lite/megatron/lite/primitive/modules/lora.py b/experimental/lite/megatron/lite/primitive/modules/lora.py new file mode 100644 index 00000000000..943e5066eb5 --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/modules/lora.py @@ -0,0 +1,556 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""LoRA helpers for Megatron Lite native model implementations. + +This module is intentionally narrow: it supports the Qwen3-MoE lite path's +Megatron-style sharded linear surfaces, not arbitrary PEFT injection. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +import torch +import torch.distributed as dist +import torch.nn as nn +import torch.nn.functional as F + +_DEFAULT_TARGET_MODULES = ("linear_qkv", "linear_proj", "linear_fc1", "linear_fc2") +_TARGET_ALIASES = { + "qkv": "linear_qkv", + "proj": "linear_proj", + "fc1": "linear_fc1", + "fc2": "linear_fc2", +} + + +@dataclass(frozen=True) +class LoraConfig: + rank: int = 0 + alpha: int | None = None + dropout: float = 0.0 + target_modules: tuple[str, ...] = field(default_factory=lambda: _DEFAULT_TARGET_MODULES) + + @property + def enabled(self) -> bool: + return self.rank > 0 + + @property + def scale(self) -> float: + return float(self.rank if self.alpha is None else self.alpha) / float(self.rank) + + def targets(self) -> set[str]: + out = set() + for target in self.target_modules: + out.add(_TARGET_ALIASES.get(target, target)) + return out + + def targets_module(self, name: str) -> bool: + canonical = _TARGET_ALIASES.get(name, name) + return canonical in self.targets() + + +def normalize_lora_config(config: LoraConfig | dict[str, Any] | None) -> LoraConfig: + if config is None: + return LoraConfig() + if isinstance(config, LoraConfig): + return config + if not isinstance(config, dict): + raise TypeError(f"LoRA config must be LoraConfig, dict, or None, got {type(config)!r}.") + values = dict(config) + enabled = values.pop("enabled", None) + if enabled is False: + values["rank"] = 0 + if "targets" in values and "target_modules" not in values: + values["target_modules"] = values.pop("targets") + else: + values.pop("targets", None) + if "target_modules" in values and not isinstance(values["target_modules"], tuple): + values["target_modules"] = tuple(values["target_modules"]) + return LoraConfig(**values) + + +def freeze_non_lora_params(model: nn.Module) -> dict[str, int]: + """Freeze base parameters and leave adapter parameters trainable.""" + + lora_tensors = 0 + lora_numel = 0 + frozen_tensors = 0 + frozen_numel = 0 + for name, param in model.named_parameters(): + if "lora" in name.lower() or "adapter" in name.lower(): + param.requires_grad_(True) + lora_tensors += 1 + lora_numel += param.numel() + else: + param.requires_grad_(False) + frozen_tensors += 1 + frozen_numel += param.numel() + return { + "lora_tensors": lora_tensors, + "lora_numel": lora_numel, + "frozen_tensors": frozen_tensors, + "frozen_numel": frozen_numel, + } + + +def trainable_param_stats(model: nn.Module) -> dict[str, int]: + tensors = 0 + numel = 0 + for param in model.parameters(): + if param.requires_grad: + tensors += 1 + numel += param.numel() + return {"trainable_tensors": tensors, "trainable_numel": numel} + + +def _gather_sequence_parallel(x: torch.Tensor, group) -> torch.Tensor: + if group is None or dist.get_world_size(group) == 1: + return x + return _AllGatherSequence.apply(x, group) + + +def _reduce_scatter_sequence_parallel(x: torch.Tensor, group) -> torch.Tensor: + if group is None or dist.get_world_size(group) == 1: + return x + return _ReduceScatterSequence.apply(x, group) + + +def _scatter_sequence_parallel(x: torch.Tensor, group, group_rank: int) -> torch.Tensor: + if group is None or dist.get_world_size(group) == 1: + return x + return _ScatterSequence.apply(x, group, group_rank) + + +def _all_reduce_sum(x: torch.Tensor, group) -> torch.Tensor: + if group is None or dist.get_world_size(group) == 1: + return x + return _AllReduceSum.apply(x, group) + + +class _AllGatherSequence(torch.autograd.Function): + @staticmethod + def forward(ctx, x: torch.Tensor, group) -> torch.Tensor: + world_size = dist.get_world_size(group) + ctx.group = group + ctx.local_seq = x.shape[0] + out = torch.empty((x.shape[0] * world_size, *x.shape[1:]), dtype=x.dtype, device=x.device) + dist.all_gather_into_tensor(out, x.contiguous(), group=group) + return out + + @staticmethod + def backward(ctx, grad: torch.Tensor): + out = torch.empty((ctx.local_seq, *grad.shape[1:]), dtype=grad.dtype, device=grad.device) + dist.reduce_scatter_tensor(out, grad.contiguous(), group=ctx.group) + return out, None + + +class _ReduceScatterSequence(torch.autograd.Function): + @staticmethod + def forward(ctx, x: torch.Tensor, group) -> torch.Tensor: + world_size = dist.get_world_size(group) + if x.shape[0] % world_size != 0: + raise ValueError( + f"Cannot reduce-scatter sequence dim {x.shape[0]} over TP={world_size}." + ) + ctx.group = group + ctx.world_size = world_size + out = torch.empty((x.shape[0] // world_size, *x.shape[1:]), dtype=x.dtype, device=x.device) + dist.reduce_scatter_tensor(out, x.contiguous(), group=group) + return out + + @staticmethod + def backward(ctx, grad: torch.Tensor): + out = torch.empty( + (grad.shape[0] * ctx.world_size, *grad.shape[1:]), dtype=grad.dtype, device=grad.device + ) + dist.all_gather_into_tensor(out, grad.contiguous(), group=ctx.group) + return out, None + + +class _ScatterSequence(torch.autograd.Function): + @staticmethod + def forward(ctx, x: torch.Tensor, group, group_rank: int) -> torch.Tensor: + world_size = dist.get_world_size(group) + if x.shape[0] % world_size != 0: + raise ValueError(f"Cannot scatter sequence dim {x.shape[0]} over TP={world_size}.") + ctx.group = group + ctx.world_size = world_size + local_seq = x.shape[0] // world_size + start = int(group_rank) * local_seq + return x[start : start + local_seq].contiguous() + + @staticmethod + def backward(ctx, grad: torch.Tensor): + out = torch.empty( + (grad.shape[0] * ctx.world_size, *grad.shape[1:]), dtype=grad.dtype, device=grad.device + ) + dist.all_gather_into_tensor(out, grad.contiguous(), group=ctx.group) + return out, None, None + + +class _AllReduceSum(torch.autograd.Function): + @staticmethod + def forward(ctx, x: torch.Tensor, group) -> torch.Tensor: + ctx.group = group + out = x.contiguous() + dist.all_reduce(out, op=dist.ReduceOp.SUM, group=group) + return out + + @staticmethod + def backward(ctx, grad: torch.Tensor): + out = grad.contiguous() + dist.all_reduce(out, op=dist.ReduceOp.SUM, group=ctx.group) + return out, None + + +def _all_gather_last_dim(x: torch.Tensor, group, *, reduce_backward: bool = False) -> torch.Tensor: + if group is None or dist.get_world_size(group) == 1: + return x + return _AllGatherLastDim.apply(x, group, reduce_backward) + + +class _AllGatherLastDim(torch.autograd.Function): + """All-gather last dim with Megatron tensor-parallel split backward.""" + + @staticmethod + def forward(ctx, x: torch.Tensor, group, reduce_backward: bool) -> torch.Tensor: + world_size = dist.get_world_size(group) + ctx.group = group + ctx.local_width = x.shape[-1] + ctx.group_rank = dist.get_rank(group) + ctx.reduce_backward = bool(reduce_backward) + flat = x.movedim(-1, 0).contiguous().view(ctx.local_width, -1) + gathered = torch.empty( + (ctx.local_width * world_size, flat.shape[1]), dtype=x.dtype, device=x.device + ) + dist.all_gather_into_tensor(gathered, flat, group=group) + return ( + gathered.view(ctx.local_width * world_size, *x.shape[:-1]).movedim(0, -1).contiguous() + ) + + @staticmethod + def backward(ctx, grad: torch.Tensor): + flat = grad.movedim(-1, 0).contiguous().view(grad.shape[-1], -1) + start = ctx.group_rank * ctx.local_width + out = flat.narrow(0, start, ctx.local_width).contiguous() + if ctx.reduce_backward: + dist.all_reduce(out, op=dist.ReduceOp.SUM, group=ctx.group) + return out.view(ctx.local_width, *grad.shape[:-1]).movedim(0, -1).contiguous(), None, None + + +class _SequenceParallelRankPartitionedLoRA(torch.autograd.Function): + """QKV LoRA path that recomputes gathered activations in backward. + + The ordinary composition of all-gather + matmul saves the full + sequence-parallel gathered input for every layer. For QKV LoRA that input + is much larger than the low-rank hidden activation. This function saves + only the local input plus LoRA weights, then repeats the small gather/matmul + sequence during backward. + """ + + @staticmethod + def forward( + ctx, x: torch.Tensor, lora_a: torch.Tensor, lora_b: torch.Tensor, scale: float, group + ): + world_size = dist.get_world_size(group) if group is not None else 1 + if world_size > 1: + gathered = _all_gather_sequence_forward(x, group, world_size) + else: + gathered = x + hidden_local = gathered.matmul(lora_a.t()) + hidden = _all_gather_last_dim_forward(hidden_local, group, world_size) + out = hidden.matmul(lora_b.t()) * scale + ctx.save_for_backward(x, lora_a, lora_b) + ctx.group = group + ctx.world_size = world_size + ctx.local_seq = x.shape[0] + ctx.local_rank_width = hidden_local.shape[-1] + ctx.scale = float(scale) + return out + + @staticmethod + def backward(ctx, grad_out: torch.Tensor): + x, lora_a, lora_b = ctx.saved_tensors + world_size = ctx.world_size + group = ctx.group + if world_size > 1: + gathered = _all_gather_sequence_forward(x, group, world_size) + else: + gathered = x + hidden_local = gathered.matmul(lora_a.t()) + hidden = _all_gather_last_dim_forward(hidden_local, group, world_size) + + grad_out_scaled = grad_out * ctx.scale + grad_b = ( + grad_out_scaled.reshape(-1, grad_out_scaled.shape[-1]) + .t() + .matmul(hidden.reshape(-1, hidden.shape[-1])) + ) + grad_hidden = grad_out_scaled.matmul(lora_b) + if world_size > 1: + grad_hidden_local = _split_last_dim( + grad_hidden, dist.get_rank(group), ctx.local_rank_width + ) + dist.all_reduce(grad_hidden_local, op=dist.ReduceOp.SUM, group=group) + else: + grad_hidden_local = grad_hidden + grad_a = ( + grad_hidden_local.reshape(-1, grad_hidden_local.shape[-1]) + .t() + .matmul(gathered.reshape(-1, gathered.shape[-1])) + ) + grad_gathered = grad_hidden_local.matmul(lora_a) + if world_size > 1: + grad_x = _reduce_scatter_sequence_forward(grad_gathered, group, ctx.local_seq) + else: + grad_x = grad_gathered + return grad_x, grad_a, grad_b, None, None + + +def _all_gather_sequence_forward(x: torch.Tensor, group, world_size: int) -> torch.Tensor: + out = torch.empty((x.shape[0] * world_size, *x.shape[1:]), dtype=x.dtype, device=x.device) + dist.all_gather_into_tensor(out, x.contiguous(), group=group) + return out + + +def _reduce_scatter_sequence_forward(x: torch.Tensor, group, local_seq: int) -> torch.Tensor: + out = torch.empty((local_seq, *x.shape[1:]), dtype=x.dtype, device=x.device) + dist.reduce_scatter_tensor(out, x.contiguous(), group=group) + return out + + +def _all_gather_last_dim_forward(x: torch.Tensor, group, world_size: int) -> torch.Tensor: + if world_size == 1: + return x + local_width = x.shape[-1] + flat = x.movedim(-1, 0).contiguous().view(local_width, -1) + gathered = torch.empty( + (local_width * world_size, flat.shape[1]), dtype=x.dtype, device=x.device + ) + dist.all_gather_into_tensor(gathered, flat, group=group) + return gathered.view(local_width * world_size, *x.shape[:-1]).movedim(0, -1).contiguous() + + +def _split_last_dim(x: torch.Tensor, group_rank: int, local_width: int) -> torch.Tensor: + start = int(group_rank) * local_width + return x.narrow(-1, start, local_width).contiguous() + + +class LinearLoRA(nn.Module): + """Low-rank delta for a sharded linear layer. + + `a` is replicated unless the caller feeds a row-parallel local input. `b` + has the local output shard for column-parallel surfaces, and the replicated + full output for row-parallel surfaces. + """ + + def __init__( + self, + in_features: int, + out_features: int, + rank: int, + *, + alpha: int | None = None, + dropout: float = 0.0, + sequence_parallel_input: bool = False, + row_parallel_output: bool = False, + sequence_parallel_scatter_output: bool = False, + tp_group=None, + tp_rank: int = 0, + rank_partition_size: int | None = None, + rank_partitioned_a: bool = False, + input_parallel_reduce: bool = False, + output_partition_size: int | None = None, + output_partitioned_b: bool = False, + a_tensor_model_parallel: bool = False, + b_tensor_model_parallel: bool = False, + ): + super().__init__() + if rank <= 0: + raise ValueError("LoRA rank must be positive for LinearLoRA.") + self.rank = int(rank) + self.rank_partitioned_a = bool(rank_partitioned_a) + if self.rank_partitioned_a: + partition_size = ( + int(rank_partition_size) + if rank_partition_size is not None + else (dist.get_world_size(tp_group) if tp_group is not None else 1) + ) + if partition_size <= 0: + raise ValueError("LoRA rank partition size must be positive.") + if self.rank % partition_size != 0: + raise ValueError( + f"LoRA rank {self.rank} must be divisible by rank partition size {partition_size}." + ) + self.rank_partition_size = partition_size + self.local_rank = self.rank // partition_size + else: + self.rank_partition_size = 1 + self.local_rank = self.rank + self.scale = float(rank if alpha is None else alpha) / float(rank) + self.dropout_p = float(dropout) + self.sequence_parallel_input = bool(sequence_parallel_input) + self.row_parallel_output = bool(row_parallel_output) + self.sequence_parallel_scatter_output = bool(sequence_parallel_scatter_output) + if self.row_parallel_output and self.sequence_parallel_scatter_output: + raise ValueError( + "Use either row_parallel_output or sequence_parallel_scatter_output, not both." + ) + self.tp_group = tp_group + self.tp_rank = int(tp_rank) + self.input_parallel_reduce = bool(input_parallel_reduce) + self.output_partitioned_b = bool(output_partitioned_b) + if self.output_partitioned_b: + partition_size = ( + int(output_partition_size) + if output_partition_size is not None + else (dist.get_world_size(tp_group) if tp_group is not None else 1) + ) + if partition_size <= 0: + raise ValueError("LoRA output partition size must be positive.") + if out_features % partition_size != 0: + raise ValueError( + f"LoRA output features {out_features} must be divisible by {partition_size}." + ) + self.output_partition_size = partition_size + self.local_out_features = out_features // partition_size + else: + self.output_partition_size = 1 + self.local_out_features = out_features + self.lora_a = nn.Parameter(torch.empty(self.local_rank, in_features)) + self.lora_b = nn.Parameter(torch.empty(self.local_out_features, rank)) + self.lora_a.tensor_model_parallel = bool(a_tensor_model_parallel) + self.lora_b.tensor_model_parallel = bool(b_tensor_model_parallel) + nn.init.kaiming_uniform_(self.lora_a, a=5**0.5) + nn.init.zeros_(self.lora_b) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + if self.sequence_parallel_input and self.rank_partitioned_a and not self.training: + # Keep eval/inference on the simple path; the memory optimization + # matters only when autograd needs to retain forward activations. + pass + elif ( + self.sequence_parallel_input + and self.rank_partitioned_a + and not self.input_parallel_reduce + and not self.output_partitioned_b + and not self.row_parallel_output + and not self.sequence_parallel_scatter_output + and self.dropout_p == 0.0 + ): + return _SequenceParallelRankPartitionedLoRA.apply( + x, self.lora_a, self.lora_b, self.scale, self.tp_group + ) + if self.sequence_parallel_input: + x = _gather_sequence_parallel(x, self.tp_group) + dropped = F.dropout(x, p=self.dropout_p, training=self.training) if self.dropout_p else x + hidden = dropped.matmul(self.lora_a.t()) + if self.rank_partitioned_a: + hidden = _all_gather_last_dim(hidden, self.tp_group, reduce_backward=True) + if self.input_parallel_reduce: + hidden = _all_reduce_sum(hidden, self.tp_group) + out = hidden.matmul(self.lora_b.t()) * self.scale + if self.output_partitioned_b: + out = _all_gather_last_dim(out, self.tp_group) + if self.row_parallel_output: + out = _reduce_scatter_sequence_parallel(out, self.tp_group) + if self.sequence_parallel_scatter_output: + out = _scatter_sequence_parallel(out, self.tp_group, self.tp_rank) + return out + + +class GroupedLinearLoRA(nn.Module): + """Per-local-expert LoRA delta for `te.GroupedLinear` expert surfaces.""" + + def __init__( + self, + num_local_experts: int, + in_features: int, + out_features: int, + rank: int, + *, + alpha: int | None = None, + dropout: float = 0.0, + ): + super().__init__() + if rank <= 0: + raise ValueError("LoRA rank must be positive for GroupedLinearLoRA.") + self.num_local_experts = int(num_local_experts) + self.rank = int(rank) + self.scale = float(rank if alpha is None else alpha) / float(rank) + self.dropout_p = float(dropout) + self.lora_a = nn.Parameter(torch.empty(num_local_experts, rank, in_features)) + self.lora_b = nn.Parameter(torch.empty(num_local_experts, out_features, rank)) + nn.init.kaiming_uniform_(self.lora_a, a=5**0.5) + nn.init.zeros_(self.lora_b) + + def forward(self, x: torch.Tensor, splits: list[int]) -> torch.Tensor: + if len(splits) != self.num_local_experts: + raise ValueError( + f"GroupedLinearLoRA expected {self.num_local_experts} splits, got {len(splits)}." + ) + outputs = [] + offset = 0 + for expert_idx, size in enumerate(splits): + x_i = x[offset : offset + size] + if size == 0: + outputs.append(x_i.new_empty((0, self.lora_b.shape[1]))) + else: + dropped = ( + F.dropout(x_i, p=self.dropout_p, training=self.training) + if self.dropout_p + else x_i + ) + h_i = dropped.matmul(self.lora_a[expert_idx].t()) + outputs.append(h_i.matmul(self.lora_b[expert_idx].t()) * self.scale) + offset += size + return torch.cat(outputs, dim=0) if outputs else x.new_empty((0, self.lora_b.shape[1])) + + +class SharedGroupedLinearLoRA(nn.Module): + """LoRA delta shared by all local experts in a GroupedLinear.""" + + def __init__( + self, + num_local_experts: int, + in_features: int, + out_features: int, + rank: int, + *, + alpha: int | None = None, + dropout: float = 0.0, + ): + super().__init__() + if rank <= 0: + raise ValueError("LoRA rank must be positive for SharedGroupedLinearLoRA.") + self.num_local_experts = int(num_local_experts) + self.rank = int(rank) + self.scale = float(rank if alpha is None else alpha) / float(rank) + self.dropout_p = float(dropout) + self.shared_across_experts = True + self.lora_a = nn.Parameter(torch.empty(rank, in_features)) + self.lora_b = nn.Parameter(torch.empty(out_features, rank)) + self.lora_a.tensor_model_parallel = False + self.lora_b.tensor_model_parallel = False + nn.init.kaiming_uniform_(self.lora_a, a=5**0.5) + nn.init.zeros_(self.lora_b) + + def forward(self, x: torch.Tensor, splits: list[int]) -> torch.Tensor: + if len(splits) != self.num_local_experts: + raise ValueError( + f"SharedGroupedLinearLoRA expected {self.num_local_experts} splits, got {len(splits)}." + ) + dropped = F.dropout(x, p=self.dropout_p, training=self.training) if self.dropout_p else x + return dropped.matmul(self.lora_a.t()).matmul(self.lora_b.t()) * self.scale + + +__all__ = [ + "GroupedLinearLoRA", + "LinearLoRA", + "LoraConfig", + "SharedGroupedLinearLoRA", + "freeze_non_lora_params", + "normalize_lora_config", + "trainable_param_stats", +] diff --git a/experimental/lite/megatron/lite/primitive/modules/mlp.py b/experimental/lite/megatron/lite/primitive/modules/mlp.py new file mode 100644 index 00000000000..9d3977d6901 --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/modules/mlp.py @@ -0,0 +1,23 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +import torch +import torch.nn as nn + +from megatron.lite.primitive.modules.experts import swiglu_with_probs + + +class SwiGLUMLP(nn.Module): + def __init__( + self, + hidden_size: int, + intermediate_size: int, + *, + swiglu_limit: float = 0.0, + ): + super().__init__() + self.gate_up = nn.Linear(hidden_size, 2 * intermediate_size, bias=False) + self.down = nn.Linear(intermediate_size, hidden_size, bias=False) + self.swiglu_limit = float(swiglu_limit or 0.0) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + y = swiglu_with_probs(self.gate_up(x), None, self.swiglu_limit) + return self.down(y.to(dtype=x.dtype)) diff --git a/experimental/lite/megatron/lite/primitive/modules/moe.py b/experimental/lite/megatron/lite/primitive/modules/moe.py new file mode 100644 index 00000000000..f80b328dbdb --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/modules/moe.py @@ -0,0 +1,77 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Shared MoE utilities: _AllToAll and MoEAuxLossAutoScaler. + +Extracted from models/*/moe.py (Level 0 Option C — pure extraction, no behavior change). +All three models (qwen3_moe, qwen3_5, deepseek_v3) had identical _AllToAll +implementations and functionally identical MoEAuxLossAutoScaler implementations. +The qwen3_moe version is used as the canonical form (adds docstring and named +intermediate variable for clarity). + +Note: this is megatron.lite's own MoEAuxLossAutoScaler. The native MLite +runtime calls `set_loss_scale` on this class to apply the +1/num_microbatches aux-loss gradient scale. +""" + +from __future__ import annotations + +import torch # pyright: ignore[reportMissingImports] +import torch.distributed as dist # pyright: ignore[reportMissingImports] + +__all__ = ["MoEAuxLossAutoScaler", "_AllToAll"] + + +class MoEAuxLossAutoScaler(torch.autograd.Function): + """Piggyback aux_loss onto autograd so main_loss.backward() triggers it.""" + + main_loss_backward_scale: torch.Tensor | None = None + + @staticmethod + def forward(ctx, output: torch.Tensor, aux_loss: torch.Tensor) -> torch.Tensor: + ctx.save_for_backward(aux_loss) + return output + + @staticmethod + def backward(ctx, grad_output: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + (aux_loss,) = ctx.saved_tensors + scale = ( + MoEAuxLossAutoScaler.main_loss_backward_scale.to(aux_loss.device) + if MoEAuxLossAutoScaler.main_loss_backward_scale is not None + else torch.ones(1, device=aux_loss.device) + ) + scaled_aux_loss_grad = torch.ones_like(aux_loss) * scale + return grad_output, scaled_aux_loss_grad + + @staticmethod + def set_loss_scale(scale: torch.Tensor) -> None: + MoEAuxLossAutoScaler.main_loss_backward_scale = scale + + +class _AllToAll(torch.autograd.Function): + @staticmethod + def forward(ctx, input_tensor, input_splits, output_splits, group): + ctx.input_splits = input_splits + ctx.output_splits = output_splits + ctx.group = group + input_tensor = input_tensor.contiguous() + output = input_tensor.new_empty([sum(output_splits)] + list(input_tensor.shape[1:])) + dist.all_to_all_single( + output, + input_tensor, + output_split_sizes=output_splits, + input_split_sizes=input_splits, + group=group, + ) + return output + + @staticmethod + def backward(ctx, grad_output): + grad_output = grad_output.contiguous() + grad_input = grad_output.new_empty([sum(ctx.input_splits)] + list(grad_output.shape[1:])) + dist.all_to_all_single( + grad_input, + grad_output, + output_split_sizes=ctx.input_splits, + input_split_sizes=ctx.output_splits, + group=ctx.group, + ) + return grad_input, None, None, None diff --git a/experimental/lite/megatron/lite/primitive/modules/mrope.py b/experimental/lite/megatron/lite/primitive/modules/mrope.py new file mode 100644 index 00000000000..0efde0006b2 --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/modules/mrope.py @@ -0,0 +1,53 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Multimodal rotary embedding primitive.""" + +from __future__ import annotations + +import torch +import torch.distributed as dist +import torch.nn as nn + +from megatron.lite.primitive.utils.rope import get_pos_emb_on_this_cp_rank + +__all__ = ["MultimodalRotaryEmbedding"] + + +class MultimodalRotaryEmbedding(nn.Module): + """Qwen-style multimodal RoPE embedding with optional CP slicing.""" + + def __init__( + self, + *, + kv_channels: int, + rotary_percent: float, + rotary_base: float, + cp_group: dist.ProcessGroup | None, + ): + super().__init__() + dim = int(kv_channels * rotary_percent) + inv_freq = 1.0 / (rotary_base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim)) + self.register_buffer("inv_freq", inv_freq, persistent=False) + self.cp_group = cp_group + + @staticmethod + def _apply_interleaved_mrope(freqs: torch.Tensor, mrope_section: list[int]) -> torch.Tensor: + freqs_t = freqs[0].clone() + for dim, offset in enumerate((1, 2), start=1): + length = mrope_section[dim] * 3 + freqs_t[..., offset:length:3] = freqs[dim, ..., offset:length:3] + return freqs_t + + def forward( + self, position_ids: torch.Tensor, mrope_section: list[int], *, packed_seq: bool = False + ) -> torch.Tensor: + seq = position_ids.to(device=self.inv_freq.device) + inv_freq = self.inv_freq.float() + inv = inv_freq[None, None, :, None].expand(3, seq.shape[1], -1, 1) + seq_expanded = seq[:, :, None, :].float() + freqs = (inv @ seq_expanded).transpose(2, 3) + freqs = self._apply_interleaved_mrope(freqs, mrope_section) + emb = torch.cat((freqs, freqs), dim=-1) + emb = emb[..., None, :].transpose(0, 1).contiguous() + if not packed_seq and self.cp_group is not None and dist.get_world_size(self.cp_group) > 1: + emb = get_pos_emb_on_this_cp_rank(emb, 0, self.cp_group) + return emb diff --git a/experimental/lite/megatron/lite/primitive/modules/mtp.py b/experimental/lite/megatron/lite/primitive/modules/mtp.py new file mode 100644 index 00000000000..919f027849b --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/modules/mtp.py @@ -0,0 +1,148 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Multi-token prediction primitives.""" + +from __future__ import annotations + +from collections.abc import Callable + +import torch +import torch.nn as nn +import transformer_engine.pytorch as te + +from megatron.lite.primitive.parallel import ( + ParallelState, + VanillaColumnParallelLinear, + VocabParallelEmbedding, + roll_packed_thd_left, + scatter_to_sequence_parallel, +) + +__all__ = ["MTPBlock", "MTPDecoderLayer", "MTPLossAutoScaler", "roll_mtp_tensor_left"] + + +def roll_mtp_tensor_left( + tensor: torch.Tensor, *, packed_seq_params=None, dims: int = -1 +) -> tuple[torch.Tensor, torch.Tensor]: + """Roll MTP inputs/labels one token left for dense or packed THD batches.""" + if packed_seq_params is not None: + return roll_packed_thd_left(tensor, packed_seq_params=packed_seq_params, dims=dims) + + dim = dims if dims >= 0 else tensor.dim() + dims + rolled = torch.roll(tensor, shifts=-1, dims=dim) + index = [slice(None)] * tensor.dim() + index[dim] = slice(-1, None) + rolled[tuple(index)] = 0 + return rolled, rolled.sum() + + +class MTPLossAutoScaler(torch.autograd.Function): + main_loss_backward_scale: float = 1.0 + + @staticmethod + def forward(ctx, output: torch.Tensor, mtp_loss: torch.Tensor): + ctx.save_for_backward(mtp_loss) + return output + + @staticmethod + def backward(ctx, grad_output: torch.Tensor): + (mtp_loss,) = ctx.saved_tensors + scaled_mtp_grad = torch.ones_like(mtp_loss) * MTPLossAutoScaler.main_loss_backward_scale + return grad_output, scaled_mtp_grad + + @staticmethod + def set_loss_scale(scale: torch.Tensor | float) -> None: + if isinstance(scale, torch.Tensor): + scale = float(scale.detach().float().item()) + MTPLossAutoScaler.main_loss_backward_scale = float(scale) + + +class MTPDecoderLayer(nn.Module): + def __init__( + self, + *, + hidden_size: int, + rms_norm_eps: float, + ps: ParallelState, + embedding: VocabParallelEmbedding, + transformer_layer: nn.Module, + detach_encoder: bool, + ): + super().__init__() + self.ps = ps + self.embedding = embedding + self.detach_encoder = detach_encoder + self.enorm = te.RMSNorm(hidden_size, eps=rms_norm_eps, zero_centered_gamma=True) + self.hnorm = te.RMSNorm(hidden_size, eps=rms_norm_eps, zero_centered_gamma=True) + self.eh_proj = VanillaColumnParallelLinear( + hidden_size * 2, hidden_size, ps, sp=ps.tp_size > 1, gather_output=True + ) + self.transformer_layer = transformer_layer + self.final_layernorm = te.RMSNorm(hidden_size, eps=rms_norm_eps, zero_centered_gamma=True) + + def forward( + self, + *, + input_ids: torch.Tensor, + position_ids: torch.Tensor | None, + hidden_states: torch.Tensor, + rotary_position_ids: torch.Tensor | None = None, + packed_seq_params=None, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]: + attention_position_ids = ( + rotary_position_ids if rotary_position_ids is not None else position_ids + ) + input_ids, _ = roll_mtp_tensor_left(input_ids, packed_seq_params=packed_seq_params, dims=-1) + if position_ids is not None: + position_ids, _ = roll_mtp_tensor_left( + position_ids, packed_seq_params=packed_seq_params, dims=-1 + ) + decoder_input = scatter_to_sequence_parallel(self.embedding(input_ids), self.ps) + if self.detach_encoder: + decoder_input = decoder_input.detach() + hidden_states = hidden_states.detach() + decoder_input = self.enorm(decoder_input) + hidden_states = self.hnorm(hidden_states) + hidden_states = torch.cat((decoder_input, hidden_states), dim=-1) + hidden_states = scatter_to_sequence_parallel(self.eh_proj(hidden_states), self.ps) + hidden_states = self.transformer_layer( + hidden_states, position_ids=attention_position_ids, packed_seq_params=packed_seq_params + ) + hidden_states = self.final_layernorm(hidden_states) + return hidden_states, input_ids, position_ids + + +class MTPBlock(nn.Module): + def __init__( + self, + *, + num_layers: int, + repeated_layer: bool, + layer_factory: Callable[[int], MTPDecoderLayer], + ): + super().__init__() + self.num_layers = num_layers + self.repeated_layer = repeated_layer + layers_to_build = 1 if repeated_layer else num_layers + self.layers = nn.ModuleList([layer_factory(idx) for idx in range(layers_to_build)]) + + def forward( + self, + *, + input_ids: torch.Tensor, + position_ids: torch.Tensor | None, + hidden_states: torch.Tensor, + packed_seq_params=None, + ) -> list[torch.Tensor]: + outputs: list[torch.Tensor] = [] + rotary_position_ids = position_ids + for depth in range(self.num_layers): + layer = self.layers[0] if self.repeated_layer else self.layers[depth] + hidden_states, input_ids, position_ids = layer( + input_ids=input_ids, + position_ids=position_ids, + hidden_states=hidden_states, + rotary_position_ids=rotary_position_ids, + packed_seq_params=packed_seq_params, + ) + outputs.append(hidden_states) + return outputs diff --git a/experimental/lite/megatron/lite/primitive/modules/router.py b/experimental/lite/megatron/lite/primitive/modules/router.py new file mode 100644 index 00000000000..66ebbb2a7cf --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/modules/router.py @@ -0,0 +1,203 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""MoE router implementations: TopKRouter (softmax) and SigmoidTopKRouter.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import torch # pyright: ignore[reportMissingImports] +import torch.distributed as dist # pyright: ignore[reportMissingImports] +import torch.nn as nn # pyright: ignore[reportMissingImports] + +from megatron.lite.primitive.modules.moe import MoEAuxLossAutoScaler +from megatron.lite.primitive.utils.moe import ( + compute_routing_scores_for_aux_loss, + router_gating_linear, + switch_load_balancing_loss_func, + topk_routing_with_score_function, +) + +if TYPE_CHECKING: + from megatron.lite.primitive.parallel import ParallelState + + +def _ordered_topk_from_routing_map( + probs_dense: torch.Tensor, routing_map: torch.Tensor, topk: int +) -> tuple[torch.Tensor, torch.Tensor]: + expert_ids = torch.arange( + probs_dense.size(-1), device=probs_dense.device, dtype=torch.long + ).expand_as(routing_map) + masked_ids = torch.where( + routing_map, expert_ids, torch.full_like(expert_ids, probs_dense.size(-1)) + ) + topk_indices = torch.sort(masked_ids, dim=-1).values[:, :topk] + topk_scores = torch.gather(probs_dense, dim=-1, index=topk_indices) + return topk_scores, topk_indices + + +class TopKRouter(nn.Module): + """TopK gating with optional high-precision router logits/probabilities.""" + + def __init__( + self, + config, + ps: ParallelState, + *, + router_bias_rate: float = 0.0, + compute_aux_loss: bool = True, + use_pre_softmax: bool = False, + moe_router_fusion: bool = False, + router_dtype: torch.dtype | None = None, + ): + super().__init__() + if router_bias_rate > 0: + raise NotImplementedError( + "expert-bias EMA is not implemented in the primitive router; " + "use load_balancing_type='none' or extend ParallelState." + ) + self.topk = config.num_experts_per_tok + self.num_experts = config.num_experts + self.aux_loss_coeff = config.router_aux_loss_coef + self.router_bias_rate = router_bias_rate + self.compute_aux_loss = compute_aux_loss + self.use_pre_softmax = use_pre_softmax + self.moe_router_fusion = moe_router_fusion + self.router_dtype = router_dtype + + self.gate = nn.Linear(config.hidden_size, config.num_experts, bias=False) + self.register_buffer( + "expert_bias", torch.zeros(config.num_experts, dtype=torch.float32), persistent=False + ) + + self._aux_loss_group = ps.tp_group if ps.tp_size > 1 else None + + def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + router_dtype = self.router_dtype or x.dtype + logits = router_gating_linear(x, self.gate.weight, None, router_dtype) + logits = logits.view(-1, self.num_experts) + num_tokens = logits.size(0) + if self.moe_router_fusion: + probs_dense, _ = topk_routing_with_score_function( + logits, + self.topk, + use_pre_softmax=self.use_pre_softmax, + score_function="softmax", + fused=True, + ) + topk_scores, topk_indices = torch.topk(probs_dense, k=self.topk, dim=-1) + else: + probs_dense, routing_map = topk_routing_with_score_function( + logits, + self.topk, + use_pre_softmax=self.use_pre_softmax, + score_function="softmax", + fused=False, + ) + topk_scores, topk_indices = _ordered_topk_from_routing_map( + probs_dense, routing_map, self.topk + ) + if self.router_dtype is None: + topk_scores = topk_scores.to(x.dtype) + + if self.compute_aux_loss and self.training and torch.is_grad_enabled(): + routing_map, aux_scores = compute_routing_scores_for_aux_loss( + logits, self.topk, score_function="softmax", fused=self.moe_router_fusion + ) + tokens_per_expert = routing_map.sum(dim=0).to(torch.int64) + total_num_tokens = num_tokens + if self._aux_loss_group is not None: + dist.all_reduce(tokens_per_expert, group=self._aux_loss_group) + total_num_tokens = num_tokens * dist.get_world_size(group=self._aux_loss_group) + aux_loss = switch_load_balancing_loss_func( + aux_scores, + tokens_per_expert, + total_num_tokens, + self.topk, + self.num_experts, + self.aux_loss_coeff, + fused=False, + ) + topk_scores = MoEAuxLossAutoScaler.apply(topk_scores, aux_loss) + + return topk_scores, topk_indices + + +class SigmoidTopKRouter(nn.Module): + """Sigmoid-family TopK router for DeepSeek-style MoE.""" + + def __init__( + self, + config, + ps: ParallelState, + *, + router_bias_rate: float = 0.0, + compute_aux_loss: bool = True, + use_pre_softmax: bool = False, + moe_router_fusion: bool = False, + ): + super().__init__() + if router_bias_rate > 0: + raise NotImplementedError( + "expert-bias EMA is not implemented in the primitive router; " + "use load_balancing_type='none' or extend ParallelState." + ) + self.topk = config.num_experts_per_tok + self.num_experts = config.n_routed_experts + self.aux_loss_coeff = getattr(config, "aux_loss_alpha", 0.0) + self.scaling_factor = config.routed_scaling_factor + self.score_function = getattr(config, "scoring_func", "sigmoid") + self.router_bias_rate = router_bias_rate + self.compute_aux_loss = compute_aux_loss + self.use_pre_softmax = use_pre_softmax + self.moe_router_fusion = moe_router_fusion + + self.gate = nn.Linear(config.hidden_size, config.n_routed_experts, bias=False) + self.register_buffer( + "expert_bias", + torch.zeros(config.n_routed_experts, dtype=torch.float32), + persistent=False, + ) + + self._aux_loss_group = ps.tp_group if ps.tp_size > 1 else None + + def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + logits = self.gate(x) + logits = logits.view(-1, self.num_experts) + num_tokens = logits.size(0) + probs_dense, routing_map = topk_routing_with_score_function( + logits, + self.topk, + score_function=self.score_function, + expert_bias=self.expert_bias.to(logits.dtype), + scaling_factor=(self.scaling_factor or None), + fused=self.moe_router_fusion, + ) + topk_scores, topk_indices = _ordered_topk_from_routing_map( + probs_dense, routing_map, self.topk + ) + topk_scores = topk_scores.to(logits.dtype) + + if self.compute_aux_loss and self.training and torch.is_grad_enabled(): + _, aux_scores = compute_routing_scores_for_aux_loss( + logits, self.topk, score_function=self.score_function, fused=self.moe_router_fusion + ) + tokens_per_expert = routing_map.sum(dim=0).to(torch.int64) + total_num_tokens = num_tokens + if self._aux_loss_group is not None: + dist.all_reduce(tokens_per_expert, group=self._aux_loss_group) + total_num_tokens = num_tokens * dist.get_world_size(group=self._aux_loss_group) + aux_loss = switch_load_balancing_loss_func( + aux_scores, + tokens_per_expert, + total_num_tokens, + self.topk, + self.num_experts, + self.aux_loss_coeff, + fused=False, + ) + topk_scores = MoEAuxLossAutoScaler.apply(topk_scores, aux_loss) + + return topk_scores, topk_indices + + +__all__ = ["SigmoidTopKRouter", "TopKRouter"] diff --git a/experimental/lite/megatron/lite/primitive/ops/__init__.py b/experimental/lite/megatron/lite/primitive/ops/__init__.py new file mode 100644 index 00000000000..473fa29a7c0 --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/ops/__init__.py @@ -0,0 +1,27 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Runtime math primitives for Megatron Lite.""" + +from megatron.lite.primitive.ops.cross_entropy import vocab_parallel_cross_entropy +from megatron.lite.primitive.ops.gated_delta_rule import l2norm, torch_chunk_gated_delta_rule +from megatron.lite.primitive.ops.logprob import ( + vocab_parallel_entropy, + vocab_parallel_log_probs_from_logits, +) +from megatron.lite.primitive.ops.sp_ops import ( + AllGatherDim0, + AllGatherDim0ForNonSPConsumer, + ReduceScatterDim0, + ScatterToSP, +) + +__all__ = [ + "AllGatherDim0", + "AllGatherDim0ForNonSPConsumer", + "ReduceScatterDim0", + "ScatterToSP", + "l2norm", + "torch_chunk_gated_delta_rule", + "vocab_parallel_cross_entropy", + "vocab_parallel_entropy", + "vocab_parallel_log_probs_from_logits", +] diff --git a/experimental/lite/megatron/lite/primitive/ops/cross_entropy.py b/experimental/lite/megatron/lite/primitive/ops/cross_entropy.py new file mode 100644 index 00000000000..9bdc40d3867 --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/ops/cross_entropy.py @@ -0,0 +1,113 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Vocab-parallel cross entropy loss (copied from Megatron-Core). + +Computes cross entropy when logits are split across TP ranks. +With TP=1 the all-reduce calls are no-ops and this degenerates to a +standard cross-entropy with in-place ops and a memory-efficient custom backward. +""" + +from __future__ import annotations + +import torch # pyright: ignore[reportMissingImports] +import torch.distributed as dist # pyright: ignore[reportMissingImports] + + +def _vocab_range(partition_vocab_size: int, rank: int, world_size: int): + start = rank * partition_vocab_size + return start, start + partition_vocab_size + + +class _VocabParallelCrossEntropy(torch.autograd.Function): + + @staticmethod + def forward(ctx, vocab_parallel_logits, target, tp_group): + # Cast to float32 and compute max for numerical stability. + vocab_parallel_logits = vocab_parallel_logits.float() + logits_max = torch.max(vocab_parallel_logits, dim=-1)[0] + + if tp_group is not None and dist.get_world_size(tp_group) > 1: + dist.all_reduce(logits_max, op=dist.ReduceOp.MAX, group=tp_group) + + # In-place subtract max. + vocab_parallel_logits -= logits_max.unsqueeze(dim=-1) + + # Partition info. + partition_vocab_size = vocab_parallel_logits.size(-1) + if tp_group is not None and dist.get_world_size(tp_group) > 1: + rank = dist.get_rank(tp_group) + world_size = dist.get_world_size(tp_group) + else: + rank = 0 + world_size = 1 + vocab_start_index, vocab_end_index = _vocab_range(partition_vocab_size, rank, world_size) + + # Mask targets outside this partition's vocab range. + target_mask = (target < vocab_start_index) | (target >= vocab_end_index) + masked_target = target.clone() - vocab_start_index + masked_target[target_mask] = 0 + + # Get predicted logits = logits[target]. + logits_2d = vocab_parallel_logits.view(-1, partition_vocab_size) + masked_target_1d = masked_target.view(-1) + arange_1d = torch.arange(logits_2d.size(0), device=logits_2d.device) + predicted_logits_1d = logits_2d[arange_1d, masked_target_1d] + predicted_logits_1d = predicted_logits_1d.clone().contiguous() + predicted_logits = predicted_logits_1d.view_as(target) + predicted_logits[target_mask] = 0.0 + + # Sum of exp(logits). + exp_logits = vocab_parallel_logits + torch.exp(vocab_parallel_logits, out=exp_logits) + sum_exp_logits = exp_logits.sum(dim=-1) + + # All-reduce predicted_logits and sum_exp_logits across TP. + if tp_group is not None and dist.get_world_size(tp_group) > 1: + dist.all_reduce(predicted_logits, op=dist.ReduceOp.SUM, group=tp_group) + dist.all_reduce(sum_exp_logits, op=dist.ReduceOp.SUM, group=tp_group) + + # Loss = log(sum(exp(logits))) - predicted_logit. + loss = torch.log(sum_exp_logits) - predicted_logits + + # Normalize exp_logits to get softmax (reused in backward). + exp_logits.div_(sum_exp_logits.unsqueeze(dim=-1)) + + # Save for backward. + ctx.save_for_backward(exp_logits, target_mask, masked_target_1d) + + return loss + + @staticmethod + def backward(ctx, grad_output): + softmax, target_mask, masked_target_1d = ctx.saved_tensors + + # grad_input = softmax (copy is implicit since softmax is saved). + grad_input = softmax + partition_vocab_size = softmax.size(-1) + grad_2d = grad_input.view(-1, partition_vocab_size) + + arange_1d = torch.arange(grad_2d.size(0), device=grad_2d.device) + softmax_update = 1.0 - target_mask.view(-1).float() + + grad_2d[arange_1d, masked_target_1d] -= softmax_update + + # Scale by upstream gradient. + grad_input.mul_(grad_output.unsqueeze(dim=-1)) + + return grad_input, None, None + + +def vocab_parallel_cross_entropy(vocab_parallel_logits, target, tp_group=None): + """Cross entropy loss for vocab-parallel logits. + + Args: + vocab_parallel_logits: [S, B, V/tp] logits split across TP ranks. + target: [S, B] integer target token ids. + tp_group: TP process group (None or single-rank group → no communication). + + Returns: + Per-token loss tensor of shape [S, B]. + """ + return _VocabParallelCrossEntropy.apply(vocab_parallel_logits, target, tp_group) + + +__all__ = ["vocab_parallel_cross_entropy"] diff --git a/experimental/lite/megatron/lite/primitive/ops/gated_delta_rule.py b/experimental/lite/megatron/lite/primitive/ops/gated_delta_rule.py new file mode 100644 index 00000000000..62c21e441f0 --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/ops/gated_delta_rule.py @@ -0,0 +1,108 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Gated Delta Rule math helpers.""" + +from __future__ import annotations + +import torch +import torch.nn.functional as F + +try: + from fla.modules.l2norm import l2norm as _fla_l2norm # pyright: ignore[reportMissingImports] + + _HAS_FLA_L2NORM = True +except ImportError: + _HAS_FLA_L2NORM = False + +__all__ = ["l2norm", "torch_chunk_gated_delta_rule"] + + +def l2norm(x: torch.Tensor, dim: int = -1, eps: float = 1e-6) -> torch.Tensor: + if _HAS_FLA_L2NORM and dim == -1 and eps == 1e-6: + return _fla_l2norm(x) + return x * torch.rsqrt((x * x).sum(dim=dim, keepdim=True) + eps) + + +def torch_chunk_gated_delta_rule( + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + chunk_size: int = 64, + initial_state: torch.Tensor | None = None, + output_final_state: bool = False, + use_qk_l2norm_in_kernel: bool = False, +) -> tuple[torch.Tensor, torch.Tensor | None]: + """Pure PyTorch Gated Delta Rule fallback for correctness smoke tests.""" + initial_dtype = query.dtype + if use_qk_l2norm_in_kernel: + query = l2norm(query) + key = l2norm(key) + query, key, value, beta, g = [ + x.transpose(1, 2).contiguous().to(torch.float32) for x in (query, key, value, beta, g) + ] + + batch_size, num_heads, sequence_length, key_dim = key.shape + value_dim = value.shape[-1] + pad_size = (chunk_size - sequence_length % chunk_size) % chunk_size + query = F.pad(query, (0, 0, 0, pad_size)) + key = F.pad(key, (0, 0, 0, pad_size)) + value = F.pad(value, (0, 0, 0, pad_size)) + beta = F.pad(beta, (0, pad_size)) + g = F.pad(g, (0, pad_size)) + total_sequence_length = sequence_length + pad_size + scale = 1 / (query.shape[-1] ** 0.5) + query = query * scale + + v_beta = value * beta.unsqueeze(-1) + k_beta = key * beta.unsqueeze(-1) + query, key, value, k_beta, v_beta = [ + x.reshape(x.shape[0], x.shape[1], -1, chunk_size, x.shape[-1]) + for x in (query, key, value, k_beta, v_beta) + ] + g = g.reshape(g.shape[0], g.shape[1], -1, chunk_size) + + mask = torch.triu( + torch.ones(chunk_size, chunk_size, dtype=torch.bool, device=query.device), diagonal=0 + ) + g = g.cumsum(dim=-1) + decay_mask = ((g.unsqueeze(-1) - g.unsqueeze(-2)).tril().exp().float()).tril() + attn = -((k_beta @ key.transpose(-1, -2)) * decay_mask).masked_fill(mask, 0) + for i in range(1, chunk_size): + row = attn[..., i, :i].clone() + sub = attn[..., :i, :i].clone() + attn[..., i, :i] = row + (row.unsqueeze(-1) * sub).sum(-2) + attn = attn + torch.eye(chunk_size, dtype=attn.dtype, device=attn.device) + + value = attn @ v_beta + k_cumdecay = attn @ (k_beta * g.exp().unsqueeze(-1)) + last_recurrent_state = ( + torch.zeros(batch_size, num_heads, key_dim, value_dim).to(value) + if initial_state is None + else initial_state.to(value) + ) + core_attn_out = torch.zeros_like(value) + mask = torch.triu( + torch.ones(chunk_size, chunk_size, dtype=torch.bool, device=query.device), diagonal=1 + ) + + for i in range(0, total_sequence_length // chunk_size): + q_i, k_i, v_i = query[:, :, i], key[:, :, i], value[:, :, i] + attn = (q_i @ k_i.transpose(-1, -2) * decay_mask[:, :, i]).masked_fill_(mask, 0) + v_prime = (k_cumdecay[:, :, i]) @ last_recurrent_state + v_new = v_i - v_prime + attn_inter = (q_i * g[:, :, i, :, None].exp()) @ last_recurrent_state + core_attn_out[:, :, i] = attn_inter + attn @ v_new + last_recurrent_state = ( + last_recurrent_state * g[:, :, i, -1, None, None].exp() + + (k_i * (g[:, :, i, -1, None] - g[:, :, i]).exp()[..., None]).transpose(-1, -2) @ v_new + ) + + if not output_final_state: + last_recurrent_state = None + core_attn_out = core_attn_out.reshape( + core_attn_out.shape[0], core_attn_out.shape[1], -1, core_attn_out.shape[-1] + ) + core_attn_out = core_attn_out[:, :, :sequence_length] + core_attn_out = core_attn_out.transpose(1, 2).contiguous().to(initial_dtype) + return core_attn_out, last_recurrent_state diff --git a/experimental/lite/megatron/lite/primitive/ops/linear_cross_entropy.py b/experimental/lite/megatron/lite/primitive/ops/linear_cross_entropy.py new file mode 100644 index 00000000000..453702dc05e --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/ops/linear_cross_entropy.py @@ -0,0 +1,70 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Linear + vocab-parallel cross entropy helpers. + +The fast path delegates to VERL's Triton-backed fused kernel when available. +The fallback keeps the same contract and is used by unit/smoke tests that do +not have the fused extension on ``PYTHONPATH``. +""" + +from __future__ import annotations + +import torch +import torch.distributed as dist + +from megatron.lite.primitive.ops.cross_entropy import vocab_parallel_cross_entropy + + +def _all_reduce_if_needed(tensor: torch.Tensor, group, op=dist.ReduceOp.SUM) -> torch.Tensor: + if group is not None and dist.is_initialized() and dist.get_world_size(group) > 1: + dist.all_reduce(tensor, op=op, group=group) + return tensor + + +def _vocab_parallel_entropy(logits: torch.Tensor, tp_group=None) -> torch.Tensor: + logits = logits.float() + logits_max = logits.max(dim=-1).values + _all_reduce_if_needed(logits_max, tp_group, op=dist.ReduceOp.MAX) + + shifted = logits - logits_max.unsqueeze(-1) + exp_logits = torch.exp(shifted) + sum_exp = exp_logits.sum(dim=-1) + _all_reduce_if_needed(sum_exp, tp_group) + + weighted_logits = (exp_logits * logits).sum(dim=-1) + _all_reduce_if_needed(weighted_logits, tp_group) + expected_logits = weighted_logits / sum_exp + return torch.log(sum_exp) + logits_max - expected_logits + + +def _reshape_like_labels(values: torch.Tensor, labels: torch.Tensor) -> torch.Tensor: + if values.shape != labels.shape and values.numel() == labels.numel(): + return values.reshape(labels.shape) + return values + + +def linear_cross_entropy( + hidden: torch.Tensor, + weight: torch.Tensor, + labels: torch.Tensor, + temperature: float = 1.0, + tp_group=None, +) -> tuple[torch.Tensor, torch.Tensor]: + """Return token log-probs and entropy without changing VERL's fused API.""" + try: + from verl.utils.kernel.linear_cross_entropy import linear_cross_entropy as _verl_lce + except Exception: + _verl_lce = None + + if _verl_lce is not None and hidden.is_cuda: + log_probs, entropy = _verl_lce(hidden, weight, labels, float(temperature), "none", tp_group) + return _reshape_like_labels(log_probs, labels), _reshape_like_labels(entropy, labels) + + logits = torch.matmul(hidden, weight.t()) + if temperature != 1.0: + logits = logits / float(temperature) + loss = vocab_parallel_cross_entropy(logits.clone(), labels, tp_group) + entropy = _vocab_parallel_entropy(logits, tp_group) + return _reshape_like_labels(-loss, labels), _reshape_like_labels(entropy, labels) + + +__all__ = ["linear_cross_entropy"] diff --git a/experimental/lite/megatron/lite/primitive/ops/logprob.py b/experimental/lite/megatron/lite/primitive/ops/logprob.py new file mode 100644 index 00000000000..c2fc295ef8b --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/ops/logprob.py @@ -0,0 +1,73 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Probability-first primitives for Megatron Lite phase 1.""" + +from __future__ import annotations + +import torch +import torch.distributed as dist + + +def vocab_parallel_log_probs_from_logits( + logits: torch.Tensor, labels: torch.Tensor | None = None +) -> torch.Tensor: + """Compute log-probabilities from already materialized logits. + + Phase 1 assumes logits are already full-vocab tensors on the local rank. + If `labels` are provided, this returns token-selected log-probabilities with + the same shape as `labels` (modulo an internal transpose when logits are + `[S, B, V]` and labels are `[B, S]`). + """ + + log_probs = torch.log_softmax(logits.float(), dim=-1) + if labels is None: + return log_probs + + aligned_labels, transposed = _align_labels_to_logits(logits, labels) + gathered = log_probs.gather(dim=-1, index=aligned_labels.unsqueeze(-1)).squeeze(-1) + return gathered.transpose(0, 1).contiguous() if transposed else gathered + + +def _all_reduce_if_needed(tensor: torch.Tensor, group=None, op=dist.ReduceOp.SUM) -> torch.Tensor: + if group is not None and dist.is_initialized() and dist.get_world_size(group) > 1: + dist.all_reduce(tensor, op=op, group=group) + return tensor + + +def vocab_parallel_entropy(logits: torch.Tensor, tp_group=None) -> torch.Tensor: + """Compute per-token entropy from logits.""" + + logits = logits.float() + logits_max = logits.max(dim=-1).values + _all_reduce_if_needed(logits_max, tp_group, op=dist.ReduceOp.MAX) + + shifted = logits - logits_max.unsqueeze(-1) + exp_logits = torch.exp(shifted) + sum_exp = exp_logits.sum(dim=-1) + _all_reduce_if_needed(sum_exp, tp_group) + + weighted_logits = (exp_logits * logits).sum(dim=-1) + _all_reduce_if_needed(weighted_logits, tp_group) + expected_logits = weighted_logits / sum_exp + return torch.log(sum_exp) + logits_max - expected_logits + + +def _align_labels_to_logits( + logits: torch.Tensor, labels: torch.Tensor +) -> tuple[torch.Tensor, bool]: + if logits.ndim != labels.ndim + 1: + raise ValueError( + f"logits rank must be labels rank + 1, got logits={logits.shape}, labels={labels.shape}." + ) + + if logits.shape[:-1] == labels.shape: + return labels, False + + if ( + logits.ndim == 3 + and labels.ndim == 2 + and logits.shape[0] == labels.shape[1] + and logits.shape[1] == labels.shape[0] + ): + return labels.transpose(0, 1).contiguous(), True + + raise ValueError(f"Could not align labels {labels.shape} with logits {logits.shape}.") diff --git a/experimental/lite/megatron/lite/primitive/ops/sp_ops.py b/experimental/lite/megatron/lite/primitive/ops/sp_ops.py new file mode 100644 index 00000000000..0d52cd0daaf --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/ops/sp_ops.py @@ -0,0 +1,90 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Sequence-parallel autograd primitives for non-TE layers (embedding, scatter/gather). + +AllGather/ReduceScatter operate on the sequence dimension (dim 0) with layout +[S, B, H]. NCCL operates on dim 0 directly — no transpose needed. +TE-based layers use TE's native sequence_parallel=True instead. +""" + +from __future__ import annotations + +import torch # pyright: ignore[reportMissingImports] +import torch.distributed as dist # pyright: ignore[reportMissingImports] + + +def _ag_dim0(x: torch.Tensor, tp_size: int, group: dist.ProcessGroup) -> torch.Tensor: + """AllGather on dim 0: [S/tp, B, H] → [S, B, H].""" + out = torch.empty(tp_size * x.shape[0], x.shape[1], x.shape[2], dtype=x.dtype, device=x.device) + dist.all_gather_into_tensor(out, x.contiguous(), group=group) + return out + + +def _rs_dim0(x: torch.Tensor, local_seq: int, group: dist.ProcessGroup) -> torch.Tensor: + """ReduceScatter on dim 0: [S, B, H] → [S/tp, B, H].""" + out = torch.empty(local_seq, x.shape[1], x.shape[2], dtype=x.dtype, device=x.device) + dist.reduce_scatter_tensor(out, x.contiguous(), group=group) + return out + + +class AllGatherDim0(torch.autograd.Function): + """AllGather [S/tp, B, H] → [S, B, H]. Backward: ReduceScatter.""" + + @staticmethod + def forward(ctx, x, tp_size, tp_rank, group): + ctx.group = group + ctx.local_seq = x.shape[0] + return _ag_dim0(x, tp_size, group) + + @staticmethod + def backward(ctx, grad): + out = _rs_dim0(grad, ctx.local_seq, ctx.group) + return out, None, None, None + + +class ReduceScatterDim0(torch.autograd.Function): + """ReduceScatter [S, B, H] → [S/tp, B, H]. Backward: AllGather.""" + + @staticmethod + def forward(ctx, x, tp_size, tp_rank, group): + ctx.tp_size = tp_size + ctx.group = group + local_seq = x.shape[0] // tp_size + return _rs_dim0(x, local_seq, ctx.group) + + @staticmethod + def backward(ctx, grad): + return _ag_dim0(grad, ctx.tp_size, ctx.group), None, None, None + + +class AllGatherDim0ForNonSPConsumer(torch.autograd.Function): + """AllGather [S/tp, B, H] → [S, B, H]. Backward: Scatter (no reduce).""" + + @staticmethod + def forward(ctx, x, tp_size, tp_rank, group): + ctx.tp_rank = tp_rank + ctx.local_seq = x.shape[0] + return _ag_dim0(x, tp_size, group) + + @staticmethod + def backward(ctx, grad): + start = ctx.tp_rank * ctx.local_seq + return grad[start : start + ctx.local_seq].contiguous(), None, None, None + + +class ScatterToSP(torch.autograd.Function): + """Scatter [S, B, H] → [S/tp, B, H] (no comm). Backward: AllGather.""" + + @staticmethod + def forward(ctx, x, tp_size, tp_rank, group): + ctx.tp_size = tp_size + ctx.group = group + local_seq = x.shape[0] // tp_size + start = tp_rank * local_seq + return x[start : start + local_seq, :, :].contiguous() + + @staticmethod + def backward(ctx, grad): + return _ag_dim0(grad, ctx.tp_size, ctx.group), None, None, None + + +__all__ = ["AllGatherDim0", "AllGatherDim0ForNonSPConsumer", "ReduceScatterDim0", "ScatterToSP"] diff --git a/experimental/lite/megatron/lite/primitive/optimizers/__init__.py b/experimental/lite/megatron/lite/primitive/optimizers/__init__.py new file mode 100644 index 00000000000..669dd8eae9d --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/optimizers/__init__.py @@ -0,0 +1,20 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Optimizer backend registry.""" + +from __future__ import annotations + +import importlib + +BACKENDS = { + "dist_opt": "megatron.lite.primitive.optimizers.megatron_wrap", + "fsdp2": "megatron.lite.primitive.optimizers.fsdp2", +} + + +def get_optimizer_backend(name: str): + if name not in BACKENDS: + raise ValueError(f"Unknown Megatron Lite optimizer backend: {name!r}.") + return importlib.import_module(BACKENDS[name]).BACKEND + + +__all__ = ["get_optimizer_backend"] diff --git a/experimental/lite/megatron/lite/primitive/optimizers/fsdp2/__init__.py b/experimental/lite/megatron/lite/primitive/optimizers/fsdp2/__init__.py new file mode 100644 index 00000000000..b6dd3404d14 --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/optimizers/fsdp2/__init__.py @@ -0,0 +1,54 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""FSDP2 primitive surface.""" + +from __future__ import annotations + +from megatron.lite.primitive.optimizers.fsdp2.grad_clip import ( + all_reduce_scalar_, + clip_grads_with_sharded_norm_, + resolve_torch_dtype, + sharded_grad_abs_max, + sharded_grad_norm, + sharded_grad_sq_sum, +) +from megatron.lite.primitive.optimizers.fsdp2.optimizer import ( + BACKEND, + FSDP2Optimizer, + FSDP2OptimizerBackend, + build_fsdp2_adamw, + build_fsdp2_training_optimizer, +) +from megatron.lite.primitive.optimizers.fsdp2.wrap import ( + FSDP2Config, + build_fsdp2_device_mesh, + build_fsdp2_process_group_mesh, + build_fsdp2_shard_placement_fn, + fsdp2_available, + promote_fsdp2_trainable_params_to_fp32, + set_fsdp2_requires_gradient_sync, + wrap_fsdp2, + wrap_fsdp2_module, +) + +__all__ = [ + "BACKEND", + "FSDP2Config", + "FSDP2Optimizer", + "FSDP2OptimizerBackend", + "all_reduce_scalar_", + "build_fsdp2_adamw", + "build_fsdp2_training_optimizer", + "build_fsdp2_device_mesh", + "build_fsdp2_process_group_mesh", + "build_fsdp2_shard_placement_fn", + "clip_grads_with_sharded_norm_", + "fsdp2_available", + "promote_fsdp2_trainable_params_to_fp32", + "resolve_torch_dtype", + "set_fsdp2_requires_gradient_sync", + "sharded_grad_abs_max", + "sharded_grad_norm", + "sharded_grad_sq_sum", + "wrap_fsdp2", + "wrap_fsdp2_module", +] diff --git a/experimental/lite/megatron/lite/primitive/optimizers/fsdp2/adamw.py b/experimental/lite/megatron/lite/primitive/optimizers/fsdp2/adamw.py new file mode 100644 index 00000000000..3bdcf2bc253 --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/optimizers/fsdp2/adamw.py @@ -0,0 +1,546 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""AdamW helpers for the FSDP2 optimizer primitive.""" + +from __future__ import annotations + +import inspect +from collections.abc import Callable, Iterable +from typing import Any + +import torch +import torch.distributed as dist +import torch.nn as nn + + +def local_grad_sq_sum( + params: Iterable[nn.Parameter], + *, + dtype: torch.dtype, + default_device: torch.device | None = None, +) -> torch.Tensor: + total: torch.Tensor | None = None + for param in params: + grad = param.grad + if grad is None: + continue + grad = to_local_tensor(grad) + if total is None: + total = torch.zeros((), device=grad.device, dtype=dtype) + total += grad.detach().to(dtype).pow(2).sum() + if total is None: + return torch.zeros((), device=default_device or torch.device("cpu"), dtype=dtype) + return total + + +def to_local_tensor(tensor): + local_tensor = getattr(tensor, "_local_tensor", None) + if isinstance(local_tensor, torch.Tensor): + return local_tensor + to_local = getattr(tensor, "to_local", None) + if callable(to_local): + return to_local() + return tensor + + +def fsdp2_model_param_dtype(param: nn.Parameter) -> torch.dtype | None: + dtype = getattr(param, "_fsdp2_model_param_dtype", None) + return dtype if isinstance(dtype, torch.dtype) else None + + +def has_dtensor_grad_or_param(param: nn.Parameter) -> bool: + grad = param.grad + return is_dtensor_like(param) or (grad is not None and is_dtensor_like(grad)) + + +def is_dtensor_like(tensor: Any) -> bool: + return ( + callable(getattr(tensor, "to_local", None)) + and hasattr(tensor, "device_mesh") + and hasattr(tensor, "placements") + ) + + +def copy_local_tensor_to_param_(param: nn.Parameter, local_tensor: torch.Tensor) -> None: + if not is_dtensor_like(param): + param.detach().copy_(local_tensor.to(device=param.device, dtype=param.dtype)) + return + + # Copy straight into the param's local shard. Reconstructing via + # DTensor.from_local mis-sizes an unevenly-sharded param (it infers global = + # local * mesh, e.g. a (3,) param over 8 ranks -> 0 or 8), so copy local->local + # (master is init'd from this same local shard, so shapes match). + local_param = to_local_tensor(param) + local_param.copy_(local_tensor.to(device=local_param.device, dtype=local_param.dtype)) + + +def all_reduce_grad_(grad: torch.Tensor, *, group: dist.ProcessGroup) -> None: + # ``to_local_tensor`` returns the DTensor's local shard storage, so the + # in-place all-reduce updates the grad directly -- no DTensor.from_local + # round-trip (which mis-sizes unevenly-sharded grads, e.g. (3,) over 8 ranks). + local_grad = to_local_tensor(grad) + dist.all_reduce(local_grad, op=dist.ReduceOp.SUM, group=group) + + +class ChainedOptimizer: + def __init__(self, optimizers: Iterable[torch.optim.Optimizer]): + self.optimizers = list(optimizers) + + @property + def param_groups(self) -> list[dict[str, Any]]: + groups: list[dict[str, Any]] = [] + for optimizer in self.optimizers: + groups.extend(optimizer.param_groups) + return groups + + def zero_grad(self, *args, **kwargs) -> None: + for optimizer in self.optimizers: + optimizer.zero_grad(*args, **kwargs) + + def step(self) -> None: + for optimizer in self.optimizers: + optimizer.step() + + def state_dict(self) -> dict[str, Any]: + return { + "type": "chained_torch_optimizer", + "optimizers": [optimizer.state_dict() for optimizer in self.optimizers], + } + + def load_state_dict(self, state_dict: dict[str, Any]) -> None: + optimizer_states = state_dict.get("optimizers") + if not isinstance(optimizer_states, list) or len(optimizer_states) != len(self.optimizers): + raise ValueError("Invalid chained torch optimizer state_dict.") + for optimizer, optimizer_state in zip(self.optimizers, optimizer_states, strict=True): + optimizer.load_state_dict(optimizer_state) + + +class FP32AdamW: + """AdamW with FP32 master params for BF16/DTensor model weights.""" + + def __init__( + self, + params: Iterable[nn.Parameter] | Iterable[dict[str, Any]], + *, + lr: float, + weight_decay: float, + betas: tuple[float, float], + eps: float, + cpu_update: bool = False, + model_param_dtypes: dict[int, torch.dtype] | None = None, + ): + self.param_groups = normalize_param_groups(params, default_weight_decay=weight_decay) + self.params: list[nn.Parameter] = [] + self.lr = lr + self.weight_decay = weight_decay + self.betas = betas + self.eps = eps + self.cpu_update = bool(cpu_update) + self.step_count = 0 + self.state: dict[nn.Parameter, dict[str, torch.Tensor]] = {} + self._master_for_param: dict[nn.Parameter, torch.Tensor] = {} + self._model_param_dtypes_by_id = dict(model_param_dtypes or {}) + self._model_dtype_for_param: dict[nn.Parameter, torch.dtype] = {} + + for group in self.param_groups: + group.setdefault("lr", lr) + group.setdefault("wd_mult", 1.0) + group_weight_decay = float(group.get("weight_decay", weight_decay)) + group["weight_decay"] = group_weight_decay + for param in group["params"]: + self.params.append(param) + model_dtype = self._model_param_dtypes_by_id.get(id(param)) + if model_dtype is not None: + self._model_dtype_for_param[param] = model_dtype + master = self._init_master_param(param) + self.state[param] = { + "master_param": master, + "exp_avg": torch.zeros_like(master, dtype=torch.float32), + "exp_avg_sq": torch.zeros_like(master, dtype=torch.float32), + "step": 0, + } + self._master_for_param[param] = master + + def _init_master_param(self, param: nn.Parameter) -> torch.Tensor: + if self.cpu_update: + local_param = to_local_tensor(param.detach()) + return local_param.detach().to(device="cpu", dtype=torch.float32).clone() + if self._model_param_dtype(param) is not None: + return param.detach().to(dtype=torch.float32).clone() + return ( + param.detach() + if param.dtype is torch.float32 + else param.detach().to(dtype=torch.float32).clone() + ) + + def _model_param_dtype(self, param: nn.Parameter) -> torch.dtype | None: + return self._model_dtype_for_param.get(param) or fsdp2_model_param_dtype(param) + + def zero_grad(self, *args, **kwargs) -> None: + set_to_none = kwargs.get("set_to_none", False) + if args: + set_to_none = bool(args[0]) + for param in self.params: + if set_to_none: + param.grad = None + elif param.grad is not None: + param.grad.detach_() + param.grad.zero_() + + def step(self) -> None: + self._step_param_groups() + + def _step_param_groups(self) -> None: + self.step_count += 1 + beta1, beta2 = self.betas + + for group in self.param_groups: + group_lr = float(group.get("lr", self.lr)) + group_weight_decay = float(group.get("weight_decay", self.weight_decay)) + for param in group["params"]: + grad = param.grad + if grad is None: + continue + state = self.state[param] + state["step"] = int(state["step"]) + 1 + param_step = int(state["step"]) + bias_correction1 = 1.0 - beta1**param_step + bias_correction2_sqrt = (1.0 - beta2**param_step) ** 0.5 + group_step_size = group_lr / bias_correction1 + master = state["master_param"] + exp_avg = state["exp_avg"] + exp_avg_sq = state["exp_avg_sq"] + if group_weight_decay != 0.0: + master.mul_(1.0 - group_lr * group_weight_decay) + grad = self._prepare_grad(grad, master) + exp_avg.mul_(beta1).add_(grad, alpha=1.0 - beta1) + exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1.0 - beta2) + denom = exp_avg_sq.sqrt().div_(bias_correction2_sqrt).add_(self.eps) + master.addcdiv_(exp_avg.to(dtype=torch.float32), denom, value=-group_step_size) + self._copy_master_to_param(param, master) + + def _prepare_grad(self, grad: torch.Tensor, master: torch.Tensor) -> torch.Tensor: + if self.cpu_update: + grad = to_local_tensor(grad) + return grad.detach().to(device=master.device, dtype=torch.float32) + return grad.detach().to(dtype=torch.float32) + + def _copy_master_to_param(self, param: nn.Parameter, master: torch.Tensor) -> None: + model_dtype = self._model_param_dtype(param) + if model_dtype is not None: + master = master.to(dtype=model_dtype).to(dtype=param.dtype) + if not self.cpu_update: + param.detach().copy_(master.to(dtype=param.dtype)) + return + copy_local_tensor_to_param_(param, master) + + def state_dict(self) -> dict[str, Any]: + return { + "type": "fp32_adamw", + "step_count": self.step_count, + "master_params": [self.state[param]["master_param"] for param in self.params], + "exp_avgs": [self.state[param]["exp_avg"] for param in self.params], + "exp_avg_sqs": [self.state[param]["exp_avg_sq"] for param in self.params], + "steps": [int(self.state[param]["step"]) for param in self.params], + "weight_decays": [ + float(group.get("weight_decay", self.weight_decay)) + for group in self.param_groups + for _param in group["params"] + ], + } + + def load_state_dict(self, state_dict: dict[str, Any]) -> None: + if state_dict.get("type") != "fp32_adamw": + raise ValueError("Invalid FP32 AdamW state_dict.") + self.step_count = int(state_dict.get("step_count", 0)) + for target_name, key in ( + ("master_params", "master_param"), + ("exp_avgs", "exp_avg"), + ("exp_avg_sqs", "exp_avg_sq"), + ): + loaded = state_dict.get(target_name) + if not isinstance(loaded, list) or len(loaded) != len(self.params): + raise ValueError(f"Invalid FP32 AdamW {target_name} state.") + for param, src in zip(self.params, loaded, strict=True): + target = self.state[param][key] + local_target = to_local_tensor(target) + local_src = to_local_tensor(src).to( + device=local_target.device, dtype=local_target.dtype + ) + local_target.copy_(local_src) + loaded_steps = state_dict.get("steps") + if loaded_steps is not None: + if not isinstance(loaded_steps, list) or len(loaded_steps) != len(self.params): + raise ValueError("Invalid FP32 AdamW steps state.") + for param, step in zip(self.params, loaded_steps, strict=True): + self.state[param]["step"] = int(step) + else: + for param in self.params: + self.state[param]["step"] = self.step_count + loaded_weight_decays = state_dict.get("weight_decays") + if loaded_weight_decays is not None: + if not isinstance(loaded_weight_decays, list) or len(loaded_weight_decays) != len( + self.params + ): + raise ValueError("Invalid FP32 AdamW weight_decay state.") + idx = 0 + for group in self.param_groups: + if not group["params"]: + continue + group["weight_decay"] = float(loaded_weight_decays[idx]) + idx += len(group["params"]) + + for param in self.params: + self._copy_master_to_param(param, self.state[param]["master_param"]) + + +def build_adamw_optimizer( + params: Iterable[nn.Parameter] | Iterable[dict[str, Any]], + *, + all_params: Iterable[nn.Parameter], + lr: float, + weight_decay: float, + betas: tuple[float, float], + eps: float, + foreach: bool | str, + use_fp32_master: bool, + cpu_update: bool, + model_param_dtypes: dict[int, torch.dtype] | None, + opt, +) -> Any: + param_groups = normalize_param_groups(params, default_weight_decay=weight_decay) + fused_adam = maybe_build_te_fused_adam_optimizer( + param_groups, + all_params=all_params, + lr=lr, + weight_decay=weight_decay, + betas=betas, + eps=eps, + opt=opt, + use_fp32_master=use_fp32_master, + ) + if fused_adam is not None: + return fused_adam + if use_fp32_master: + return FP32AdamW( + param_groups, + lr=lr, + weight_decay=weight_decay, + betas=betas, + eps=eps, + cpu_update=cpu_update, + model_param_dtypes=model_param_dtypes, + ) + if foreach not in {True, False, "auto"}: + raise ValueError(f"adamw_foreach must be True, False, or 'auto', got {foreach!r}.") + if foreach is False: + return torch.optim.AdamW( + param_groups, lr=lr, weight_decay=weight_decay, betas=betas, eps=eps, foreach=False + ) + + dtensor_param_groups, tensor_param_groups = split_dtensor_and_tensor_param_groups( + param_groups, default_weight_decay=weight_decay + ) + split_param_groups = [group for group in (dtensor_param_groups, tensor_param_groups) if group] + if foreach == "auto" and not dtensor_param_groups: + return torch.optim.AdamW( + param_groups, lr=lr, weight_decay=weight_decay, betas=betas, eps=eps, foreach=False + ) + if len(split_param_groups) <= 1: + return torch.optim.AdamW( + split_param_groups[0] if split_param_groups else param_groups, + lr=lr, + weight_decay=weight_decay, + betas=betas, + eps=eps, + foreach=True, + ) + return ChainedOptimizer( + torch.optim.AdamW( + group, lr=lr, weight_decay=weight_decay, betas=betas, eps=eps, foreach=True + ) + for group in split_param_groups + ) + + +def maybe_build_te_fused_adam_optimizer( + param_groups: list[dict[str, Any]], + *, + all_params: Iterable[nn.Parameter], + lr: float, + weight_decay: float, + betas: tuple[float, float], + eps: float, + opt, + use_fp32_master: bool, +) -> Any | None: + if not get_bool_opt(opt, "fsdp2_use_te_fused_adam", default=False): + return None + from transformer_engine.pytorch.optimizers.fused_adam import FusedAdam + + all_param_list = list(all_params) + master_weights = get_bool_opt( + opt, "master_weights", default=use_fp32_master and should_use_master_weights(all_param_list) + ) + kwargs = dict( + lr=lr, + weight_decay=weight_decay, + betas=betas, + eps=eps, + adam_w_mode=True, + master_weights=master_weights, + master_weight_dtype=get_dtype_opt(opt, "master_weight_dtype", default=torch.float32), + store_param_remainders=get_bool_opt(opt, "store_param_remainders", default=master_weights), + exp_avg_dtype=get_dtype_opt(opt, "exp_avg_dtype", default=torch.float32), + exp_avg_sq_dtype=get_dtype_opt(opt, "exp_avg_sq_dtype", default=torch.float32), + ) + return FusedAdam(param_groups, **filter_supported_kwargs(FusedAdam.__init__, kwargs)) + + +def filter_supported_kwargs(fn: Callable[..., Any], kwargs: dict[str, Any]) -> dict[str, Any]: + try: + params = inspect.signature(fn).parameters + except (TypeError, ValueError): + return kwargs + if any(param.kind is inspect.Parameter.VAR_KEYWORD for param in params.values()): + return kwargs + return {key: value for key, value in kwargs.items() if key in params} + + +def should_use_master_weights(params: Iterable[nn.Parameter]) -> bool: + return any(param.is_floating_point() and param.dtype is not torch.float32 for param in params) + + +def get_bool_opt(opt, attr: str, *, default: bool) -> bool: + value = get_opt_value(opt, attr) + if value is None: + return bool(default) + if isinstance(value, bool): + return value + if isinstance(value, (int, float)): + return bool(value) + return str(value).strip().lower() in {"1", "true", "yes", "on"} + + +def get_dtype_opt(opt, attr: str, *, default: torch.dtype) -> torch.dtype: + value = get_opt_value(opt, attr) + if value is None: + return default + if isinstance(value, torch.dtype): + return value + name = str(value).removeprefix("torch.") + resolved = getattr(torch, name, None) + if not isinstance(resolved, torch.dtype): + raise ValueError(f"Unsupported dtype for FSDP2 TE FusedAdam: {value!r}.") + return resolved + + +def get_opt_value(opt, attr: str): + if opt is None: + return None + if isinstance(opt, dict): + value = opt.get(attr) + override = opt.get("override_optimizer_config") + else: + value = getattr(opt, attr, None) + override = getattr(opt, "override_optimizer_config", None) + if value is not None: + return value + if isinstance(override, dict): + return override.get(attr) + return None + + +def normalize_param_groups( + params: Iterable[nn.Parameter] | Iterable[dict[str, Any]], *, default_weight_decay: float +) -> list[dict[str, Any]]: + items = list(params) + if not items: + return [] + if all(isinstance(item, dict) for item in items): + groups: list[dict[str, Any]] = [] + for item in items: + group = dict(item) + group_params = list(group.get("params", ())) + if not group_params: + continue + group["params"] = group_params + group.setdefault("weight_decay", default_weight_decay) + groups.append(group) + return groups + return [{"params": items, "weight_decay": default_weight_decay}] + + +def split_dtensor_and_tensor_param_groups( + param_groups: Iterable[dict[str, Any]], *, default_weight_decay: float +) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + dtensor_groups: list[dict[str, Any]] = [] + tensor_groups: list[dict[str, Any]] = [] + for group in param_groups: + dtensor_params, tensor_params = split_dtensor_and_tensor_params(group["params"]) + metadata = {key: value for key, value in group.items() if key != "params"} + metadata.setdefault("weight_decay", default_weight_decay) + if dtensor_params: + dtensor_groups.append({**metadata, "params": dtensor_params}) + if tensor_params: + tensor_groups.append({**metadata, "params": tensor_params}) + return dtensor_groups, tensor_groups + + +def split_dtensor_and_tensor_params( + params: Iterable[nn.Parameter], +) -> tuple[list[nn.Parameter], list[nn.Parameter]]: + dtensor_params: list[nn.Parameter] = [] + tensor_params: list[nn.Parameter] = [] + for param in params: + if is_dtensor_like(param): + dtensor_params.append(param) + else: + tensor_params.append(param) + return dtensor_params, tensor_params + + +def iter_torch_optimizers(optimizer: Any) -> Iterable[torch.optim.Optimizer]: + if isinstance(optimizer, ChainedOptimizer): + yield from optimizer.optimizers + else: + yield optimizer + + +def dtensor_from_local( + local_tensor: torch.Tensor, + device_mesh: Any, + placements: Any, + *, + shape: Any = None, + stride: Any = None, +) -> torch.Tensor: + from torch.distributed.tensor import DTensor + + # ``DTensor.from_local`` infers the global shape as local_shard * mesh, which + # is WRONG for unevenly-sharded params (FSDP2 pads the last shard). Pass the + # original global shape/stride so the round-trip is exact for any dim not + # divisible by the mesh size. + if shape is not None: + return DTensor.from_local( + local_tensor, device_mesh, placements, shape=shape, stride=stride + ) + return DTensor.from_local(local_tensor, device_mesh, placements) + + +__all__ = [ + "all_reduce_grad_", + "build_adamw_optimizer", + "copy_local_tensor_to_param_", + "dtensor_from_local", + "filter_supported_kwargs", + "fsdp2_model_param_dtype", + "get_bool_opt", + "get_dtype_opt", + "get_opt_value", + "has_dtensor_grad_or_param", + "is_dtensor_like", + "iter_torch_optimizers", + "local_grad_sq_sum", + "normalize_param_groups", + "to_local_tensor", +] diff --git a/experimental/lite/megatron/lite/primitive/optimizers/fsdp2/grad_clip.py b/experimental/lite/megatron/lite/primitive/optimizers/fsdp2/grad_clip.py new file mode 100644 index 00000000000..d93149008ac --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/optimizers/fsdp2/grad_clip.py @@ -0,0 +1,327 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Sharded gradient norm and clipping primitive for FSDP2 optimizers.""" + +from __future__ import annotations + +import math +from collections import defaultdict +from collections.abc import Callable, Iterable +from typing import Any + +import torch +import torch.distributed as dist +import torch.nn as nn + +try: # pragma: no cover - import availability is PyTorch-version dependent. + from torch.distributed.tensor import DTensor, Partial, Replicate +except ImportError: # pragma: no cover + DTensor = Partial = Replicate = None # type: ignore[assignment] + + +def sharded_grad_sq_sum( + params: Iterable[nn.Parameter], + *, + accum_dtype: str | torch.dtype = torch.float32, + default_device: torch.device | None = None, + chunk_size_numel: int = 0, + scalar_all_reduce: ( + Callable[[torch.Tensor, dist.ProcessGroup, dist.ReduceOp], None] | None + ) = None, +) -> torch.Tensor: + """Return global L2 grad squared-sum for Tensor/DTensor parameters. + + The primitive reduces one scalar per DTensor sharding group over mesh + dimensions whose placement is not replicated. Pipeline/expert reductions + are intentionally left to the runtime adapter because they are model-layout + policy, not a DTensor property. + """ + + dtype = resolve_torch_dtype(accum_dtype) + groups = _group_grads(params) + total: torch.Tensor | None = None + for group in groups.values(): + local_sq = _group_local_sq_sum(group, dtype=dtype, chunk_size_numel=chunk_size_numel) + meta = group[0][2] + if meta is not None and not _has_partial_placement(meta) and dist.is_initialized(): + _reduce_dtensor_scalar_( + local_sq, meta, op=dist.ReduceOp.SUM, scalar_all_reduce=scalar_all_reduce + ) + total = local_sq if total is None else total.to(local_sq.device) + local_sq + + if total is None: + return torch.zeros((), device=default_device or torch.device("cpu"), dtype=dtype) + return total + + +def sharded_grad_norm( + params: Iterable[nn.Parameter], + *, + norm_type: float = 2.0, + pp_group: dist.ProcessGroup | None = None, + accum_dtype: str | torch.dtype = torch.float32, + default_device: torch.device | None = None, +) -> torch.Tensor: + """Return global grad norm for Tensor/DTensor parameters. + + Only L2 and infinity norms are implemented because they cover Megatron Lite's + optimizer path today and avoid ambiguous cross-placement semantics for + arbitrary p-norms. + """ + + if math.isinf(float(norm_type)): + total = sharded_grad_abs_max( + params, pp_group=pp_group, accum_dtype=accum_dtype, default_device=default_device + ) + return total + if float(norm_type) != 2.0: + raise ValueError(f"sharded_grad_norm supports norm_type=2.0 or inf, got {norm_type!r}.") + sq_sum = sharded_grad_sq_sum(params, accum_dtype=accum_dtype, default_device=default_device) + if pp_group is not None and dist.is_initialized() and dist.get_world_size(pp_group) > 1: + all_reduce_scalar_(sq_sum, op=dist.ReduceOp.SUM, group=pp_group) + return sq_sum.sqrt() + + +def sharded_grad_abs_max( + params: Iterable[nn.Parameter], + *, + pp_group: dist.ProcessGroup | None = None, + accum_dtype: str | torch.dtype = torch.float32, + default_device: torch.device | None = None, +) -> torch.Tensor: + """Return global infinity grad norm for Tensor/DTensor parameters.""" + + dtype = resolve_torch_dtype(accum_dtype) + groups = _group_grads(params) + total: torch.Tensor | None = None + for group in groups.values(): + local_max = _group_local_abs_max(group, dtype=dtype) + meta = group[0][2] + if meta is not None and not _has_partial_placement(meta) and dist.is_initialized(): + _reduce_dtensor_scalar_(local_max, meta, op=dist.ReduceOp.MAX) + total = local_max if total is None else torch.maximum(total.to(local_max.device), local_max) + + if total is None: + total = torch.zeros((), device=default_device or torch.device("cpu"), dtype=dtype) + if pp_group is not None and dist.is_initialized() and dist.get_world_size(pp_group) > 1: + all_reduce_scalar_(total, op=dist.ReduceOp.MAX, group=pp_group) + return total + + +def all_reduce_scalar_( + value: torch.Tensor, + *, + op: dist.ReduceOp, + group: dist.ProcessGroup, +) -> None: + """All-reduce a scalar on a device compatible with the process group backend.""" + + reduced = _scalar_for_process_group(value, group) + dist.all_reduce(reduced, op=op, group=group) + if reduced is not value: + value.copy_(reduced.to(device=value.device, dtype=value.dtype)) + + +@torch.no_grad() +def clip_grads_with_sharded_norm_( + params: Iterable[nn.Parameter], max_norm: float, total_norm: torch.Tensor | float +) -> None: + """Scale gradients in-place using a precomputed global norm.""" + + max_norm = float(max_norm) + if max_norm <= 0: + return + if isinstance(total_norm, torch.Tensor): + if not bool(torch.isfinite(total_norm).item()): + return + clip_coef = (max_norm / (total_norm + 1.0e-6)).clamp(max=1.0) + if float(clip_coef.item()) >= 1.0: + return + else: + norm_value = float(total_norm) + if not math.isfinite(norm_value): + return + clip_coef = max_norm / (norm_value + 1.0e-6) + if clip_coef >= 1.0: + return + for param in params: + if param.grad is not None: + _scale_grad_(param.grad, clip_coef) + + +def resolve_torch_dtype(dtype: str | torch.dtype) -> torch.dtype: + if isinstance(dtype, torch.dtype): + resolved = dtype + else: + name = dtype.removeprefix("torch.") + resolved = getattr(torch, name, None) + if not isinstance(resolved, torch.dtype): + raise ValueError(f"Unsupported torch dtype for grad norm accumulation: {dtype!r}") + if not torch.empty((), dtype=resolved).is_floating_point(): + raise ValueError(f"Grad norm accumulation dtype must be floating point: {dtype!r}") + return resolved + + +def _group_grads( + params: Iterable[nn.Parameter], +) -> dict[tuple[Any, ...], list[tuple[nn.Parameter, torch.Tensor, Any | None]]]: + groups: dict[tuple[Any, ...], list[tuple[nn.Parameter, torch.Tensor, Any | None]]] = ( + defaultdict(list) + ) + for param in params: + grad = param.grad + if grad is None: + continue + meta = _dtensor_meta(param, grad) + if meta is None: + key = ("tensor", grad.device) + else: + key = ( + "dtensor", + id(meta.device_mesh), + tuple((type(placement).__name__, repr(placement)) for placement in meta.placements), + ) + groups[key].append((param, grad, meta)) + return groups + + +def _group_local_sq_sum( + group: list[tuple[nn.Parameter, torch.Tensor, Any | None]], + *, + dtype: torch.dtype, + chunk_size_numel: int = 0, +) -> torch.Tensor: + device = _local_grad(group[0][1], group[0][2]).device + total = torch.zeros((), device=device, dtype=dtype) + for _param, grad, meta in group: + local_grad = _local_grad(grad, meta) + total += _tensor_sq_sum(local_grad.detach(), dtype=dtype, chunk_size_numel=chunk_size_numel) + return total + + +def _tensor_sq_sum( + tensor: torch.Tensor, *, dtype: torch.dtype, chunk_size_numel: int = 0 +) -> torch.Tensor: + if chunk_size_numel <= 0 or tensor.numel() <= chunk_size_numel: + return tensor.to(dtype).pow(2).sum() + try: + flat = tensor.view(-1) + except RuntimeError: + flat = tensor.reshape(-1) + total = torch.zeros((), device=tensor.device, dtype=dtype) + for start in range(0, flat.numel(), chunk_size_numel): + chunk = flat.narrow(0, start, min(chunk_size_numel, flat.numel() - start)) + total += chunk.to(dtype).pow(2).sum() + return total + + +def _group_local_abs_max( + group: list[tuple[nn.Parameter, torch.Tensor, Any | None]], *, dtype: torch.dtype +) -> torch.Tensor: + device = _local_grad(group[0][1], group[0][2]).device + total = torch.zeros((), device=device, dtype=dtype) + for _param, grad, meta in group: + local_grad = _local_grad(grad, meta) + if local_grad.numel() > 0: + total = torch.maximum(total, local_grad.detach().to(dtype).abs().max()) + return total + + +def _local_grad(grad: torch.Tensor, meta: Any | None) -> torch.Tensor: + if meta is not None and _has_partial_placement(meta): + full_tensor = getattr(grad, "full_tensor", None) + if callable(full_tensor): + return full_tensor() + to_local = getattr(grad, "to_local", None) + if callable(to_local): + return to_local() + return grad + + +def _scale_grad_(grad: torch.Tensor, scale: float | torch.Tensor) -> None: + # clip_coef is a scalar; scale every shard by it in place. A plain scalar mul_ + # is correct for ANY DTensor placement (Shard/Replicate/Partial) and avoids a + # to_local()/from_local() round-trip, which mis-reconstructs the global shape of + # an unevenly-sharded param -- e.g. a (3,) mHC scale FSDP-sharded over 8 ranks: + # from_local assumes even sharding and infers dim0=8, so copy_ raises + # "tensor a (3) must match tensor b (8) at dim 0". + grad.mul_(float(scale) if isinstance(scale, torch.Tensor) else scale) + + +def _dtensor_meta(param: nn.Parameter, grad: torch.Tensor) -> Any | None: + if _is_dtensor_like(grad): + return grad + if _is_dtensor_like(param): + return param + return None + + +def _is_dtensor_like(tensor: Any) -> bool: + if DTensor is not None and isinstance(tensor, DTensor): + return True + return ( + callable(getattr(tensor, "to_local", None)) + and hasattr(tensor, "device_mesh") + and hasattr(tensor, "placements") + ) + + +def _has_partial_placement(dtensor: Any) -> bool: + return any(_placement_name(placement) == "Partial" for placement in dtensor.placements) + + +def _is_replicate_placement(placement: Any) -> bool: + return _placement_name(placement) == "Replicate" + + +def _placement_name(placement: Any) -> str: + return type(placement).__name__ + + +def _reduce_dtensor_scalar_( + value: torch.Tensor, + dtensor: Any, + *, + op: dist.ReduceOp, + scalar_all_reduce: ( + Callable[[torch.Tensor, dist.ProcessGroup, dist.ReduceOp], None] | None + ) = None, +) -> None: + for mesh_dim, placement in enumerate(dtensor.placements): + if _is_replicate_placement(placement): + continue + group = dtensor.device_mesh.get_group(mesh_dim) + if dist.get_world_size(group) > 1: + if scalar_all_reduce is None: + all_reduce_scalar_(value, op=op, group=group) + else: + scalar_all_reduce(value, group, op) + + +def _scalar_for_process_group( + value: torch.Tensor, group: dist.ProcessGroup +) -> torch.Tensor: + backend = _process_group_backend(group) + if "nccl" in backend and value.device.type != "cuda": + if not torch.cuda.is_available(): + raise RuntimeError("NCCL scalar all_reduce requires a CUDA tensor.") + return value.to(device=torch.device("cuda", torch.cuda.current_device())) + if "gloo" in backend and value.device.type != "cpu": + return value.to(device=torch.device("cpu")) + return value + + +def _process_group_backend(group: dist.ProcessGroup) -> str: + try: + return str(dist.get_backend(group)).lower() + except (RuntimeError, ValueError, TypeError): + return "" + + +__all__ = [ + "all_reduce_scalar_", + "clip_grads_with_sharded_norm_", + "resolve_torch_dtype", + "sharded_grad_abs_max", + "sharded_grad_norm", + "sharded_grad_sq_sum", +] diff --git a/experimental/lite/megatron/lite/primitive/optimizers/fsdp2/optimizer.py b/experimental/lite/megatron/lite/primitive/optimizers/fsdp2/optimizer.py new file mode 100644 index 00000000000..63f2935c372 --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/optimizers/fsdp2/optimizer.py @@ -0,0 +1,680 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""FSDP2 optimizer adapter for Megatron Lite runtime contracts.""" + +from __future__ import annotations + +import math +from collections.abc import Callable, Iterable +from dataclasses import dataclass +from types import SimpleNamespace +from typing import Any + +import torch +import torch.distributed as dist +import torch.nn as nn + +from megatron.lite.primitive.optimizers.fsdp2.adamw import ( + all_reduce_grad_, + build_adamw_optimizer, + fsdp2_model_param_dtype, + get_bool_opt, + has_dtensor_grad_or_param, + local_grad_sq_sum, +) +from megatron.lite.primitive.optimizers.fsdp2.grad_clip import ( + all_reduce_scalar_, + clip_grads_with_sharded_norm_, + resolve_torch_dtype, + sharded_grad_sq_sum, +) +from megatron.lite.primitive.optimizers.fsdp2.state import ( + OffloadedStateEntry, + move_offloaded_optimizer_state_to_device, + move_optimizer_state_to_cpu, +) +from megatron.lite.primitive.optimizers.fsdp2.wrap import ( + FSDP2Config, + build_fsdp2_process_group_mesh, + build_fsdp2_shard_placement_fn, + promote_fsdp2_trainable_params_to_fp32, + wrap_fsdp2, + wrap_fsdp2_module, +) +from megatron.lite.primitive.parallel.state import ParallelState + +_DEFAULT_RESHARD_AFTER_FORWARD: bool | int | None = True +_DEFAULT_WRAP_ROOT = True +_DEFAULT_LEAF_MODULE_NAMES = ("embed", "head") +_DEFAULT_FORWARD_PREFETCH_DEPTH = 1 +_DEFAULT_BACKWARD_PREFETCH_DEPTH = 0 +_DEFAULT_PARAM_DTYPE: str | None = "bfloat16" +_DEFAULT_REDUCE_DTYPE: str | None = "float32" +_DEFAULT_USE_FP32_SHARDS = True +_DEFAULT_USE_FP32_MASTER = True +_DEFAULT_ADAMW_FOREACH: bool | str = "auto" + + +class FSDP2Optimizer: + """Adapt an optimizer to Megatron Lite's FSDP2 optimizer contract.""" + + name = "fsdp2" + + def __init__( + self, + optimizer: Any, + params: Iterable[nn.Parameter], + ps: ParallelState | None = None, + *, + clip_grad: float = 1.0, + replicated_grad_params: Iterable[nn.Parameter] | None = None, + replicated_grad_sync_group: dist.ProcessGroup | None = None, + replicated_grad_sync_divisor: float | None = None, + replicated_grad_norm_group: dist.ProcessGroup | None = None, + expert_sharded_grad_params: Iterable[nn.Parameter] | None = None, + expert_sharded_grad_scale: float | None = None, + expert_sharded_grad_norm_group: dist.ProcessGroup | None = None, + tp_replicated_grad_params: Iterable[nn.Parameter] | None = None, + tp_replicated_grad_sync_group: dist.ProcessGroup | None = None, + grad_norm_accum_dtype: str | torch.dtype = torch.float32, + param_names: dict[int, str] | None = None, + ): + self.optimizer = optimizer + self.params = list(params) + self.param_names = dict(param_names or {}) + self.ps = ps + self.clip_grad = float(clip_grad) + self.grad_norm_accum_dtype = resolve_torch_dtype(grad_norm_accum_dtype) + self.replicated_grad_params = list(replicated_grad_params or ()) + self._replicated_grad_param_ids = {id(param) for param in self.replicated_grad_params} + self.replicated_grad_sync_group = replicated_grad_sync_group + self.replicated_grad_sync_divisor = replicated_grad_sync_divisor + self.replicated_grad_norm_group = replicated_grad_norm_group + self.expert_sharded_grad_params = list(expert_sharded_grad_params or ()) + self._expert_sharded_grad_param_ids = { + id(param) for param in self.expert_sharded_grad_params + } + self.expert_sharded_grad_scale = ( + 1.0 if expert_sharded_grad_scale is None else float(expert_sharded_grad_scale) + ) + self.expert_sharded_grad_norm_group = expert_sharded_grad_norm_group + self.tp_replicated_grad_params = list(tp_replicated_grad_params or ()) + self._tp_replicated_grad_param_ids = {id(param) for param in self.tp_replicated_grad_params} + self.tp_replicated_grad_sync_group = tp_replicated_grad_sync_group + self._cpu_offloaded_state: dict[tuple[int, str], OffloadedStateEntry] = {} + self.grad_sync_enabled = False + + @property + def param_groups(self): + return self.optimizer.param_groups + + def zero_grad(self) -> None: + self.grad_sync_enabled = False + self.optimizer.zero_grad(set_to_none=True) + + def step(self) -> tuple[bool, float, int]: + self.sync_tp_replicated_grads() + self.sync_replicated_grads() + self.scale_expert_sharded_grads() + grad_norm = self.clip_grad_norm() + if not math.isfinite(grad_norm): + self.grad_sync_enabled = False + return False, float(grad_norm), 0 + self.optimizer.step() + self.grad_sync_enabled = False + return True, float(grad_norm), 0 + + def sync_replicated_grads(self) -> None: + group = self.replicated_grad_sync_group + if not self.replicated_grad_params: + return + group_size = 1 + if group is not None and dist.is_initialized(): + group_size = dist.get_world_size(group) + if group is None and self.replicated_grad_sync_divisor is None: + return + divisor = self.replicated_grad_sync_divisor + if divisor is None: + divisor = float(group_size) + for param in self.replicated_grad_params: + grad = param.grad + if grad is None: + continue + if group is not None and dist.is_initialized() and group_size > 1: + dist.all_reduce(grad, op=dist.ReduceOp.SUM, group=group) + if divisor != 1.0: + grad.div_(divisor) + + def sync_tp_replicated_grads(self) -> None: + group = self.tp_replicated_grad_sync_group + if not self.tp_replicated_grad_params: + return + if group is None or not dist.is_initialized() or dist.get_world_size(group) <= 1: + return + for param in self.tp_replicated_grad_params: + grad = param.grad + if grad is not None: + all_reduce_grad_(grad, group=group) + + def scale_expert_sharded_grads(self) -> None: + if not self.expert_sharded_grad_params or self.expert_sharded_grad_scale == 1.0: + return + for param in self.expert_sharded_grad_params: + grad = param.grad + if grad is not None: + grad.mul_(self.expert_sharded_grad_scale) + + def clip_grad_norm(self) -> float: + excluded_sharded_param_ids = ( + self._replicated_grad_param_ids + | self._tp_replicated_grad_param_ids + | self._expert_sharded_grad_param_ids + ) + sharded_params = [ + param for param in self.params if id(param) not in excluded_sharded_param_ids + ] + dtensor_sharded_params = [ + param for param in sharded_params if has_dtensor_grad_or_param(param) + ] + plain_sharded_params = [ + param for param in sharded_params if not has_dtensor_grad_or_param(param) + ] + total_sq = sharded_grad_sq_sum( + dtensor_sharded_params, accum_dtype=self.grad_norm_accum_dtype + ) + plain_sharded_sq = local_grad_sq_sum( + plain_sharded_params, + dtype=resolve_torch_dtype(self.grad_norm_accum_dtype), + default_device=total_sq.device, + ) + plain_sharded_group = None + if self.ps is not None: + plain_sharded_group = self.ps.dp_cp_group or self.ps.dp_group + if ( + plain_sharded_group is not None + and dist.is_initialized() + and dist.get_world_size(plain_sharded_group) > 1 + ): + all_reduce_scalar_(plain_sharded_sq, op=dist.ReduceOp.SUM, group=plain_sharded_group) + total_sq = total_sq.to(plain_sharded_sq.device) + plain_sharded_sq + if ( + self.ps is not None + and self.ps.tp_group is not None + and dist.is_initialized() + and dist.get_world_size(self.ps.tp_group) > 1 + ): + all_reduce_scalar_(total_sq, op=dist.ReduceOp.SUM, group=self.ps.tp_group) + + tp_replicated_sq = sharded_grad_sq_sum( + self.tp_replicated_grad_params, + accum_dtype=self.grad_norm_accum_dtype, + default_device=total_sq.device, + ) + replicated_sq = local_grad_sq_sum( + self.replicated_grad_params, + dtype=self.grad_norm_accum_dtype, + default_device=total_sq.device, + ) + if ( + self.replicated_grad_norm_group is not None + and dist.is_initialized() + and dist.get_world_size(self.replicated_grad_norm_group) > 1 + ): + all_reduce_scalar_( + replicated_sq, op=dist.ReduceOp.SUM, group=self.replicated_grad_norm_group + ) + + expert_sharded_sq = sharded_grad_sq_sum( + self.expert_sharded_grad_params, + accum_dtype=self.grad_norm_accum_dtype, + default_device=total_sq.device, + ) + if ( + self.expert_sharded_grad_norm_group is not None + and dist.is_initialized() + and dist.get_world_size(self.expert_sharded_grad_norm_group) > 1 + ): + all_reduce_scalar_( + expert_sharded_sq, + op=dist.ReduceOp.SUM, + group=self.expert_sharded_grad_norm_group, + ) + + total_sq = ( + total_sq + + tp_replicated_sq.to(total_sq.device) + + replicated_sq.to(total_sq.device) + + expert_sharded_sq.to(total_sq.device) + ) + if ( + self.ps is not None + and self.ps.pp_group is not None + and dist.is_initialized() + and dist.get_world_size(self.ps.pp_group) > 1 + ): + all_reduce_scalar_(total_sq, op=dist.ReduceOp.SUM, group=self.ps.pp_group) + + grad_norm = total_sq.sqrt() + if torch.isfinite(grad_norm): + clip_grads_with_sharded_norm_(self.params, self.clip_grad, grad_norm) + return float(grad_norm.float().item()) + + def state_dict(self) -> dict[str, Any]: + return self.optimizer.state_dict() + + def load_state_dict(self, state_dict: dict[str, Any]) -> None: + self.optimizer.load_state_dict(state_dict) + + def offload_state_to_cpu(self) -> None: + move_optimizer_state_to_cpu( + self.optimizer, self._cpu_offloaded_state, include_dtensor_state=True + ) + + def load_state_to_device(self) -> None: + move_offloaded_optimizer_state_to_device(self.optimizer, self._cpu_offloaded_state) + + +def build_fsdp2_adamw( + model_chunks: list[nn.Module], + opt, + ps: ParallelState, + *, + replicated_grad_params: Iterable[nn.Parameter] | None = None, + replicated_grad_sync_group: dist.ProcessGroup | None = None, + replicated_grad_sync_divisor: float | None = None, + replicated_grad_norm_group: dist.ProcessGroup | None = None, + expert_sharded_grad_params: Iterable[nn.Parameter] | None = None, + expert_sharded_grad_scale: float | None = None, + expert_sharded_grad_norm_group: dist.ProcessGroup | None = None, + tp_replicated_grad_params: Iterable[nn.Parameter] | None = None, + tp_replicated_grad_sync_group: dist.ProcessGroup | None = None, + grad_norm_accum_dtype: str | torch.dtype = torch.float32, + adamw_foreach: bool | str = "auto", + use_fp32_master: bool = False, + model_param_dtypes: dict[tuple[int, str], torch.dtype] | None = None, +) -> FSDP2Optimizer: + """Build AdamW from Megatron Lite's shared OptimizerConfig-like object.""" + + optimizer_name = getattr(opt, "optimizer", "adam") + if optimizer_name not in {"adam", "adamw"}: + raise ValueError(f"fsdp2 supports adam/adamw, got {optimizer_name!r}.") + + params, param_groups, param_names, param_model_dtypes = _build_adamw_param_groups( + model_chunks, + weight_decay=float(getattr(opt, "weight_decay", 0.01)), + apply_wd_to_qk_layernorm=bool(getattr(opt, "apply_wd_to_qk_layernorm", False)), + model_param_dtypes=model_param_dtypes, + ) + beta1 = getattr(opt, "adam_beta1", None) + beta2 = getattr(opt, "adam_beta2", None) + eps = getattr(opt, "adam_eps", None) + offload_fraction = getattr(opt, "offload_fraction", None) or 0.0 + optimizer = build_adamw_optimizer( + param_groups, + all_params=params, + lr=float(getattr(opt, "lr", 1.0e-4)), + weight_decay=float(getattr(opt, "weight_decay", 0.01)), + betas=(0.9 if beta1 is None else beta1, 0.999 if beta2 is None else beta2), + eps=1.0e-8 if eps is None else eps, + foreach=adamw_foreach, + use_fp32_master=use_fp32_master, + cpu_update=use_fp32_master and float(offload_fraction) > 0.0, + model_param_dtypes=param_model_dtypes, + opt=opt, + ) + return FSDP2Optimizer( + optimizer, + params, + ps, + clip_grad=float(getattr(opt, "clip_grad", 1.0)), + replicated_grad_params=replicated_grad_params, + replicated_grad_sync_group=replicated_grad_sync_group, + replicated_grad_sync_divisor=replicated_grad_sync_divisor, + replicated_grad_norm_group=replicated_grad_norm_group, + expert_sharded_grad_params=expert_sharded_grad_params, + expert_sharded_grad_scale=expert_sharded_grad_scale, + expert_sharded_grad_norm_group=expert_sharded_grad_norm_group, + tp_replicated_grad_params=tp_replicated_grad_params, + tp_replicated_grad_sync_group=tp_replicated_grad_sync_group, + grad_norm_accum_dtype=grad_norm_accum_dtype, + param_names=param_names, + ) + + +def build_fsdp2_training_optimizer( + model_chunks: list[nn.Module], + opt, + ps: ParallelState, + *, + unit_modules: tuple[type[nn.Module] | str, ...], + expert_classifier: Callable[[str], bool] | None = None, + expert_module_leaf_name: str = "experts", + deterministic: bool | None = None, + vpp: int | None = 1, + leaf_module_names: Iterable[str] = _DEFAULT_LEAF_MODULE_NAMES, + reshard_after_forward: bool | int | None = _DEFAULT_RESHARD_AFTER_FORWARD, + wrap_root: bool = _DEFAULT_WRAP_ROOT, + forward_prefetch_depth: int = _DEFAULT_FORWARD_PREFETCH_DEPTH, + backward_prefetch_depth: int = _DEFAULT_BACKWARD_PREFETCH_DEPTH, + param_dtype: str | torch.dtype | None = _DEFAULT_PARAM_DTYPE, + reduce_dtype: str | torch.dtype | None = _DEFAULT_REDUCE_DTYPE, + use_fp32_shards: bool | None = None, + use_fp32_master: bool | None = None, + adamw_foreach: bool | str = _DEFAULT_ADAMW_FOREACH, +) -> FSDP2Optimizer: + """Wrap model chunks with FSDP2 and build the matching AdamW adapter.""" + + if (vpp or 1) > 1 and ps.pp_size <= 1: + raise ValueError("optimizer='fsdp2' requires pp>1 when vpp>1.") + + expert_params = _collect_expert_params(model_chunks, ps, expert_classifier) + expert_modules = _collect_expert_modules( + model_chunks, ps, expert_classifier, expert_module_leaf_name=expert_module_leaf_name + ) + if expert_params and not expert_modules: + raise RuntimeError("FSDP2 expert parameters were found but no expert module was found.") + + if opt is None: + opt = SimpleNamespace( + optimizer="adam", + lr=1e-4, + weight_decay=0.01, + clip_grad=1.0, + adam_beta1=None, + adam_beta2=None, + adam_eps=None, + ) + if deterministic is None: + from megatron.lite.primitive.deterministic import deterministic_requested + + deterministic = deterministic_requested() + effective_use_fp32_shards = ( + bool(use_fp32_shards) + if use_fp32_shards is not None + else get_bool_opt(opt, "fsdp2_use_fp32_shards", default=_DEFAULT_USE_FP32_SHARDS) + ) + effective_use_fp32_master = ( + bool(use_fp32_master) + if use_fp32_master is not None + else get_bool_opt(opt, "fsdp2_use_fp32_master", default=_DEFAULT_USE_FP32_MASTER) + ) + + unit_reshard_after_forward = _fsdp2_unit_reshard_after_forward( + ps, reshard_after_forward=reshard_after_forward + ) + fsdp2_config = FSDP2Config( + unit_modules=unit_modules, + leaf_module_names=tuple(leaf_module_names), + reshard_after_forward=unit_reshard_after_forward, + last_unit_reshard_after_forward=unit_reshard_after_forward, + root_reshard_after_forward=False, + wrap_root=wrap_root, + forward_prefetch_depth=_fsdp2_prefetch_depth(ps, default_depth=forward_prefetch_depth), + backward_prefetch_depth=_fsdp2_prefetch_depth(ps, default_depth=backward_prefetch_depth), + param_dtype=param_dtype, + reduce_dtype=reduce_dtype, + ) + if effective_use_fp32_shards: + model_param_dtypes = _collect_model_param_dtypes(model_chunks) + for chunk in model_chunks: + promote_fsdp2_trainable_params_to_fp32(chunk) + else: + model_param_dtypes = {} + + tp_replicated_grad_param_names = _collect_tp_replicated_grad_param_names(model_chunks) + + dense_shard_placement_fn = build_fsdp2_shard_placement_fn(ps.dp_cp_size) + expert_shard_placement_fn = build_fsdp2_shard_placement_fn(ps.expert_dp_size) + + if expert_modules: + if ps.ep_dp_group is None: + raise RuntimeError("FSDP2 expert sharding requires ParallelState.ep_dp_group.") + expert_mesh = build_fsdp2_process_group_mesh( + ps.ep_dp_group, mesh_dim_name="expert_dp", device_type=fsdp2_config.device_type + ) + for module in expert_modules: + wrap_fsdp2_module( + module, + ps, + fsdp2_config, + mesh=expert_mesh, + shard_placement_fn=expert_shard_placement_fn, + reshard_after_forward=unit_reshard_after_forward, + ) + + ignored_expert_params = _collect_module_params(expert_modules) + for chunk in model_chunks: + wrap_fsdp2( + chunk, + ps, + fsdp2_config, + ignored_params=ignored_expert_params or None, + shard_placement_fn=dense_shard_placement_fn, + ) + if model_param_dtypes: + _restore_model_param_dtypes(model_chunks, model_param_dtypes) + + tp_replicated_grad_params = _collect_tp_replicated_grad_params( + model_chunks, param_names=tp_replicated_grad_param_names + ) + return build_fsdp2_adamw( + model_chunks, + opt, + ps, + expert_sharded_grad_params=list(ignored_expert_params), + expert_sharded_grad_scale=( + float(ps.expert_dp_size) / float(ps.dp_cp_size) if ignored_expert_params else None + ), + expert_sharded_grad_norm_group=ps.ep_group if ignored_expert_params else None, + tp_replicated_grad_params=tp_replicated_grad_params, + tp_replicated_grad_sync_group=ps.tp_group if tp_replicated_grad_params else None, + grad_norm_accum_dtype="float32", + adamw_foreach=False if deterministic else adamw_foreach, + use_fp32_master=effective_use_fp32_master, + model_param_dtypes=model_param_dtypes, + ) + + +def _collect_expert_params( + chunks: Iterable[nn.Module], ps: ParallelState, expert_classifier: Callable[[str], bool] | None +) -> set[nn.Parameter]: + if ps.ep_size <= 1 or expert_classifier is None: + return set() + return { + param + for chunk in chunks + for name, param in chunk.named_parameters() + if expert_classifier(name) + } + + +def _collect_expert_modules( + chunks: Iterable[nn.Module], + ps: ParallelState, + expert_classifier: Callable[[str], bool] | None, + *, + expert_module_leaf_name: str, +) -> list[nn.Module]: + if ps.ep_size <= 1 or expert_classifier is None: + return [] + modules: list[nn.Module] = [] + seen: set[int] = set() + for chunk in chunks: + for module_name, module in chunk.named_modules(): + if module_name.rsplit(".", 1)[-1] != expert_module_leaf_name: + continue + prefix = f"{module_name}." + if not any( + expert_classifier(prefix + name) + for name, _param in module.named_parameters(recurse=True) + ): + continue + module_id = id(module) + if module_id not in seen: + modules.append(module) + seen.add(module_id) + return modules + + +def _collect_module_params(modules: Iterable[nn.Module]) -> set[nn.Parameter]: + return {param for module in modules for param in module.parameters()} + + +def _collect_model_param_dtypes(chunks: Iterable[nn.Module]) -> dict[tuple[int, str], torch.dtype]: + return { + (chunk_idx, name): param.dtype + for chunk_idx, chunk in enumerate(chunks) + for name, param in chunk.named_parameters() + if param.requires_grad and param.is_floating_point() and param.dtype != torch.float32 + } + + +def _restore_model_param_dtypes( + chunks: Iterable[nn.Module], model_param_dtypes: dict[tuple[int, str], torch.dtype] +) -> None: + for chunk_idx, chunk in enumerate(chunks): + for name, param in chunk.named_parameters(): + model_dtype = model_param_dtypes.get((chunk_idx, name)) + if model_dtype is not None: + param._fsdp2_model_param_dtype = model_dtype + + +def _collect_tp_replicated_grad_param_names(chunks: Iterable[nn.Module]) -> list[tuple[int, str]]: + names: list[tuple[int, str]] = [] + for chunk_idx, chunk in enumerate(chunks): + sp_param_ids = {id(param) for param in getattr(chunk, "sp_params", ())} + if not sp_param_ids: + continue + for name, param in chunk.named_parameters(): + if id(param) in sp_param_ids: + names.append((chunk_idx, name)) + return names + + +def _collect_tp_replicated_grad_params( + chunks: Iterable[nn.Module], *, param_names: Iterable[tuple[int, str]] | None = None +) -> list[nn.Parameter]: + chunk_list = list(chunks) + params: list[nn.Parameter] = [] + seen: set[int] = set() + if param_names is not None: + named_by_chunk = [dict(chunk.named_parameters()) for chunk in chunk_list] + for chunk_idx, name in param_names: + if chunk_idx >= len(named_by_chunk): + continue + param = named_by_chunk[chunk_idx].get(name) + if param is None or not param.requires_grad or id(param) in seen: + continue + params.append(param) + seen.add(id(param)) + return params + for chunk in chunk_list: + for param in getattr(chunk, "sp_params", ()): + if not param.requires_grad or id(param) in seen: + continue + params.append(param) + seen.add(id(param)) + return params + + +def _fsdp2_unit_reshard_after_forward( + ps: ParallelState, *, reshard_after_forward: bool | int | None +) -> bool | int | None: + if ps.pp_size > 1: + return False + return reshard_after_forward + + +def _fsdp2_prefetch_depth(ps: ParallelState, *, default_depth: int) -> int: + if ps.pp_size > 1: + return 0 + return default_depth + + +def _build_adamw_param_groups( + model_chunks: Iterable[nn.Module], + *, + weight_decay: float, + apply_wd_to_qk_layernorm: bool, + model_param_dtypes: dict[tuple[int, str], torch.dtype] | None = None, +) -> tuple[list[nn.Parameter], list[dict[str, Any]], dict[int, str], dict[int, torch.dtype]]: + params: list[nn.Parameter] = [] + decay_params: list[nn.Parameter] = [] + no_decay_params: list[nn.Parameter] = [] + param_names: dict[int, str] = {} + param_model_dtypes: dict[int, torch.dtype] = {} + seen_param_ids: set[int] = set() + + for chunk_idx, chunk in enumerate(model_chunks): + for name, param in chunk.named_parameters(): + if not param.requires_grad or id(param) in seen_param_ids: + continue + seen_param_ids.add(id(param)) + params.append(param) + param_names[id(param)] = f"chunk{chunk_idx}.{name}" + model_dtype = None + if model_param_dtypes is not None: + model_dtype = model_param_dtypes.get((chunk_idx, name)) + if model_dtype is None: + model_dtype = fsdp2_model_param_dtype(param) + if model_dtype is not None: + param_model_dtypes[id(param)] = model_dtype + if _matches_megatron_no_weight_decay(name, param, apply_wd_to_qk_layernorm): + no_decay_params.append(param) + else: + decay_params.append(param) + + # Match Megatron's default get_megatron_optimizer(config_overrides=None): + # 1D params and bias skip AdamW decay unless the Q/K layernorm override is set. + param_groups: list[dict[str, Any]] = [] + if decay_params: + param_groups.append({"params": decay_params, "weight_decay": weight_decay, "wd_mult": 1.0}) + if no_decay_params: + param_groups.append({"params": no_decay_params, "weight_decay": 0.0, "wd_mult": 0.0}) + return params, param_groups, param_names, param_model_dtypes + + +def _matches_megatron_no_weight_decay( + name: str, param: nn.Parameter, apply_wd_to_qk_layernorm: bool +) -> bool: + if len(param.shape) != 1 and not name.endswith(".bias"): + return False + if not apply_wd_to_qk_layernorm: + return True + return not ( + "q_layernorm." in name or "k_layernorm." in name or "q_norm." in name or "k_norm." in name + ) + + +@dataclass(frozen=True, slots=True) +class FSDP2OptimizerBackend: + name: str = "fsdp2" + runtime_backend: str = "fsdp2" + + def zero_grad(self, optimizer: FSDP2Optimizer) -> None: + optimizer.zero_grad() + + def finish_grad_sync(self, optimizer: FSDP2Optimizer) -> None: + return None + + def clip_grad_norm(self, optimizer: FSDP2Optimizer): + return optimizer.clip_grad_norm() + + def step(self, optimizer: FSDP2Optimizer): + return optimizer.step() + + def state_dict(self, optimizer: FSDP2Optimizer) -> dict: + return optimizer.state_dict() + + def load_state_dict(self, optimizer: FSDP2Optimizer, state_dict: dict) -> None: + optimizer.load_state_dict(state_dict) + + +BACKEND = FSDP2OptimizerBackend() + +__all__ = [ + "BACKEND", + "FSDP2OptimizerBackend", + "FSDP2Optimizer", + "build_fsdp2_adamw", + "build_fsdp2_training_optimizer", +] diff --git a/experimental/lite/megatron/lite/primitive/optimizers/fsdp2/state.py b/experimental/lite/megatron/lite/primitive/optimizers/fsdp2/state.py new file mode 100644 index 00000000000..b8efb65d0fc --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/optimizers/fsdp2/state.py @@ -0,0 +1,106 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Optimizer state movement helpers for the FSDP2 primitive.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import torch + +from megatron.lite.primitive.optimizers.fsdp2.adamw import ( + dtensor_from_local, + is_dtensor_like, + iter_torch_optimizers, +) + + +@dataclass +class OffloadedStateEntry: + device: torch.device + is_dtensor: bool = False + device_mesh: Any | None = None + placements: Any | None = None + # Global shape/stride captured so an unevenly-sharded DTensor reconstructs + # exactly on reload (from_local would otherwise infer local_shard * mesh). + global_shape: Any | None = None + global_stride: Any | None = None + + +def move_optimizer_state_to_cpu( + optimizer: Any, + offloaded: dict[tuple[int, str], OffloadedStateEntry], + *, + include_dtensor_state: bool, +) -> None: + for child in iter_torch_optimizers(optimizer): + state = getattr(child, "state", None) + if not isinstance(state, dict): + continue + for param, param_state in state.items(): + if not isinstance(param_state, dict): + continue + for key, value in list(param_state.items()): + if is_dtensor_like(value): + if not include_dtensor_state: + continue + local_value = value.to_local() + if not isinstance(local_value, torch.Tensor) or not local_value.is_cuda: + continue + offloaded[(id(param), key)] = OffloadedStateEntry( + device=local_value.device, + is_dtensor=True, + device_mesh=value.device_mesh, + placements=value.placements, + global_shape=tuple(value.shape), + global_stride=tuple(value.stride()), + ) + param_state[key] = local_value.detach().to("cpu") + continue + + if not isinstance(value, torch.Tensor) or not value.is_cuda: + continue + offloaded[(id(param), key)] = OffloadedStateEntry(device=value.device) + param_state[key] = value.detach().to("cpu") + + +def move_offloaded_optimizer_state_to_device( + optimizer: Any, offloaded: dict[tuple[int, str], OffloadedStateEntry] +) -> None: + if not offloaded: + return + remaining = dict(offloaded) + for child in iter_torch_optimizers(optimizer): + state = getattr(child, "state", None) + if not isinstance(state, dict): + continue + for param, param_state in state.items(): + if not isinstance(param_state, dict): + continue + for key, entry in list(remaining.items()): + param_id, state_key = key + if param_id != id(param) or state_key not in param_state: + continue + value = param_state[state_key] + if isinstance(value, torch.Tensor) and not is_dtensor_like(value): + device_value = value.to(entry.device, non_blocking=True) + if entry.is_dtensor: + param_state[state_key] = dtensor_from_local( + device_value, + entry.device_mesh, + entry.placements, + shape=entry.global_shape, + stride=entry.global_stride, + ) + else: + param_state[state_key] = device_value + remaining.pop(key, None) + for key in set(offloaded) - set(remaining): + offloaded.pop(key, None) + + +__all__ = [ + "OffloadedStateEntry", + "move_offloaded_optimizer_state_to_device", + "move_optimizer_state_to_cpu", +] diff --git a/experimental/lite/megatron/lite/primitive/optimizers/fsdp2/wrap.py b/experimental/lite/megatron/lite/primitive/optimizers/fsdp2/wrap.py new file mode 100644 index 00000000000..149920d8a4a --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/optimizers/fsdp2/wrap.py @@ -0,0 +1,498 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""PyTorch FSDP2 wrapping primitive. + +This module is intentionally independent from Megatron Lite model and runtime +packages. Model protocols can call it after building modules and before +building the optimizer. +""" + +from __future__ import annotations + +import importlib +import warnings +from collections.abc import Callable, Iterable +from dataclasses import dataclass, field +from typing import Any + +import torch +import torch.distributed as dist +import torch.nn as nn + +from megatron.lite.primitive.parallel.state import ParallelState + +UnitModule = type[nn.Module] | str + +_WARNED_TP_NOT_RECOMMENDED = False + + +@dataclass(frozen=True) +class FSDP2Config: + """Configuration for ``wrap_fsdp2``.""" + + unit_modules: tuple[UnitModule, ...] = field(default_factory=tuple) + leaf_module_names: tuple[str, ...] = field(default_factory=tuple) + reshard_after_forward: bool | int | None = None + last_unit_reshard_after_forward: bool | int | None = False + root_reshard_after_forward: bool | int | None = False + wrap_root: bool = True + preserve_param_attrs: bool = True + forward_prefetch_depth: int = 1 + backward_prefetch_depth: int = 0 + mesh_dim_name: str = "dp_cp" + device_type: str = "cuda" + param_dtype: str | torch.dtype | None = None + reduce_dtype: str | torch.dtype | None = None + output_dtype: str | torch.dtype | None = None + cast_forward_inputs: bool | None = True + + def __post_init__(self) -> None: + object.__setattr__(self, "unit_modules", tuple(self.unit_modules)) + object.__setattr__(self, "leaf_module_names", tuple(self.leaf_module_names)) + if not self.wrap_root and not self.unit_modules and not self.leaf_module_names: + raise ValueError( + "FSDP2Config requires wrap_root=True, at least one unit module, " + "or at least one leaf module name." + ) + for name in self.leaf_module_names: + if not isinstance(name, str) or not name: + raise ValueError("leaf_module_names entries must be non-empty strings.") + if not self.mesh_dim_name: + raise ValueError("mesh_dim_name must be non-empty.") + if not self.device_type: + raise ValueError("device_type must be non-empty.") + if self.forward_prefetch_depth < 0: + raise ValueError("forward_prefetch_depth must be >= 0.") + if self.backward_prefetch_depth < 0: + raise ValueError("backward_prefetch_depth must be >= 0.") + + +def fsdp2_available() -> bool: + """Return whether the installed PyTorch exposes FSDP2 ``fully_shard``.""" + + try: + from torch.distributed import DeviceMesh # noqa: F401 + from torch.distributed.fsdp import fully_shard # noqa: F401 + except ImportError: + return False + return True + + +def build_fsdp2_device_mesh(ps: ParallelState, config: FSDP2Config | None = None) -> Any: + """Build the default one-dimensional FSDP2 DeviceMesh from ``ParallelState``.""" + + cfg = config or FSDP2Config() + if not dist.is_initialized(): + raise RuntimeError("FSDP2 requires torch.distributed to be initialized.") + + group = ps.dp_cp_group or ps.dp_group + if group is None: + raise RuntimeError("FSDP2 requires ParallelState.dp_cp_group or dp_group.") + + from torch.distributed import DeviceMesh + + return DeviceMesh.from_group( + group, device_type=cfg.device_type, mesh_dim_names=(cfg.mesh_dim_name,) + ) + + +def build_fsdp2_process_group_mesh( + group: dist.ProcessGroup, *, mesh_dim_name: str, device_type: str = "cuda" +) -> Any: + """Build a one-dimensional FSDP2 DeviceMesh from an explicit process group.""" + + if not dist.is_initialized(): + raise RuntimeError("FSDP2 requires torch.distributed to be initialized.") + if group is None: + raise RuntimeError("FSDP2 requires a non-null process group.") + if not mesh_dim_name: + raise ValueError("mesh_dim_name must be non-empty.") + if not device_type: + raise ValueError("device_type must be non-empty.") + + from torch.distributed import DeviceMesh + + return DeviceMesh.from_group(group, device_type=device_type, mesh_dim_names=(mesh_dim_name,)) + + +def build_fsdp2_shard_placement_fn(fsdp_size: int) -> Callable[[nn.Parameter], Any]: + """Build AutoModel-style FSDP2 shard placement. + + Choose the first tensor dimension divisible by the FSDP group size to avoid + padded DTensor shards when possible. + """ + + if fsdp_size <= 0: + raise ValueError(f"fsdp_size must be positive, got {fsdp_size}.") + + def shard_placement_fn(param: nn.Parameter) -> Any: + from torch.distributed.tensor import Shard + + for dim, size in enumerate(param.shape): + if int(size) % fsdp_size == 0: + return Shard(dim) + return Shard(0) + + return shard_placement_fn + + +def wrap_fsdp2( + model: nn.Module, + ps: ParallelState, + config: FSDP2Config | None = None, + *, + mesh: Any | None = None, + ignored_params: set[nn.Parameter] | None = None, + mp_policy: Any | None = None, + offload_policy: Any | None = None, + shard_placement_fn: Callable[[nn.Parameter], Any] | None = None, +) -> nn.Module: + """Apply PyTorch FSDP2 ``fully_shard`` to selected modules and the root. + + The model is mutated in place and returned for call-site convenience. + ``wrap_fsdp2`` must run before constructing a regular PyTorch optimizer. + """ + + cfg = config or FSDP2Config() + _warn_tp_not_recommended(ps) + fully_shard = _load_fully_shard() + fsdp_mesh = mesh if mesh is not None else build_fsdp2_device_mesh(ps, cfg) + unit_types = _resolve_unit_module_types(cfg.unit_modules) + + saved_attrs = _save_param_attrs(model) if cfg.preserve_param_attrs else {} + common_kwargs = _fully_shard_kwargs( + mesh=fsdp_mesh, + reshard_after_forward=None, + ignored_params=ignored_params, + mp_policy=mp_policy or _mixed_precision_policy_from_config(cfg), + offload_policy=offload_policy, + shard_placement_fn=shard_placement_fn, + ) + + wrapped_units: list[nn.Module] = [] + unit_modules = list(_iter_fsdp2_unit_modules(model, unit_types, cfg.leaf_module_names)) + for idx, sub_module in enumerate(unit_modules): + kwargs = dict(common_kwargs) + _set_optional_reshard_after_forward( + kwargs, _unit_reshard_after_forward(cfg, idx, len(unit_modules)) + ) + fully_shard(sub_module, **kwargs) + wrapped_units.append(sub_module) + + if cfg.wrap_root: + kwargs = dict(common_kwargs) + _set_optional_reshard_after_forward(kwargs, cfg.root_reshard_after_forward) + fully_shard(model, **kwargs) + + _apply_fsdp2_prefetch( + model, + wrapped_units, + forward_depth=cfg.forward_prefetch_depth, + backward_depth=cfg.backward_prefetch_depth, + ) + if saved_attrs: + _restore_param_attrs(model, saved_attrs) + return model + + +def wrap_fsdp2_module( + module: nn.Module, + ps: ParallelState, + config: FSDP2Config | None = None, + *, + mesh: Any | None = None, + ignored_params: set[nn.Parameter] | None = None, + mp_policy: Any | None = None, + offload_policy: Any | None = None, + shard_placement_fn: Callable[[nn.Parameter], Any] | None = None, + reshard_after_forward: bool | int | None = None, +) -> nn.Module: + """Apply FSDP2 ``fully_shard`` to exactly one module. + + This is used for nested modules that need a different sharding mesh from + their parent, e.g. EP-local MoE experts sharded over expert-DP while the + transformer block is sharded over dense DP/CP. + """ + + cfg = config or FSDP2Config() + _warn_tp_not_recommended(ps) + fully_shard = _load_fully_shard() + fsdp_mesh = mesh if mesh is not None else build_fsdp2_device_mesh(ps, cfg) + + saved_attrs = _save_param_attrs(module) if cfg.preserve_param_attrs else {} + kwargs = _fully_shard_kwargs( + mesh=fsdp_mesh, + reshard_after_forward=( + cfg.root_reshard_after_forward + if reshard_after_forward is None + else reshard_after_forward + ), + ignored_params=ignored_params, + mp_policy=mp_policy or _mixed_precision_policy_from_config(cfg), + offload_policy=offload_policy, + shard_placement_fn=shard_placement_fn, + ) + fully_shard(module, **kwargs) + + if saved_attrs: + _restore_param_attrs(module, saved_attrs) + return module + + +def _warn_tp_not_recommended(ps: ParallelState) -> None: + global _WARNED_TP_NOT_RECOMMENDED + if _WARNED_TP_NOT_RECOMMENDED or ps.tp_size <= 1: + return + if dist.is_available() and dist.is_initialized() and dist.get_rank() != 0: + return + _WARNED_TP_NOT_RECOMMENDED = True + warnings.warn( + f"FSDP2 with tp={ps.tp_size} is supported, but TP is not recommended " + "for FSDP2 V1; prefer tp=1 etp=1 for precision and speed signoff.", + RuntimeWarning, + stacklevel=3, + ) + + +def promote_fsdp2_trainable_params_to_fp32( + model: nn.Module, *, ignored_params: set[nn.Parameter] | None = None +) -> int: + """Promote FSDP2-owned trainable floating parameters to FP32 shards. + + Model protocols still use ``FSDP2Config.param_dtype`` to run compute in + BF16. Keeping the sharded parameters in FP32 makes the torch optimizer path + closer to MCore dist-opt's main-param semantics and avoids BF16 grad-norm + and update drift before FSDP2 wrapping. + """ + + ignored_param_ids = {id(param) for param in ignored_params or ()} + promoted = 0 + with torch.no_grad(): + for param in model.parameters(): + if id(param) in ignored_param_ids: + continue + if not param.requires_grad or not param.is_floating_point(): + continue + if param.dtype == torch.float32: + continue + param._fsdp2_model_param_dtype = param.dtype + param.data = param.data.to(torch.float32) + if param.grad is not None: + param.grad = param.grad.to(torch.float32) + promoted += 1 + return promoted + + +def set_fsdp2_requires_gradient_sync( + module: nn.Module, requires_gradient_sync: bool, *, recurse: bool = True +) -> int: + """Set FSDP2 gradient sync state and return the number of touched roots.""" + + setter = getattr(module, "set_requires_gradient_sync", None) + if callable(setter): + setter(requires_gradient_sync, recurse=recurse) + return 1 + if not recurse: + return 0 + + touched = 0 + for child in module.modules(): + if child is module: + continue + setter = getattr(child, "set_requires_gradient_sync", None) + if callable(setter): + setter(requires_gradient_sync, recurse=False) + touched += 1 + return touched + + +def _load_fully_shard(): + try: + from torch.distributed.fsdp import fully_shard + except ImportError as exc: + raise RuntimeError( + "PyTorch FSDP2 is unavailable; install a PyTorch build with " + "torch.distributed.fsdp.fully_shard." + ) from exc + return fully_shard + + +def _resolve_unit_module_types(unit_modules: Iterable[UnitModule]) -> tuple[type[nn.Module], ...]: + resolved: list[type[nn.Module]] = [] + for item in unit_modules: + if isinstance(item, str): + item = _import_module_type(item) + if not isinstance(item, type) or not issubclass(item, nn.Module): + raise TypeError( + "FSDP2 unit_modules entries must be nn.Module subclasses or import paths." + ) + resolved.append(item) + return tuple(resolved) + + +def _import_module_type(path: str) -> type[nn.Module]: + module_name, sep, attr_name = path.rpartition(".") + if not sep or not module_name or not attr_name: + raise ValueError(f"Invalid FSDP2 unit module path: {path!r}") + module = importlib.import_module(module_name) + obj = getattr(module, attr_name) + if not isinstance(obj, type) or not issubclass(obj, nn.Module): + raise TypeError(f"FSDP2 unit module path does not resolve to nn.Module: {path!r}") + return obj + + +def _iter_fsdp2_unit_modules( + root: nn.Module, unit_types: tuple[type[nn.Module], ...], leaf_module_names: tuple[str, ...] +) -> Iterable[nn.Module]: + leaf_names = set(leaf_module_names) + if not unit_types and not leaf_names: + return () + ordered_units: list[nn.Module] = [] + seen: set[int] = set() + + def visit(module: nn.Module, module_name: str) -> None: + leaf_name = module_name.rsplit(".", 1)[-1] if module_name else "" + selected = module is not root and ( + isinstance(module, unit_types) or leaf_name in leaf_names + ) + if selected and _module_has_trainable_param(module): + module_id = id(module) + if module_id not in seen: + ordered_units.append(module) + seen.add(module_id) + return + for child_name, child in _iter_ordered_named_children(module): + full_name = child_name if not module_name else f"{module_name}.{child_name}" + visit(child, full_name) + + visit(root, "") + return tuple(ordered_units) + + +def _iter_ordered_named_children(module: nn.Module) -> Iterable[tuple[str, nn.Module]]: + if isinstance(module, nn.ModuleDict): + return module.items() + if isinstance(module, nn.ModuleList): + return ((str(idx), module[idx]) for idx in range(len(module))) + return module.named_children() + + +def _module_has_trainable_param(module: nn.Module) -> bool: + return any(param.requires_grad for param in module.parameters()) + + +def _is_fsdp2_module(module: nn.Module) -> bool: + return hasattr(module, "set_modules_to_forward_prefetch") and hasattr( + module, "set_modules_to_backward_prefetch" + ) + + +def _apply_fsdp2_prefetch( + root: nn.Module, wrapped_units: list[nn.Module], *, forward_depth: int, backward_depth: int +) -> None: + fsdp_units = [module for module in wrapped_units if _is_fsdp2_module(module)] + fsdp_root = root if _is_fsdp2_module(root) else None + + if fsdp_units and forward_depth > 0: + if fsdp_root is not None: + fsdp_root.set_modules_to_forward_prefetch(fsdp_units[:forward_depth]) + for idx, current in enumerate(fsdp_units[:-1]): + targets = fsdp_units[idx + 1 : idx + 1 + forward_depth] + if targets: + current.set_modules_to_forward_prefetch(targets) + + if len(fsdp_units) > 1 and backward_depth > 0: + for idx in range(1, len(fsdp_units)): + start = max(0, idx - backward_depth) + targets = list(reversed(fsdp_units[start:idx])) + if targets: + fsdp_units[idx].set_modules_to_backward_prefetch(targets) + + +def _unit_reshard_after_forward(cfg: FSDP2Config, idx: int, total_units: int) -> bool | int | None: + if total_units > 0 and idx == total_units - 1: + return cfg.last_unit_reshard_after_forward + return cfg.reshard_after_forward + + +def _set_optional_reshard_after_forward( + kwargs: dict[str, Any], reshard_after_forward: bool | int | None +) -> None: + if reshard_after_forward is not None: + kwargs["reshard_after_forward"] = reshard_after_forward + + +def _fully_shard_kwargs( + *, + mesh: Any, + reshard_after_forward: bool | int | None, + ignored_params: set[nn.Parameter] | None, + mp_policy: Any | None, + offload_policy: Any | None, + shard_placement_fn: Callable[[nn.Parameter], Any] | None, +) -> dict[str, Any]: + kwargs: dict[str, Any] = {"mesh": mesh} + if reshard_after_forward is not None: + kwargs["reshard_after_forward"] = reshard_after_forward + if ignored_params is not None: + kwargs["ignored_params"] = ignored_params + if mp_policy is not None: + kwargs["mp_policy"] = mp_policy + if offload_policy is not None: + kwargs["offload_policy"] = offload_policy + if shard_placement_fn is not None: + kwargs["shard_placement_fn"] = shard_placement_fn + return kwargs + + +def _mixed_precision_policy_from_config(cfg: FSDP2Config) -> Any | None: + if cfg.param_dtype is None and cfg.reduce_dtype is None and cfg.output_dtype is None: + return None + try: + from torch.distributed.fsdp import MixedPrecisionPolicy + except ImportError as exc: + raise RuntimeError("FSDP2 mixed precision policy is unavailable.") from exc + kwargs = dict( + param_dtype=_resolve_torch_dtype(cfg.param_dtype), + reduce_dtype=_resolve_torch_dtype(cfg.reduce_dtype), + output_dtype=_resolve_torch_dtype(cfg.output_dtype), + ) + if cfg.cast_forward_inputs is not None: + kwargs["cast_forward_inputs"] = cfg.cast_forward_inputs + try: + return MixedPrecisionPolicy(**kwargs) + except TypeError: + kwargs.pop("cast_forward_inputs", None) + return MixedPrecisionPolicy(**kwargs) + + +def _resolve_torch_dtype(dtype: str | torch.dtype | None) -> torch.dtype | None: + if dtype is None or isinstance(dtype, torch.dtype): + return dtype + name = dtype.removeprefix("torch.") + resolved = getattr(torch, name, None) + if not isinstance(resolved, torch.dtype): + raise ValueError(f"Unsupported torch dtype for FSDP2 mixed precision: {dtype!r}") + return resolved + + +def _save_param_attrs(module: nn.Module) -> dict[str, dict[str, Any]]: + return {name: dict(vars(param)) for name, param in module.named_parameters()} + + +def _restore_param_attrs(module: nn.Module, saved_attrs: dict[str, dict[str, Any]]) -> None: + for name, param in module.named_parameters(): + for attr_name, attr_value in saved_attrs.get(name, {}).items(): + setattr(param, attr_name, attr_value) + + +__all__ = [ + "FSDP2Config", + "build_fsdp2_device_mesh", + "build_fsdp2_process_group_mesh", + "build_fsdp2_shard_placement_fn", + "fsdp2_available", + "promote_fsdp2_trainable_params_to_fp32", + "set_fsdp2_requires_gradient_sync", + "wrap_fsdp2", + "wrap_fsdp2_module", +] diff --git a/experimental/lite/megatron/lite/primitive/optimizers/megatron_wrap.py b/experimental/lite/megatron/lite/primitive/optimizers/megatron_wrap.py new file mode 100644 index 00000000000..8545a07d22d --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/optimizers/megatron_wrap.py @@ -0,0 +1,452 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Megatron-Core optimizer wrap backend for Megatron Lite.""" + +from __future__ import annotations + +from dataclasses import dataclass, fields +from types import SimpleNamespace +from typing import Any + +import torch # pyright: ignore[reportMissingImports] +import torch.nn as nn # pyright: ignore[reportMissingImports] + +from megatron.lite.primitive.protocols import ExpertClassifierFn, default_expert_classifier + + +def validate_dist_opt_config(engine_cfg) -> None: + """Validate dist_opt constraints owned by this optimizer primitive.""" + p = engine_cfg.parallel + if p.vpp > 1 and p.pp == 1: + raise ValueError("dist_opt requires pp>1 when vpp>1.") + + +validate_dist_opt_session = validate_dist_opt_config + + +def _effective_etp(parallel) -> int: + return int(parallel.etp if parallel.etp is not None else 1) + + +def _ensure_dist_opt_mpu_parallel_state(engine_cfg) -> None: + """Initialize Megatron-Core mpu globals when dist_opt fallback groups are used.""" + + from megatron.core import parallel_state as mpu # pyright: ignore[reportMissingImports] + + p = engine_cfg.parallel + expected = (int(p.tp), int(p.ep), _effective_etp(p), int(p.pp), int(p.cp)) + if mpu.is_initialized(): + current = ( + int(mpu.get_tensor_model_parallel_world_size()), + int(mpu.get_expert_model_parallel_world_size()), + int(mpu.get_expert_tensor_parallel_world_size() or 1), + int(mpu.get_pipeline_model_parallel_world_size()), + int(mpu.get_context_parallel_world_size()), + ) + if current != expected: + raise RuntimeError( + "dist_opt found an incompatible existing Megatron-Core parallel state: " + f"current={current}, expected={expected}." + ) + return + + mpu.initialize_model_parallel( + tensor_model_parallel_size=p.tp, + pipeline_model_parallel_size=p.pp, + virtual_pipeline_model_parallel_size=None if int(p.vpp or 1) <= 1 else p.vpp, + context_parallel_size=p.cp, + expert_model_parallel_size=p.ep, + expert_tensor_parallel_size=_effective_etp(p), + create_gloo_process_groups=bool(getattr(engine_cfg, "deterministic", False)), + ) + + +def build_dist_opt_optimizer_config( + opt, *, override_optimizer_config: dict[str, Any] | None = None +): + """Build Megatron-Core OptimizerConfig from user's OptimizerConfig (duck-typed). + + Single source of truth for Megatron Lite's Megatron-Core optimizer stack. + + Works on either `runtime.contracts.config.OptimizerConfig` (real dataclass) + or a `SimpleNamespace` with the same field names (legacy lite path). + """ + from megatron.core.optimizer.optimizer_config import ( + OptimizerConfig as CoreOptimizerConfig, # pyright: ignore[reportMissingImports] + ) + + offload = getattr(opt, "offload_fraction", None) or 0.0 + args: dict[str, Any] = { + "optimizer": opt.optimizer, + "lr": opt.lr, + "min_lr": getattr(opt, "min_lr", 0.0), + "weight_decay": opt.weight_decay, + "clip_grad": opt.clip_grad, + "use_distributed_optimizer": True, + "bf16": True, + "params_dtype": torch.bfloat16, + } + if offload > 0: + args["optimizer_offload_fraction"] = offload + args["overlap_cpu_optimizer_d2h_h2d"] = True + args["optimizer_cpu_offload"] = True + if getattr(opt, "adam_beta1", None) is not None: + args["adam_beta1"] = opt.adam_beta1 + if getattr(opt, "adam_beta2", None) is not None: + args["adam_beta2"] = opt.adam_beta2 + if getattr(opt, "adam_eps", None) is not None: + args["adam_eps"] = opt.adam_eps + if getattr(opt, "use_precision_aware_optimizer", None) is not None: + args["use_precision_aware_optimizer"] = opt.use_precision_aware_optimizer + if getattr(opt, "decoupled_weight_decay", None) is not None: + args["decoupled_weight_decay"] = opt.decoupled_weight_decay + if override_optimizer_config: + args.update(override_optimizer_config) + return CoreOptimizerConfig(**args) + + +def build_dist_opt_stack( + model_chunks: list[nn.Module], + *, + model_cfg, + engine_cfg, + ps, + is_expert: ExpertClassifierFn | None = None, + proto=None, + skip_ddp_wrap: bool = False, +): + """Wrap ML model chunks with Megatron-Core DDP and build the matching dist_opt optimizer. + + Args: + skip_ddp_wrap: when True, ``model_chunks`` are assumed to already be + Megatron-Core ``DistributedDataParallel``-wrapped; we skip our own wrapping + and feed them directly to the optimizer. The bucket layout + influences optimizer master-grad sharding, so callers that prewrap + chunks own the DDP config compatibility. + """ + from megatron.core.distributed import DistributedDataParallel, DistributedDataParallelConfig + from megatron.core.distributed.finalize_model_grads import finalize_model_grads + from megatron.core.optimizer import get_megatron_optimizer + from megatron.core.transformer.enums import ModelType + + validate_dist_opt_config(engine_cfg) + + p = engine_cfg.parallel + opt = engine_cfg.optimizer + + dist_opt_transformer_cfg = _build_transformer_config(model_cfg, engine_cfg) + dist_opt_transformer_cfg.finalize_model_grads_func = finalize_model_grads + if is_expert is not None: + is_expert_param = is_expert + elif proto is not None and hasattr(proto, "EXPERT_CLASSIFIER"): + is_expert_param = proto.EXPERT_CLASSIFIER + else: + is_expert_param = default_expert_classifier + use_mpu_groups = bool(getattr(engine_cfg, "deterministic", False)) + if use_mpu_groups: + _ensure_dist_opt_mpu_parallel_state(engine_cfg) + pg_collection = None if use_mpu_groups else _build_pg_collection(ps, engine_cfg) + + if skip_ddp_wrap: + # Caller already wrapped and marked every param. Our helper setting + # `param.allreduce` on dense params could clash with Megatron-Core code paths that + # distinguish `hasattr(param,'allreduce')` from `getattr(..., True)`. + wrapped_chunks = list(model_chunks) + else: + ddp_config = DistributedDataParallelConfig( + use_distributed_optimizer=True, overlap_grad_reduce=False, grad_reduce_in_fp32=True + ) + wrapped_chunks = [] + for chunk_idx, chunk in enumerate(model_chunks): + chunk.model_type = ModelType.encoder_or_decoder + _mark_dist_opt_parallel_attrs(chunk, is_expert_param, tp_size=p.tp) + ddp_kwargs = {} + if pg_collection is not None: + ddp_kwargs["pg_collection"] = pg_collection + wrapped_chunks.append( + DistributedDataParallel( + dist_opt_transformer_cfg, + ddp_config, + chunk, + disable_bucketing=(chunk_idx > 0), + **ddp_kwargs, + ) + ) + + # Single-source-of-truth OptimizerConfig construction for native lite + # model protocols. + opt_config = build_dist_opt_optimizer_config(opt) + + # This branch falls back to Megatron-Core mpu globals for the optimizer's process + # groups. Long term, this primitive should always pass its own + # `pg_collection`. + if skip_ddp_wrap or use_mpu_groups: + optimizer = get_megatron_optimizer(config=opt_config, model_chunks=wrapped_chunks) + optimizer._dist_opt_pg_collection = None # pyright: ignore[reportAttributeAccessIssue] + else: + optimizer = get_megatron_optimizer( + config=opt_config, + model_chunks=wrapped_chunks, + use_gloo_process_groups=False, + pg_collection=pg_collection, + ) + optimizer._dist_opt_pg_collection = ( + pg_collection # pyright: ignore[reportAttributeAccessIssue] + ) + return wrapped_chunks, optimizer + + +def build_dist_opt_training_optimizer( + model_chunks: list[nn.Module], + *, + model_cfg, + impl_cfg, + ps, + model_name: str, + is_expert: ExpertClassifierFn | None = None, + skip_ddp_wrap: bool = False, + deterministic: bool | None = None, +): + """Build the dist_opt DDP+optimizer stack from a Megatron Lite model ImplConfig.""" + + opt = impl_cfg.optimizer_config + if opt is None: + opt = SimpleNamespace( + optimizer="adam", + lr=1e-4, + weight_decay=0.01, + clip_grad=1.0, + offload_fraction=None, + adam_beta1=None, + adam_beta2=None, + adam_eps=None, + ) + if deterministic is None: + from megatron.lite.primitive.deterministic import deterministic_requested + + deterministic = deterministic_requested() + + engine_cfg = SimpleNamespace( + model_name=model_name, + parallel=impl_cfg.parallel, + optimizer=opt, + deterministic=bool(deterministic), + ) + model_chunks[:], optimizer = build_dist_opt_stack( + model_chunks, + model_cfg=model_cfg, + engine_cfg=engine_cfg, + ps=ps, + is_expert=is_expert, + skip_ddp_wrap=skip_ddp_wrap, + ) + + def finalize_grads() -> None: + finalize_dist_opt_grads(model_chunks, optimizer) + + return optimizer, finalize_grads + + +def finalize_dist_opt_grads(model_chunks: list[nn.Module], optimizer) -> None: + """Run Megatron-Core gradient finalization to match the optimizer's expected contract.""" + from megatron.core.distributed.finalize_model_grads import finalize_model_grads + + finalize_model_grads(model_chunks, pg_collection=optimizer._dist_opt_pg_collection) + + +def _build_transformer_config(model_cfg, engine_cfg): + from megatron.core.transformer.transformer_config import TransformerConfig + + p = engine_cfg.parallel + kwargs = dict( + num_layers=max(getattr(model_cfg, "num_hidden_layers", 1), 1), + hidden_size=max(getattr(model_cfg, "hidden_size", 1), 1), + num_attention_heads=max(getattr(model_cfg, "num_attention_heads", 1), 1), + num_query_groups=getattr(model_cfg, "num_key_value_heads", None), + num_moe_experts=getattr(model_cfg, "num_experts", None), + moe_ffn_hidden_size=getattr(model_cfg, "moe_intermediate_size", None), + tensor_model_parallel_size=p.tp, + pipeline_model_parallel_size=p.pp, + context_parallel_size=p.cp, + expert_model_parallel_size=p.ep, + expert_tensor_parallel_size=p.etp if p.etp is not None else 1, + sequence_parallel=p.tp > 1, + bf16=True, + params_dtype=torch.bfloat16, + ) + if hasattr(model_cfg, "add_bias_linear"): + kwargs["add_bias_linear"] = bool(model_cfg.add_bias_linear) + elif kwargs["num_moe_experts"] is not None: + kwargs["add_bias_linear"] = False + if p.pp > 1: + kwargs["pipeline_dtype"] = torch.bfloat16 + return TransformerConfig(**kwargs) + + +def _mark_dist_opt_parallel_attrs( + model: nn.Module, is_expert_param: ExpertClassifierFn, *, tp_size: int +) -> None: + """Mark per-param optimizer metadata (allreduce / tensor_model_parallel / sequence_parallel). + + IMPORTANT: respect attrs that are already set. Prewrapped Megatron-Core models may + mark these correctly per-param (e.g. `moe.router.weight` is 2D but + TP-replicated, and must NOT have `tensor_model_parallel=True`). Blind + override would cause dist_opt grad-norm to over-count replicated params. + """ + sp_param_ids = {id(param) for param in getattr(model, "sp_params", [])} + for name, param in model.named_parameters(): + # Megatron-Core uses `allreduce=False` to route expert params into expert-DP buffers. + if not hasattr(param, "allreduce"): + param.allreduce = not is_expert_param(name) + if tp_size > 1 and id(param) not in sp_param_ids and param.ndim > 1: + # vision params are replicated across TP (AVG all-reduce, not TP-split). + # tensor_model_parallel=True would cause dist_opt to wrong-account their grad-norm. + if getattr(param, "average_gradients_across_tp_domain", False): + continue + # Skip params already marked sequence_parallel=True: they are TP-replicated + # with SP-sharded input (e.g. shared_experts.gate_weight, RMSNorm weights). + # Stacking tensor_model_parallel=True on top would cause double all-reduce. + if getattr(param, "sequence_parallel", False): + continue + # Distopt excludes TP replicas from grad-norm accounting via this metadata. + if not hasattr(param, "tensor_model_parallel"): + param.tensor_model_parallel = True + + for param in getattr(model, "sp_params", []): + if not hasattr(param, "sequence_parallel"): + param.sequence_parallel = True + param.allreduce = True + param.tensor_model_parallel = False + + +def _build_pg_collection(ps, engine_cfg): + import torch.distributed as dist # pyright: ignore[reportMissingImports] + + from megatron.core.process_groups_config import ProcessGroupCollection + + if ps.pp_group is None: + raise ValueError("dist_opt requires a local pp_group.") + + def _dense_rank(tp_i: int, cp_i: int, dp_i: int, pp_i: int) -> int: + return ((pp_i * ps.dp_size + dp_i) * ps.cp_size + cp_i) * ps.tp_size + tp_i + + def _expert_rank(etp_i: int, ep_i: int, edp_i: int, pp_i: int) -> int: + return ((pp_i * ps.expert_dp_size + edp_i) * ps.ep_size + ep_i) * ps.etp_size + etp_i + + rank = dist.get_rank() + world = dist.get_world_size() + + singleton_group = None + for singleton_rank in range(world): + group = dist.new_group([singleton_rank]) + if rank == singleton_rank: + singleton_group = group + if singleton_group is None: + raise RuntimeError( + "Failed to construct singleton process group for optional dist_opt reductions." + ) + + if engine_cfg.parallel.pp == 1: + mp_group = ps.tp_group + tp_ep_pp_group = ps.tp_ep_group + else: + mp_group = None + for dp_idx in range(ps.dp_size): + for cp_idx in range(ps.cp_size): + ranks = [ + _dense_rank(tp_idx, cp_idx, dp_idx, pp_idx) + for pp_idx in range(ps.pp_size) + for tp_idx in range(ps.tp_size) + ] + group = dist.new_group(ranks) + if rank in ranks: + mp_group = group + + tp_ep_pp_group = None + for expert_dp_idx in range(ps.expert_dp_size): + ranks = [ + _expert_rank(etp_idx, ep_idx, expert_dp_idx, pp_idx) + for pp_idx in range(ps.pp_size) + for ep_idx in range(ps.ep_size) + for etp_idx in range(ps.etp_size) + ] + group = dist.new_group(ranks) + if rank in ranks: + tp_ep_pp_group = group + + if mp_group is None or tp_ep_pp_group is None: + raise RuntimeError("Failed to construct dist_opt pipeline-aware process groups.") + + pg_kwargs = dict( + tp=ps.tp_group, + cp=ps.cp_group, + pp=ps.pp_group, + ep=ps.ep_group, + mp=mp_group, + dp=ps.dp_group, + dp_cp=ps.dp_cp_group, + expt_dp=ps.ep_dp_group, + expt_tp=ps.etp_group, + tp_ep=ps.tp_ep_group, + tp_ep_pp=tp_ep_pp_group, + # For dist_opt, grad stats are reduced over the full optimizer instance. + # With a single dist-opt instance in this benchmark proof, that is the global world group. + intra_dist_opt=dist.group.WORLD, + # ML models do not expose Megatron-Core's embedding/position-embedding sharing surface. + # Use singleton groups so optional embedding reductions become no-ops + # without falling back to the global MCore embedding group. + embd=singleton_group, + pos_embd=singleton_group, + ) + supported_fields = {field.name for field in fields(ProcessGroupCollection)} + return ProcessGroupCollection( + **{key: value for key, value in pg_kwargs.items() if key in supported_fields} + ) + + +# --------------------------------------------------------------------------- +# Backend adapter (consumed by runtime/session.py) +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True, slots=True) +class DistOptBackend: + name: str = "dist_opt" + runtime_backend: str = "dist_opt" + + def zero_grad(self, optimizer: Any) -> None: + optimizer.zero_grad() + + def finish_grad_sync(self, optimizer: Any) -> None: + if hasattr(optimizer, "finish_grad_sync"): + optimizer.finish_grad_sync() + + def clip_grad_norm(self, optimizer: Any): + if hasattr(optimizer, "clip_grad_norm"): + return optimizer.clip_grad_norm() + return None + + def step(self, optimizer: Any): + return optimizer.step() + + def state_dict(self, optimizer: Any) -> dict: + return optimizer.state_dict() + + def load_state_dict(self, optimizer: Any, state_dict: dict) -> None: + optimizer.load_state_dict(state_dict) + + def finalize_grads(self, finalize_fn, model_chunks: list[Any], optimizer: Any) -> None: + finalize_fn(model_chunks, optimizer) + + +BACKEND = DistOptBackend() + +__all__ = [ + "BACKEND", + "DistOptBackend", + "build_dist_opt_optimizer_config", + "build_dist_opt_stack", + "build_dist_opt_training_optimizer", + "finalize_dist_opt_grads", + "validate_dist_opt_config", + "validate_dist_opt_session", +] diff --git a/experimental/lite/megatron/lite/primitive/parallel/__init__.py b/experimental/lite/megatron/lite/primitive/parallel/__init__.py new file mode 100644 index 00000000000..8ba90b5a1de --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/parallel/__init__.py @@ -0,0 +1,91 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Megatron Lite-owned parallel runtime exports.""" + +from __future__ import annotations + +from megatron.lite.primitive.parallel.cp import ( + contiguous_to_zigzag_chunks, + split_packed_for_cp, + zigzag_position_ids_for_cp, + zigzag_reconstruct_from_cp_parts, + zigzag_slice_for_cp, + zigzag_split_for_cp, + zigzag_to_contiguous_chunks, +) +from megatron.lite.primitive.parallel.pipeline import forward_backward_pipelining +from megatron.lite.primitive.parallel.pp import ( + PipelineChunkLayout, + build_pipeline_chunk_layout, +) +from megatron.lite.primitive.parallel.sp import ( + gather_for_non_sp_head, + gather_from_sequence_parallel, + scatter_to_sequence_parallel, +) +from megatron.lite.primitive.parallel.state import ParallelState, init_parallel +from megatron.lite.primitive.parallel.thd import ( + PackedSeqParams, + PackedTHDBatch, + has_packed_thd_params, + pack_nested_thd, + parallel_state_from_model, + prepare_packed_thd_for_context_parallel, + prepare_packed_thd_kwargs_for_context_parallel, + reconstruct_packed_from_cp_parts, + roll_packed_thd_left, + split_packed_to_cp_local, + unpack_packed_thd_to_nested, +) + +_LAZY_LINEAR_EXPORTS = { + "ColumnParallelLinear", + "RowParallelLinear", + "VanillaColumnParallelLinear", + "VocabParallelEmbedding", + "VocabParallelOutput", + "pad_vocab_for_tp", +} + + +def __getattr__(name: str): + if name in _LAZY_LINEAR_EXPORTS: + from megatron.lite.primitive.parallel import linear as _linear + + return getattr(_linear, name) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +__all__ = [ + "ColumnParallelLinear", + "contiguous_to_zigzag_chunks", + "PackedSeqParams", + "PackedTHDBatch", + "PipelineChunkLayout", + "ParallelState", + "RowParallelLinear", + "VanillaColumnParallelLinear", + "VocabParallelEmbedding", + "VocabParallelOutput", + "build_pipeline_chunk_layout", + "forward_backward_pipelining", + "gather_for_non_sp_head", + "gather_from_sequence_parallel", + "has_packed_thd_params", + "init_parallel", + "pad_vocab_for_tp", + "pack_nested_thd", + "parallel_state_from_model", + "prepare_packed_thd_for_context_parallel", + "prepare_packed_thd_kwargs_for_context_parallel", + "reconstruct_packed_from_cp_parts", + "roll_packed_thd_left", + "scatter_to_sequence_parallel", + "split_packed_to_cp_local", + "split_packed_for_cp", + "unpack_packed_thd_to_nested", + "zigzag_position_ids_for_cp", + "zigzag_reconstruct_from_cp_parts", + "zigzag_slice_for_cp", + "zigzag_split_for_cp", + "zigzag_to_contiguous_chunks", +] diff --git a/experimental/lite/megatron/lite/primitive/parallel/cp.py b/experimental/lite/megatron/lite/primitive/parallel/cp.py new file mode 100644 index 00000000000..26820e188f4 --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/parallel/cp.py @@ -0,0 +1,363 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Context parallel zigzag sequence splitting helpers.""" + +from __future__ import annotations + +from typing import Optional + +import torch +import torch.distributed as dist + + +def zigzag_split_for_cp( + tensor: torch.Tensor, + cp_rank: int, + cp_size: int, + seq_dim: int = 1, +) -> torch.Tensor: + """Split tensor along sequence dim using zigzag (striped) pattern for CP. + + Splits into 2*cp_size chunks; GPU i gets chunk[i] + chunk[2*cp_size-1-i]. + This balances causal-mask workload across CP ranks. + + Example (CP=2, seq=8): chunks [0,1,2,3] -> + GPU0: chunk[0]+chunk[3] = tokens [0,1,6,7] + GPU1: chunk[1]+chunk[2] = tokens [2,3,4,5] + """ + if cp_size <= 1: + return tensor + seq_len = tensor.shape[seq_dim] + assert ( + seq_len % (2 * cp_size) == 0 + ), f"seq_len={seq_len} must be divisible by 2*cp_size={2 * cp_size}" + shape = list(tensor.shape) + shape[seq_dim : seq_dim + 1] = [2 * cp_size, seq_len // (2 * cp_size)] + tensor = tensor.view(*shape) + idx = torch.tensor( + [cp_rank, 2 * cp_size - cp_rank - 1], + dtype=torch.long, + device=tensor.device, + ) + tensor = tensor.index_select(seq_dim, idx) + shape[seq_dim : seq_dim + 2] = [seq_len // cp_size] + return tensor.reshape(*shape) + + +def zigzag_reconstruct_from_cp_parts( + parts: list[torch.Tensor] | tuple[torch.Tensor, ...], seq_dim: int = 1 +) -> torch.Tensor: + """Reconstruct a full sequence from per-rank zigzag CP shards.""" + cp_size = len(parts) + if cp_size <= 1: + return parts[0] + local_len = parts[0].shape[seq_dim] + assert ( + local_len % 2 == 0 + ), f"local seq_len={local_len} must be divisible by 2 for zigzag CP reconstruction" + for idx, part in enumerate(parts): + assert ( + part.shape == parts[0].shape + ), f"CP part {idx} shape {tuple(part.shape)} != {tuple(parts[0].shape)}" + + chunk = local_len // 2 + full_len = local_len * cp_size + out_shape = list(parts[0].shape) + out_shape[seq_dim] = full_len + full = torch.zeros(out_shape, dtype=parts[0].dtype, device=parts[0].device) + for rank, part in enumerate(parts): + first = part.narrow(seq_dim, 0, chunk) + second = part.narrow(seq_dim, chunk, chunk) + full.narrow(seq_dim, rank * chunk, chunk).copy_(first) + full.narrow(seq_dim, full_len - (rank + 1) * chunk, chunk).copy_(second) + return full + + +def zigzag_slice_for_cp( + tensor: torch.Tensor, cp_rank: int, cp_size: int, seq_dim: int = 1 +) -> torch.Tensor: + """Return one rank's zigzag CP shard from a full sequence tensor.""" + if cp_size <= 1: + return tensor + seq_len = tensor.shape[seq_dim] + assert ( + seq_len % (2 * cp_size) == 0 + ), f"seq_len={seq_len} must be divisible by 2*cp_size={2 * cp_size}" + chunk = seq_len // (2 * cp_size) + first = tensor.narrow(seq_dim, cp_rank * chunk, chunk) + second_start = seq_len - (cp_rank + 1) * chunk + second = tensor.narrow(seq_dim, second_start, chunk) + return torch.cat((first, second), dim=seq_dim).contiguous() + + +def contiguous_slice_for_cp( + tensor: torch.Tensor, cp_rank: int, cp_size: int, seq_dim: int = 1 +) -> torch.Tensor: + """Return one rank's contiguous CP shard from a full sequence tensor.""" + if cp_size <= 1: + return tensor + seq_len = tensor.shape[seq_dim] + if seq_len % cp_size != 0: + raise ValueError(f"seq_len={seq_len} must be divisible by cp_size={cp_size}") + local_len = seq_len // cp_size + return tensor.narrow(seq_dim, cp_rank * local_len, local_len).contiguous() + + +def contiguous_position_ids_for_cp( + seq_len: int, + cp_rank: int, + cp_size: int, + device: torch.device, +) -> torch.Tensor: + """Return global position IDs for this CP rank under contiguous splitting.""" + if cp_size <= 1: + return torch.arange(seq_len, device=device).unsqueeze(0) + if seq_len % cp_size != 0: + raise ValueError(f"seq_len={seq_len} must be divisible by cp_size={cp_size}") + local_len = seq_len // cp_size + start = cp_rank * local_len + return torch.arange(start, start + local_len, device=device).unsqueeze(0) + + +def local_position_ids_for_cp(position_ids, *, batch, local_seq_len, cp_rank, cp_size): + """Validate and contiguous-slice full-length position_ids to this CP rank.""" + if position_ids.dim() == 1: + position_ids = position_ids.unsqueeze(0) + if position_ids.dim() != 2: + raise ValueError("position_ids must have shape (S,) or (B, S).") + if position_ids.size(0) == 1 and batch > 1: + position_ids = position_ids.expand(batch, -1) + if position_ids.size(0) != batch: + raise ValueError( + f"position_ids batch={position_ids.size(0)} does not match input batch={batch}." + ) + if cp_size <= 1 or position_ids.size(1) == local_seq_len: + return position_ids + + full_seq_len = local_seq_len * cp_size + if position_ids.size(1) != full_seq_len: + raise ValueError( + "CP expects position_ids to be either CP-local or full-length; " + f"got {position_ids.size(1)} for local_seq_len={local_seq_len}, cp={cp_size}." + ) + return contiguous_slice_for_cp(position_ids, cp_rank, cp_size, seq_dim=1) + + +def local_sequence_tensor_for_cp( + tensor, + *, + local_seq_len, + cp_rank, + cp_size, + seq_dim=1, + name: str = "tensor", + unsqueeze_1d: bool = True, +): + """Validate and contiguous-slice a full-length sequence tensor to this CP rank.""" + if tensor is None or cp_size <= 1: + return tensor + if unsqueeze_1d and tensor.dim() == 1: + tensor = tensor.unsqueeze(0) + full_seq_len = local_seq_len * cp_size + seq_len = tensor.size(seq_dim) + if seq_len == local_seq_len: + return tensor + if seq_len != full_seq_len: + raise ValueError( + f"CP expects {name} to be either CP-local or full-length; " + f"got {seq_len} for local_seq_len={local_seq_len}, cp={cp_size}." + ) + return contiguous_slice_for_cp(tensor, cp_rank, cp_size, seq_dim=seq_dim) + + +def zigzag_to_contiguous_chunks( + tensor: torch.Tensor, + cp_group: dist.ProcessGroup | None, + seq_dim: int = 1, +) -> torch.Tensor: + """Swap a CP-local tensor from Megatron zigzag layout to contiguous chunks. + + Zigzag CP layout assigns rank ``r`` global chunks ``[r, 2*cp-r-1]``. + Linear-attention all-gather CP kernels expect rank ``r`` to hold chunks + ``[2*r, 2*r+1]``. The conversion is a chunk-level all-to-all and preserves + the local tensor shape. + """ + return _zigzag_contiguous_chunk_swap(tensor, cp_group, seq_dim, to_contiguous=True) + + +def contiguous_to_zigzag_chunks( + tensor: torch.Tensor, + cp_group: dist.ProcessGroup | None, + seq_dim: int = 1, +) -> torch.Tensor: + """Inverse of :func:`zigzag_to_contiguous_chunks`.""" + return _zigzag_contiguous_chunk_swap(tensor, cp_group, seq_dim, to_contiguous=False) + + +def _zigzag_contiguous_chunk_swap( + tensor: torch.Tensor, + cp_group: Optional[dist.ProcessGroup], + seq_dim: int, + *, + to_contiguous: bool, +) -> torch.Tensor: + cp_size = dist.get_world_size(cp_group) if cp_group is not None else 1 + if cp_size <= 1: + return tensor + cp_rank = dist.get_rank(cp_group) + + if seq_dim != 0: + tensor = tensor.movedim(seq_dim, 0) + tensor = tensor.contiguous() + + local_len = tensor.size(0) + if local_len % 2 != 0: + raise ValueError( + f"zigzag/contiguous CP chunk swap requires even local sequence length, got {local_len}." + ) + chunk_len = local_len // 2 + + def rank_to_chunks(rank: int, in_zigzag: bool) -> tuple[int, int]: + if in_zigzag: + return rank, 2 * cp_size - rank - 1 + return 2 * rank, 2 * rank + 1 + + def chunk_to_dest(chunk_idx: int, target_zigzag: bool) -> tuple[int, int]: + if target_zigzag: + if chunk_idx < cp_size: + return chunk_idx, 0 + return 2 * cp_size - chunk_idx - 1, 1 + return chunk_idx // 2, chunk_idx % 2 + + source_in_zigzag = to_contiguous + target_in_zigzag = not to_contiguous + local_chunks = [tensor[:chunk_len], tensor[chunk_len:]] + local_chunk_indices = rank_to_chunks(cp_rank, source_in_zigzag) + local_dests = [chunk_to_dest(chunk_idx, target_in_zigzag) for chunk_idx in local_chunk_indices] + local_slot_order = sorted(range(2), key=lambda slot: local_dests[slot]) + send_buf = torch.cat([local_chunks[slot] for slot in local_slot_order], dim=0).contiguous() + + input_split_chunks = [0] * cp_size + for dst_rank, _dst_slot in local_dests: + input_split_chunks[dst_rank] += 1 + + output_split_chunks = [0] * cp_size + recv_dst_slots_per_source: list[list[int]] = [[] for _ in range(cp_size)] + for src_rank in range(cp_size): + src_chunks = rank_to_chunks(src_rank, source_in_zigzag) + src_dests = [chunk_to_dest(chunk_idx, target_in_zigzag) for chunk_idx in src_chunks] + src_slot_order = sorted(range(2), key=lambda slot: src_dests[slot]) + for slot in src_slot_order: + dst_rank, dst_slot = src_dests[slot] + if dst_rank == cp_rank: + output_split_chunks[src_rank] += 1 + recv_dst_slots_per_source[src_rank].append(dst_slot) + + input_split_sizes = [count * chunk_len for count in input_split_chunks] + output_split_sizes = [count * chunk_len for count in output_split_chunks] + recv_shape = (sum(output_split_sizes), *send_buf.shape[1:]) + recv_buf = torch.empty(recv_shape, dtype=send_buf.dtype, device=send_buf.device) + from torch.distributed.nn.functional import all_to_all_single + + recv_buf = all_to_all_single( + recv_buf, + send_buf, + output_split_sizes=output_split_sizes, + input_split_sizes=input_split_sizes, + group=cp_group, + ) + + target_slots: list[torch.Tensor | None] = [None, None] + offset = 0 + for src_rank in range(cp_size): + for dst_slot in recv_dst_slots_per_source[src_rank]: + target_slots[dst_slot] = recv_buf[offset : offset + chunk_len] + offset += chunk_len + if any(slot is None for slot in target_slots): + raise RuntimeError("Incomplete CP chunk reassembly.") + + out = torch.cat([slot for slot in target_slots if slot is not None], dim=0) + if seq_dim != 0: + out = out.movedim(0, seq_dim) + return out.contiguous() + + +def zigzag_position_ids_for_cp( + seq_len: int, + cp_rank: int, + cp_size: int, + device: torch.device, +) -> torch.Tensor: + """Return global position IDs for this CP rank under zigzag splitting. + + Returns shape [1, seq_len // cp_size] matching batch dim convention. + """ + if cp_size <= 1: + return torch.arange(seq_len, device=device).unsqueeze(0) + chunk = seq_len // (2 * cp_size) + first = torch.arange(cp_rank * chunk, (cp_rank + 1) * chunk, device=device) + second_start = (2 * cp_size - cp_rank - 1) * chunk + second = torch.arange(second_start, second_start + chunk, device=device) + return torch.cat([first, second]).unsqueeze(0) + + +def split_packed_for_cp( + input_ids: torch.Tensor, + position_ids: torch.Tensor, + cu_seqlens: torch.Tensor, + max_seqlen: int, + cp_rank: int, + cp_size: int, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, int]: + """Zigzag-split a packed sequence batch for context parallelism. + + Each sample defined by *cu_seqlens* is split with the same zigzag + (striped) pattern as :func:`zigzag_split_for_cp`: tokens are divided + into ``2 * cp_size`` chunks, and this rank keeps + ``chunk[cp_rank] + chunk[2*cp_size - 1 - cp_rank]``. + + Returns: + ``(input_ids, position_ids, cu_seqlens, max_seqlen)`` for this CP rank. + """ + if cp_size <= 1: + return input_ids, position_ids, cu_seqlens, max_seqlen + + num_seqs = cu_seqlens.size(0) - 1 + ids_parts: list[torch.Tensor] = [] + pos_parts: list[torch.Tensor] = [] + new_lengths: list[int] = [] + + for i in range(num_seqs): + start = int(cu_seqlens[i].item()) + end = int(cu_seqlens[i + 1].item()) + seq_len = end - start + assert ( + seq_len % (2 * cp_size) == 0 + ), f"Sample {i} length {seq_len} not divisible by 2*cp_size={2 * cp_size}" + chunk = seq_len // (2 * cp_size) + c1 = start + cp_rank * chunk + c2 = start + (2 * cp_size - cp_rank - 1) * chunk + ids_parts.append(input_ids[c1 : c1 + chunk]) + ids_parts.append(input_ids[c2 : c2 + chunk]) + pos_parts.append(position_ids[c1 : c1 + chunk]) + pos_parts.append(position_ids[c2 : c2 + chunk]) + new_lengths.append(2 * chunk) + + new_ids = torch.cat(ids_parts) + new_pos = torch.cat(pos_parts) + lens = torch.tensor(new_lengths, dtype=torch.int32, device=cu_seqlens.device) + new_cu = torch.zeros(num_seqs + 1, dtype=torch.int32, device=cu_seqlens.device) + torch.cumsum(lens, dim=0, out=new_cu[1:]) + return new_ids, new_pos, new_cu, max(new_lengths) + + +__all__ = [ + "contiguous_position_ids_for_cp", + "contiguous_slice_for_cp", + "contiguous_to_zigzag_chunks", + "split_packed_for_cp", + "zigzag_to_contiguous_chunks", + "zigzag_reconstruct_from_cp_parts", + "zigzag_position_ids_for_cp", + "zigzag_slice_for_cp", + "zigzag_split_for_cp", +] diff --git a/experimental/lite/megatron/lite/primitive/parallel/linear.py b/experimental/lite/megatron/lite/primitive/parallel/linear.py new file mode 100644 index 00000000000..7ab907b8afd --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/parallel/linear.py @@ -0,0 +1,414 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""TP-parallel linear layers and vocab-parallel embedding/output.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import torch # pyright: ignore[reportMissingImports] +import torch.distributed as dist # pyright: ignore[reportMissingImports] +import torch.nn as nn # pyright: ignore[reportMissingImports] +import transformer_engine.pytorch as te # pyright: ignore[reportMissingImports] + +from megatron.lite.primitive.utils import ensure_divisible + +if TYPE_CHECKING: + from megatron.lite.primitive.parallel.state import ParallelState + + +# --------------------------------------------------------------------------- +# Vanilla column-parallel linear (torch.matmul kernel, NOT TE). +# +# Matches Megatron-Core `tensor_parallel.ColumnParallelLinear` bit-for-bit +# in bf16: same `torch.matmul(input, weight.t())` forward, same +# all-reduce-on-backward-grad_input pattern. Use this for heads like the +# vocab LM projection where MC's GPT model uses vanilla torch matmul +# (hardcoded in `LinearCrossEntropyModule(tensor_parallel.ColumnParallelLinear)`) +# — TE's `te.Linear` uses a different cuBLAS algo selection that introduces +# ~3e-4 loss-level drift under bf16 vs torch.matmul. +# +# For QKV / MoE experts we still prefer the TE path (fused LN+linear, FP8 +# readiness) — this vanilla path is a drop-in substitute only when kernel +# parity with the reference backend is required. +# --------------------------------------------------------------------------- + + +class _VanillaColParallelMatmul(torch.autograd.Function): + """forward: output = matmul(input, weight.t()) — replicated input, sharded output. + backward: grad_input = grad_output @ weight (+ all-reduce across TP); + grad_weight = grad_output.T @ input. + Mirrors MC `LinearWithGradAccumulationAndAsyncCommunication`. + """ + + @staticmethod + def forward(ctx, input_, weight, tp_group): + ctx.save_for_backward(input_, weight) + ctx.tp_group = tp_group + return torch.matmul(input_, weight.t()) + + @staticmethod + def backward(ctx, grad_output): + input_, weight = ctx.saved_tensors + grad_input = grad_output.matmul(weight) + if ctx.tp_group is not None and dist.get_world_size(ctx.tp_group) > 1: + dist.all_reduce(grad_input, group=ctx.tp_group) + # grad_weight = grad_output^T @ input (sum over leading dims). + gi = grad_output.reshape(-1, grad_output.shape[-1]) + xi = input_.reshape(-1, input_.shape[-1]) + grad_weight = gi.t().matmul(xi) + return grad_input, grad_weight, None + + +class _VanillaColParallelMatmulSP(torch.autograd.Function): + """SP-aware column-parallel matmul matching MC's + `ColumnParallelLinear(sequence_parallel=True)` kernel bit-for-bit. + + forward: + - all-gather input along dim-0 from [S/tp, B, H] → [S, B, H] + - matmul(gathered, weight.t()) → [S, B, V/tp] + backward: + - grad_input_full = grad_output @ weight → [S, B, H] + - reduce-scatter dim-0 → [S/tp, B, H] + - grad_weight = grad_output^T @ gathered_input + """ + + @staticmethod + def forward(ctx, input_, weight, tp_group): + ws = dist.get_world_size(tp_group) if tp_group is not None else 1 + if ws > 1: + s_local = input_.shape[0] + total_shape = (s_local * ws, *input_.shape[1:]) + total_input = torch.empty(total_shape, dtype=input_.dtype, device=input_.device) + dist.all_gather_into_tensor(total_input, input_.contiguous(), group=tp_group) + else: + total_input = input_ + ctx.save_for_backward(total_input, weight) + ctx.tp_group = tp_group + ctx.tp_size = ws + ctx.local_s = input_.shape[0] + return torch.matmul(total_input, weight.t()) + + @staticmethod + def backward(ctx, grad_output): + total_input, weight = ctx.saved_tensors + grad_input_full = grad_output.matmul(weight) + if ctx.tp_size > 1: + out_shape = (ctx.local_s, *grad_input_full.shape[1:]) + grad_input = torch.empty( + out_shape, dtype=grad_input_full.dtype, device=grad_input_full.device + ) + dist.reduce_scatter_tensor(grad_input, grad_input_full.contiguous(), group=ctx.tp_group) + else: + grad_input = grad_input_full + gi = grad_output.reshape(-1, grad_output.shape[-1]) + xi = total_input.reshape(-1, total_input.shape[-1]) + grad_weight = gi.t().matmul(xi) + return grad_input, grad_weight, None + + +class _VanillaColLinear(nn.Module): + """Drop-in for `te.Linear(parallel_mode='column')` using torch.matmul. + + Shape: `self.weight` is (out_features_per_tp, in_features). Exposes + `.weight` at the same attribute path as TE Linear for checkpoint-loader + compatibility. + + When `sp=True`, input is assumed SP-sharded on dim-0; forward gathers + before matmul and backward reduce-scatters grad_input — matching MC's + `ColumnParallelLinear(sequence_parallel=True)` bit-for-bit. When + `sp=False`, input is assumed replicated and grad_input is all-reduced. + """ + + def __init__(self, in_features: int, out_features: int, ps: ParallelState, *, sp: bool = False): + super().__init__() + self.tp_group = ps.tp_group + self.tp_size = ps.tp_size + self.sp = sp + local_out = ensure_divisible(out_features, ps.tp_size) + self.weight = nn.Parameter(torch.empty(local_out, in_features, dtype=torch.bfloat16)) + nn.init.xavier_uniform_(self.weight) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + if self.sp: + return _VanillaColParallelMatmulSP.apply(x, self.weight, self.tp_group) + return _VanillaColParallelMatmul.apply(x, self.weight, self.tp_group) + + +class VanillaColumnParallelLinear(nn.Module): + """Public vanilla column-parallel linear matching MCore torch matmul.""" + + def __init__( + self, + in_features: int, + out_features: int, + ps: ParallelState, + *, + sp: bool = False, + gather_output: bool = False, + ): + super().__init__() + self.linear = _VanillaColLinear(in_features, out_features, ps, sp=sp) + self.tp_size = ps.tp_size + self.tp_group = ps.tp_group + self.gather_output = gather_output + + @property + def weight(self) -> torch.nn.Parameter: + return self.linear.weight + + def forward(self, x: torch.Tensor) -> torch.Tensor: + out = self.linear(x) + if self.gather_output and self.tp_size > 1: + out = _AllGatherLastDim.apply(out, self.tp_size, self.tp_group) + return out + + +class ColumnParallelLinear(nn.Module): + """TE-based column-parallel linear. Splits output dim across TP.""" + + def __init__( + self, + in_features: int, + out_features: int, + ps: ParallelState, + bias: bool = False, + gather_output: bool = False, + normalization: str | None = None, + eps: float = 1e-6, + zero_centered_gamma: bool = False, + sequence_parallel: bool | None = None, + ): + super().__init__() + self.tp_size = ps.tp_size + self.tp_rank = ps.tp_rank + self.tp_group = ps.tp_group + self.local_out = ensure_divisible(out_features, ps.tp_size) + self.use_sp = ( + ps.tp_size > 1 and not gather_output if sequence_parallel is None else sequence_parallel + ) + if normalization is not None: + self.linear = te.LayerNormLinear( + in_features, + out_features, + bias=bias, + normalization=normalization, + eps=eps, + zero_centered_gamma=zero_centered_gamma, + params_dtype=torch.bfloat16, + parallel_mode="column", + sequence_parallel=self.use_sp, + tp_group=ps.tp_group, + tp_size=ps.tp_size, + ) + else: + self.linear = te.Linear( + in_features, + out_features, + bias=bias, + params_dtype=torch.bfloat16, + parallel_mode="column", + sequence_parallel=self.use_sp, + tp_group=ps.tp_group, + tp_size=ps.tp_size, + ) + self.gather_output = gather_output + + def forward(self, x: torch.Tensor) -> torch.Tensor: + out = self.linear(x) + if self.gather_output and self.tp_size > 1: + out = _AllGatherLastDim.apply(out, self.tp_size, self.tp_group) + return out + + +class RowParallelLinear(nn.Module): + """TE-based row-parallel linear. Splits input dim across TP.""" + + def __init__( + self, + in_features: int, + out_features: int, + ps: ParallelState, + bias: bool = False, + input_is_parallel: bool = True, + ): + super().__init__() + self.tp_size = ps.tp_size + self.tp_rank = ps.tp_rank + self.tp_group = ps.tp_group + self.use_sp = ps.tp_size > 1 + self.local_in = ensure_divisible(in_features, ps.tp_size) + self.linear = te.Linear( + in_features, + out_features, + bias=bias, + params_dtype=torch.bfloat16, + parallel_mode="row", + sequence_parallel=self.use_sp, + tp_group=ps.tp_group, + tp_size=ps.tp_size, + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.linear(x) + + +def pad_vocab_for_tp(vocab_size: int, tp_size: int) -> int: + """Round vocab up to be divisible by `lcm(128, tp_size)`. + + Matches MC's `_vocab_size_with_padding(..., make_vocab_size_divisible_by=128)`: + pad to 128-multiple for GEMM alignment, and also require tp-divisibility. + For typical `tp_size ∈ {1,2,4,...,128}`, `lcm = 128` so a vocab already + divisible by 128 (e.g. Qwen3-MoE's 151936) stays unchanged — which is + what MC's `output_layer` sees. Using `128 * tp_size` instead would + over-pad (e.g. 151936 -> 152064 + at tp=2), introducing 128 extra logits into the vocab-parallel cross- + entropy log-sum-exp and driving a ~3e-4 loss drift. + """ + import math + + divisor = math.lcm(128, tp_size) + return ((vocab_size + divisor - 1) // divisor) * divisor + + +class VocabParallelEmbedding(nn.Module): + """Embedding table split across TP on the vocab dimension.""" + + def __init__( + self, vocab_size: int, hidden_size: int, ps: ParallelState, *, deterministic: bool = False + ): + super().__init__() + self.tp_size = ps.tp_size + self.tp_rank = ps.tp_rank + self.deterministic = bool(deterministic) + padded_vocab = pad_vocab_for_tp(vocab_size, ps.tp_size) + self.local_vocab = ensure_divisible(padded_vocab, ps.tp_size) + self.vocab_start = self.tp_rank * self.local_vocab + self.vocab_end = self.vocab_start + self.local_vocab + self.embedding = nn.Embedding(self.local_vocab, hidden_size) + self.tp_group = ps.tp_group + + def forward(self, input_ids: torch.Tensor) -> torch.Tensor: + # input_ids: [B, S] → out: [S, B, H] + mask = (input_ids >= self.vocab_start) & (input_ids < self.vocab_end) + local_ids = (input_ids - self.vocab_start).clamp(min=0, max=self.local_vocab - 1) + if self.deterministic: + out = self.embedding.weight[local_ids] + else: + out = self.embedding(local_ids) + out = out * mask.unsqueeze(-1) + if self.tp_size > 1: + out = _ReduceFromTP.apply(out, self.tp_group) + return out.transpose(0, 1).contiguous() + + +class _ColForLMHead(nn.Module): + """Thin wrapper exposing `.linear` (with `.weight`) for checkpoint-loader + compat, switchable between TE and vanilla torch.matmul kernel. + + `sp=True` threads to the underlying linear: vanilla uses the SP-aware + matmul (gather-in / reduce-scatter-on-bwd); TE uses `sequence_parallel=True` + on `te.Linear`. Both match MC's `output_layer(sequence_parallel=True)` + semantics so the upstream final_layernorm can run on SP-sharded input. + """ + + def __init__( + self, + in_features: int, + out_features: int, + ps: ParallelState, + *, + backend: str = "vanilla", + sp: bool = False, + ): + super().__init__() + if backend == "vanilla": + # Matches MC's LinearCrossEntropyModule → tensor_parallel.ColumnParallelLinear + # kernel bit-for-bit in bf16. Preferred for LM head parity with the reference backend. + self.linear = _VanillaColLinear(in_features, out_features, ps, sp=sp) + elif backend == "te": + # TE-backed path — cuBLAS algo may differ from MC's torch.matmul + # under bf16, producing ~3e-4 loss-level drift. Keep for future + # FP8 / fused-norm paths. + _wrapper = ColumnParallelLinear( + in_features, out_features, ps, bias=False, gather_output=not sp + ) + self.linear = _wrapper.linear + else: + raise ValueError(f"Unknown LM-head backend: {backend!r}") + + +class VocabParallelOutput(nn.Module): + """Output projection split across TP on the vocab dimension (column parallel). + + Default backend is `"vanilla"` (torch.matmul) to match the reference backend's + `tensor_parallel.ColumnParallelLinear` bit-for-bit. Pass `backend="te"` + to use TE's `te.Linear` kernel (e.g. for FP8 inference paths). + """ + + def __init__( + self, vocab_size: int, hidden_size: int, ps: ParallelState, *, backend: str = "vanilla" + ): + super().__init__() + padded_vocab = pad_vocab_for_tp(vocab_size, ps.tp_size) + # SP-aware head: when tp>1 we run on SP-sharded input (matches MC + # GPTModel where final_layernorm runs on SP-sharded hiddens and + # output_layer gathers internally + reduce-scatters on backward). + self.col = _ColForLMHead(hidden_size, padded_vocab, ps, backend=backend, sp=ps.tp_size > 1) + self.padded_vocab = padded_vocab + self.local_vocab = padded_vocab // ps.tp_size + self.vocab_size = vocab_size + self.ps = ps + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.col.linear(x) + + def gather(self, logits: torch.Tensor) -> torch.Tensor: + """All-gather TP-sharded logits and trim to actual vocab_size.""" + if self.ps.tp_size > 1: + chunks = [torch.empty_like(logits) for _ in range(self.ps.tp_size)] + dist.all_gather(chunks, logits, group=self.ps.tp_group) + logits = torch.cat(chunks, dim=-1) + return logits[..., : self.vocab_size] + + +class _AllGatherLastDim(torch.autograd.Function): + """all-gather along last dim; backward = take local shard.""" + + @staticmethod + def forward(ctx, x, tp_size, group): + ctx.tp_size = tp_size + ctx.group = group + ctx.local_dim = x.shape[-1] + ctx.rank = dist.get_rank(group) + chunks = [torch.empty_like(x) for _ in range(tp_size)] + dist.all_gather(chunks, x.contiguous(), group=group) + return torch.cat(chunks, dim=-1) + + @staticmethod + def backward(ctx, grad_output): + start = ctx.rank * ctx.local_dim + return grad_output[..., start : start + ctx.local_dim].contiguous(), None, None + + +class _ReduceFromTP(torch.autograd.Function): + """all-reduce forward; identity backward.""" + + @staticmethod + def forward(ctx, x, group): + out = x.clone() + dist.all_reduce(out, group=group) + return out + + @staticmethod + def backward(ctx, grad_output): + return grad_output, None + + +__all__ = [ + "ColumnParallelLinear", + "RowParallelLinear", + "VanillaColumnParallelLinear", + "VocabParallelEmbedding", + "VocabParallelOutput", + "pad_vocab_for_tp", +] diff --git a/experimental/lite/megatron/lite/primitive/parallel/mhc.py b/experimental/lite/megatron/lite/primitive/parallel/mhc.py new file mode 100644 index 00000000000..9a48022a300 --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/parallel/mhc.py @@ -0,0 +1,52 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +from typing import Any + +import torch + + +def expand_mhc_hidden_for_pipeline(hidden: torch.Tensor, *, hc_mult: int) -> torch.Tensor: + if hidden.dim() == 3: + return hidden.unsqueeze(2).expand(-1, -1, hc_mult, -1).contiguous() + return hidden + + +def fold_mhc_hidden_for_pipeline(hidden: torch.Tensor) -> torch.Tensor: + """Fold the 4-D hc streams [B, S, hc_mult, H] into [B, S, hc_mult * H] for PP P2P. + + The pipeline P2P recv buffer is 3-D, so the hc_mult parallel residual streams must be + flattened into the hidden dimension before crossing a stage boundary. No-op for 3-D input. + """ + if hidden.dim() == 4: + b, s, m, h = hidden.shape + return hidden.reshape(b, s, m * h).contiguous() + return hidden + + +def unfold_mhc_hidden_from_pipeline(hidden: torch.Tensor, *, hc_mult: int) -> torch.Tensor: + """Inverse of :func:`fold_mhc_hidden_for_pipeline`: [B, S, hc_mult * H] -> [B, S, hc_mult, H]. + + Used on non-first PP stages to restore the hc_mult streams received over P2P. No-op when the + tensor is already 4-D. + """ + if hidden.dim() == 4: + return hidden + b, s, mh = hidden.shape + return hidden.reshape(b, s, hc_mult, mh // hc_mult).contiguous() + + +def contract_mhc_hidden_for_pipeline( + hidden: torch.Tensor, + *, + norm: Any, + head: Any, + return_source: bool = False, +): + if head is None or norm is None: + if return_source: + return hidden, None + return hidden + source = hidden + contracted = norm(head(hidden)) + if return_source: + return contracted, source + return contracted diff --git a/experimental/lite/megatron/lite/primitive/parallel/pipeline.py b/experimental/lite/megatron/lite/primitive/parallel/pipeline.py new file mode 100644 index 00000000000..5cee84b543a --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/parallel/pipeline.py @@ -0,0 +1,763 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Pipeline parallel: 1F1B schedule with correct backward handling.""" + +from __future__ import annotations + +import os +from collections.abc import Callable +from typing import TYPE_CHECKING + +import torch # pyright: ignore[reportMissingImports] +import torch.distributed as dist # pyright: ignore[reportMissingImports] + +from megatron.lite.primitive.utils import ensure_divisible +from megatron.lite.runtime.contracts.loss import split_loss_context, use_loss_context + +if TYPE_CHECKING: + from megatron.lite.primitive.parallel.state import ParallelState + + +def forward_backward_pipelining( + forward_step_fn: Callable, + model_chunks: list, + data_iter, + config, + ps: ParallelState, + tensor_shape: tuple[int, ...] | None = None, + grad_sync_fn: Callable[[], None] | None = None, + pre_forward_hook: Callable[[torch.Tensor], None] | None = None, + loss_fn: Callable | None = None, + forward_only: bool = False, +) -> list[dict]: + """ + Run forward and backward passes with pipeline parallelism. + + Args: + forward_step_fn: Callable(model, batch) -> output_dict with "loss" and "hidden_states" + model_chunks: local model chunks. ``len>1`` enables interleaved VPP. + data_iter: iterator yielding micro-batches + config: training config + ps: parallel state + tensor_shape: shape of hidden states passed between stages [B, S, H] + grad_sync_fn: called before the last micro-batch's backward to enable + overlapped gradient ReduceScatter in DistributedOptimizer. + + Returns: + List of output dicts from forward passes. + """ + if ps.pp_size <= 1: + if forward_only: + return _forward_only_no_pipeline( + forward_step_fn, + model_chunks[0], + data_iter, + config, + ps, + pre_forward_hook=pre_forward_hook, + loss_fn=loss_fn, + ) + return _no_pipeline( + forward_step_fn, + model_chunks[0], + data_iter, + config, + ps, + grad_sync_fn=grad_sync_fn, + pre_forward_hook=pre_forward_hook, + loss_fn=loss_fn, + ) + + if tensor_shape is None: + raise ValueError("tensor_shape is required when PP > 1") + + num_microbatches = _num_microbatches_from_config(config, ps) + + if forward_only: + return _forward_only_pipeline_schedule( + forward_step_fn, + model_chunks, + data_iter, + num_microbatches, + ps, + tensor_shape, + pre_forward_hook=pre_forward_hook, + loss_fn=loss_fn, + ) + + if len(model_chunks) > 1: + return _interleaved_1f1b_schedule( + forward_step_fn, + model_chunks, + data_iter, + num_microbatches, + config, + ps, + tensor_shape, + grad_sync_fn=grad_sync_fn, + pre_forward_hook=pre_forward_hook, + loss_fn=loss_fn, + ) + + return _1f1b_schedule( + forward_step_fn, + model_chunks[0], + data_iter, + num_microbatches, + config, + ps, + tensor_shape, + grad_sync_fn=grad_sync_fn, + pre_forward_hook=pre_forward_hook, + loss_fn=loss_fn, + ) + + +# ══════════════════════════════════════════════════════════════════════ +# No pipeline (PP=1) +# ══════════════════════════════════════════════════════════════════════ +def _num_microbatches_from_config(config, ps: ParallelState) -> int: + explicit = getattr(config, "num_microbatches", None) + if explicit is not None: + return int(explicit) + return ensure_divisible(config.gbs, config.mbs * ps.dp_size) + + +def _set_aux_loss_scale(pre_forward_hook, num_microbatches: int) -> None: + if pre_forward_hook is not None: + scale = torch.tensor(1.0 / num_microbatches, device="cuda") + pre_forward_hook(scale) + + +def _batch_get(batch, key: str): + if isinstance(batch, dict): + return batch.get(key) + return getattr(batch, key, None) + + +def _apply_external_loss( + out: dict, batch, loss_fn, loss_context=None +) -> tuple[torch.Tensor, dict] | tuple[None, None]: + if loss_fn is None: + return None, None + # Mirror run_microbatch_loop: pass loss_context as 3rd arg when present. + if loss_context is None: + loss, metrics = loss_fn(out, batch) + else: + loss, metrics = loss_fn(out, batch, loss_context) + out["loss"] = loss + out["_loss_fn_metrics"] = metrics + return loss, metrics + + +def _compact_pipeline_output(out: dict | None) -> dict: + if not out: + return {} + compact: dict = {} + if "model_output" in out: + compact["model_output"] = out["model_output"] + if "loss" in out and out["loss"] is not None: + loss = out["loss"] + compact["loss"] = loss.detach().item() if isinstance(loss, torch.Tensor) else float(loss) + if "_loss_fn_metrics" in out: + compact["metrics"] = out["_loss_fn_metrics"] + elif "metrics" in out: + compact["metrics"] = out["metrics"] + return compact + + +def _no_pipeline( + forward_step_fn, + model, + data_iter, + config, + ps, + *, + grad_sync_fn=None, + pre_forward_hook=None, + loss_fn=None, +): + num_microbatches = _num_microbatches_from_config(config, ps) + outputs = [] + for i in range(num_microbatches): + if grad_sync_fn and i == num_microbatches - 1: + grad_sync_fn() + batch = next(data_iter) + _set_aux_loss_scale(pre_forward_hook, num_microbatches) + output = forward_step_fn(model, batch) + if loss_fn is not None: + loss, _metrics = _apply_external_loss(output, batch, loss_fn) + assert loss is not None + else: + loss = output["loss"] + loss = loss / num_microbatches + loss.backward() + outputs.append(_compact_pipeline_output(output)) + return outputs + + +def _forward_only_no_pipeline( + forward_step_fn, model, data_iter, config, ps, *, pre_forward_hook=None, loss_fn=None +): + num_microbatches = _num_microbatches_from_config(config, ps) + del ps + outputs = [] + for _ in range(num_microbatches): + batch = next(data_iter) + _set_aux_loss_scale(pre_forward_hook, num_microbatches) + output = forward_step_fn(model, batch) + _apply_external_loss(output, batch, loss_fn) + outputs.append(_compact_pipeline_output(output)) + return outputs + + +# ══════════════════════════════════════════════════════════════════════ +# 1F1B Schedule +# ══════════════════════════════════════════════════════════════════════ +def _1f1b_schedule( + forward_step_fn, + model, + data_iter, + num_microbatches: int, + config, + ps: ParallelState, + tensor_shape: tuple[int, ...], + *, + grad_sync_fn=None, + pre_forward_hook=None, + loss_fn=None, +): + """ + 1-Forward-1-Backward pipeline schedule using batch_isend_irecv. + + Communication is always done via combined send+recv to avoid deadlocks. + """ + num_warmup = min(ps.pp_size - ps.pp_rank - 1, num_microbatches) + num_steady = num_microbatches - num_warmup + + # Split each microbatch into (PackedBatch, LossContext) like run_microbatch_loop; the connector + # yields (batch, loss_context) tuples, so forward_step must receive the unwrapped batch. + batches = [split_loss_context(next(data_iter)) for _ in range(num_microbatches)] + mb_idx = 0 + + input_tensors: list[torch.Tensor | None] = [] + output_hiddens: list[torch.Tensor | None] = [] + losses: list[torch.Tensor | None] = [] + outputs: list[dict] = [] + + # Fix 3: Pre-allocate recv buffers to avoid torch.empty per P2P call. + _fwd_recv_buf = ( + torch.empty(tensor_shape, dtype=_PIPELINE_TENSOR_DTYPE, device="cuda") + if not ps.pp_is_first + else None + ) + _bwd_recv_buf = ( + torch.empty(tensor_shape, dtype=_PIPELINE_TENSOR_DTYPE, device="cuda") + if not ps.pp_is_last + else None + ) + + def _run_forward(input_tensor, batch, loss_context=None): + _set_aux_loss_scale(pre_forward_hook, num_microbatches) + if not ps.pp_is_first: + # `model` is the dist_opt DDP-wrapped chunk; set_input_tensor lives on the base lite model. + from megatron.lite.primitive.ckpt.hf_weights import unwrap_model + + unwrap_model(model).set_input_tensor(input_tensor) + with use_loss_context(loss_context): + out = forward_step_fn(model, batch) + if ps.pp_is_last: + _apply_external_loss(out, batch, loss_fn, loss_context) + return out + + def _run_backward(inp_t, hid_t, loss_t, grad_t): + if ps.pp_is_last: + if loss_t is not None: + loss_t.backward() + else: + if hid_t is not None and hid_t.requires_grad: + torch.autograd.backward(hid_t, grad_t) + return inp_t.grad if inp_t is not None else None + + def _p2p(send_fwd=None, send_bwd=None, recv_fwd=False, recv_bwd=False): + return _send_recv_pipeline( + send_fwd, + send_bwd, + recv_fwd, + recv_bwd, + ps, + tensor_shape, + fwd_recv_buf=_fwd_recv_buf, + bwd_recv_buf=_bwd_recv_buf, + ) + + # ── Warmup: pure forward passes ── + fwd_input: torch.Tensor | None = None + for k in range(num_warmup): + if not ps.pp_is_first and k == 0: + fwd_input, _ = _p2p(recv_fwd=True) + + batch, loss_ctx = batches[mb_idx] + mb_idx += 1 + current_input = fwd_input + out = _run_forward(fwd_input, batch, loss_ctx) + hidden = out.get("hidden_states") + loss_s = out["loss"] / num_microbatches if "loss" in out and ps.pp_is_last else None + + need_recv_next = not ps.pp_is_first and k < num_warmup - 1 + if not ps.pp_is_last: + fwd_input, _ = _p2p(send_fwd=hidden, recv_fwd=need_recv_next) + elif need_recv_next: + fwd_input, _ = _p2p(recv_fwd=True) + + input_tensors.append(current_input if not ps.pp_is_first else None) + output_hiddens.append(hidden) + losses.append(loss_s) + # Fix 4: only keep loss from output, drop logits/hidden references + outputs.append(_compact_pipeline_output(out)) + + # ── Steady: interleaved forward + backward ── + for k in range(num_steady): + if grad_sync_fn and num_warmup == 0 and k == num_steady - 1: + grad_sync_fn() + + if not ps.pp_is_first and k == 0 and num_warmup == 0: + fwd_input, _ = _p2p(recv_fwd=True) + + batch, loss_ctx = batches[mb_idx] + mb_idx += 1 + out = _run_forward(fwd_input, batch, loss_ctx) + hidden = out.get("hidden_states") + loss_s = out["loss"] / num_microbatches if "loss" in out and ps.pp_is_last else None + + input_tensors.append(fwd_input if not ps.pp_is_first else None) + output_hiddens.append(hidden) + losses.append(loss_s) + outputs.append(_compact_pipeline_output(out)) + + send_fwd = hidden if not ps.pp_is_last else None + need_bwd = not ps.pp_is_last + _, bwd_grad = _p2p(send_fwd=send_fwd, recv_bwd=need_bwd) + + old_inp = input_tensors.pop(0) + old_hid = output_hiddens.pop(0) + old_loss = losses.pop(0) + in_grad = _run_backward(old_inp, old_hid, old_loss, bwd_grad) + + send_bwd = in_grad if not ps.pp_is_first else None + need_fwd = not ps.pp_is_first and k < num_steady - 1 + fwd_input, _ = _p2p(send_bwd=send_bwd, recv_fwd=need_fwd) + + # ── Cooldown: drain remaining backwards ── + for k in range(num_warmup): + if grad_sync_fn and k == num_warmup - 1: + grad_sync_fn() + + need_bwd = not ps.pp_is_last + _, bwd_grad = _p2p(recv_bwd=need_bwd) + + old_inp = input_tensors.pop(0) + old_hid = output_hiddens.pop(0) + old_loss = losses.pop(0) + in_grad = _run_backward(old_inp, old_hid, old_loss, bwd_grad) + + send_bwd = in_grad if not ps.pp_is_first else None + if send_bwd is not None: + _p2p(send_bwd=send_bwd) + + return outputs + + +# ══════════════════════════════════════════════════════════════════════ +# Pipeline communication helpers +# ══════════════════════════════════════════════════════════════════════ +_PIPELINE_TENSOR_DTYPE = torch.bfloat16 + + +def _deallocate_output_tensor(tensor: torch.Tensor | None) -> None: + """Free a large output tensor after it has been sent to the next stage.""" + if tensor is not None: + tensor.data = torch.empty(1, device=tensor.device, dtype=tensor.dtype) + + +# ══════════════════════════════════════════════════════════════════════ +# Interleaved 1F1B (VPP) Schedule +# ══════════════════════════════════════════════════════════════════════ +def _build_schedule_table( + num_microbatches: int, num_chunks: int, group_size: int +) -> list[tuple[int, int]]: + """Build (microbatch_id, model_chunk_id) table for VPP scheduling.""" + table: list[tuple[int, int]] = [] + for start in range(0, num_microbatches, group_size): + end = min(start + group_size, num_microbatches) + for chunk in range(num_chunks): + for mb in range(start, end): + table.append((mb, chunk)) + return table + + +def _send_recv_pipeline( + send_fwd: torch.Tensor | None, + send_bwd: torch.Tensor | None, + recv_fwd: bool, + recv_bwd: bool, + ps: ParallelState, + tensor_shape: tuple[int, ...], + *, + fwd_recv_buf: torch.Tensor | None = None, + bwd_recv_buf: torch.Tensor | None = None, + batch_p2p: bool = True, + clone_recv: bool = False, +) -> tuple[torch.Tensor | None, torch.Tensor | None]: + """P2P communication between pipeline stages.""" + _dbg = int(os.environ.get("MEGATRON_LITE_PP_DEBUG", "0")) + rank = dist.get_rank() + + ops: list[dist.P2POp] = [] + fwd_buf: torch.Tensor | None = None + bwd_buf: torch.Tensor | None = None + + p2p_group = ps.pp_group + + if send_fwd is not None: + t = send_fwd.to(_PIPELINE_TENSOR_DTYPE) + ops.append(dist.P2POp(dist.isend, t, ps.pp_next_rank, p2p_group)) + if recv_fwd: + fwd_buf = ( + fwd_recv_buf + if fwd_recv_buf is not None + else torch.empty(tensor_shape, dtype=_PIPELINE_TENSOR_DTYPE, device="cuda") + ) + ops.append(dist.P2POp(dist.irecv, fwd_buf, ps.pp_prev_rank, p2p_group)) + if send_bwd is not None: + t = send_bwd.to(_PIPELINE_TENSOR_DTYPE) + ops.append(dist.P2POp(dist.isend, t, ps.pp_prev_rank, p2p_group)) + if recv_bwd: + bwd_buf = ( + bwd_recv_buf + if bwd_recv_buf is not None + else torch.empty(tensor_shape, dtype=_PIPELINE_TENSOR_DTYPE, device="cuda") + ) + ops.append(dist.P2POp(dist.irecv, bwd_buf, ps.pp_next_rank, p2p_group)) + + if ops: + if _dbg: + desc = [] + if send_fwd is not None: + desc.append(f"send_fwd→{ps.pp_next_rank}({list(send_fwd.shape)})") + if recv_fwd: + desc.append(f"recv_fwd←{ps.pp_prev_rank}") + if send_bwd is not None: + desc.append(f"send_bwd→{ps.pp_prev_rank}") + if recv_bwd: + desc.append(f"recv_bwd←{ps.pp_next_rank}") + op_name = "batch_isend_irecv" if batch_p2p else "isend_irecv" + print(f"[P2P r{rank}] {op_name}: {' '.join(desc)}", flush=True) + if batch_p2p: + reqs = dist.batch_isend_irecv(ops) + else: + direct_tensors = [] + reqs = [] + if send_fwd is not None: + t = send_fwd.to(_PIPELINE_TENSOR_DTYPE) + direct_tensors.append(t) + reqs.append(dist.isend(t, ps.pp_next_rank, group=p2p_group)) + if recv_fwd: + reqs.append(dist.irecv(fwd_buf, ps.pp_prev_rank, group=p2p_group)) + if send_bwd is not None: + t = send_bwd.to(_PIPELINE_TENSOR_DTYPE) + direct_tensors.append(t) + reqs.append(dist.isend(t, ps.pp_prev_rank, group=p2p_group)) + if recv_bwd: + reqs.append(dist.irecv(bwd_buf, ps.pp_next_rank, group=p2p_group)) + for req in reqs: + req.wait() + if _dbg: + print(f"[P2P r{rank}] batch done", flush=True) + + if fwd_buf is not None: + if clone_recv: + fwd_buf = fwd_buf.clone() + fwd_buf.grad = None + fwd_buf.requires_grad_() + if bwd_buf is not None and clone_recv: + bwd_buf = bwd_buf.clone() + return fwd_buf, bwd_buf + + +def _pipeline_stage_barrier(ps: ParallelState) -> None: + if ps.pp_cpu_group is not None and ps.pp_size > 1: + dist.barrier(group=ps.pp_cpu_group) + + +def _set_virtual_pipeline_rank(ps: ParallelState, chunk_id: int | None, num_chunks: int) -> None: + if chunk_id is None or num_chunks <= 1: + ps.virtual_pipeline_size = None + ps.virtual_pipeline_rank = None + return + ps.virtual_pipeline_size = num_chunks + ps.virtual_pipeline_rank = chunk_id + + +def _run_pipeline_chunk_forward( + forward_step_fn, + model, + batch, + input_tensor: torch.Tensor | None, + *, + is_first_stage: bool, + is_last_stage: bool, + num_microbatches: int, + pre_forward_hook=None, + loss_fn=None, +) -> dict: + _set_aux_loss_scale(pre_forward_hook, num_microbatches) + if not is_first_stage: + model.set_input_tensor(input_tensor) + out = forward_step_fn(model, batch) + if is_last_stage: + _apply_external_loss(out, batch, loss_fn) + return out + + +def _forward_only_pipeline_schedule( + forward_step_fn, + model_chunks: list, + data_iter, + num_microbatches: int, + ps: ParallelState, + tensor_shape: tuple[int, ...], + *, + pre_forward_hook=None, + loss_fn=None, +): + """Simple PP/VPP forward-only schedule used for log-prob inference.""" + num_chunks = len(model_chunks) + total_stages = ps.pp_size * num_chunks + outputs: list[dict] = [] + + for _mb in range(num_microbatches): + batch = next(data_iter) + _set_aux_loss_scale(pre_forward_hook, num_microbatches) + pending_activation: torch.Tensor | None = None + last_output: dict | None = None + for stage_id in range(total_stages): + stage_pp_rank = stage_id % ps.pp_size + is_local_stage = stage_pp_rank == ps.pp_rank + hidden: torch.Tensor | None = None + if is_local_stage: + chunk_id = stage_id // ps.pp_size + _set_virtual_pipeline_rank(ps, chunk_id, num_chunks) + model = model_chunks[chunk_id] + is_first_stage = stage_id == 0 + is_last_stage = stage_id == total_stages - 1 + activation = None if is_first_stage else pending_activation + pending_activation = None + out = _run_pipeline_chunk_forward( + forward_step_fn, + model, + batch, + activation, + is_first_stage=is_first_stage, + is_last_stage=is_last_stage, + num_microbatches=num_microbatches, + pre_forward_hook=None, + loss_fn=loss_fn, + ) + if is_last_stage: + last_output = out + else: + hidden = out.get("hidden_states") + + if stage_id < total_stages - 1: + recv_next = (stage_id + 1) % ps.pp_size == ps.pp_rank + _pipeline_stage_barrier(ps) + fwd_buf, _ = _send_recv_pipeline( + hidden if is_local_stage else None, + None, + recv_next, + False, + ps, + tensor_shape, + batch_p2p=False, + clone_recv=True, + ) + if recv_next: + pending_activation = fwd_buf + + outputs.append(_compact_pipeline_output(last_output) if last_output is not None else {}) + + _set_virtual_pipeline_rank(ps, None, num_chunks) + return outputs + + +def _interleaved_1f1b_schedule( + forward_step_fn, + model_chunks: list, + data_iter, + num_microbatches: int, + config, + ps: ParallelState, + tensor_shape: tuple[int, ...], + *, + grad_sync_fn=None, + pre_forward_hook=None, + loss_fn=None, +): + """ + Correct non-overlapped schedule for Virtual Pipeline Parallelism (VPP). + + Local chunks are laid out in global layer order as + ``chunk_id * pp_size + pp_rank``. The previous interleaved 1F1B bringup + schedule could ask the first physical PP stage to receive the next virtual + chunk before the last physical stage had produced it. This schedule keeps + the same VPP semantics but runs one micro-batch at a time in global stage + order, which is slower but deterministic and easy to validate against + Megatron. + """ + num_chunks = len(model_chunks) + total_stages = ps.pp_size * num_chunks + rank = dist.get_rank() + outputs: list[dict] = [] + + _dbg = int(os.environ.get("MEGATRON_LITE_PP_DEBUG", "0")) + if _dbg: + print( + f"[VPP r{rank}] entered simple schedule pp_rank={ps.pp_rank} " + f"microbatches={num_microbatches} chunks={num_chunks} total_stages={total_stages}", + flush=True, + ) + + for mb_id in range(num_microbatches): + batch = next(data_iter) + _set_aux_loss_scale(pre_forward_hook, num_microbatches) + saved: dict[ + int, tuple[torch.Tensor | None, torch.Tensor | None, torch.Tensor | None, dict] + ] = {} + pending_activation: torch.Tensor | None = None + + # Forward in true virtual-stage order: + # chunk0/rank0 -> chunk0/rank1 -> ... -> chunkN/rankP. + for stage_id in range(total_stages): + stage_pp_rank = stage_id % ps.pp_size + is_local_stage = stage_pp_rank == ps.pp_rank + hidden: torch.Tensor | None = None + if is_local_stage: + chunk_id = stage_id // ps.pp_size + _set_virtual_pipeline_rank(ps, chunk_id, num_chunks) + model = model_chunks[chunk_id] + is_first_stage = stage_id == 0 + is_last_stage = stage_id == total_stages - 1 + activation = None if is_first_stage else pending_activation + pending_activation = None + if _dbg: + activation_shape = None if activation is None else tuple(activation.shape) + print( + f"[VPP r{rank}] mb={mb_id} fwd stage={stage_id} " + f"chunk={chunk_id} activation_shape={activation_shape}", + flush=True, + ) + out = _run_pipeline_chunk_forward( + forward_step_fn, + model, + batch, + activation, + is_first_stage=is_first_stage, + is_last_stage=is_last_stage, + num_microbatches=num_microbatches, + pre_forward_hook=None, + loss_fn=loss_fn, + ) + + hidden = out.get("hidden_states") + if _dbg: + hidden_shape = None if hidden is None else tuple(hidden.shape) + print( + f"[VPP r{rank}] mb={mb_id} fwd stage={stage_id} " + f"chunk={chunk_id} hidden_shape={hidden_shape}", + flush=True, + ) + loss = out["loss"] / num_microbatches if is_last_stage and "loss" in out else None + saved[stage_id] = (activation, hidden, loss, out) + + if is_last_stage: + outputs.append(_compact_pipeline_output(out)) + + if stage_id < total_stages - 1: + recv_next = (stage_id + 1) % ps.pp_size == ps.pp_rank + if _dbg and (is_local_stage or recv_next): + print( + f"[VPP r{rank}] mb={mb_id} fwd boundary={stage_id} " + f"send={is_local_stage and hidden is not None} recv_next={recv_next}", + flush=True, + ) + _pipeline_stage_barrier(ps) + fwd_buf, _ = _send_recv_pipeline( + hidden if is_local_stage else None, + None, + recv_next, + False, + ps, + tensor_shape, + batch_p2p=False, + clone_recv=True, + ) + if recv_next: + pending_activation = fwd_buf + + if grad_sync_fn and mb_id == num_microbatches - 1: + grad_sync_fn() + + # Backward in the reverse virtual-stage order. + pending_grad: torch.Tensor | None = None + for stage_id in range(total_stages - 1, -1, -1): + stage_pp_rank = stage_id % ps.pp_size + is_local_stage = stage_pp_rank == ps.pp_rank + inp_grad: torch.Tensor | None = None + if is_local_stage: + chunk_id = stage_id // ps.pp_size + _set_virtual_pipeline_rank(ps, chunk_id, num_chunks) + is_first_stage = stage_id == 0 + is_last_stage = stage_id == total_stages - 1 + inp, out_t, loss, _out = saved[stage_id] + grad = None if is_last_stage else pending_grad + pending_grad = None + if _dbg: + print(f"[VPP r{rank}] mb={mb_id} bwd stage={stage_id}", flush=True) + if is_last_stage: + if loss is not None: + loss.backward() + elif out_t is not None and out_t.requires_grad: + torch.autograd.backward(out_t, grad) + if not is_first_stage: + inp_grad = inp.grad if inp is not None else None + + if stage_id > 0: + recv_prev = (stage_id - 1) % ps.pp_size == ps.pp_rank + if _dbg and (is_local_stage or recv_prev): + print( + f"[VPP r{rank}] mb={mb_id} bwd boundary={stage_id} " + f"send={is_local_stage and inp_grad is not None} recv_prev={recv_prev}", + flush=True, + ) + _pipeline_stage_barrier(ps) + _, bwd_buf = _send_recv_pipeline( + None, + inp_grad if is_local_stage else None, + False, + recv_prev, + ps, + tensor_shape, + batch_p2p=False, + clone_recv=True, + ) + if recv_prev: + pending_grad = bwd_buf + + if _dbg: + print(f"[VPP r{rank}] mb={mb_id} complete", flush=True) + + _set_virtual_pipeline_rank(ps, None, num_chunks) + return outputs + + +__all__ = ["forward_backward_pipelining"] diff --git a/experimental/lite/megatron/lite/primitive/parallel/pp.py b/experimental/lite/megatron/lite/primitive/parallel/pp.py new file mode 100644 index 00000000000..61077f07d56 --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/parallel/pp.py @@ -0,0 +1,99 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Pipeline parallel layer layout — a thin wrapper over Megatron-core's +``PipelineParallelLayerLayout`` (authorized mcore-reuse), which owns the per-stage +split for non-divisible decoder counts plus MTP. Two pp-only modes: + +* **auto** (default, only ``pp`` set): balance ``[E, decoder*N, mtp*K, loss]`` across stages. +* **custom** (``ParallelConfig.pp_layout``): an explicit mcore layout string/list, + e.g. ``"E|t*5|t*6|t,m,L"``. + +Not supported (raise, never mis-place): VPP, and standalone MTP (``m`` off the +final/loss stage — mlite's MTP shares the head there; cross-stage MTP is a follow-up). +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from megatron.lite.primitive.parallel.state import ParallelState + + +@dataclass +class PipelineChunkLayout: + layer_indices: list[int] = field(default_factory=list) + has_embed: bool = False + has_head: bool = False + has_mtp: bool = False + + +def _auto_layout(num_hidden_layers: int, pp_size: int, num_mtp_layers: int): + """Balance ``[E, decoder*N, mtp*K, loss]`` into even contiguous chunks; the + embedding/MTP/loss slots make their stages carry fewer decoders (e.g. 6/pp4 -> + [1,2,2,1]) — Megatron's embedding/loss split accounting.""" + from megatron.core.transformer.pipeline_parallel_layer_layout import ( + PipelineParallelLayerLayout, + ) + + units = ["embedding"] + ["decoder"] * num_hidden_layers + ["mtp"] * max(num_mtp_layers, 0) + ["loss"] + base, remainder = divmod(len(units), pp_size) + rows, pos = [], 0 + for size in (base + (1 if s < remainder else 0) for s in range(pp_size)): + rows.append(units[pos : pos + size]) + pos += size + return PipelineParallelLayerLayout(rows, pipeline_model_parallel_size=pp_size) + + +def build_pipeline_chunk_layout( + num_hidden_layers: int, + ps: ParallelState, + vpp: int | None = None, + vpp_chunk_id: int | None = None, + *, + num_mtp_layers: int = 0, +) -> PipelineChunkLayout: + """``layer_indices`` / ``has_embed`` / ``has_head`` / ``has_mtp`` for this PP rank, + from ``ps.pp_layout`` (custom) or an auto-balanced layout. ``has_mtp`` follows the + layout's ``m`` placement, so MTP is built where the layout says, not a fixed rank.""" + if (vpp is not None and vpp > 1) or vpp_chunk_id is not None: + raise NotImplementedError("VPP / interleaved pipeline layout is not supported (use vpp=1).") + + if ps.pp_size <= 1: # no pipeline: this stage owns everything + return PipelineChunkLayout( + layer_indices=list(range(num_hidden_layers)), + has_embed=True, + has_head=True, + has_mtp=num_mtp_layers > 0, + ) + + from megatron.core.transformer.enums import LayerType + from megatron.core.transformer.pipeline_parallel_layer_layout import ( + PipelineParallelLayerLayout, + ) + + pp_layout = getattr(ps, "pp_layout", None) + if pp_layout is not None: + layout = PipelineParallelLayerLayout(pp_layout, pipeline_model_parallel_size=ps.pp_size) + if layout.virtual_pipeline_model_parallel_size > 1: + raise NotImplementedError("VPP pp_layout is not supported (one stage per pp rank).") + else: + layout = _auto_layout(num_hidden_layers, ps.pp_size, num_mtp_layers) + + # validate_layer_layout checks legality and returns mtp_standalone=True when `m` is + # off the final stage — which mlite's head-coupled MTP cannot run. + if layout.validate_layer_layout(num_hidden_layers, num_mtp_layers or None): + raise NotImplementedError( + "Standalone MTP (pp_layout with `m` off the final/loss stage) is not " + "implemented yet — mlite's MTP shares the output head there. Use the auto " + "layout (set only `pp`), or place `m` on the same stage as `L`." + ) + return PipelineChunkLayout( + layer_indices=layout.get_layer_id_list(LayerType.decoder, vp_stage=0, pp_rank=ps.pp_rank), + has_embed=ps.pp_is_first, + has_head=ps.pp_is_last, + has_mtp=layout.get_num_layers_to_build(LayerType.mtp, vp_stage=0, pp_rank=ps.pp_rank) > 0, + ) + + +__all__ = ["PipelineChunkLayout", "build_pipeline_chunk_layout"] diff --git a/experimental/lite/megatron/lite/primitive/parallel/sp.py b/experimental/lite/megatron/lite/primitive/parallel/sp.py new file mode 100644 index 00000000000..1c85c4b58cb --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/parallel/sp.py @@ -0,0 +1,45 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Sequence parallel scatter/gather helpers.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import torch # pyright: ignore[reportMissingImports] + +from megatron.lite.primitive.ops.sp_ops import ( + AllGatherDim0, + AllGatherDim0ForNonSPConsumer, + ScatterToSP, +) + +if TYPE_CHECKING: + from megatron.lite.primitive.parallel.state import ParallelState + + +def scatter_to_sequence_parallel(x: torch.Tensor, ps: ParallelState) -> torch.Tensor: + """Scatter [S, B, H] → [S/tp, B, H] for sequence parallel. No-op when tp=1.""" + if ps.tp_size == 1: + return x + return ScatterToSP.apply(x, ps.tp_size, ps.tp_rank, ps.tp_group) + + +def gather_from_sequence_parallel(x: torch.Tensor, ps: ParallelState) -> torch.Tensor: + """Gather [S/tp, B, H] → [S, B, H] from sequence parallel. No-op when tp=1.""" + if ps.tp_size == 1: + return x + return AllGatherDim0.apply(x, ps.tp_size, ps.tp_rank, ps.tp_group) + + +def gather_for_non_sp_head(x: torch.Tensor, ps: ParallelState) -> torch.Tensor: + """AllGather for non-SP consumer (e.g. vocab parallel head).""" + if ps.tp_size == 1: + return x + return AllGatherDim0ForNonSPConsumer.apply(x, ps.tp_size, ps.tp_rank, ps.tp_group) + + +__all__ = [ + "gather_for_non_sp_head", + "gather_from_sequence_parallel", + "scatter_to_sequence_parallel", +] diff --git a/experimental/lite/megatron/lite/primitive/parallel/state.py b/experimental/lite/megatron/lite/primitive/parallel/state.py new file mode 100644 index 00000000000..93481126953 --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/parallel/state.py @@ -0,0 +1,197 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""ParallelState and process group initialization.""" + +from __future__ import annotations + +from dataclasses import dataclass + +import torch.distributed as dist # pyright: ignore[reportMissingImports] + +from megatron.lite.primitive.utils import ensure_divisible + + +@dataclass +class ParallelState: + tp_group: dist.ProcessGroup | None = None + ep_group: dist.ProcessGroup | None = None + etp_group: dist.ProcessGroup | None = None + cp_group: dist.ProcessGroup | None = None + pp_group: dist.ProcessGroup | None = None + pp_cpu_group: dist.ProcessGroup | None = None + dp_group: dist.ProcessGroup | None = None + dp_cp_group: dist.ProcessGroup | None = None + tp_ep_group: dist.ProcessGroup | None = None + ep_dp_group: dist.ProcessGroup | None = None + cp_global_ranks: list[int] | None = None + pp_global_ranks: list[int] | None = None + + tp_size: int = 1 + ep_size: int = 1 + etp_size: int = 1 + cp_size: int = 1 + pp_size: int = 1 + dp_size: int = 1 + dp_cp_size: int = 1 + expert_dp_size: int = 1 + + tp_rank: int = 0 + ep_rank: int = 0 + etp_rank: int = 0 + cp_rank: int = 0 + pp_rank: int = 0 + dp_rank: int = 0 + dp_cp_rank: int = 0 + expert_dp_rank: int = 0 + + pp_is_first: bool = True + pp_is_last: bool = True + pp_next_rank: int = -1 + pp_prev_rank: int = -1 + virtual_pipeline_size: int | None = None + virtual_pipeline_rank: int | None = None + + # Optional explicit mcore pipeline layout (custom mode); None -> auto-infer. + pp_layout: str | list | None = None + + +def init_parallel(config) -> ParallelState: + """ + Initialize all process groups using dual rank decomposition. + + Dense layers: world = TP × CP × PP × DP + Expert layers: world = ETP × EP × PP × expert_DP + """ + assert dist.is_initialized(), "Call torch.distributed.init_process_group first" + + world = dist.get_world_size() + rank = dist.get_rank() + tp, ep, etp, cp, pp = config.tp, config.ep, config.etp, config.cp, config.pp + + dense_dp = ensure_divisible(world, tp * cp * pp) + expert_dp = ensure_divisible(world, etp * ep * pp) + + ps = ParallelState() + ps.pp_layout = getattr(config, "pp_layout", None) + ps.tp_size, ps.ep_size, ps.etp_size = tp, ep, etp + ps.cp_size, ps.pp_size, ps.dp_size = cp, pp, dense_dp + ps.expert_dp_size = expert_dp + ps.dp_cp_size = dense_dp * cp + + def _d(tp_i, cp_i, dp_i, pp_i): + return ((pp_i * dense_dp + dp_i) * cp + cp_i) * tp + tp_i + + def _e(etp_i, ep_i, edp_i, pp_i): + return ((pp_i * expert_dp + edp_i) * ep + ep_i) * etp + etp_i + + t = rank + my_tp = t % tp + t //= tp + my_cp = t % cp + t //= cp + my_ddp = t % dense_dp + t //= dense_dp + my_pp = t + + t = rank + my_etp = t % etp + t //= etp + my_ep = t % ep + t //= ep + my_edp = t % expert_dp + t //= expert_dp + assert t == my_pp, "PP rank must agree between dense and expert decompositions" + + ps.tp_rank, ps.cp_rank, ps.dp_rank, ps.pp_rank = my_tp, my_cp, my_ddp, my_pp + ps.dp_cp_rank = my_ddp * cp + my_cp + ps.ep_rank, ps.etp_rank, ps.expert_dp_rank = my_ep, my_etp, my_edp + ps.pp_is_first = my_pp == 0 + ps.pp_is_last = my_pp == pp - 1 + + for d in range(dense_dp): + for p in range(pp): + for c in range(cp): + ranks = [_d(t, c, d, p) for t in range(tp)] + g = dist.new_group(ranks) + if rank in ranks: + ps.tp_group = g + + for d in range(dense_dp): + for p in range(pp): + for t in range(tp): + ranks = [_d(t, c, d, p) for c in range(cp)] + g = dist.new_group(ranks) + if rank in ranks: + ps.cp_group = g + ps.cp_global_ranks = ranks + + for d in range(dense_dp): + for c in range(cp): + for t in range(tp): + ranks = [_d(t, c, d, p) for p in range(pp)] + g = dist.new_group(ranks) + try: + cpu_g = dist.new_group(ranks, backend="gloo") + except (RuntimeError, ValueError): + cpu_g = None + if rank in ranks: + ps.pp_group = g + ps.pp_cpu_group = cpu_g + ps.pp_global_ranks = ranks + + for p in range(pp): + for c in range(cp): + for t in range(tp): + ranks = [_d(t, c, d, p) for d in range(dense_dp)] + g = dist.new_group(ranks) + if rank in ranks: + ps.dp_group = g + + for p in range(pp): + for t in range(tp): + ranks = [_d(t, c, d, p) for d in range(dense_dp) for c in range(cp)] + g = dist.new_group(ranks) + if rank in ranks: + ps.dp_cp_group = g + + for d in range(expert_dp): + for p in range(pp): + for t in range(etp): + ranks = [_e(t, e, d, p) for e in range(ep)] + g = dist.new_group(ranks) + if rank in ranks: + ps.ep_group = g + + if etp > 1: + for d in range(expert_dp): + for p in range(pp): + for e in range(ep): + ranks = [_e(t, e, d, p) for t in range(etp)] + g = dist.new_group(ranks) + if rank in ranks: + ps.etp_group = g + + for d in range(expert_dp): + for p in range(pp): + ranks = [_e(t, e, d, p) for e in range(ep) for t in range(etp)] + g = dist.new_group(ranks) + if rank in ranks: + ps.tp_ep_group = g + + for p in range(pp): + for e in range(ep): + for t in range(etp): + ranks = [_e(t, e, d, p) for d in range(expert_dp)] + g = dist.new_group(ranks) + if rank in ranks: + ps.ep_dp_group = g + + pp_ranks = ps.pp_global_ranks + if pp_ranks is None: + raise RuntimeError("Pipeline ranks were not initialized.") + ps.pp_next_rank = pp_ranks[(my_pp + 1) % pp] if pp > 1 else rank + ps.pp_prev_rank = pp_ranks[(my_pp - 1) % pp] if pp > 1 else rank + + return ps + + +__all__ = ["ParallelState", "init_parallel"] diff --git a/experimental/lite/megatron/lite/primitive/parallel/thd.py b/experimental/lite/megatron/lite/primitive/parallel/thd.py new file mode 100644 index 00000000000..65a61bc0e01 --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/parallel/thd.py @@ -0,0 +1,613 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Packed sequence helpers for variable-length (THD) attention.""" + +from __future__ import annotations + +from copy import copy +from dataclasses import dataclass +from typing import Any + +import torch +import torch.distributed as dist + +from megatron.lite.primitive.utils.packed_seq import PackedSeqParams + + +@dataclass(frozen=True) +class PackedTHDBatch: + """Dense packed representation of a jagged no-padding batch.""" + + input_ids: torch.Tensor + labels: torch.Tensor | None + loss_mask: torch.Tensor | None + position_ids: torch.Tensor + packed_seq_params: Any + cu_seqlens_padded: torch.Tensor + lengths: torch.Tensor + padded_lengths: torch.Tensor + cp_size: int = 1 + cp_rank: int = 0 + cp_group: Any | None = None + + +def _make_packed_seq_params( + *, + cu_seqlens_padded: torch.Tensor, + max_seqlen: int, + cp_size: int = 1, + cp_rank: int = 0, + cp_group: Any | None = None, +): + extra_args = {} + if cp_size > 1: + extra_args["local_cp_size"] = cp_size + if cp_group is not None: + extra_args["cp_group"] = cp_group + return PackedSeqParams( + qkv_format="thd", + cu_seqlens_q=cu_seqlens_padded, + cu_seqlens_kv=cu_seqlens_padded, + max_seqlen_q=max_seqlen, + max_seqlen_kv=max_seqlen, + cu_seqlens_q_padded=cu_seqlens_padded, + cu_seqlens_kv_padded=cu_seqlens_padded, + cp_rank=cp_rank, + **extra_args, + ) + + +def _slice_along_dim(tensor: torch.Tensor, dim: int, start: int, end: int) -> torch.Tensor: + index = [slice(None)] * tensor.dim() + index[dim] = slice(start, end) + return tensor[tuple(index)] + + +def _assign_along_dim(dst: torch.Tensor, dim: int, start: int, src: torch.Tensor) -> None: + index = [slice(None)] * dst.dim() + index[dim] = slice(start, start + src.size(dim)) + dst[tuple(index)] = src + + +def _split_full_to_cp_local( + tensor: torch.Tensor, *, cu_seqlens_padded: torch.Tensor, cp_size: int, cp_rank: int, dim: int +) -> torch.Tensor: + if cp_size <= 1: + return tensor + + total_local = int(cu_seqlens_padded[-1].item()) // cp_size + out_shape = list(tensor.shape) + out_shape[dim] = total_local + local = torch.zeros(out_shape, dtype=tensor.dtype, device=tensor.device) + + for idx in range(int(cu_seqlens_padded.numel()) - 1): + full_start = int(cu_seqlens_padded[idx].item()) + full_end = int(cu_seqlens_padded[idx + 1].item()) + padded_len = full_end - full_start + if padded_len <= 0: + continue + chunk = padded_len // (2 * cp_size) + local_start = full_start // cp_size + + first = _slice_along_dim( + tensor, dim, full_start + cp_rank * chunk, full_start + (cp_rank + 1) * chunk + ) + second = _slice_along_dim( + tensor, dim, full_end - (cp_rank + 1) * chunk, full_end - cp_rank * chunk + ) + _assign_along_dim(local, dim, local_start, first) + _assign_along_dim(local, dim, local_start + chunk, second) + + return local + + +def _reconstruct_full_from_cp_parts( + parts: list[torch.Tensor], *, cu_seqlens_padded: torch.Tensor, cp_size: int, dim: int +) -> torch.Tensor: + if cp_size <= 1: + return parts[0] + + total_full = int(cu_seqlens_padded[-1].item()) + out_shape = list(parts[0].shape) + out_shape[dim] = total_full + full = torch.zeros(out_shape, dtype=parts[0].dtype, device=parts[0].device) + + for idx in range(int(cu_seqlens_padded.numel()) - 1): + full_start = int(cu_seqlens_padded[idx].item()) + full_end = int(cu_seqlens_padded[idx + 1].item()) + padded_len = full_end - full_start + if padded_len <= 0: + continue + chunk = padded_len // (2 * cp_size) + local_start = full_start // cp_size + + for rank, part in enumerate(parts): + first = _slice_along_dim(part, dim, local_start, local_start + chunk) + second = _slice_along_dim(part, dim, local_start + chunk, local_start + 2 * chunk) + _assign_along_dim(full, dim, full_start + rank * chunk, first) + _assign_along_dim(full, dim, full_end - (rank + 1) * chunk, second) + + return full + + +def reconstruct_packed_from_cp_parts( + parts: list[torch.Tensor], *, cu_seqlens_padded: torch.Tensor, cp_size: int, dim: int +) -> torch.Tensor: + """Reconstruct a full packed THD tensor from CP-local zigzag parts.""" + return _reconstruct_full_from_cp_parts( + parts, cu_seqlens_padded=cu_seqlens_padded, cp_size=cp_size, dim=dim + ) + + +def split_packed_to_cp_local( + tensor: torch.Tensor, *, cu_seqlens_padded: torch.Tensor, cp_size: int, cp_rank: int, dim: int +) -> torch.Tensor: + """Slice a full packed THD tensor back to one CP rank's zigzag shard.""" + return _split_full_to_cp_local( + tensor, cu_seqlens_padded=cu_seqlens_padded, cp_size=cp_size, cp_rank=cp_rank, dim=dim + ) + + +def _packed_cu_seqlens(packed_seq_params: Any) -> torch.Tensor | None: + if packed_seq_params is None: + return None + cu_seqlens = getattr(packed_seq_params, "cu_seqlens_q_padded", None) + if cu_seqlens is None: + cu_seqlens = getattr(packed_seq_params, "cu_seqlens_q", None) + return cu_seqlens + + +def has_packed_thd_params(packed_seq_params: Any) -> bool: + return _packed_cu_seqlens(packed_seq_params) is not None + + +def _sequence_dim(tensor: torch.Tensor) -> int: + return tensor.dim() - 1 if tensor.dim() > 1 else 0 + + +def _with_cp_metadata(packed_seq_params: Any, *, cp_size: int, cp_rank: int, cp_group: Any): + updated = copy(packed_seq_params) + updated.local_cp_size = cp_size + updated.cp_rank = cp_rank + updated.cp_group = cp_group + return updated + + +def parallel_state_from_model(model: Any) -> Any: + """Return the MLite parallel state from a raw model or wrapper.""" + + current = model + seen: set[int] = set() + while current is not None and id(current) not in seen: + seen.add(id(current)) + ps = getattr(current, "ps", None) + if ps is not None: + return ps + ps = getattr(current, "_parallel_state", None) + if ps is not None: + return ps + current = getattr(current, "module", None) + return None + + +def prepare_packed_thd_for_context_parallel( + packed_seq_params: Any, + tensors: tuple[torch.Tensor | None, ...], + *, + cp_size: int, + cp_rank: int, + cp_group: Any = None, + dims: tuple[int | None, ...] | None = None, +) -> tuple[Any, tuple[torch.Tensor | None, ...]]: + """Split plain packed THD tensors to one CP rank without knowing batch keys.""" + + tensor_tuple = tuple(tensors) + cu_seqlens = _packed_cu_seqlens(packed_seq_params) + if cu_seqlens is None or cp_size <= 1: + return packed_seq_params, tensor_tuple + if int(getattr(packed_seq_params, "local_cp_size", None) or 1) > 1: + return packed_seq_params, tensor_tuple + if dims is not None and len(dims) != len(tensor_tuple): + raise ValueError( + f"dims length {len(dims)} does not match tensors length {len(tensor_tuple)}." + ) + + local_tensors: list[torch.Tensor | None] = [] + for idx, tensor in enumerate(tensor_tuple): + if tensor is None: + local_tensors.append(None) + continue + dim = _sequence_dim(tensor) if dims is None or dims[idx] is None else int(dims[idx]) + local_tensors.append( + _split_full_to_cp_local( + tensor, + cu_seqlens_padded=cu_seqlens, + cp_size=cp_size, + cp_rank=cp_rank, + dim=dim, + ) + ) + return ( + _with_cp_metadata(packed_seq_params, cp_size=cp_size, cp_rank=cp_rank, cp_group=cp_group), + tuple(local_tensors), + ) + + +def prepare_packed_thd_kwargs_for_context_parallel( + model: Any, + kwargs: dict[str, Any], + *, + tensor_keys: tuple[str, ...] = ("input_ids", "labels", "loss_mask", "position_ids"), +) -> None: + packed_seq_params = kwargs.get("packed_seq_params") + if not has_packed_thd_params(packed_seq_params): + return + + ps = parallel_state_from_model(model) + packed_seq_params, tensors = prepare_packed_thd_for_context_parallel( + packed_seq_params, + tuple(kwargs.get(key) for key in tensor_keys), + cp_size=int(getattr(ps, "cp_size", 1) or 1), + cp_rank=int(getattr(ps, "cp_rank", 0) or 0), + cp_group=getattr(ps, "cp_group", None), + ) + if packed_seq_params is not None or "packed_seq_params" in kwargs: + kwargs["packed_seq_params"] = packed_seq_params + for key, tensor in zip(tensor_keys, tensors, strict=True): + if tensor is not None or key in kwargs: + kwargs[key] = tensor + + +@dataclass(frozen=True) +class ThdPackMeta: + """Per-sequence THD padding layout, recomputable from true seq lengths. + + Shared by a model's pack/unpack pair so the connector never needs to hold + ``PackedSeqParams`` or padded token tensors to reverse a model output. + """ + + lengths: torch.Tensor + padded_lengths: torch.Tensor + cu_seqlens_padded: torch.Tensor + cp_size: int + cp_group: Any | None + + +def thd_pack_meta( + seq_lens: torch.Tensor, + *, + tp_size: int = 1, + cp_size: int = 1, + cp_group: Any | None = None, + contiguous: bool = False, +) -> ThdPackMeta: + """Compute the padded THD layout for ``seq_lens`` without copying tokens. + + ``contiguous=False`` aligns to Megatron/TE zigzag CP (``2*cp``); ``True`` + aligns to contiguous CP (``cp``). Mirrors :func:`pack_nested_thd` padding. + """ + lengths = seq_lens.to(dtype=torch.int32) + cp_align = (cp_size if contiguous else 2 * cp_size) if cp_size > 1 else 1 + align_size = max(int(tp_size), 1) * cp_align + pad_size = (align_size - lengths % align_size) % align_size + padded_lengths = lengths + pad_size + cu_seqlens_padded = torch.zeros( + lengths.numel() + 1, dtype=torch.int32, device=lengths.device + ) + cu_seqlens_padded[1:] = torch.cumsum(padded_lengths, dim=0) + return ThdPackMeta(lengths, padded_lengths, cu_seqlens_padded, cp_size, cp_group) + + +def unpack_thd_to_nested( + output: torch.Tensor, meta: ThdPackMeta, *, contiguous: bool = False +) -> torch.Tensor: + """Reverse a model output back to jagged true-length form using ``meta``. + + Gathers CP-local shards (zigzag or contiguous reconstruct) then slices each + sequence's true length out of the padded layout. + """ + if output.dim() >= 2 and output.shape[0] == 1: + flat = output[0] + elif output.dim() >= 2 and output.shape[1] == 1: + flat = output[:, 0] + else: + flat = output + + if meta.cp_size > 1: + parts = _all_gather_cp_tensor(flat, cp_size=meta.cp_size, cp_group=meta.cp_group) + if contiguous: + flat = torch.cat(parts, dim=0) + else: + flat = _reconstruct_full_from_cp_parts( + parts, cu_seqlens_padded=meta.cu_seqlens_padded, cp_size=meta.cp_size, dim=0 + ) + + pieces = [] + for idx, length_t in enumerate(meta.lengths): + length = int(length_t.item()) + start = int(meta.cu_seqlens_padded[idx].item()) + pieces.append(flat[start : start + length]) + return torch.nested.as_nested_tensor(pieces, layout=torch.jagged) + + +def _all_gather_cp_tensor( + tensor: torch.Tensor, *, cp_size: int, cp_group: Any +) -> list[torch.Tensor]: + if cp_size <= 1: + return [tensor] + if cp_group is None: + raise ValueError("CP THD gather requires cp_group.") + try: + from torch.distributed.nn.functional import all_gather + + return list(all_gather(tensor, group=cp_group)) + except Exception: + parts = [torch.empty_like(tensor) for _ in range(cp_size)] + dist.all_gather(parts, tensor, group=cp_group) + return parts + + +def _roll_packed_thd_left_local( + tensor: torch.Tensor, *, cu_seqlens_padded: torch.Tensor, dims: int = -1 +) -> tuple[torch.Tensor, torch.Tensor]: + dim = dims if dims >= 0 else tensor.dim() + dims + if dim < 0 or dim >= tensor.dim(): + raise ValueError(f"Invalid roll dim {dims} for tensor with {tensor.dim()} dims.") + + rolled = tensor.clone() + for idx in range(int(cu_seqlens_padded.numel()) - 1): + start = int(cu_seqlens_padded[idx].item()) + end = int(cu_seqlens_padded[idx + 1].item()) + if end <= start: + continue + + index = [slice(None)] * tensor.dim() + index[dim] = slice(start, end) + seq = torch.roll(tensor[tuple(index)], shifts=-1, dims=dim) + + zero_index = [slice(None)] * seq.dim() + zero_index[dim] = slice(-1, None) + seq[tuple(zero_index)] = 0 + rolled[tuple(index)] = seq + + return rolled, rolled.sum() + + +def roll_packed_thd_left( + tensor: torch.Tensor, + *, + cu_seqlens_padded: torch.Tensor | None = None, + packed_seq_params: Any | None = None, + dims: int = -1, +) -> tuple[torch.Tensor, torch.Tensor]: + """Roll a THD packed tensor left without crossing sequence boundaries.""" + + cp_size = 1 + cp_rank = 0 + cp_group = None + if packed_seq_params is not None: + cu_seqlens_padded = getattr(packed_seq_params, "cu_seqlens_q", None) + cp_size = int(getattr(packed_seq_params, "local_cp_size", None) or 1) + cp_rank = int(getattr(packed_seq_params, "cp_rank", 0) or 0) + cp_group = getattr(packed_seq_params, "cp_group", None) + if cu_seqlens_padded is None: + raise ValueError("THD packed roll requires cu_seqlens.") + + dim = dims if dims >= 0 else tensor.dim() + dims + if cp_size <= 1: + return _roll_packed_thd_left_local(tensor, cu_seqlens_padded=cu_seqlens_padded, dims=dim) + + parts = _all_gather_cp_tensor(tensor, cp_size=cp_size, cp_group=cp_group) + full = _reconstruct_full_from_cp_parts( + parts, cu_seqlens_padded=cu_seqlens_padded, cp_size=cp_size, dim=dim + ) + rolled_full, token_sum = _roll_packed_thd_left_local( + full, cu_seqlens_padded=cu_seqlens_padded, dims=dim + ) + local = _split_full_to_cp_local( + rolled_full, cu_seqlens_padded=cu_seqlens_padded, cp_size=cp_size, cp_rank=cp_rank, dim=dim + ) + return local, token_sum + + +def pack_nested_thd( + input_ids: torch.Tensor, + *, + tp_size: int = 1, + cp_size: int = 1, + cp_rank: int = 0, + cp_group: Any | None = None, + split_cp: bool = True, + labels: torch.Tensor | None = None, + roll_labels: bool = False, + loss_mask: torch.Tensor | None = None, + roll_loss_mask: bool = False, +) -> PackedTHDBatch: + """Pack a jagged no-padding batch into Megatron Lite's THD model input. + + Mirrors VERL/Megatron's THD engine convention: each sequence is + padded to the tensor-parallel alignment, concatenated, then represented as + a single ``[1, local_padded_tokens]`` token row plus ``PackedSeqParams``. + For CP>1 the local row uses Megatron/TE zigzag chunking unless + ``split_cp=False``. The latter keeps full packed tokens and plain + ``PackedSeqParams`` while still padding each sample to CP-compatible + alignment, matching the external VERL runtime contract. + """ + + if cp_size < 1: + raise ValueError(f"cp_size must be >= 1, got {cp_size}") + if cp_rank < 0 or cp_rank >= cp_size: + raise ValueError(f"cp_rank must be in [0, {cp_size}), got {cp_rank}") + if not getattr(input_ids, "is_nested", False): + raise TypeError("pack_nested_thd expects a jagged NestedTensor input_ids.") + if labels is not None and not getattr(labels, "is_nested", False): + raise TypeError( + "pack_nested_thd expects jagged NestedTensor labels when labels are provided." + ) + if loss_mask is not None and not getattr(loss_mask, "is_nested", False): + raise TypeError( + "pack_nested_thd expects jagged NestedTensor loss_mask when loss_mask is provided." + ) + + align_size = max(int(tp_size), 1) * (2 * cp_size if cp_size > 1 else 1) + device = input_ids.device + offsets = input_ids.offsets().to(device=device) + lengths = offsets.diff().to(dtype=torch.int32) + pad_size = (align_size - lengths % align_size) % align_size + padded_lengths = lengths + pad_size + + cu_seqlens_padded = torch.zeros(lengths.numel() + 1, dtype=torch.int32, device=device) + cu_seqlens_padded[1:] = torch.cumsum(padded_lengths, dim=0) + total_padded = int(cu_seqlens_padded[-1].item()) + total_local = total_padded // cp_size if split_cp else total_padded + max_seqlen = int(padded_lengths.max().item()) if padded_lengths.numel() else 0 + + packed_input = torch.zeros(total_local, dtype=input_ids.dtype, device=device) + packed_labels = ( + torch.zeros(total_local, dtype=labels.dtype, device=device) if labels is not None else None + ) + packed_loss_mask = ( + torch.zeros(total_local, dtype=loss_mask.dtype, device=device) + if loss_mask is not None + else None + ) + # Megatron's rotary embedding slices position embeddings on the CP rank. + # Keep packed position ids full-length while CP-slicing tokens/labels/masks. + position_ids = torch.zeros(total_padded, dtype=torch.long, device=device) + + for idx, length_t in enumerate(lengths): + length = int(length_t.item()) + padded_length = int(padded_lengths[idx].item()) + full_start = int(cu_seqlens_padded[idx].item()) + local_start = full_start // cp_size if split_cp else full_start + + seq_input = torch.zeros(padded_length, dtype=input_ids.dtype, device=device) + seq_input[:length] = input_ids[idx] + seq_labels = None + if labels is not None: + assert packed_labels is not None + seq_labels = torch.zeros(padded_length, dtype=labels.dtype, device=device) + seq_labels[:length] = labels[idx] + if roll_labels and length > 0: + seq_labels[:length] = torch.roll(seq_labels[:length], shifts=-1, dims=0) + seq_labels[length - 1] = 0 + seq_loss_mask = None + if loss_mask is not None: + assert packed_loss_mask is not None + seq_loss_mask = torch.zeros(padded_length, dtype=loss_mask.dtype, device=device) + seq_loss_mask[:length] = loss_mask[idx] + if roll_loss_mask and length > 0: + seq_loss_mask[:length] = torch.roll(seq_loss_mask[:length], shifts=-1, dims=0) + seq_loss_mask[length - 1] = 0 + seq_positions = torch.zeros(padded_length, dtype=torch.long, device=device) + seq_positions[:length] = torch.arange(length, dtype=torch.long, device=device) + + local_input = ( + _split_full_to_cp_local( + seq_input, + cu_seqlens_padded=torch.tensor( + [0, padded_length], dtype=torch.int32, device=device + ), + cp_size=cp_size, + cp_rank=cp_rank, + dim=0, + ) + if split_cp + else seq_input + ) + packed_input[local_start : local_start + local_input.numel()] = local_input + if seq_labels is not None: + local_labels = ( + _split_full_to_cp_local( + seq_labels, + cu_seqlens_padded=torch.tensor( + [0, padded_length], dtype=torch.int32, device=device + ), + cp_size=cp_size, + cp_rank=cp_rank, + dim=0, + ) + if split_cp + else seq_labels + ) + assert packed_labels is not None + packed_labels[local_start : local_start + local_labels.numel()] = local_labels + if seq_loss_mask is not None: + local_loss_mask = ( + _split_full_to_cp_local( + seq_loss_mask, + cu_seqlens_padded=torch.tensor( + [0, padded_length], dtype=torch.int32, device=device + ), + cp_size=cp_size, + cp_rank=cp_rank, + dim=0, + ) + if split_cp + else seq_loss_mask + ) + assert packed_loss_mask is not None + packed_loss_mask[local_start : local_start + local_loss_mask.numel()] = local_loss_mask + position_ids[full_start : full_start + padded_length] = seq_positions + + return PackedTHDBatch( + input_ids=packed_input.unsqueeze(0), + labels=packed_labels.unsqueeze(0) if packed_labels is not None else None, + loss_mask=(packed_loss_mask.unsqueeze(0) if packed_loss_mask is not None else None), + position_ids=position_ids.unsqueeze(0), + packed_seq_params=_make_packed_seq_params( + cu_seqlens_padded=cu_seqlens_padded, + max_seqlen=max_seqlen, + cp_size=cp_size if split_cp else 1, + cp_rank=cp_rank if split_cp else 0, + cp_group=cp_group if split_cp else None, + ), + cu_seqlens_padded=cu_seqlens_padded, + lengths=lengths, + padded_lengths=padded_lengths, + cp_size=cp_size, + cp_rank=cp_rank, + cp_group=cp_group, + ) + + +def unpack_packed_thd_to_nested(output: torch.Tensor, batch: PackedTHDBatch) -> torch.Tensor: + """Unpack ``[1, total_padded, ...]`` THD model output back to jagged form.""" + + if output.dim() >= 2 and output.shape[0] == 1: + flat = output[0] + elif output.dim() >= 2 and output.shape[1] == 1: + flat = output[:, 0] + else: + flat = output + + if batch.cp_size > 1: + dim = 0 + parts = _all_gather_cp_tensor(flat, cp_size=batch.cp_size, cp_group=batch.cp_group) + flat = _reconstruct_full_from_cp_parts( + parts, cu_seqlens_padded=batch.cu_seqlens_padded, cp_size=batch.cp_size, dim=dim + ) + + pieces = [] + for idx, length_t in enumerate(batch.lengths): + length = int(length_t.item()) + start = int(batch.cu_seqlens_padded[idx].item()) + pieces.append(flat[start : start + length]) + return torch.nested.as_nested_tensor(pieces, layout=torch.jagged) + + +__all__ = [ + "PackedSeqParams", + "PackedTHDBatch", + "ThdPackMeta", + "has_packed_thd_params", + "pack_nested_thd", + "parallel_state_from_model", + "prepare_packed_thd_for_context_parallel", + "prepare_packed_thd_kwargs_for_context_parallel", + "reconstruct_packed_from_cp_parts", + "roll_packed_thd_left", + "split_packed_to_cp_local", + "thd_pack_meta", + "unpack_packed_thd_to_nested", + "unpack_thd_to_nested", +] diff --git a/experimental/lite/megatron/lite/primitive/protocols.py b/experimental/lite/megatron/lite/primitive/protocols.py new file mode 100644 index 00000000000..d000864bc4a --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/protocols.py @@ -0,0 +1,24 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Train-side lightweight protocols and defaults.""" + +from __future__ import annotations + +from collections.abc import Callable + +from torch.distributed.tensor import Replicate # pyright: ignore[reportMissingImports] + +ExpertClassifierFn = Callable[[str], bool] +PlacementFn = Callable[[str], list] + + +def default_expert_classifier(name: str) -> bool: + """Default: params with 'experts' (but not 'router' or 'shared') are expert params.""" + return "experts" in name and "router" not in name and "shared" not in name + + +def default_placement_fn(name: str) -> list: + """Default: all Replicate (safe but no resharding benefit).""" + return [Replicate(), Replicate(), Replicate(), Replicate()] + + +__all__ = ["ExpertClassifierFn", "PlacementFn", "default_expert_classifier", "default_placement_fn"] diff --git a/experimental/lite/megatron/lite/primitive/recompute.py b/experimental/lite/megatron/lite/primitive/recompute.py new file mode 100644 index 00000000000..a83a209b0f7 --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/recompute.py @@ -0,0 +1,381 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Model-agnostic activation recompute and offload wrappers.""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +import torch # pyright: ignore[reportMissingImports] +import torch.nn as nn # pyright: ignore[reportMissingImports] + +# ── CheckpointWithoutOutput ─────────────────────────────────────────────────── +# Zero-copy C++ extension: makes dst's UntypedStorage point to src's data. +# Operates below TensorImpl level → ALL views/reshapes that share dst's StorageImpl +# (e.g. TE GroupedLinear's inp.reshape() saved for backward) see the restored data. +# Equivalent to MC's share_storage in megatron/core/tensor_parallel/random.py. +_SHARE_STORAGE_SRC = r""" +#include + +void share_storage(at::Tensor dst, at::Tensor src) { + auto* dst_impl = dst.storage().unsafeGetStorageImpl(); + auto* src_ref = new c10::Storage(src.storage()); + void* data = src_ref->data_ptr().get(); + size_t nbytes = src_ref->nbytes(); + c10::Device device = src_ref->device(); + c10::DataPtr shared( + data, + static_cast(src_ref), + [](void* ctx) { delete static_cast(ctx); }, + device); + dst_impl->set_data_ptr(std::move(shared)); + dst_impl->set_nbytes(nbytes); +} +""" + +_share_storage_ext = None + + +def _get_share_storage() -> Callable: + global _share_storage_ext + if _share_storage_ext is None: + from torch.utils.cpp_extension import load_inline + + _share_storage_ext = load_inline( + name="share_storage_ext", + cpp_sources=_SHARE_STORAGE_SRC, + functions=["share_storage"], + verbose=False, + ) + return _share_storage_ext.share_storage + + +class _CheckpointWithoutOutputFn(torch.autograd.Function): + """Autograd Function for CheckpointWithoutOutput. + + Forward: runs function with no_grad, saves inputs. + Backward: uses outputs/inputs set by CheckpointWithoutOutput._recompute(). + """ + + @staticmethod + def forward(ctx, run_function, ckpt_obj, *args): + ctx.run_function = run_function + ctx.preserve_rng_state = ckpt_obj.preserve_rng_state + if ckpt_obj.preserve_rng_state: + ctx.cpu_rng_state = torch.get_rng_state() + ctx.cuda_rng_state = torch.cuda.get_rng_state() + + ctx.tensor_indices = [i for i, a in enumerate(args) if isinstance(a, torch.Tensor)] + ctx.non_tensor_args = [ + (i, a) for i, a in enumerate(args) if not isinstance(a, torch.Tensor) + ] + ctx.num_args = len(args) + ctx.save_for_backward(*[a for a in args if isinstance(a, torch.Tensor)]) + + with torch.no_grad(): + outputs = run_function(*args) + + ckpt_obj.ctx = ctx + return outputs + + @staticmethod + def backward(ctx, *grad_outputs): + # inputs and outputs are set by CheckpointWithoutOutput._recompute() + # before this backward runs (via the hook registered on the downstream tensor). + inputs = ctx.inputs + outputs = ctx.outputs + torch.autograd.backward(outputs, grad_outputs) + ctx.outputs = None + ctx.inputs = None + grads = tuple(inp.grad if isinstance(inp, torch.Tensor) else None for inp in inputs) + return (None, None) + grads + + +class CheckpointWithoutOutput: + """Checkpoint a function and discard its output to save memory. + + Equivalent to MC's CheckpointWithoutOutput from megatron/core/tensor_parallel/random.py. + The output tensor's storage is freed immediately after downstream computation; + it is recomputed just-in-time during backward via a hook on the downstream output. + + The C++ share_storage extension restores the output at the StorageImpl level so + that ALL aliases (including views saved by TE GroupedLinear's backward) see the data. + + Usage (mirrors MC's moe_act pattern):: + + ckpt = CheckpointWithoutOutput() + h = ckpt.checkpoint(activation_func, fc1_out, probs) + fc2_out = fc2(h, m_splits) + ckpt.discard_output_and_register_recompute(fc2_out) + """ + + def __init__(self, preserve_rng_state: bool = True): + self.preserve_rng_state = preserve_rng_state + self.run_function: Callable | None = None + self._cpu_rng: torch.Tensor | None = None + self._cuda_rng: torch.Tensor | None = None + self.ctx: Any | None = None + self.outputs: tuple[torch.Tensor, ...] | None = None + + def checkpoint(self, run_function: Callable, *args) -> Any: + self.run_function = run_function + if self.preserve_rng_state: + self._cpu_rng = torch.get_rng_state() + self._cuda_rng = torch.cuda.get_rng_state() + + outputs = _CheckpointWithoutOutputFn.apply(run_function, self, *args) + self.outputs = (outputs,) if isinstance(outputs, torch.Tensor) else tuple(outputs) + return outputs + + def _recompute(self, _) -> None: + if self.ctx is None: + return + + # Reconstruct args from saved context. + tensors = list(self.ctx.saved_tensors) + args: list = [None] * self.ctx.num_args + t_it = iter(tensors) + for i in self.ctx.tensor_indices: + t = next(t_it) + args[i] = t.detach().requires_grad_(t.requires_grad) + for i, val in self.ctx.non_tensor_args: + args[i] = val + + # Recompute with forward-time RNG states. + if self.preserve_rng_state: + saved_cpu = torch.get_rng_state() + saved_cuda = torch.cuda.get_rng_state() + torch.set_rng_state(self._cpu_rng) + torch.cuda.set_rng_state(self._cuda_rng) + with torch.enable_grad(): + outputs = self.run_function(*args) + if self.preserve_rng_state: + torch.set_rng_state(saved_cpu) + torch.cuda.set_rng_state(saved_cuda) + + if isinstance(outputs, torch.Tensor): + outputs = (outputs,) + + # Zero-copy: make original output's StorageImpl point to recomputed data. + share_storage = _get_share_storage() + for orig, new in zip(self.outputs, outputs, strict=False): + share_storage(orig, new) + + self.ctx.outputs = list(outputs) + self.ctx.inputs = args + self.run_function = None + self._cpu_rng = None + self._cuda_rng = None + self.outputs = None + self.ctx = None + + def reset(self) -> None: + """Reset state so the instance can be reused across forward passes.""" + self.run_function = None + self._cpu_rng = None + self._cuda_rng = None + self.ctx = None + self.outputs = None + + def discard_output_and_register_recompute(self, hook_tensor: torch.Tensor) -> None: + """Free output tensor storage; recompute when hook_tensor's grad is computed.""" + for out in self.outputs: + out.untyped_storage().resize_(0) + if hook_tensor.requires_grad: + hook_tensor.register_hook(self._recompute) + + +ModuleMap = dict[str, Callable[[nn.Module], nn.Module | None]] +"""Maps module name → lambda that extracts a sub-module from a layer.""" + + +def apply_recompute( + layers: nn.ModuleList, + module_names: list[str], + module_map: ModuleMap, + no_rng_modules: set[str] | None = None, +) -> None: + """Wrap specified sub-modules with activation checkpointing for recomputation.""" + if not module_names: + return + no_rng = no_rng_modules or set() + for layer in layers: + if "full" in module_names: + wrap_checkpoint(layer) + else: + for mod_name in module_names: + if mod_name in module_map: + submod = module_map[mod_name](layer) + if submod is not None: + wrap_checkpoint(submod, preserve_rng_state=mod_name not in no_rng) + + +def apply_offload(layers: nn.ModuleList, module_names: list[str], module_map: ModuleMap) -> None: + """Wrap specified sub-modules with activation offloading to CPU.""" + if not module_names: + return + try: + from torch.utils.checkpoint import CheckpointPolicy # noqa: F401 + except ImportError: + log_rank0("WARNING: torch.utils.checkpoint policy_fn not available, skipping offload") + return + for layer in layers: + for mod_name in module_names: + if mod_name in module_map: + submod = module_map[mod_name](layer) + if submod is not None: + wrap_offload(submod) + + +class CheckpointFunction(torch.autograd.Function): + """Reentrant activation checkpoint using custom autograd.Function. + + Adapted from Megatron-Core's CheckpointFunction. Key differences: + - No distribute_saved_activations (not needed for our TP implementation) + - No model-parallel RNG tracker (we use TE's built-in RNG management) + - Handles expert_bias restore for MoE router determinism + """ + + @staticmethod + def forward( + ctx: Any, run_function: Callable, preserve_rng_state: bool, *args: torch.Tensor + ) -> Any: + ctx.run_function = run_function + ctx.preserve_rng_state = preserve_rng_state + + # Save RNG states for deterministic recomputation. + if preserve_rng_state: + ctx.cpu_rng_state = torch.get_rng_state() + ctx.cuda_rng_state = torch.cuda.get_rng_state() + + # Run forward without gradient tracking — discard intermediate activations. + with torch.no_grad(): + outputs = run_function(*args) + + # Save inputs for recomputation in backward. + ctx.save_for_backward(*args) + return outputs + + @staticmethod + def backward(ctx: Any, *grad_outputs: torch.Tensor) -> tuple: + if not torch.autograd._is_checkpoint_valid(): + raise RuntimeError( + "Checkpointing is not compatible with .grad(), use .backward() instead" + ) + + inputs = ctx.saved_tensors + + # Fork RNG: restore forward-time states, then reset to current after recompute. + if ctx.preserve_rng_state: + current_cpu_rng = torch.get_rng_state() + current_cuda_rng = torch.cuda.get_rng_state() + torch.set_rng_state(ctx.cpu_rng_state) + torch.cuda.set_rng_state(ctx.cuda_rng_state) + + # Recompute forward pass with gradients enabled. + detached = tuple( + t.detach().requires_grad_(t.requires_grad) if isinstance(t, torch.Tensor) else t + for t in inputs + ) + with torch.enable_grad(): + outputs = ctx.run_function(*detached) + + # Restore RNG states. + if ctx.preserve_rng_state: + torch.set_rng_state(current_cpu_rng) + torch.cuda.set_rng_state(current_cuda_rng) + + if isinstance(outputs, torch.Tensor): + outputs = (outputs,) + + # Filter to outputs that need gradients. + outputs_with_grad = [] + grad_for_outputs = [] + for out, grad in zip(outputs, grad_outputs, strict=False): + if torch.is_tensor(out) and out.requires_grad: + outputs_with_grad.append(out) + grad_for_outputs.append(grad) + + if outputs_with_grad: + torch.autograd.backward(outputs_with_grad, grad_for_outputs) + + grads = tuple(inp.grad if isinstance(inp, torch.Tensor) else None for inp in detached) + # None for run_function, None for preserve_rng_state, then grads for each input. + return (None, None) + grads + + +def wrap_checkpoint(module: nn.Module, *, preserve_rng_state: bool = True) -> None: + """Wrap a module's forward with reentrant activation checkpointing.""" + original_forward = module.forward + _routers = [m for m in module.modules() if hasattr(m, "expert_bias")] + + def _checkpointed_forward(*args, **kwargs): + # expert_bias is modified in-place by the router during forward. + # Save and restore it so the recomputation in backward sees the same values. + if _routers: + saved = [r.expert_bias.clone() for r in _routers] + call_count = [0] + + def _fwd(*a, **kw): + call_count[0] += 1 + if call_count[0] > 1: + for r, s in zip(_routers, saved, strict=False): + r.expert_bias.copy_(s) + return original_forward(*a, **kw) + + else: + _fwd = original_forward + + # CheckpointFunction.apply only accepts positional tensor args. + # Wrap kwargs into the function closure. + if kwargs: + + def _fn(*a): + return _fwd(*a, **kwargs) + + else: + _fn = _fwd + + return CheckpointFunction.apply(_fn, preserve_rng_state, *args) + + module.forward = _checkpointed_forward + + +def wrap_offload(module: nn.Module) -> None: + """Wrap a module's forward with activation offloading.""" + original_forward = module.forward + + def _offloaded_forward(*args, **kwargs): + return torch.utils.checkpoint.checkpoint( + original_forward, *args, use_reentrant=False, **kwargs + ) + + module.forward = _offloaded_forward + + +def log_rank0(msg: str) -> None: + if not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0: + print(f"[megatron.lite] {msg}", flush=True) + + +def parse_recompute_spec(recompute: str | list[str] | None) -> list[str]: + """Parse a recompute spec into a list of module names.""" + if recompute is None or recompute == "none": + return [] + if recompute == "full": + return ["full"] + if isinstance(recompute, list): + return recompute + return recompute.split(",") + + +__all__ = [ + "CheckpointFunction", + "CheckpointWithoutOutput", + "ModuleMap", + "apply_offload", + "apply_recompute", + "parse_recompute_spec", + "wrap_checkpoint", + "wrap_offload", +] diff --git a/experimental/lite/megatron/lite/primitive/train_step.py b/experimental/lite/megatron/lite/primitive/train_step.py new file mode 100644 index 00000000000..88eeb896945 --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/train_step.py @@ -0,0 +1,192 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Reusable train-step primitives owned by Megatron Lite.""" + +from __future__ import annotations + +from collections.abc import Callable + +import torch +import torch.distributed as dist +from megatron.lite.primitive.parallel import ParallelState +from megatron.lite.primitive.protocols import ExpertClassifierFn, default_expert_classifier +from megatron.lite.runtime.contracts.loss import split_loss_context, use_loss_context + + +def run_microbatch_loop( + model, + data_iter, + num_microbatches: int, + forward_fn, + optimizer=None, + dist_opt: bool = False, + pre_forward_hook: Callable[[torch.Tensor], None] | None = None, + loss_fn: Callable | None = None, + forward_only: bool = False, +): + """Run forward-backward over microbatches with loss accumulation. + + Args: + forward_fn: ``forward_fn(model, batch) -> dict`` with at least ``"loss"`` key + (when ``loss_fn`` is None) or model outputs (when ``loss_fn`` is provided). + pre_forward_hook: Optional callable ``hook(scale: torch.Tensor) -> None`` + invoked once per microbatch, right before ``forward_fn``. ``scale`` is + ``1.0 / num_microbatches`` (matches MC's ``schedules.forward_step``, + see `pipeline_parallel/schedules.py`). Used e.g. by MoE aux-loss + scale-setting; runtime stays model-agnostic by passing the hook + through from the model bundle's extras. + # TODO: once CP is supported, align with MC's + # `schedules:297`-style scale of `cp_group_size / num_microbatches` + # (currently assumes ``cp_group_size == 1``). + loss_fn: Optional external loss function. + ``loss_fn(model_output: dict, batch) -> (loss: Tensor, metrics: dict)``. + When provided, ``forward_fn`` output is passed to ``loss_fn`` instead of + reading ``out["loss"]`` directly. This enables RLHF policy/value losses. + forward_only: When True, run forward without backward (e.g. validation / + ``infer_batch``). The caller (verl) runs the forward under ``no_grad`` so the + loss has no ``grad_fn``; calling ``.backward()`` then raises. Mirrors the + pipeline path, which already threads ``forward_only`` to skip backward. + """ + last_out = None + all_metrics: list[dict] = [] + for mb in range(num_microbatches): + batch, loss_context = split_loss_context(next(data_iter)) + if pre_forward_hook is not None: + scale = torch.tensor(1.0 / num_microbatches, device="cuda") + pre_forward_hook(scale) + with use_loss_context(loss_context): + out = forward_fn(model, batch) + if dist_opt and optimizer is not None and mb == num_microbatches - 1: + optimizer.grad_sync_enabled = True + if loss_fn is not None: + if loss_context is None: + loss, metrics = loss_fn(out, batch) + else: + loss, metrics = loss_fn(out, batch, loss_context) + if not forward_only: + (loss / num_microbatches).backward() + out["loss"] = loss.detach() + all_metrics.append(metrics) + elif not forward_only: + (out["loss"] / num_microbatches).backward() + last_out = out + if last_out is not None and all_metrics: + last_out["_loss_fn_metrics"] = all_metrics + return last_out + + +def compute_and_clip_grad_norm( + model, + optimizer, + max_norm: float, + use_dist_opt: bool, + sp_params=None, + sp_group=None, + *, + report_global_norm: bool = False, + ps: ParallelState | None = None, + is_expert_param: ExpertClassifierFn = default_expert_classifier, +): + """SP AllReduce + finish grad sync + clip grad norm. Returns grad_norm.""" + if sp_params: + sp_grads = [p.grad for p in sp_params if p.grad is not None] + if sp_grads: + flat = torch.cat([g.view(-1) for g in sp_grads]) + dist.all_reduce(flat, op=dist.ReduceOp.SUM, group=sp_group) + offset = 0 + for g in sp_grads: + n = g.numel() + g.copy_(flat[offset : offset + n].view_as(g)) + offset += n + report_norm = None + if report_global_norm: + if ps is None: + raise ValueError("`ps` is required when `report_global_norm=True`.") + report_norm = compute_global_grad_norm(model, ps, is_expert_param=is_expert_param) + if use_dist_opt: + optimizer.finish_grad_sync() + return optimizer.clip_grad_norm() + local_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm) + return report_norm if report_norm is not None else local_norm + + +def compute_global_grad_norm( + model, ps: ParallelState, *, is_expert_param: ExpertClassifierFn = default_expert_classifier +) -> torch.Tensor: + """Compute benchmark global grad norm with dist-opt-aligned reduction order.""" + dense_sq = _bucketed_grad_sq_sum( + model, + include_param=lambda name: not is_expert_param(name), + replica_group=ps.dp_cp_group, + replica_size=ps.dp_cp_size, + ) + expert_sq = _bucketed_grad_sq_sum( + model, + include_param=is_expert_param, + replica_group=ps.ep_dp_group, + replica_size=ps.expert_dp_size, + ) + + if ps.tp_size > 1 and ps.tp_group is not None: + dist.all_reduce(dense_sq, group=ps.tp_group) + + if ps.ep_size > 1 and ps.ep_group is not None: + dist.all_reduce(expert_sq, group=ps.ep_group) + if ps.etp_size > 1 and ps.etp_group is not None: + dist.all_reduce(expert_sq, group=ps.etp_group) + + total_sq = dense_sq + expert_sq + if ps.pp_size > 1 and ps.pp_group is not None: + dist.all_reduce(total_sq, group=ps.pp_group) + return total_sq.sqrt() + + +def _bucketed_grad_sq_sum( + model, + *, + include_param: Callable[[str], bool], + replica_group, + replica_size: int, + max_bucket_bytes: int = 80 * 1024 * 1024, +) -> torch.Tensor: + """Accumulate squared norm after averaging replica grads within each bucket.""" + total_sq = torch.zeros(1, device="cuda") + bucket: list[torch.Tensor] = [] + bucket_bytes = 0 + + def flush_bucket() -> None: + nonlocal bucket_bytes, bucket, total_sq + if not bucket: + return + flat = torch.cat(bucket) + if replica_size > 1 and replica_group is not None: + dist.all_reduce(flat, group=replica_group) + flat.div_(replica_size) + total_sq += flat.float().norm().pow(2) + bucket = [] + bucket_bytes = 0 + + for name, param in model.named_parameters(): + if param.grad is None or not include_param(name): + continue + grad = param.grad.view(-1) + grad_bytes = grad.numel() * grad.element_size() + if bucket and bucket_bytes + grad_bytes > max_bucket_bytes: + flush_bucket() + bucket.append(grad) + bucket_bytes += grad_bytes + + flush_bucket() + return total_sq + + +def optimizer_step(optimizer) -> None: + """Execute optimizer step.""" + optimizer.step() + + +__all__ = [ + "compute_and_clip_grad_norm", + "compute_global_grad_norm", + "optimizer_step", + "run_microbatch_loop", +] diff --git a/experimental/lite/megatron/lite/primitive/utils/__init__.py b/experimental/lite/megatron/lite/primitive/utils/__init__.py new file mode 100644 index 00000000000..3ab4a5c33a7 --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/utils/__init__.py @@ -0,0 +1,26 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +from __future__ import annotations + +import torch # pyright: ignore[reportMissingImports] + + +def build_fp8_recipe(train_config=None): + """Build the standard TE FP8 recipe (DelayedScaling, HYBRID format, H100).""" + from transformer_engine.common.recipe import DelayedScaling, Format + + return DelayedScaling(margin=0, fp8_format=Format.HYBRID) + + +def ensure_divisible(numerator: int, denominator: int, msg: str = "") -> int: + if numerator % denominator != 0: + detail = f" ({msg})" if msg else "" + raise ValueError(f"{numerator} is not divisible by {denominator}{detail}") + return numerator // denominator + + +def log_rank0(msg: str) -> None: + if not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0: + print(f"[megatron.lite] {msg}", flush=True) + + +__all__ = ["build_fp8_recipe", "ensure_divisible", "log_rank0"] diff --git a/experimental/lite/megatron/lite/primitive/utils/moe.py b/experimental/lite/megatron/lite/primitive/utils/moe.py new file mode 100644 index 00000000000..c219d4d6228 --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/utils/moe.py @@ -0,0 +1,426 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""MoE routing, permutation, and router GEMM helpers for MLite primitives.""" + +from __future__ import annotations + +from typing import Optional, Tuple + +import torch + +from transformer_engine.pytorch.cpp_extensions import general_gemm +from transformer_engine.pytorch.module import base as te_module_base +from transformer_engine.pytorch.permutation import moe_permute as fused_permute +from transformer_engine.pytorch.permutation import ( + moe_permute_and_pad_with_probs as fused_permute_and_pad_with_probs, +) +from transformer_engine.pytorch.permutation import ( + moe_permute_with_probs as fused_permute_with_probs, +) +from transformer_engine.pytorch.permutation import moe_unpermute as fused_unpermute +from transformer_engine.pytorch.router import ( + fused_compute_score_for_moe_aux_loss, + fused_moe_aux_loss, + fused_topk_with_score_function, +) + + +def _te_general_gemm( + a: torch.Tensor, + b: torch.Tensor, + out_dtype: torch.dtype | None = None, + *, + layout: str = "TN", + out: torch.Tensor | None = None, + bias: torch.Tensor | None = None, + grad: bool = False, +): + if (get_workspace := getattr(te_module_base, "get_workspace", None)) is None: + return None + kwargs = dict( + workspace=get_workspace(), + out_dtype=out_dtype, + quantization_params=None, + gelu=None, + gelu_in=None, + accumulate=False, + layout=layout, + out=out, + bias=bias, + use_split_accumulator=False, + grad=grad, + ub=None, + ub_type=None, + extra_output=None, + bulk_overlap=False, + ) + return general_gemm(a, b, **kwargs) + + +def switch_load_balancing_loss_func( + probs: torch.Tensor, + tokens_per_expert: torch.Tensor, + total_num_tokens: int, + topk: int, + num_experts: int, + moe_aux_loss_coeff: float, + *, + fused: bool = False, + padding_mask: torch.Tensor | None = None, +) -> torch.Tensor: + if padding_mask is not None: + probs = probs * padding_mask.unsqueeze(-1) + + if fused: + return fused_moe_aux_loss( + probs=probs, + tokens_per_expert=tokens_per_expert, + total_num_tokens=total_num_tokens, + topk=topk, + num_experts=num_experts, + coeff=moe_aux_loss_coeff, + ) + + aggregated_probs_per_expert = probs.sum(dim=0) + return torch.sum(aggregated_probs_per_expert * tokens_per_expert) * ( + num_experts * moe_aux_loss_coeff / (topk * total_num_tokens * total_num_tokens) + ) + + +def permute( + tokens: torch.Tensor, + routing_map: torch.Tensor, + probs: Optional[torch.Tensor] = None, + num_out_tokens: Optional[int] = None, + fused: bool = False, + drop_and_pad: bool = False, + tokens_per_expert: Optional[torch.Tensor] = None, + align_size: int = 0, +) -> Tuple[ + torch.Tensor, + Optional[torch.Tensor], + torch.Tensor, + Optional[torch.Tensor], + Optional[torch.Tensor], +]: + if fused and probs is None: + permuted_input, sorted_indices = fused_permute( + tokens, routing_map, num_out_tokens=num_out_tokens + ) + return permuted_input, None, sorted_indices, None, tokens_per_expert + + if fused and probs is not None: + if tokens_per_expert is not None and align_size > 0: + return fused_permute_and_pad_with_probs( + tokens, probs, routing_map, tokens_per_expert, align_size + ) + output, permuted_probs, row_id_map = fused_permute_with_probs( + tokens, probs, routing_map, num_out_tokens=num_out_tokens + ) + return output, permuted_probs, row_id_map, None, tokens_per_expert + + num_tokens, _hidden = tokens.shape + num_experts = routing_map.shape[1] + permuted_probs = None + if drop_and_pad and num_out_tokens is not None: + capacity = num_out_tokens // num_experts + assert not routing_map.requires_grad + routing_map = routing_map.to(dtype=torch.int8).T.contiguous() + sorted_indices = routing_map.argsort(dim=-1, descending=True, stable=True)[ + :, :capacity + ].contiguous() + sorted_indices = sorted_indices.view(-1) + + if probs is not None: + probs_t_1d = probs.T.contiguous().view(-1) + indices_dim0 = torch.arange(num_experts, device=routing_map.device).unsqueeze(-1) + indices_dim1 = sorted_indices.view(num_experts, capacity) + indices_1d = (indices_dim0 * num_tokens + indices_dim1).view(-1) + permuted_probs = probs_t_1d.index_select(0, indices_1d) + else: + if num_out_tokens is None: + raise AssertionError("num_out_tokens is required for argsort-based permute") + + routing_map = routing_map.bool().T.contiguous() + flat_sorted = routing_map.reshape(-1).argsort(descending=True, stable=True) + flat_sorted = flat_sorted[:num_out_tokens] + sorted_indices = flat_sorted % num_tokens + + if probs is not None: + permuted_probs = probs.T.contiguous().reshape(-1)[flat_sorted] + + return ( + tokens.index_select(0, sorted_indices), + permuted_probs, + sorted_indices, + None, + tokens_per_expert, + ) + + +def unpermute( + permuted_tokens: torch.Tensor, + sorted_indices: torch.Tensor, + restore_shape: torch.Size, + probs: Optional[torch.Tensor] = None, + routing_map: Optional[torch.Tensor] = None, + fused: bool = False, + drop_and_pad: bool = False, + pad_offsets: Optional[torch.Tensor] = None, +) -> torch.Tensor: + if fused: + kwargs = {} + if pad_offsets is not None: + kwargs["pad_offsets"] = pad_offsets + return fused_unpermute( + permuted_tokens, + sorted_indices, + merging_probs=probs, + restore_shape=restore_shape, + **kwargs, + ) + + _, hidden = restore_shape + input_dtype = permuted_tokens.dtype + + if probs is not None: + assert routing_map is not None, "Mask must be provided to permute the probs." + if drop_and_pad: + num_experts = routing_map.size(1) + num_permuted_tokens = sorted_indices.size(0) + capacity = num_permuted_tokens // num_experts + num_unpermuted_tokens = probs.size(0) + probs_t_1d = probs.T.contiguous().view(-1) + indices_dim0 = torch.arange(num_experts, device=routing_map.device).unsqueeze(-1) + indices_dim1 = sorted_indices.view(num_experts, capacity) + indices_1d = (indices_dim0 * num_unpermuted_tokens + indices_dim1).view(-1) + permuted_probs = probs_t_1d.index_select(0, indices_1d) + else: + permuted_probs = probs.T.contiguous().masked_select(routing_map.T.contiguous()) + permuted_tokens = permuted_tokens * permuted_probs.unsqueeze(-1) + + output_tokens = torch.zeros( + restore_shape, dtype=permuted_tokens.dtype, device=permuted_tokens.device + ) + if torch.are_deterministic_algorithms_enabled(): + output_tokens.index_add_(0, sorted_indices, permuted_tokens) + else: + output_tokens.scatter_add_( + 0, sorted_indices.unsqueeze(1).expand(-1, hidden), permuted_tokens + ) + return output_tokens.to(dtype=input_dtype) + + +def group_limited_topk( + scores: torch.Tensor, + topk: int, + num_tokens: int, + num_experts: int, + num_groups: int, + group_topk: int, +) -> Tuple[torch.Tensor, torch.Tensor]: + group_scores = ( + scores.view(num_tokens, num_groups, -1).topk(topk // group_topk, dim=-1)[0].sum(dim=-1) + ) + group_idx = torch.topk(group_scores, k=group_topk, dim=-1, sorted=False)[1] + group_mask = torch.zeros_like(group_scores) + group_mask.scatter_(1, group_idx, 1) + score_mask = ( + group_mask.unsqueeze(-1) + .expand(num_tokens, num_groups, num_experts // num_groups) + .reshape(num_tokens, -1) + ) + masked_scores = scores.masked_fill(~score_mask.bool(), float("-inf")) + return torch.topk(masked_scores, k=topk, dim=-1) + + +def topk_routing_with_score_function( + logits: torch.Tensor, + topk: int, + use_pre_softmax: bool = False, + num_groups: Optional[int] = None, + group_topk: Optional[int] = None, + scaling_factor: Optional[float] = None, + score_function: str = "softmax", + expert_bias: Optional[torch.Tensor] = None, + fused: bool = False, + dense_output: bool = False, +) -> Tuple[torch.Tensor, torch.Tensor]: + assert logits.dim() == 2, f"Expected 2D logits [num_tokens, num_experts], got {logits.dim()}." + num_tokens, num_experts = logits.shape + if fused: + return fused_topk_with_score_function( + logits=logits, + topk=topk, + use_pre_softmax=use_pre_softmax, + num_groups=num_groups, + group_topk=group_topk, + scaling_factor=scaling_factor, + score_function=score_function, + expert_bias=expert_bias, + ) + + def compute_topk( + scores: torch.Tensor, + k: int, + groups: Optional[int] = None, + groups_topk: Optional[int] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + if groups_topk: + assert groups is not None + return group_limited_topk( + scores=scores, + topk=k, + num_tokens=num_tokens, + num_experts=num_experts, + num_groups=groups, + group_topk=groups_topk, + ) + return torch.topk(scores, k=k, dim=1, sorted=torch.is_grad_enabled()) + + if score_function == "softmax": + if use_pre_softmax: + scores = torch.softmax(logits, dim=-1, dtype=torch.float32) + probs, top_indices = compute_topk(scores, topk, num_groups, group_topk) + else: + scores, top_indices = compute_topk(logits, topk, num_groups, group_topk) + probs = torch.softmax(scores, dim=-1, dtype=torch.float32) + elif score_function in ("sigmoid", "sqrtsoftplus"): + if score_function == "sigmoid": + scores = torch.sigmoid(logits.float()) + else: + scores = torch.nn.functional.softplus(logits.float()).sqrt() + if expert_bias is not None: + scores_for_routing = scores + expert_bias.float() + _, top_indices = compute_topk(scores_for_routing, topk, num_groups, group_topk) + scores = torch.gather(scores, dim=1, index=top_indices) + else: + scores, top_indices = compute_topk(scores, topk, num_groups, group_topk) + probs = scores / (scores.sum(dim=-1, keepdim=True) + 1e-20) if topk > 1 else scores + else: + raise ValueError(f"Invalid score_function: {score_function}") + + if scaling_factor: + probs = probs * scaling_factor + probs = probs.type_as(logits) + + if dense_output: + return probs, top_indices + + if torch.are_deterministic_algorithms_enabled(): + routing_probs = torch.zeros_like(logits) + rows = torch.arange(num_tokens, device=logits.device).unsqueeze(1) + routing_probs.index_put_((rows, top_indices), probs, accumulate=False) + routing_map = torch.zeros_like(logits, dtype=logits.dtype) + routing_map.index_put_( + (rows, top_indices), torch.ones_like(probs, dtype=routing_map.dtype), accumulate=False + ) + routing_map = routing_map.bool() + else: + routing_probs = torch.zeros_like(logits).scatter(1, top_indices, probs) + routing_map = torch.zeros_like(logits).int().scatter(1, top_indices, 1).bool() + return routing_probs, routing_map + + +def compute_routing_scores_for_aux_loss( + logits: torch.Tensor, + topk: int, + score_function: str, + *, + fused: bool = False, + padding_mask: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + if fused: + routing_map, scores = fused_compute_score_for_moe_aux_loss( + logits=logits, topk=topk, score_function=score_function + ) + else: + if score_function == "softmax": + scores = torch.softmax(logits, dim=-1, dtype=torch.float32) + elif score_function == "sigmoid": + scores = torch.sigmoid(logits.float()) + scores = scores / (scores.sum(dim=-1, keepdim=True) + 1e-20) + elif score_function == "sqrtsoftplus": + scores = torch.nn.functional.softplus(logits.float()).sqrt() + scores = scores / (scores.sum(dim=-1, keepdim=True) + 1e-20) + else: + raise ValueError(f"Invalid score_function: {score_function}") + _, top_indices = torch.topk(scores, k=topk, dim=1) + routing_map = torch.zeros_like(logits).int().scatter(1, top_indices, 1).bool() + + if padding_mask is not None: + valid_mask = (~padding_mask).unsqueeze(-1) + routing_map = routing_map * valid_mask + scores = scores * valid_mask + return routing_map, scores + + +class RouterGatingLinearFunction(torch.autograd.Function): + @staticmethod + def forward( + ctx, + inp: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor | None, + router_dtype: torch.dtype, + ) -> torch.Tensor: + ctx.save_for_backward(inp, weight, bias) + ctx.router_dtype = router_dtype + ctx.input_dtype = inp.dtype + ctx.weight_dtype = weight.dtype + inp_shape = inp.shape + inp = inp.view(-1, inp_shape[-1]) + + gemm_out = None + if router_dtype != torch.float64: + gemm_out = _te_general_gemm(weight, inp, router_dtype, layout="TN", bias=bias) + if gemm_out is not None: + output = gemm_out[0] + elif bias is None: + output = torch.mm(inp.to(router_dtype), weight.to(router_dtype).t()) + else: + output = torch.addmm( + bias.to(router_dtype), inp.to(router_dtype), weight.to(router_dtype).t() + ) + return output.view(*inp_shape[:-1], -1) + + @staticmethod + def backward(ctx, grad_output: torch.Tensor): + inp, weight, bias = ctx.saved_tensors + inp_shape = inp.shape + grad_shape = grad_output.shape + inp = inp.view(-1, inp_shape[-1]) + grad_output = grad_output.view(-1, grad_shape[-1]) + + grad_input_out = grad_weight_out = None + if ctx.router_dtype != torch.float64: + grad_input_out = _te_general_gemm( + weight.to(ctx.router_dtype), grad_output, ctx.router_dtype, layout="NN", grad=True + ) + grad_weight_out = _te_general_gemm( + inp.to(ctx.router_dtype), grad_output, ctx.router_dtype, layout="NT", grad=True + ) + if grad_input_out is not None and grad_weight_out is not None: + grad_input = grad_input_out[0].to(ctx.input_dtype) + grad_weight = grad_weight_out[0].to(ctx.weight_dtype) + else: + grad_input = torch.mm(grad_output, weight.to(ctx.router_dtype)).to(ctx.input_dtype) + grad_weight = torch.mm(grad_output.t(), inp.to(ctx.router_dtype)).to(ctx.weight_dtype) + grad_bias = grad_output.sum(dim=0).to(ctx.weight_dtype) if bias is not None else None + return grad_input.view(*inp_shape), grad_weight, grad_bias, None + + +def router_gating_linear( + inp: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None, router_dtype: torch.dtype +) -> torch.Tensor: + return RouterGatingLinearFunction.apply(inp, weight, bias, router_dtype) + + +__all__ = [ + "compute_routing_scores_for_aux_loss", + "group_limited_topk", + "permute", + "router_gating_linear", + "switch_load_balancing_loss_func", + "topk_routing_with_score_function", + "unpermute", +] diff --git a/experimental/lite/megatron/lite/primitive/utils/packed_seq.py b/experimental/lite/megatron/lite/primitive/utils/packed_seq.py new file mode 100644 index 00000000000..d28b28fb850 --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/utils/packed_seq.py @@ -0,0 +1,61 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Packed sequence parameter containers for THD-format primitives.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import torch +from torch import Tensor + + +@dataclass +class PackedSeqParams: + """Parameters consumed by TE DotProductAttention and MLite THD helpers.""" + + qkv_format: str | None = "thd" + cu_seqlens_q: Tensor | None = None + cu_seqlens_kv: Tensor | None = None + cu_seqlens_q_padded: Tensor | None = None + cu_seqlens_kv_padded: Tensor | None = None + max_seqlen_q: int | None = None + max_seqlen_kv: int | None = None + local_cp_size: int | None = None + cp_group: Any | None = None + total_tokens: int | None = None + seq_idx: Tensor | None = None + cp_rank: int | None = None + + def __post_init__(self) -> None: + cu_seqlens = ( + self.cu_seqlens_q_padded if self.cu_seqlens_q_padded is not None else self.cu_seqlens_q + ) + if isinstance(cu_seqlens, Tensor) and self.total_tokens is not None: + total_tokens_tensor = torch.tensor( + [self.total_tokens], dtype=cu_seqlens.dtype, device=cu_seqlens.device + ) + cu_seqlens_with_max = torch.cat([cu_seqlens, total_tokens_tensor]) + seq_lengths = (cu_seqlens_with_max[1:] - cu_seqlens_with_max[:-1]).clamp(min=0) + self.seq_idx = ( + torch.repeat_interleave( + torch.arange(seq_lengths.numel(), device=cu_seqlens.device), seq_lengths + ) + .to(torch.int32) + .unsqueeze(0) + ) + + @staticmethod + def from_cu_seqlens(cu_seqlens: Tensor, max_seqlen: int) -> PackedSeqParams: + return PackedSeqParams( + qkv_format="thd", + cu_seqlens_q=cu_seqlens, + cu_seqlens_kv=cu_seqlens, + max_seqlen_q=max_seqlen, + max_seqlen_kv=max_seqlen, + cu_seqlens_q_padded=cu_seqlens, + cu_seqlens_kv_padded=cu_seqlens, + ) + + +__all__ = ["PackedSeqParams"] diff --git a/experimental/lite/megatron/lite/primitive/utils/rope.py b/experimental/lite/megatron/lite/primitive/utils/rope.py new file mode 100644 index 00000000000..115ede80f67 --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/utils/rope.py @@ -0,0 +1,144 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""RoPE helpers for MLite attention primitives.""" + +from __future__ import annotations + +import warnings +from typing import Optional + +import torch +from torch import Tensor + + +def get_pos_emb_on_this_cp_rank( + pos_emb: Tensor, seq_dim: int, cp_group: torch.distributed.ProcessGroup +) -> Tensor: + if cp_group is None: + raise ValueError("cp_group must be provided to get positional embedding per CP rank") + cp_size = cp_group.size() + cp_rank = cp_group.rank() + cp_idx = torch.tensor( + [cp_rank, (2 * cp_size - cp_rank - 1)], device=pos_emb.device, dtype=torch.long + ) + pos_emb = pos_emb.view( + *pos_emb.shape[:seq_dim], 2 * cp_size, -1, *pos_emb.shape[(seq_dim + 1) :] + ) + pos_emb = pos_emb.index_select(seq_dim, cp_idx) + return pos_emb.view(*pos_emb.shape[:seq_dim], -1, *pos_emb.shape[(seq_dim + 2) :]) + + +def _rotate_half(x: Tensor, rotary_interleaved: bool) -> Tensor: + if not rotary_interleaved: + x1, x2 = torch.chunk(x, 2, dim=-1) + return torch.cat((-x2, x1), dim=-1) + x1 = x[:, :, :, ::2] + x2 = x[:, :, :, 1::2] + x_new = torch.stack((-x2, x1), dim=-1) + return x_new.view(x_new.shape[0], x_new.shape[1], x_new.shape[2], -1) + + +def _apply_rotary_pos_emb_bshd( + t: Tensor, + freqs: Tensor, + rotary_interleaved: bool = False, + mla_rotary_interleaved: bool = False, + mscale: float = 1.0, + multi_latent_attention: Optional[bool] = None, +) -> Tensor: + if multi_latent_attention is not None: + warnings.warn( + "multi_latent_attention is deprecated. Use mla_rotary_interleaved instead.", + DeprecationWarning, + stacklevel=2, + ) + mla_rotary_interleaved = multi_latent_attention + + rot_dim = freqs.shape[-1] + t, t_pass = t[..., :rot_dim], t[..., rot_dim:] + + if mla_rotary_interleaved: + x1 = t[..., 0::2] + x2 = t[..., 1::2] + t = torch.cat((x1, x2), dim=-1) + + cos_ = (torch.cos(freqs) * mscale).to(t.dtype) + sin_ = (torch.sin(freqs) * mscale).to(t.dtype) + t = (t * cos_) + (_rotate_half(t, rotary_interleaved) * sin_) + return torch.cat((t, t_pass), dim=-1) + + +def _get_thd_freqs_on_this_cp_rank( + cp_rank: int, cp_size: int, x: Tensor, freqs: Tensor, offset: int = 0 +) -> Tensor: + if cp_size > 1: + cp_seg = x.size(0) // 2 + full_seqlen = cp_size * x.size(0) + return torch.cat( + [ + freqs[offset + cp_rank * cp_seg : offset + (cp_rank + 1) * cp_seg], + freqs[ + offset + + full_seqlen + - (cp_rank + 1) * cp_seg : offset + + full_seqlen + - cp_rank * cp_seg + ], + ] + ) + return freqs[offset : offset + x.size(0)] + + +def _apply_rotary_pos_emb_thd( + t: Tensor, + cu_seqlens: Tensor, + freqs: Tensor, + rotary_interleaved: bool = False, + mla_rotary_interleaved: bool = False, + mscale: float = 1.0, + cp_group: torch.distributed.ProcessGroup = None, + multi_latent_attention: Optional[bool] = None, +) -> Tensor: + if multi_latent_attention is not None: + warnings.warn( + "multi_latent_attention is deprecated. Use mla_rotary_interleaved instead.", + DeprecationWarning, + stacklevel=2, + ) + mla_rotary_interleaved = multi_latent_attention + + if cp_group is None: + raise ValueError("cp_group must be provided for THD format RoPE") + cp_size = cp_group.size() + cp_rank = cp_group.rank() + seqlens = ((cu_seqlens[1:] - cu_seqlens[:-1]) // cp_size).tolist() + + sequence_splits = torch.split(t, seqlens) + if freqs.dim() >= 1 and freqs.size(0) == cu_seqlens[-1]: + freq_slices = [] + for i, x in enumerate(sequence_splits): + seq_start_offset = cu_seqlens[i].item() + freq_slices.append( + _get_thd_freqs_on_this_cp_rank(cp_rank, cp_size, x, freqs, seq_start_offset) + ) + freqs_packed = torch.cat(freq_slices, dim=0) + else: + freqs_packed = torch.cat( + [_get_thd_freqs_on_this_cp_rank(cp_rank, cp_size, x, freqs) for x in sequence_splits], + dim=0, + ) + + return _apply_rotary_pos_emb_bshd( + t.unsqueeze(1), + freqs_packed, + rotary_interleaved=rotary_interleaved, + mla_rotary_interleaved=mla_rotary_interleaved, + mscale=mscale, + ).squeeze(1) + + +__all__ = [ + "_apply_rotary_pos_emb_bshd", + "_apply_rotary_pos_emb_thd", + "_rotate_half", + "get_pos_emb_on_this_cp_rank", +] diff --git a/experimental/lite/megatron/lite/primitive/utils/rotary.py b/experimental/lite/megatron/lite/primitive/utils/rotary.py new file mode 100644 index 00000000000..df5ecbee508 --- /dev/null +++ b/experimental/lite/megatron/lite/primitive/utils/rotary.py @@ -0,0 +1,299 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Rotary embedding modules for MLite primitives.""" + +from __future__ import annotations + +import math +from functools import lru_cache +from typing import Optional + +import torch +from torch import Tensor, nn + +from megatron.lite.primitive.utils.rope import get_pos_emb_on_this_cp_rank + + +def _default_rope_device(use_cpu_initialization: bool) -> str | torch.device: + if use_cpu_initialization or not torch.cuda.is_available(): + return "cpu" + return torch.device("cuda", torch.cuda.current_device()) + + +class RotaryEmbedding(nn.Module): + """Rotary embedding with optional context-parallel slicing.""" + + def __init__( + self, + kv_channels: int, + rotary_percent: float = 1.0, + rotary_interleaved: bool = False, + seq_len_interpolation_factor: float | None = None, + rotary_base: float = 10000, + rope_scaling: bool = False, + rope_scaling_factor: float = 8.0, + use_cpu_initialization: bool = False, + cp_group: Optional[torch.distributed.ProcessGroup] = None, + ) -> None: + super().__init__() + dim = kv_channels + if rotary_percent < 1.0: + dim = int(dim * rotary_percent) + self.rotary_interleaved = rotary_interleaved + self.seq_len_interpolation_factor = seq_len_interpolation_factor + device = _default_rope_device(use_cpu_initialization) + self.inv_freq = 1.0 / ( + rotary_base ** (torch.arange(0, dim, 2, dtype=torch.float32, device=device) / dim) + ) + if rope_scaling: + self.inv_freq = self._apply_scaling(self.inv_freq, factor=rope_scaling_factor) + self.cp_group = cp_group + + def _apply_scaling( + self, + freqs: Tensor, + factor: float = 8, + low_freq_factor: float = 1, + high_freq_factor: float = 4, + original_max_position_embeddings: int = 8192, + ) -> Tensor: + low_freq_wavelen = original_max_position_embeddings / low_freq_factor + high_freq_wavelen = original_max_position_embeddings / high_freq_factor + + wavelen = 2 * math.pi / freqs + inv_freq_llama = torch.where(wavelen > low_freq_wavelen, freqs / factor, freqs) + smooth_factor = (original_max_position_embeddings / wavelen - low_freq_factor) / ( + high_freq_factor - low_freq_factor + ) + smoothed_inv_freq = ( + 1 - smooth_factor + ) * inv_freq_llama / factor + smooth_factor * inv_freq_llama + is_medium_freq = ~(wavelen < high_freq_wavelen) * ~(wavelen > low_freq_wavelen) + return torch.where(is_medium_freq, smoothed_inv_freq, inv_freq_llama) + + def get_freqs_non_repeated(self, max_seq_len: int, offset: int = 0) -> Tensor: + seq = ( + torch.arange(max_seq_len, device=self.inv_freq.device, dtype=self.inv_freq.dtype) + + offset + ) + if self.seq_len_interpolation_factor is not None: + seq *= 1 / self.seq_len_interpolation_factor + return torch.outer(seq, self.inv_freq) + + def get_cos_sin(self, max_seq_len: int, offset: int = 0) -> tuple[Tensor, Tensor]: + freqs = self.get_freqs_non_repeated(max_seq_len, offset) + return torch.cos(freqs), torch.sin(freqs) + + def get_emb(self, max_seq_len: int, offset: int = 0) -> Tensor: + if self.inv_freq.device.type == "cpu" and torch.cuda.is_available(): + self.inv_freq = self.inv_freq.to(device=torch.cuda.current_device()) + + freqs = self.get_freqs_non_repeated(max_seq_len, offset) + if not self.rotary_interleaved: + emb = torch.cat((freqs, freqs), dim=-1) + else: + emb = torch.stack((freqs.view(-1, 1), freqs.view(-1, 1)), dim=-1).view( + freqs.shape[0], -1 + ) + return emb[:, None, None, :] + + @lru_cache(maxsize=32) + def forward( + self, + max_seq_len: int, + offset: int = 0, + packed_seq: bool = False, + cp_group: Optional[torch.distributed.ProcessGroup] = None, + ) -> Tensor: + emb = self.get_emb(max_seq_len, offset) + if cp_group is None: + cp_group = self.cp_group + if cp_group is not None and cp_group.size() > 1 and not packed_seq: + emb = get_pos_emb_on_this_cp_rank(emb, 0, cp_group) + return emb + + def _load_from_state_dict(self, state_dict, prefix, *args, **kwargs): + state_dict.pop(f"{prefix}inv_freq", None) + return super()._load_from_state_dict(state_dict, prefix, *args, **kwargs) + + +class YarnRotaryEmbedding(RotaryEmbedding): + """YARN rotary embedding variant used by MLA-style models.""" + + def __init__( + self, + kv_channels: int, + rotary_percent: float = 1.0, + rotary_interleaved: bool = False, + seq_len_interpolation_factor: Optional[float] = None, + rotary_base: float = 10000.0, + use_cpu_initialization: bool = False, + scaling_factor: float = 1.0, + original_max_position_embeddings: int = 4096, + beta_fast: float = 32.0, + beta_slow: float = 1.0, + mscale: float = 1.0, + mscale_all_dim: float = 0.0, + correction_range_round_to_int: bool = True, + cp_group: Optional[torch.distributed.ProcessGroup] = None, + ): + self.dim = kv_channels + self.rotary_base = rotary_base + self.scaling_factor = scaling_factor + self.original_max_position_embeddings = original_max_position_embeddings + self.beta_fast = beta_fast + self.beta_slow = beta_slow + self.mscale = mscale + self.mscale_all_dim = mscale_all_dim + self.correction_range_round_to_int = correction_range_round_to_int + + device = _default_rope_device(use_cpu_initialization) + self.inv_freq_extra = 1.0 / ( + self.rotary_base + ** (torch.arange(0, self.dim, 2, dtype=torch.float32, device=device) / self.dim) + ) + self.inv_freq_inter = 1.0 / ( + self.scaling_factor + * self.rotary_base + ** (torch.arange(0, self.dim, 2, dtype=torch.float32, device=device) / self.dim) + ) + super().__init__( + kv_channels=kv_channels, + rotary_percent=rotary_percent, + rotary_interleaved=rotary_interleaved, + seq_len_interpolation_factor=seq_len_interpolation_factor, + rotary_base=rotary_base, + use_cpu_initialization=use_cpu_initialization, + cp_group=cp_group, + ) + self._set_cos_sin_cache( + self.original_max_position_embeddings, offset=0, dtype=torch.get_default_dtype() + ) + self.forward.cache_clear() + + def get_emb(self, max_seq_len: int, offset: int = 0) -> tuple[Tensor, float]: + if self.rotary_interleaved: + raise AssertionError("YARN RoPE does not support interleaved rotary embeddings") + if self.inv_freq_extra.device.type == "cpu" and torch.cuda.is_available(): + self.inv_freq_extra = self.inv_freq_extra.to(device=torch.cuda.current_device()) + if self.inv_freq_inter.device.type == "cpu" and torch.cuda.is_available(): + self.inv_freq_inter = self.inv_freq_inter.to(device=torch.cuda.current_device()) + + low, high = _yarn_find_correction_range( + self.beta_fast, + self.beta_slow, + self.dim, + self.rotary_base, + self.original_max_position_embeddings, + self.correction_range_round_to_int, + ) + inv_freq_mask = 1.0 - _yarn_linear_ramp_mask( + low, high, self.dim // 2, device=self.inv_freq_extra.device + ).to(dtype=torch.float32) + inv_freq = self.inv_freq_inter * (1 - inv_freq_mask) + self.inv_freq_extra * inv_freq_mask + seq = ( + torch.arange( + max_seq_len, device=self.inv_freq_extra.device, dtype=self.inv_freq_extra.dtype + ) + + offset + ) + freqs = torch.outer(seq, inv_freq) + concentration = _yarn_get_concentration_factor( + self.scaling_factor, self.mscale, self.mscale_all_dim + ) + emb = torch.cat((freqs, freqs), dim=-1) + return emb[:, None, None, :], concentration + + @lru_cache(maxsize=32) + def forward( + self, + max_seq_len: int, + offset: int = 0, + packed_seq: bool = False, + cp_group: Optional[torch.distributed.ProcessGroup] = None, + ) -> tuple[Tensor, float]: + emb, concentration = self.get_emb(max_seq_len, offset) + if cp_group is None: + cp_group = self.cp_group + if cp_group is not None and cp_group.size() > 1 and not packed_seq: + emb = get_pos_emb_on_this_cp_rank(emb, 0, cp_group) + return emb, concentration + + def _set_cos_sin_cache(self, seq_len, offset, dtype, packed_seq=False, cp_group=None): + self.max_seq_len_cached = seq_len + self.offset_cached = offset + self.dtype_cached = dtype + self.packed_seq_cached = packed_seq + emb, concentration = self.forward(seq_len, offset, packed_seq=packed_seq, cp_group=cp_group) + self.register_buffer( + "cos_cached", (emb.cos() * concentration).to(dtype).contiguous(), persistent=False + ) + self.register_buffer( + "sin_cached", (emb.sin() * concentration).to(dtype).contiguous(), persistent=False + ) + + def get_cached_cos_sin( + self, seq_len, offset=0, dtype=torch.get_default_dtype(), packed_seq=False, cp_group=None + ): + if ( + seq_len > self.max_seq_len_cached + or offset != self.offset_cached + or dtype != self.dtype_cached + or packed_seq != self.packed_seq_cached + ): + self._set_cos_sin_cache(seq_len, offset, dtype, packed_seq, cp_group) + return self.cos_cached[:seq_len, ...], self.sin_cached[:seq_len, ...] + + +def _yarn_find_correction_dim( + num_rotations: float, dim: int, rotary_base: float = 10000, max_position_embeddings: int = 2048 +) -> float: + return (dim * math.log(max_position_embeddings / (num_rotations * 2 * math.pi))) / ( + 2 * math.log(rotary_base) + ) + + +def _yarn_find_correction_range( + low_rot: float, + high_rot: float, + dim: int, + rotary_base: float = 10000, + max_position_embeddings: int = 2048, + round_to_int: bool = True, +) -> tuple[int, int]: + low = _yarn_find_correction_dim(low_rot, dim, rotary_base, max_position_embeddings) + high = _yarn_find_correction_dim(high_rot, dim, rotary_base, max_position_embeddings) + if round_to_int: + low = math.floor(low) + high = math.ceil(high) + return max(low, 0), min(high, dim - 1) + + +def _yarn_linear_ramp_mask( + minimum: float, maximum: float, dim: int, device: torch.device +) -> Tensor: + if minimum == maximum: + maximum += 0.001 + linear_func = (torch.arange(dim, dtype=torch.float32, device=device) - minimum) / ( + maximum - minimum + ) + return torch.clamp(linear_func, 0, 1) + + +def _yarn_get_mscale(scale: float = 1, mscale: float = 1) -> float: + if scale <= 1: + return 1.0 + return 0.1 * mscale * math.log(scale) + 1.0 + + +@lru_cache(maxsize=8) +def _yarn_get_concentration_factor( + scaling_factor: float, mscale: Optional[float], mscale_all_dim: Optional[float] +) -> float: + if mscale is None or mscale_all_dim is None: + return _yarn_get_mscale(scaling_factor) + return float( + _yarn_get_mscale(scaling_factor, mscale) / _yarn_get_mscale(scaling_factor, mscale_all_dim) + ) + + +__all__ = ["RotaryEmbedding", "YarnRotaryEmbedding", "_yarn_get_mscale"] diff --git a/experimental/lite/megatron/lite/runtime/__init__.py b/experimental/lite/megatron/lite/runtime/__init__.py new file mode 100644 index 00000000000..9f8f1a8906c --- /dev/null +++ b/experimental/lite/megatron/lite/runtime/__init__.py @@ -0,0 +1,83 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Runtime entrypoints for Megatron Lite.""" + +from __future__ import annotations + +import importlib +from typing import TYPE_CHECKING + +from megatron.lite.runtime.contracts.config import RuntimeConfig + +if TYPE_CHECKING: + from megatron.lite.runtime.backends import Runtime + from megatron.lite.runtime.backends.bridge.config import BridgeConfig + from megatron.lite.runtime.backends.mlite.config import MegatronLiteConfig + from megatron.lite.runtime.contracts.data import ( + Batch, + ForwardResult, + ModelOutputs, + PackedBatch, + TrainBatch, + ) + from megatron.lite.runtime.contracts.handle import ModelHandle + + +def _runtime_registry() -> dict[str, str]: + from megatron.lite.runtime.backends import RUNTIME_REGISTRY + + return RUNTIME_REGISTRY + + +def register_runtime(name: str, module_path: str) -> None: + """Register a custom runtime backend. + + Args: + name: Backend name (used in ``RuntimeConfig.backend``). + module_path: Dotted module path providing a ``create(hf_path, cfg)`` function. + + Example:: + + from megatron.lite.runtime import register_runtime + register_runtime("my_backend", "my_package.my_runtime") + """ + _runtime_registry()[name] = module_path + + +def create_runtime(cfg: RuntimeConfig) -> Runtime: + """Create a Runtime instance for the given config.""" + mod = importlib.import_module(_runtime_registry()[cfg.backend]) + return mod.create(cfg.hf_path, cfg.backend_cfg) + + +def __getattr__(name: str): + _lazy = { + "Batch": "megatron.lite.runtime.contracts.data", + "BridgeConfig": "megatron.lite.runtime.backends.bridge.config", + "MegatronLiteConfig": "megatron.lite.runtime.backends.mlite.config", + "ForwardResult": "megatron.lite.runtime.contracts.data", + "ModelHandle": "megatron.lite.runtime.contracts.handle", + "ModelOutputs": "megatron.lite.runtime.contracts.data", + "PackedBatch": "megatron.lite.runtime.contracts.data", + "Runtime": "megatron.lite.runtime.backends", + "TrainBatch": "megatron.lite.runtime.contracts.data", + } + if name in _lazy: + mod = importlib.import_module(_lazy[name]) + return getattr(mod, name) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +__all__ = [ + "Batch", + "BridgeConfig", + "ForwardResult", + "MegatronLiteConfig", + "ModelHandle", + "ModelOutputs", + "PackedBatch", + "Runtime", + "RuntimeConfig", + "TrainBatch", + "create_runtime", + "register_runtime", +] diff --git a/experimental/lite/megatron/lite/runtime/backends/__init__.py b/experimental/lite/megatron/lite/runtime/backends/__init__.py new file mode 100644 index 00000000000..b4f0c729759 --- /dev/null +++ b/experimental/lite/megatron/lite/runtime/backends/__init__.py @@ -0,0 +1,178 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Runtime ABC and registry. + +Runtime API tiers +----------------- +L1 — **Pretrain Ready** (9 abstract methods, must implement): + build_model, save_checkpoint, load_checkpoint, + train_mode, eval_mode, + forward_backward, zero_grad, optimizer_step, lr_scheduler_step + +L2 — **RL Ready** (+ export_weights): + Enables RL frameworks to extract weights for the inference engine. + +L3 — **RL Best** (+ to): + Enables offloading model/optimizer/grad between training and rollout + phases to free GPU memory for the inference engine. + +A new backend only needs to implement L1 to work for pretraining. +Override ``export_weights`` and/or ``to`` to unlock higher tiers. +Check ``runtime.tier`` to see which level a backend supports. +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from collections.abc import Callable, Iterator +from typing import TYPE_CHECKING, Any, Literal + +if TYPE_CHECKING: + import torch + + from megatron.lite.runtime.contracts.data import ForwardResult + from megatron.lite.runtime.contracts.handle import ModelHandle + + +class Runtime(ABC): + """Base class for all runtime implementations. + + MegatronLiteRuntime and custom impls subclass this. + """ + + # ── L1: Pretrain Ready (必须实现) ──────────────────────────── + + # Model lifecycle + + @abstractmethod + def build_model(self, hf_path: str | None = None, cfg: Any = None, **kwargs) -> ModelHandle: + """Build model state for this runtime. + + Implementations should default to the ``hf_path`` / ``backend_cfg`` + captured by ``create_runtime(RuntimeConfig(...))``. Passing arguments to + ``build_model`` is supported as an advanced override path, but public + examples should prefer ``handle = rt.build_model()``. + """ + ... + + @abstractmethod + def save_checkpoint(self, handle: ModelHandle, path: str, **kwargs) -> None: ... + + @abstractmethod + def load_checkpoint(self, handle: ModelHandle, path: str, **kwargs) -> int: ... + + # Mode switching + + @abstractmethod + def train_mode(self, handle: ModelHandle) -> Any: ... + + @abstractmethod + def eval_mode(self, handle: ModelHandle) -> Any: ... + + # Training atoms + + @abstractmethod + def forward_backward( + self, + handle: ModelHandle, + data: Any, + loss_fn: Callable | None, + *, + num_microbatches: int = 1, + forward_only: bool = False, + ) -> ForwardResult: + """Forward + backward pass over data. + + Args: + num_microbatches: Number of microbatches to accumulate inside one + logical training step. + loss_fn: Optional external loss function with signature:: + + loss_fn(model_output: dict, batch) -> (loss: Tensor, metrics: dict) + + When ``loss_fn`` is None, the model computes loss internally + (standard pretrain/SFT path). When provided, ``model_output`` + is the dict returned by the model's forward (logits, log_probs, etc.) + and ``batch`` is the current microbatch. + """ + ... + + @abstractmethod + def zero_grad(self, handle: ModelHandle) -> None: ... + + @abstractmethod + def optimizer_step(self, handle: ModelHandle) -> tuple[bool, float, int | None]: + """Run optimizer step. + + Returns: + (update_successful, grad_norm, num_zeros_in_grad) + """ + ... + + @abstractmethod + def lr_scheduler_step(self, handle: ModelHandle) -> float | list[float]: ... + + # ── Parallel state queries ─────────────────────────────────── + + def is_mp_src_rank_with_outputs(self, handle: ModelHandle) -> bool: + """True if this rank is PP-last, TP-0, CP-0 (has full output).""" + return True # default: no parallelism, every rank has outputs + + # ── L2: RL Ready (覆盖即解锁) ─────────────────────────────── + + def export_weights(self, handle: ModelHandle, **kwargs) -> Iterator[tuple[str, torch.Tensor]]: + """Iterate over (name, tensor) pairs for HF-compatible weight export. + + Required by RL frameworks to send weights to the inference engine. + Override to unlock **RL Ready** tier. + """ + raise NotImplementedError( + f"{type(self).__name__} does not implement export_weights. " + "Implement it to unlock the RL Ready tier." + ) + + # ── L3: RL Best (覆盖即解锁) ──────────────────────────────── + + def to( + self, + handle: ModelHandle, + device: str, + *, + model: bool = True, + optimizer: bool = True, + grad: bool = True, + ) -> None: + """Move model / optimizer / gradients to *device*. + + Enables offloading between training and rollout phases so the + inference engine can reclaim GPU memory. + Override to unlock **RL Best** tier. + """ + raise NotImplementedError( + f"{type(self).__name__} does not implement to(). " + "Implement it to unlock the RL Best tier." + ) + + # ── Tier introspection ────────────────────────────────────── + + @property + def tier(self) -> Literal["pretrain", "rl_ready", "rl_best"]: + """Report the highest API tier this runtime supports.""" + cls = type(self) + has_export = cls.export_weights is not Runtime.export_weights + has_to = cls.to is not Runtime.to + if has_export and has_to: + return "rl_best" + if has_export: + return "rl_ready" + return "pretrain" + + +# --------------------------------------------------------------------------- +# Registry +# --------------------------------------------------------------------------- + +RUNTIME_REGISTRY: dict[str, str] = { + "bridge": "megatron.lite.runtime.backends.bridge", + "mbridge": "megatron.lite.runtime.backends.mbridge", + "mlite": "megatron.lite.runtime.backends.mlite", +} diff --git a/experimental/lite/megatron/lite/runtime/backends/bridge/__init__.py b/experimental/lite/megatron/lite/runtime/backends/bridge/__init__.py new file mode 100644 index 00000000000..71072edbdde --- /dev/null +++ b/experimental/lite/megatron/lite/runtime/backends/bridge/__init__.py @@ -0,0 +1,18 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Megatron-Bridge backend for the Megatron Lite runtime API.""" + +from __future__ import annotations + +from typing import Any + +from megatron.lite.runtime.backends import Runtime as RuntimeBase +from megatron.lite.runtime.backends.bridge.config import BridgeConfig +from megatron.lite.runtime.backends.bridge.runtime import BridgeRuntime + + +def create(hf_path: str, cfg: BridgeConfig | dict[str, Any]) -> RuntimeBase: + """Factory called by ``megatron.lite.runtime.create_runtime``.""" + return BridgeRuntime(hf_path, cfg) + + +__all__ = ["BridgeConfig", "BridgeRuntime", "create"] diff --git a/experimental/lite/megatron/lite/runtime/backends/bridge/config.py b/experimental/lite/megatron/lite/runtime/backends/bridge/config.py new file mode 100644 index 00000000000..cdb20c3d684 --- /dev/null +++ b/experimental/lite/megatron/lite/runtime/backends/bridge/config.py @@ -0,0 +1,78 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Megatron-Bridge backend configuration.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +from megatron.lite.runtime.contracts.config import OptimizerConfig, ParallelConfig, pick_fields + + +@dataclass +class BridgeConfig: + """Config for ``BridgeRuntime``. + + The backend is intentionally thin: it lowers Megatron Lite's runtime + contract into Megatron-Bridge / Megatron-Core objects, while model-specific + mutations stay in examples as explicit benchmark hooks. + """ + + model_name: str = "auto" + + parallel: ParallelConfig = field(default_factory=ParallelConfig) + seed: int = 42 + param_offload: bool = False + optimizer_offload: bool = False + load_hf_weights: bool = True + build_optimizer: bool = True + + # When False, the bridge feeds a dense [b=1, s] forward (no THD packing). + # Used for deterministic layout-matched parity vs models whose Megatron-Core + # kernel is dense-only (e.g. GatedDeltaNet). Default True keeps THD packing. + use_thd: bool = True + + override_ddp_config: dict[str, Any] = field(default_factory=dict) + override_transformer_config: dict[str, Any] = field(default_factory=dict) + override_optimizer_config: dict[str, Any] = field(default_factory=dict) + + optimizer: OptimizerConfig = field(default_factory=OptimizerConfig) + + # Bench-only hook. Kept callable so examples can trim model configs without + # making the runtime know about benchmark profiles. + bridge_post_init: Any = None + + @classmethod + def from_dict(cls, cfg: dict[str, Any]) -> BridgeConfig: + """Construct ``BridgeConfig`` from a flat or nested mapping.""" + if "num_microbatches" in cfg: + raise ValueError( + "BridgeConfig does not accept `num_microbatches`; " + "pass it to Runtime.forward_backward(..., num_microbatches=...) instead." + ) + + parallel_src = cfg.get("parallel") + parallel_data = ( + pick_fields(ParallelConfig, parallel_src) if isinstance(parallel_src, dict) else {} + ) + parallel_data.update(pick_fields(ParallelConfig, cfg)) + parallel = ParallelConfig(**parallel_data) + + optimizer_src = cfg.get("optimizer", {}) + lr_src = cfg.get("lr_scheduler", {}) + optimizer_data: dict[str, Any] = {} + if isinstance(optimizer_src, dict): + optimizer_data.update(optimizer_src) + if isinstance(lr_src, dict): + optimizer_data.update(lr_src) + optimizer = OptimizerConfig(**pick_fields(OptimizerConfig, optimizer_data)) + + skip = {"parallel", "optimizer", "lr_scheduler"} + return cls( + **{k: v for k, v in pick_fields(cls, cfg).items() if k not in skip}, + parallel=parallel, + optimizer=optimizer, + ) + + +__all__ = ["BridgeConfig"] diff --git a/experimental/lite/megatron/lite/runtime/backends/bridge/runtime.py b/experimental/lite/megatron/lite/runtime/backends/bridge/runtime.py new file mode 100644 index 00000000000..f56bc948aee --- /dev/null +++ b/experimental/lite/megatron/lite/runtime/backends/bridge/runtime.py @@ -0,0 +1,944 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Runtime backend backed by Megatron-Bridge.""" + +from __future__ import annotations + +import logging +import os +from collections.abc import Iterator +from datetime import timedelta +from typing import Any + +import torch +import torch.distributed as dist +from megatron.lite.primitive.optimizers.megatron_wrap import build_dist_opt_optimizer_config +from megatron.lite.runtime.backends import Runtime as RuntimeBase +from megatron.lite.runtime.backends.bridge.config import BridgeConfig +from megatron.lite.runtime.contracts.data import Batch, ForwardResult, ModelOutputs, PackedBatch +from megatron.lite.runtime.contracts.handle import ModelHandle +from megatron.lite.runtime.megatron_utils import ( + build_sharded_state_dict, + is_mp_src_rank_with_outputs, + load_model_to_gpu, + load_optimizer, + offload_model_to_cpu, + offload_optimizer, + register_training_hooks, +) + +logger = logging.getLogger(__name__) + + +class _MpuParallelState: + """Adapter exposing Megatron-Core mpu through ``ModelHandle`` properties.""" + + def __init__(self, mpu): + self._mpu = mpu + + @property + def dp_rank(self) -> int: + return self._mpu.get_data_parallel_rank() + + @property + def dp_size(self) -> int: + return self._mpu.get_data_parallel_world_size() + + @property + def dp_group(self): + return self._mpu.get_data_parallel_group() + + @property + def tp_rank(self) -> int: + return self._mpu.get_tensor_model_parallel_rank() + + @property + def tp_size(self) -> int: + return self._mpu.get_tensor_model_parallel_world_size() + + @property + def pp_rank(self) -> int: + return self._mpu.get_pipeline_model_parallel_rank() + + @property + def pp_size(self) -> int: + return self._mpu.get_pipeline_model_parallel_world_size() + + @property + def cp_rank(self) -> int: + return self._mpu.get_context_parallel_rank() + + @property + def cp_size(self) -> int: + return self._mpu.get_context_parallel_world_size() + + @property + def cp_group(self): + return self._mpu.get_context_parallel_group() + + +def _lower_transformer_overrides(cfg: BridgeConfig) -> dict[str, Any]: + overrides = {"attention_backend": "flash"} + overrides.update(cfg.override_transformer_config) + return overrides + + +def _bridge_hf_config(bridge): + hf_pretrained = getattr(bridge, "hf_pretrained", None) + if hf_pretrained is None: + return None + return getattr(hf_pretrained, "config", hf_pretrained) + + +def _lower_provider_value(key: str, value: Any) -> Any: + if key == "attention_backend" and isinstance(value, str): + try: + from megatron.core.transformer.enums import AttnBackend + except ModuleNotFoundError as exc: + if exc.name != "megatron.core": + raise + return value + + return AttnBackend[value] + return value + + +def _configure_provider(provider, cfg: BridgeConfig) -> None: + from megatron.lite.primitive.deterministic import deterministic_requested + + p = cfg.parallel + provider.tensor_model_parallel_size = p.tp + provider.pipeline_model_parallel_size = p.pp + provider.context_parallel_size = p.cp + provider.expert_model_parallel_size = p.ep + if p.etp is not None: + provider.expert_tensor_parallel_size = p.etp + if p.vpp > 1: + provider.virtual_pipeline_model_parallel_size = p.vpp + provider.sequence_parallel = p.tp > 1 + provider.bf16 = True + provider.fp16 = False + provider.deterministic_mode = deterministic_requested() + if provider.deterministic_mode: + os.environ.setdefault("NVTE_ALLOW_NONDETERMINISTIC_ALGO", "0") + + for key, value in _lower_transformer_overrides(cfg).items(): + setattr(provider, key, _lower_provider_value(key, value)) + + +def _register_bridge_compat_aliases() -> None: + """Register local Megatron-Bridge aliases for supported checkpoint variants.""" + from megatron.bridge.models.conversion import model_bridge + from megatron.bridge.models.conversion.mapping_registry import MegatronMappingRegistry + from megatron.bridge.models.conversion.param_mapping import ( + AutoMapping, + GatedMLPMapping, + GDNConv1dMapping, + QKVMapping, + ReplicatedMapping, + RMSNorm2ZeroCenteredRMSNormMapping, + merge_gdn_linear_weights, + split_gdn_linear_weights, + ) + from megatron.bridge.models.qwen.qwen3_next_bridge import Qwen3NextBridge + from megatron.bridge.utils.common_utils import extract_expert_number_from_param + from megatron.core.models.gpt.gpt_model import GPTModel + + class Qwen35SplitGDNMapping(AutoMapping): + """Bridge mapping for Qwen3.5 split GDN in-proj weights.""" + + def __init__(self, megatron_param: str, qkv: str, z: str, b: str, a: str): + super().__init__( + megatron_param=megatron_param, hf_param={"qkv": qkv, "z": z, "b": b, "a": a} + ) + self._tp_mapping = AutoMapping(megatron_param, megatron_param) + + def hf_to_megatron( + self, hf_weights: dict[str, torch.Tensor], megatron_module + ) -> torch.Tensor: + if self.tp_rank == 0: + config = self._get_config(megatron_module) + qkvz = torch.cat([hf_weights["qkv"], hf_weights["z"]], dim=0) + ba = torch.cat([hf_weights["b"], hf_weights["a"]], dim=0) + merged = merge_gdn_linear_weights(config, qkvz, ba, tp_size=self.tp_size) + else: + merged = None + return self._tp_mapping.hf_to_megatron(merged, megatron_module) + + def megatron_to_hf(self, megatron_weights, megatron_module) -> dict[str, torch.Tensor]: + if megatron_weights is not None: + megatron_weights = self.maybe_dequantize(megatron_weights) + + if megatron_module is None: + config = self.broadcast_obj_from_pp_rank(None) + else: + config = self._get_config(megatron_module) + config = self.broadcast_obj_from_pp_rank(config) + + packed_dict = self._tp_mapping.megatron_to_hf(megatron_weights, megatron_module) + if not packed_dict: + return {} + + packed = next(iter(packed_dict.values())) + qkvz, ba = split_gdn_linear_weights(config, packed, tp_size=self.tp_size) + qk_dim = config.linear_key_head_dim * config.linear_num_key_heads + v_dim = config.linear_value_head_dim * config.linear_num_value_heads + qkv, z = qkvz.split([2 * qk_dim + v_dim, v_dim], dim=0) + b, a = ba.chunk(2, dim=0) + return { + self.hf_param["qkv"]: qkv, + self.hf_param["z"]: z, + self.hf_param["b"]: b, + self.hf_param["a"]: a, + } + + def resolve(self, captures): + megatron_param, hf_param = self._resolve_names(captures) + return type(self)( + megatron_param, hf_param["qkv"], hf_param["z"], hf_param["b"], hf_param["a"] + ) + + class Qwen35PackedExpertDownMapping(AutoMapping): + """Bridge mapping for Qwen3.5 packed expert down-projection weights.""" + + def __init__(self, megatron_param: str, hf_param: str, permute_dims=None): + super().__init__( + megatron_param=megatron_param, hf_param=hf_param, permute_dims=permute_dims + ) + self.allow_hf_name_mismatch = True + + def hf_to_megatron(self, hf_weights: torch.Tensor, megatron_module) -> torch.Tensor: + expert_number = extract_expert_number_from_param(self.megatron_param) + expert_weight = hf_weights[expert_number].contiguous() + return super().hf_to_megatron(expert_weight, megatron_module) + + def megatron_to_hf(self, megatron_weights, megatron_module) -> dict[str, torch.Tensor]: + converted = super().megatron_to_hf(megatron_weights, megatron_module) + return converted + + def _validate_patterns(self, *args, **kwargs): + pass + + class Qwen35RouterMapping(AutoMapping): + """Bridge mapping for Qwen3.5 router weights with bench expert truncation.""" + + def hf_to_megatron(self, hf_weights: torch.Tensor, megatron_module) -> torch.Tensor: + config = self._get_config(megatron_module) + num_experts = getattr(config, "num_moe_experts", None) + if num_experts is not None and hf_weights.shape[0] != num_experts: + hf_weights = hf_weights[:num_experts].contiguous() + return super().hf_to_megatron(hf_weights, megatron_module) + + class Qwen35PackedExpertGateUpMapping(AutoMapping): + """Bridge mapping for Qwen3.5 packed expert gate/up projection weights.""" + + def __init__(self, megatron_param: str, hf_param: str, permute_dims=None): + super().__init__( + megatron_param=megatron_param, hf_param=hf_param, permute_dims=permute_dims + ) + self.allow_hf_name_mismatch = True + GatedMLPMapping._validate_patterns = lambda *args, **kwargs: None + self._gated_mapping = GatedMLPMapping( + megatron_param=self.megatron_param, + gate=f"{self.hf_param}.gate", + up=f"{self.hf_param}.up", + ) + + def hf_to_megatron(self, hf_weights: torch.Tensor, megatron_module) -> torch.Tensor: + expert_number = extract_expert_number_from_param(self.megatron_param) + expert_weight = hf_weights[expert_number].contiguous() + gate, up = torch.chunk(expert_weight, 2, dim=0) + return self._gated_mapping.hf_to_megatron({"gate": gate, "up": up}, megatron_module) + + def megatron_to_hf(self, megatron_weights, megatron_module) -> dict[str, torch.Tensor]: + converted = self._gated_mapping.megatron_to_hf(megatron_weights, megatron_module) + if not converted: + return {} + + fused = {} + for name, tensor in converted.items(): + if not name.endswith(".gate"): + continue + base_name = name[: -len(".gate")] + up_tensor = converted.get(f"{base_name}.up") + if up_tensor is None: + continue + gate_tensor = tensor.contiguous() + up_tensor = up_tensor.contiguous() + fused[base_name] = torch.stack( + [gate_tensor, up_tensor], dim=0 if up_tensor.ndim == 2 else 1 + ) + return fused + + def _validate_patterns(self, *args, **kwargs): + pass + + class Qwen35MoEBridge(Qwen3NextBridge): + """Megatron-Bridge Qwen3-Next bridge adjusted for Qwen3.5 HF naming.""" + + def _text_config(self, hf_pretrained): + config = getattr(hf_pretrained, "config", hf_pretrained) + text_config = getattr(config, "text_config", config) + + if getattr(text_config, "intermediate_size", None) is None: + text_config.intermediate_size = 5120 + + rope_parameters = getattr(text_config, "rope_parameters", None) + if isinstance(rope_parameters, dict): + rope_theta = rope_parameters.get("rope_theta") + if rope_theta is not None: + text_config.rope_theta = rope_theta + partial_rotary_factor = rope_parameters.get("partial_rotary_factor") + if partial_rotary_factor is not None: + text_config.partial_rotary_factor = partial_rotary_factor + + if not hasattr(text_config, "tie_word_embeddings") and hasattr( + config, "tie_word_embeddings" + ): + text_config.tie_word_embeddings = config.tie_word_embeddings + + return text_config + + def provider_bridge(self, hf_pretrained): + text_config = self._text_config(hf_pretrained) + shim = type("_Qwen35TextConfigShim", (), {"config": text_config})() + provider = super().provider_bridge(shim) + + aux_loss = getattr(text_config, "router_aux_loss_coef", None) + if aux_loss is not None: + provider.moe_aux_loss_coeff = aux_loss + + return provider + + def mapping_registry(self): + prefix = "model.language_model" + param_mappings = { + "embedding.word_embeddings.weight": f"{prefix}.embed_tokens.weight", + "output_layer.weight": "lm_head.weight", + "decoder.final_layernorm.weight": f"{prefix}.norm.weight", + "decoder.layers.*.pre_mlp_layernorm.weight": f"{prefix}.layers.*.post_attention_layernorm.weight", + "decoder.layers.*.self_attention.linear_qkv.layer_norm_weight": ( + f"{prefix}.layers.*.input_layernorm.weight" + ), + "decoder.layers.*.self_attention.q_layernorm.weight": f"{prefix}.layers.*.self_attn.q_norm.weight", + "decoder.layers.*.self_attention.k_layernorm.weight": f"{prefix}.layers.*.self_attn.k_norm.weight", + "decoder.layers.*.self_attention.linear_proj.weight": f"{prefix}.layers.*.self_attn.o_proj.weight", + "decoder.layers.*.self_attention.in_proj.layer_norm_weight": ( + f"{prefix}.layers.*.input_layernorm.weight" + ), + "decoder.layers.*.self_attention.out_proj.weight": f"{prefix}.layers.*.linear_attn.out_proj.weight", + "decoder.layers.*.self_attention.A_log": f"{prefix}.layers.*.linear_attn.A_log", + "decoder.layers.*.self_attention.dt_bias": f"{prefix}.layers.*.linear_attn.dt_bias", + } + + mapping_list = [ + AutoMapping(megatron_param=megatron_param, hf_param=hf_param) + for megatron_param, hf_param in param_mappings.items() + ] + AutoMapping.register_module_type("SharedExpertMLP", "column") + AutoMapping.register_module_type("GatedDeltaNet", "column") + + mapping_list.extend( + [ + QKVMapping( + megatron_param="decoder.layers.*.self_attention.linear_qkv.weight", + q=f"{prefix}.layers.*.self_attn.q_proj.weight", + k=f"{prefix}.layers.*.self_attn.k_proj.weight", + v=f"{prefix}.layers.*.self_attn.v_proj.weight", + ), + GDNConv1dMapping( + megatron_param="decoder.layers.*.self_attention.conv1d.weight", + hf_param=f"{prefix}.layers.*.linear_attn.conv1d.weight", + ), + Qwen35SplitGDNMapping( + megatron_param="decoder.layers.*.self_attention.in_proj.weight", + qkv=f"{prefix}.layers.*.linear_attn.in_proj_qkv.weight", + z=f"{prefix}.layers.*.linear_attn.in_proj_z.weight", + b=f"{prefix}.layers.*.linear_attn.in_proj_b.weight", + a=f"{prefix}.layers.*.linear_attn.in_proj_a.weight", + ), + Qwen35RouterMapping( + megatron_param="decoder.layers.*.mlp.router.weight", + hf_param=f"{prefix}.layers.*.mlp.gate.weight", + ), + Qwen35PackedExpertGateUpMapping( + megatron_param="decoder.layers.*.mlp.experts.linear_fc1.weight*", + hf_param=f"{prefix}.layers.*.mlp.experts.gate_up_proj", + ), + Qwen35PackedExpertDownMapping( + megatron_param="decoder.layers.*.mlp.experts.linear_fc2.weight*", + hf_param=f"{prefix}.layers.*.mlp.experts.down_proj", + ), + GatedMLPMapping( + megatron_param="decoder.layers.*.mlp.shared_experts.linear_fc1.weight", + gate=f"{prefix}.layers.*.mlp.shared_expert.gate_proj.weight", + up=f"{prefix}.layers.*.mlp.shared_expert.up_proj.weight", + ), + AutoMapping( + megatron_param="decoder.layers.*.mlp.shared_experts.linear_fc2.weight", + hf_param=f"{prefix}.layers.*.mlp.shared_expert.down_proj.weight", + ), + ReplicatedMapping( + megatron_param="decoder.layers.*.mlp.shared_experts.gate_weight", + hf_param=f"{prefix}.layers.*.mlp.shared_expert_gate.weight", + ), + RMSNorm2ZeroCenteredRMSNormMapping( + "decoder.layers.*.self_attention.out_norm.weight", + f"{prefix}.layers.*.linear_attn.norm.weight", + ), + ] + ) + + return MegatronMappingRegistry(*mapping_list) + + registry = getattr(model_bridge.get_model_bridge, "_exact_types", {}) + for source in ("Qwen3_5MoeForConditionalGeneration", "Qwen3_5MoeForCausalLM"): + if source not in registry: + model_bridge.register_bridge_implementation( + source=source, target=GPTModel, bridge_class=Qwen35MoEBridge + ) + + +def _build_bridge(hf_path: str, cfg: BridgeConfig): + """Build Megatron-Bridge AutoBridge lazily from an HF model path.""" + from megatron.bridge import AutoBridge + + _register_bridge_compat_aliases() + bridge = AutoBridge.from_hf_pretrained(hf_path, trust_remote_code=True) + hf_config = _bridge_hf_config(bridge) + if hf_config is not None and not hasattr(hf_config, "rope_theta"): + hf_config.rope_theta = hf_config.to_dict().get("rope_theta", 1000000.0) + + if callable(cfg.bridge_post_init): + cfg.bridge_post_init(bridge) + + return bridge + + +def _build_optimizer(model_list: list, cfg: BridgeConfig): + from megatron.core.optimizer import get_megatron_optimizer + + return get_megatron_optimizer( + config=build_dist_opt_optimizer_config( + cfg.optimizer, override_optimizer_config=cfg.override_optimizer_config + ), + model_chunks=model_list, + ) + + +def _build_lr_scheduler(optimizer, cfg: BridgeConfig): + opt = cfg.optimizer + total_steps = opt.total_training_steps + if total_steps <= 0: + return None + + from megatron.core.optimizer_param_scheduler import OptimizerParamScheduler + + warmup_steps = opt.lr_warmup_steps + if warmup_steps <= 0 and opt.lr_warmup_steps_ratio > 0: + warmup_steps = int(opt.lr_warmup_steps_ratio * total_steps) + warmup_steps = max(warmup_steps, 0) + decay_steps = opt.lr_decay_steps if opt.lr_decay_steps is not None else total_steps + + return OptimizerParamScheduler( + optimizer, + init_lr=opt.lr_warmup_init, + max_lr=opt.lr, + min_lr=opt.min_lr, + lr_warmup_steps=warmup_steps, + lr_decay_steps=decay_steps, + lr_decay_style=opt.lr_decay_style, + start_wd=opt.weight_decay, + end_wd=opt.weight_decay, + wd_incr_steps=total_steps, + wd_incr_style=opt.weight_decay_incr_style, + use_checkpoint_opt_param_scheduler=opt.use_checkpoint_opt_param_scheduler, + override_opt_param_scheduler=not opt.use_checkpoint_opt_param_scheduler, + wsd_decay_steps=opt.lr_wsd_decay_steps, + lr_wsd_decay_style=opt.lr_wsd_decay_style, + ) + + +def _build_ddp_config(cfg: BridgeConfig): + from megatron.core.distributed import DistributedDataParallelConfig + + ddp_kwargs = { + "use_distributed_optimizer": True, + "overlap_grad_reduce": False, + "grad_reduce_in_fp32": True, + } + ddp_kwargs.update(cfg.override_ddp_config) + return DistributedDataParallelConfig(**ddp_kwargs) + + +def _resolve_benchmark_protocol(cfg: BridgeConfig, bridge) -> Any | None: + """Best-effort protocol lookup for model stats used by bench examples.""" + from megatron.lite.model.registry import get_train_runtime_module, resolve_model_type_from_hf + + model_name = cfg.model_name + if model_name == "auto": + try: + model_name = resolve_model_type_from_hf(_bridge_hf_config(bridge)) + except ValueError: + return None + + try: + return get_train_runtime_module(model_name) + except ValueError: + return None + + +def _as_data_iter(data: Any): + if hasattr(data, "__next__"): + return data + if isinstance(data, list): + return iter(data) + return iter([data]) + + +# MLITE_LAYERING_ALLOW_BRIDGE_FORWARD_METADATA_BEGIN +def _nested_from_packed(tensor: torch.Tensor | None, seq_lens: torch.Tensor): + """Split a 1-D packed (true, unpadded) tensor into a jagged nested tensor.""" + if tensor is None: + return None + flat = tensor.reshape(-1) + pieces = [] + offset = 0 + for length_t in seq_lens.tolist(): + length = int(length_t) + pieces.append(flat.narrow(0, offset, length)) + offset += length + if offset != flat.numel(): + raise ValueError( + f"PackedBatch sizes sum to {offset}, tensor has {flat.numel()} tokens." + ) + return torch.nested.as_nested_tensor(pieces, layout=torch.jagged) + + +def _bridge_forward_kwargs_from_packed_batch( + batch: PackedBatch, + *, + tp_size: int = 1, + cp_size: int = 1, + cp_rank: int = 0, + cp_group: Any = None, +) -> dict[str, Any]: + """Render transient Megatron-Core THD kwargs for BridgeRuntime.forward_step only. + + Reuses the same canonical packing the native lite protocols use + (:func:`pack_nested_thd` + :func:`prepare_packed_thd_for_context_parallel`): + each sequence is padded to the TP/CP zigzag alignment, ``labels`` are rolled + one position left, and for ``cp_size > 1`` the token rows / labels / loss + mask / position ids are zigzag-split to this CP rank while ``packed_seq_params`` + keeps full ``cu_seqlens`` plus CP metadata. Identical layout on both backends + keeps the mlite-vs-bridge comparison fair and CP-correct. + """ + from megatron.lite.primitive.parallel.thd import ( + pack_nested_thd, + prepare_packed_thd_for_context_parallel, + ) + + seq_lens = batch.seq_lens + has_loss_mask = batch.loss_mask is not None + packed = pack_nested_thd( + _nested_from_packed(batch.input_ids, seq_lens), + tp_size=tp_size, + cp_size=cp_size, + cp_rank=cp_rank, + cp_group=cp_group if cp_size > 1 else None, + split_cp=False, + labels=_nested_from_packed(batch.labels, seq_lens), + roll_labels=batch.labels is not None, + loss_mask=_nested_from_packed(batch.loss_mask, seq_lens) if has_loss_mask else None, + roll_loss_mask=has_loss_mask, + ) + sample: dict[str, Any] = { + "input_ids": packed.input_ids, + "labels": packed.labels, + "position_ids": packed.position_ids, + "packed_seq_params": packed.packed_seq_params, + } + if packed.loss_mask is not None: + sample["loss_mask"] = packed.loss_mask + + if cp_size > 1: + tensor_keys = ("input_ids", "labels", "loss_mask", "position_ids") + psp, tensors = prepare_packed_thd_for_context_parallel( + sample["packed_seq_params"], + tuple(sample.get(key) for key in tensor_keys), + cp_size=cp_size, + cp_rank=cp_rank, + cp_group=cp_group, + ) + sample["packed_seq_params"] = psp + for key, tensor in zip(tensor_keys, tensors, strict=True): + if tensor is not None or key in sample: + sample[key] = tensor + return sample +# MLITE_LAYERING_ALLOW_BRIDGE_FORWARD_METADATA_END + + +def _bridge_forward_kwargs_bshd(batch: PackedBatch) -> dict[str, Any]: + """Render dense [b=1, s] forward kwargs for a single packed sequence. + + Used when ``use_thd`` is False so the Megatron-Core reference runs its dense + kernel (e.g. GatedDeltaNet, whose THD path is unsupported / non-deterministic + in the pinned core), giving a deterministic, layout-matched parity vs the + native lite dense forward on identical tokens. Carries no THD packing + metadata (no ``packed_seq_params`` / ``position_ids``) — single unpadded + sequence, so an all-ones attention mask reproduces the full causal sequence. + CP=1 only. + """ + total = int(batch.seq_lens.sum().item()) + return { + "input_ids": batch.input_ids.reshape(1, total).contiguous(), + "labels": ( + batch.labels.reshape(1, total).contiguous() if batch.labels is not None else None + ), + "attention_mask": torch.ones((1, total), dtype=torch.long, device=batch.input_ids.device), + } + + +class BridgeRuntime(RuntimeBase): + """Megatron-Bridge training backend using Megatron-Core optimizer state.""" + + def __init__(self, hf_path: str, cfg: BridgeConfig | dict[str, Any]): + self._hf_path = hf_path + self._cfg = cfg if isinstance(cfg, BridgeConfig) else BridgeConfig.from_dict(cfg) + self._offload_param = self._cfg.param_offload + self._offload_optimizer = self._cfg.optimizer_offload + + def build_model( + self, hf_path: str | None = None, cfg: BridgeConfig | dict[str, Any] | None = None, **kwargs + ) -> ModelHandle: + from megatron.core import parallel_state as mpu + from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed + from megatron.core.transformer.enums import ModelType + + if cfg is None: + rt_cfg = self._cfg + elif isinstance(cfg, BridgeConfig): + rt_cfg = cfg + else: + rt_cfg = BridgeConfig.from_dict(cfg) + hf_path = hf_path or self._hf_path + + if not dist.is_initialized(): + dist.init_process_group("nccl", timeout=timedelta(minutes=10)) + torch.cuda.set_device(dist.get_rank() % torch.cuda.device_count()) + + p = rt_cfg.parallel + init_kwargs = dict( + tensor_model_parallel_size=p.tp, + pipeline_model_parallel_size=p.pp, + expert_model_parallel_size=p.ep, + context_parallel_size=p.cp, + ) + if p.etp is not None: + init_kwargs["expert_tensor_parallel_size"] = p.etp + if p.vpp > 1: + init_kwargs["virtual_pipeline_model_parallel_size"] = p.vpp + + if not mpu.model_parallel_is_initialized(): + mpu.initialize_model_parallel(**init_kwargs) + model_parallel_cuda_manual_seed(rt_cfg.seed) + + bridge = _build_bridge(hf_path, rt_cfg) + provider = bridge.to_megatron_provider( + load_weights=rt_cfg.load_hf_weights, hf_path=hf_path if rt_cfg.load_hf_weights else None + ) + _configure_provider(provider, rt_cfg) + if hasattr(provider, "finalize"): + provider.finalize() + + model_list = provider.provide_distributed_model( + model_type=ModelType.encoder_or_decoder, + wrap_with_ddp=True, + ddp_config=_build_ddp_config(rt_cfg), + bf16=True, + ) + + optimizer = _build_optimizer(model_list, rt_cfg) if rt_cfg.build_optimizer else None + lr_scheduler = _build_lr_scheduler(optimizer, rt_cfg) if optimizer is not None else None + register_training_hooks(model_list, optimizer) + + if self._offload_param: + offload_model_to_cpu(model_list) + if self._offload_optimizer and optimizer is not None: + offload_optimizer(optimizer) + + logger.info("BridgeRuntime: model built, tp=%d ep=%d pp=%d cp=%d", p.tp, p.ep, p.pp, p.cp) + + return ModelHandle( + model=model_list[0], + optimizer=optimizer, + lr_scheduler=lr_scheduler, + parallel_state=_MpuParallelState(mpu), + config=rt_cfg, + _extras={ + "bridge": bridge, + "provider": provider, + "model_list": model_list, + "mpu": mpu, + "model_cfg": _bridge_hf_config(bridge), + "protocol": _resolve_benchmark_protocol(rt_cfg, bridge), + "optimizer_backend": "dist_opt" if optimizer is not None else "none", + "world_size": dist.get_world_size(), + }, + ) + + def forward_backward( + self, + handle: ModelHandle, + data: Any, + loss_fn, + *, + num_microbatches: int = 1, + forward_only: bool = False, + ) -> ForwardResult: + from megatron.core import parallel_state as mpu + from megatron.core.pipeline_parallel.schedules import get_forward_backward_func + + if num_microbatches < 1: + raise ValueError("num_microbatches must be >= 1") + + model_list = handle._extras["model_list"] + data_iter = _as_data_iter(data) + last_loss: list[float | None] = [None] + last_output: list[torch.Tensor | None] = [None] + + # CP geometry for the transient PackedBatch -> THD kwargs rendering below. + cp_size = mpu.get_context_parallel_world_size() + cp_kwargs = { + "tp_size": mpu.get_tensor_model_parallel_world_size(), + "cp_size": cp_size, + "cp_rank": mpu.get_context_parallel_rank(), + "cp_group": mpu.get_context_parallel_group() if cp_size > 1 else None, + } + + def _fwd_step(data_iterator, model): + # MLITE_LAYERING_ALLOW_BRIDGE_FORWARD_METADATA_BEGIN + sample = next(data_iterator) + owns_transient_metadata = False + if isinstance(sample, PackedBatch): + if self._cfg.use_thd: + sample = _bridge_forward_kwargs_from_packed_batch(sample, **cp_kwargs) + owns_transient_metadata = True + else: + sample = _bridge_forward_kwargs_bshd(sample) + elif isinstance(sample, Batch): + sample = { + "input_ids": sample["input_ids"], + "labels": sample["labels"], + } + if not isinstance(sample, dict): + raise TypeError( + f"BridgeRuntime expected dict or Batch data, got {type(sample).__name__}." + ) + if not owns_transient_metadata: + leaked = {"packed_seq_params", "position_ids"}.intersection(sample) + if leaked: + raise ValueError( + "BridgeRuntime data must not carry model-internal keys " + f"{sorted(leaked)}; pass a raw PackedBatch instead." + ) + + output_tensor = model( + input_ids=sample["input_ids"], + position_ids=sample.get("position_ids"), + attention_mask=sample.get("attention_mask"), + labels=sample["labels"], + packed_seq_params=sample.get("packed_seq_params"), + ) + if isinstance(output_tensor, tuple): + output_tensor = output_tensor[0] + last_output[0] = output_tensor + loss_sample = { + key: value + for key, value in sample.items() + if key not in {"packed_seq_params", "position_ids"} + } + # MLITE_LAYERING_ALLOW_BRIDGE_FORWARD_METADATA_END + + def _bridge_loss_fn(output_tensor, non_loss_data=False): + if loss_fn is not None: + loss, _metrics = loss_fn({"output_tensor": output_tensor}, loss_sample) + else: + loss = output_tensor.mean() + last_loss[0] = float(loss.detach().item()) + return loss, {} + + return output_tensor, _bridge_loss_fn + + vpp_size = mpu.get_virtual_pipeline_model_parallel_world_size() + if vpp_size is not None and vpp_size > 1: + batches = [next(data_iter) for _ in range(num_microbatches)] + batch_generator = [iter(batches) for _ in range(vpp_size)] + else: + batch_generator = data_iter + + get_forward_backward_func()( + forward_step_func=_fwd_step, + data_iterator=batch_generator, + model=model_list, + num_microbatches=num_microbatches, + forward_only=forward_only, + seq_length=1, + micro_batch_size=1, + ) + + loss_val = last_loss[0] + if mpu.get_pipeline_model_parallel_world_size() > 1: + loss_t = torch.tensor([loss_val or 0.0], device="cuda") + dist.broadcast(loss_t, src=mpu.get_pipeline_model_parallel_last_rank()) + loss_val = float(loss_t.item()) + + result_loss = torch.tensor(loss_val or 0.0) + return ForwardResult( + model_output=ModelOutputs(loss=result_loss, vocab_parallel_logits=last_output[0]), + metrics={"loss": loss_val if loss_val is not None else 0.0}, + ) + + def zero_grad(self, handle: ModelHandle) -> None: + if handle._optimizer is not None: + handle._optimizer.zero_grad() + for model in handle._extras["model_list"]: + if handle._optimizer is None: + model.zero_grad(set_to_none=True) + if hasattr(model, "zero_grad_buffer"): + model.zero_grad_buffer() + + def optimizer_step(self, handle: ModelHandle) -> tuple[bool, float, int | None]: + if handle._optimizer is None: + return True, 0.0, 0 + update_successful, grad_norm, num_zeros = handle._optimizer.step() + return update_successful, float(grad_norm), num_zeros + + def lr_scheduler_step(self, handle: ModelHandle) -> float | list[float]: + if handle._lr_scheduler is not None: + handle._lr_scheduler.step(1) + return handle._optimizer.param_groups[0]["lr"] + return 0.0 + + def is_mp_src_rank_with_outputs(self, handle: ModelHandle) -> bool: + return is_mp_src_rank_with_outputs() + + def to( + self, + handle: ModelHandle, + device: str, + *, + model: bool = True, + optimizer: bool = True, + grad: bool = True, + ) -> None: + model_list = handle._extras["model_list"] + opt = handle._optimizer + if device == "cuda": + if model: + load_model_to_gpu(model_list, load_grad=grad) + if optimizer and opt is not None: + load_optimizer(opt) + elif device == "cpu": + if model: + offload_model_to_cpu(model_list) + if optimizer and opt is not None: + offload_optimizer(opt) + else: + raise ValueError(f"BridgeRuntime.to supports only 'cpu' or 'cuda', got {device!r}.") + + def train_mode(self, handle: ModelHandle): + return _BridgeTrainCtx(self, handle) + + def eval_mode(self, handle: ModelHandle): + return _BridgeEvalCtx(self, handle) + + def export_weights(self, handle: ModelHandle, **kwargs) -> Iterator[tuple[str, torch.Tensor]]: + bridge = handle._extras["bridge"] + model_list = handle._extras["model_list"] + load_model_to_gpu(model_list, load_grad=False) + return bridge.export_hf_weights(model_list, cpu=bool(kwargs.get("cpu", False))) + + def save_checkpoint(self, handle: ModelHandle, path: str, **kwargs) -> None: + model_list = handle._extras["model_list"] + opt = handle._optimizer + load_model_to_gpu(model_list, load_grad=True) + + from megatron.core import dist_checkpointing + + state = build_sharded_state_dict(model_list, opt, handle._lr_scheduler) + os.makedirs(path, exist_ok=True) + dist_checkpointing.save(state, path) + dist.barrier() + + if self._offload_param: + offload_model_to_cpu(model_list) + + def load_checkpoint(self, handle: ModelHandle, path: str, **kwargs) -> None: + model_list = handle._extras["model_list"] + opt = handle._optimizer + load_model_to_gpu(model_list, load_grad=True) + + from megatron.core import dist_checkpointing + + state = build_sharded_state_dict(model_list, opt, handle._lr_scheduler) + dist_checkpointing.load(state, path) + + if self._offload_param: + offload_model_to_cpu(model_list) + if self._offload_optimizer and opt is not None: + offload_optimizer(opt) + + +class _BridgeTrainCtx: + def __init__(self, runtime: BridgeRuntime, handle: ModelHandle): + self._runtime = runtime + self._handle = handle + + def __enter__(self): + if self._runtime._offload_param or self._runtime._offload_optimizer: + self._runtime.to( + self._handle, + "cuda", + model=self._runtime._offload_param, + optimizer=self._runtime._offload_optimizer, + grad=self._runtime._offload_param, + ) + for model in self._handle._extras["model_list"]: + model.train() + return self + + def __exit__(self, *exc): + self._runtime.zero_grad(self._handle) + if self._runtime._offload_param or self._runtime._offload_optimizer: + self._runtime.to( + self._handle, + "cpu", + model=self._runtime._offload_param, + optimizer=self._runtime._offload_optimizer, + grad=self._runtime._offload_param, + ) + return False + + +class _BridgeEvalCtx: + def __init__(self, runtime: BridgeRuntime, handle: ModelHandle): + self._runtime = runtime + self._handle = handle + self._prev_grad = torch.is_grad_enabled() + + def __enter__(self): + if self._runtime._offload_param: + self._runtime.to(self._handle, "cuda", model=True, optimizer=False, grad=False) + for model in self._handle._extras["model_list"]: + model.eval() + torch.set_grad_enabled(False) + return self + + def __exit__(self, *exc): + torch.set_grad_enabled(self._prev_grad) + if self._runtime._offload_param: + self._runtime.to(self._handle, "cpu", model=True, optimizer=False, grad=False) + return False + + +__all__ = ["BridgeRuntime"] diff --git a/experimental/lite/megatron/lite/runtime/backends/mbridge/__init__.py b/experimental/lite/megatron/lite/runtime/backends/mbridge/__init__.py new file mode 100644 index 00000000000..993aa2576c0 --- /dev/null +++ b/experimental/lite/megatron/lite/runtime/backends/mbridge/__init__.py @@ -0,0 +1,18 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""mbridge backend for the Megatron Lite runtime API.""" + +from __future__ import annotations + +from typing import Any + +from megatron.lite.runtime.backends import Runtime as RuntimeBase +from megatron.lite.runtime.backends.bridge.config import BridgeConfig +from megatron.lite.runtime.backends.mbridge.runtime import MBridgeRuntime + + +def create(hf_path: str, cfg: BridgeConfig | dict[str, Any]) -> RuntimeBase: + """Factory called by ``megatron.lite.runtime.create_runtime``.""" + return MBridgeRuntime(hf_path, cfg) + + +__all__ = ["BridgeConfig", "MBridgeRuntime", "create"] diff --git a/experimental/lite/megatron/lite/runtime/backends/mbridge/runtime.py b/experimental/lite/megatron/lite/runtime/backends/mbridge/runtime.py new file mode 100644 index 00000000000..d2424a69cb4 --- /dev/null +++ b/experimental/lite/megatron/lite/runtime/backends/mbridge/runtime.py @@ -0,0 +1,167 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Runtime backend backed by the legacy ``mbridge`` package.""" + +from __future__ import annotations + +import logging +from collections.abc import Iterator +from datetime import timedelta +from typing import Any + +import torch +import torch.distributed as dist + +from megatron.lite.runtime.backends.bridge.config import BridgeConfig +from megatron.lite.runtime.backends.bridge.runtime import ( + BridgeRuntime, + _build_lr_scheduler, + _build_optimizer, + _lower_transformer_overrides, + _MpuParallelState, +) +from megatron.lite.runtime.contracts.handle import ModelHandle +from megatron.lite.runtime.megatron_utils import ( + load_model_to_gpu, + offload_model_to_cpu, + offload_optimizer, + register_training_hooks, +) + +logger = logging.getLogger(__name__) + + +def _build_mbridge(hf_path: str, cfg: BridgeConfig): + """Build the legacy mbridge AutoBridge lazily from an HF model path.""" + from mbridge import AutoBridge + + from megatron.lite.primitive.deterministic import deterministic_requested + + bridge = AutoBridge.from_pretrained(hf_path, trust_remote_code=True) + bridge.set_extra_args(sequence_parallel=cfg.parallel.tp > 1) + + transformer_overrides = _lower_transformer_overrides(cfg) + if transformer_overrides: + bridge.set_extra_args(**transformer_overrides) + + if not hasattr(bridge.hf_config, "rope_theta"): + bridge.hf_config.rope_theta = bridge.hf_config.to_dict().get("rope_theta", 1000000.0) + + bridge.set_extra_args(bf16=True, fp16=False) + + if cfg.model_name == "qwen3_5": + bridge.set_extra_args( + deterministic_mode=deterministic_requested(), fused_single_qkv_rope=False + ) + + if callable(cfg.bridge_post_init): + cfg.bridge_post_init(bridge) + + return bridge + + +def _resolve_mbridge_benchmark_protocol(cfg: BridgeConfig, bridge) -> Any | None: + """Best-effort protocol lookup for model stats used by bench examples.""" + from megatron.lite.model.registry import get_train_runtime_module, resolve_model_type_from_hf + + model_name = cfg.model_name + if model_name == "auto": + try: + model_name = resolve_model_type_from_hf(bridge.hf_config) + except ValueError: + return None + + try: + return get_train_runtime_module(model_name) + except ValueError: + return None + + +class MBridgeRuntime(BridgeRuntime): + """mbridge training backend using Megatron-Core optimizer state.""" + + def build_model( + self, hf_path: str | None = None, cfg: BridgeConfig | dict[str, Any] | None = None, **kwargs + ) -> ModelHandle: + from megatron.core import parallel_state as mpu + from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed + from megatron.core.transformer.enums import ModelType + + if cfg is None: + rt_cfg = self._cfg + elif isinstance(cfg, BridgeConfig): + rt_cfg = cfg + else: + rt_cfg = BridgeConfig.from_dict(cfg) + hf_path = hf_path or self._hf_path + + if not dist.is_initialized(): + dist.init_process_group("nccl", timeout=timedelta(minutes=10)) + torch.cuda.set_device(dist.get_rank() % torch.cuda.device_count()) + + p = rt_cfg.parallel + init_kwargs = dict( + tensor_model_parallel_size=p.tp, + pipeline_model_parallel_size=p.pp, + expert_model_parallel_size=p.ep, + context_parallel_size=p.cp, + ) + if p.etp is not None: + init_kwargs["expert_tensor_parallel_size"] = p.etp + if p.vpp > 1: + init_kwargs["virtual_pipeline_model_parallel_size"] = p.vpp + + if not mpu.model_parallel_is_initialized(): + mpu.initialize_model_parallel(**init_kwargs) + model_parallel_cuda_manual_seed(rt_cfg.seed) + + bridge = _build_mbridge(hf_path, rt_cfg) + + ddp_config = { + "use_distributed_optimizer": True, + "overlap_grad_reduce": False, + "grad_reduce_in_fp32": True, + } + ddp_config.update(rt_cfg.override_ddp_config) + + model_list = bridge.get_model( + model_type=ModelType.encoder_or_decoder, wrap_with_ddp=True, ddp_config=ddp_config + ) + if rt_cfg.load_hf_weights: + bridge.load_weights(model_list, hf_path, memory_efficient=True) + + optimizer = _build_optimizer(model_list, rt_cfg) if rt_cfg.build_optimizer else None + lr_scheduler = _build_lr_scheduler(optimizer, rt_cfg) if optimizer is not None else None + register_training_hooks(model_list, optimizer) + + if self._offload_param: + offload_model_to_cpu(model_list) + if self._offload_optimizer and optimizer is not None: + offload_optimizer(optimizer) + + logger.info("MBridgeRuntime: model built, tp=%d ep=%d pp=%d cp=%d", p.tp, p.ep, p.pp, p.cp) + + return ModelHandle( + model=model_list[0], + optimizer=optimizer, + lr_scheduler=lr_scheduler, + parallel_state=_MpuParallelState(mpu), + config=rt_cfg, + _extras={ + "bridge": bridge, + "model_list": model_list, + "mpu": mpu, + "model_cfg": bridge.hf_config, + "protocol": _resolve_mbridge_benchmark_protocol(rt_cfg, bridge), + "optimizer_backend": "dist_opt" if optimizer is not None else "none", + "world_size": dist.get_world_size(), + }, + ) + + def export_weights(self, handle: ModelHandle, **kwargs) -> Iterator[tuple[str, torch.Tensor]]: + bridge = handle._extras["bridge"] + model_list = handle._extras["model_list"] + load_model_to_gpu(model_list, load_grad=False) + return bridge.export_weights(model_list) + + +__all__ = ["MBridgeRuntime"] diff --git a/experimental/lite/megatron/lite/runtime/backends/mlite/__init__.py b/experimental/lite/megatron/lite/runtime/backends/mlite/__init__.py new file mode 100644 index 00000000000..cc514f2858e --- /dev/null +++ b/experimental/lite/megatron/lite/runtime/backends/mlite/__init__.py @@ -0,0 +1,18 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Megatron Lite backend — Megatron Lite's own training engine.""" + +from __future__ import annotations + +from typing import Any + +from megatron.lite.runtime.backends import Runtime as RuntimeBase +from megatron.lite.runtime.backends.mlite.config import MegatronLiteConfig +from megatron.lite.runtime.backends.mlite.runtime import MegatronLiteRuntime + + +def create(hf_path: str, cfg: MegatronLiteConfig | dict[str, Any]) -> RuntimeBase: + """Factory called by create_runtime. + + create_runtime escape hatch is checked inside MegatronLiteRuntime.build_model(). + """ + return MegatronLiteRuntime(hf_path, cfg) diff --git a/experimental/lite/megatron/lite/runtime/backends/mlite/config.py b/experimental/lite/megatron/lite/runtime/backends/mlite/config.py new file mode 100644 index 00000000000..edff0336517 --- /dev/null +++ b/experimental/lite/megatron/lite/runtime/backends/mlite/config.py @@ -0,0 +1,96 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Megatron Lite backend configuration.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +from megatron.lite.runtime.contracts.config import OptimizerConfig, ParallelConfig, pick_fields + + +@dataclass(slots=True) +class DebugConfig: + """Megatron Lite backend debug flags. Not exposed to end users.""" + + param_update: bool = False + optimizer_state: bool = False + grad_phases: bool = False + router_summary: bool = False + moe_io: bool = False + attn_io: bool = False + + +@dataclass +class MegatronLiteConfig: + """Config for MegatronLiteRuntime (Megatron Lite's default 5D parallel runtime). + + Megatron Lite-specific training features live in ``impl_cfg`` (a plain dict — + each impl reads the keys it needs via its own typed ImplConfig). + """ + + # ── identity ── + model_name: str = "auto" + impl: str = "lite" + hf_path: str = "" + + # ── parallelism and optimizer ── + parallel: ParallelConfig = field(default_factory=ParallelConfig) + optimizer: OptimizerConfig = field(default_factory=OptimizerConfig) + + # ── common runtime/model fields ── + attention_backend_override: str | None = "flash" + router_aux_loss_coef: float | None = None + load_hf_weights: bool = True + + # ── impl-specific (each impl reads its own keys) ── + impl_cfg: dict[str, Any] = field(default_factory=dict) + + # ── debug ── + debug: DebugConfig = field(default_factory=DebugConfig) + + # ── bench-only hook: mutate model_cfg after build (e.g. expert truncation) ── + model_config_hook: Any = None + + @classmethod + def from_dict(cls, hf_path: str, cfg: dict[str, Any]) -> MegatronLiteConfig: + """Construct MegatronLiteConfig from a flat dict (legacy / OmegaConf path).""" + if "num_microbatches" in cfg: + raise ValueError( + "MegatronLiteConfig no longer accepts `num_microbatches`; " + "pass it to Runtime.forward_backward(..., num_microbatches=...) instead" + ) + parallel = ParallelConfig(**pick_fields(ParallelConfig, cfg)) + + opt_d = cfg.get("optimizer", {}) + optimizer = ( + OptimizerConfig(**pick_fields(OptimizerConfig, opt_d)) + if isinstance(opt_d, dict) + else OptimizerConfig() + ) + if isinstance(opt_d, dict) and isinstance(opt_d.get("override_optimizer_config"), dict): + optimizer.override_optimizer_config = dict(opt_d["override_optimizer_config"]) + + # impl_cfg: merge nested dict + top-level overrides + impl_cfg: dict[str, Any] = {} + nested = cfg.get("impl_cfg") + if isinstance(nested, dict): + impl_cfg.update(nested) + for k in list(impl_cfg): + if k in cfg: + impl_cfg[k] = cfg[k] + for k in ("recompute", "use_thd", "use_deepep", "precision_aware_opt"): + if k in cfg and k not in impl_cfg: + impl_cfg[k] = cfg[k] + + skip = {"parallel", "optimizer", "impl_cfg", "debug"} + return cls( + **{k: v for k, v in pick_fields(cls, cfg).items() if k not in skip}, + hf_path=hf_path, + parallel=parallel, + optimizer=optimizer, + impl_cfg=impl_cfg, + ) + + +__all__ = ["MegatronLiteConfig", "DebugConfig"] diff --git a/experimental/lite/megatron/lite/runtime/backends/mlite/runtime.py b/experimental/lite/megatron/lite/runtime/backends/mlite/runtime.py new file mode 100644 index 00000000000..da59ccdb981 --- /dev/null +++ b/experimental/lite/megatron/lite/runtime/backends/mlite/runtime.py @@ -0,0 +1,586 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""MegatronLiteRuntime — Megatron Lite's default training backend.""" + +from __future__ import annotations + +import os +from collections.abc import Callable, Iterator +from dataclasses import fields as dc_fields +from datetime import timedelta +from itertools import chain +from typing import Any + +import torch +import torch.distributed as dist +from megatron.lite.runtime.backends import Runtime as RuntimeBase +from megatron.lite.runtime.backends.mlite.config import MegatronLiteConfig +from megatron.lite.runtime.contracts.data import ForwardResult, ModelOutputs, PackedBatch +from megatron.lite.runtime.contracts.handle import ModelHandle +from megatron.lite.runtime.contracts.loss import get_loss_context, split_loss_context, use_loss_context + + +def _build_impl_cfg(proto, rt_cfg: MegatronLiteConfig): + """Construct typed impl config, backfilling hf_path + optimizer_config.""" + init_fields = {f.name for f in dc_fields(proto.ImplConfig) if f.init} + # Only forward impl_cfg keys this model's ImplConfig declares: the connector + # may pass knobs (e.g. cross_entropy_fusion) that some models don't model. + impl_cfg_kwargs = {key: value for key, value in rt_cfg.impl_cfg.items() if key in init_fields} + impl_cfg_kwargs["parallel"] = rt_cfg.parallel + if ( + "attention_backend_override" in init_fields + and impl_cfg_kwargs.get("attention_backend_override") is None + ): + impl_cfg_kwargs["attention_backend_override"] = rt_cfg.attention_backend_override + if "hf_path" in init_fields and impl_cfg_kwargs.get("hf_path") in (None, "") and rt_cfg.hf_path: + impl_cfg_kwargs["hf_path"] = rt_cfg.hf_path + # Thread the user-level OptimizerConfig so the protocol can pass it to + # optimizer primitives without reading runtime internals. + if ( + "optimizer_config" in init_fields + and impl_cfg_kwargs.get("optimizer_config") is None + and getattr(rt_cfg, "optimizer", None) is not None + ): + impl_cfg_kwargs["optimizer_config"] = rt_cfg.optimizer + return proto.ImplConfig(**impl_cfg_kwargs) + + +def _apply_attention_backend_env(backend: str | None, *, tag: str) -> None: + if backend is None: + return + + env_overrides = { + "auto": ("1", "1", "1"), + "flash": ("1", "0", "0"), + "fused": ("0", "1", "0"), + "unfused": ("0", "0", "1"), + "local": ("0", "0", "0"), + } + try: + flash, fused, unfused = env_overrides[backend] + except KeyError as exc: + raise ValueError( + "attention_backend_override must be one of {'auto', 'flash', 'fused', 'unfused', 'local'}" + ) from exc + + os.environ["NVTE_FLASH_ATTN"] = flash + os.environ["NVTE_FUSED_ATTN"] = fused + os.environ["NVTE_UNFUSED_ATTN"] = unfused + + +def _infer_pipeline_tensor_shape(batch: PackedBatch, model_cfg: Any, ps) -> tuple[int, int, int]: + if model_cfg is None or not hasattr(model_cfg, "hidden_size"): + raise ValueError("Megatron Lite pipeline runtime requires model_cfg.hidden_size.") + if not isinstance(batch, PackedBatch): + raise TypeError("Megatron Lite pipeline runtime requires PackedBatch inputs.") + + input_ids = batch.input_ids + if input_ids.dim() == 1: + batch_size = 1 + local_seq_len = int(input_ids.size(0)) + elif input_ids.dim() == 2: + batch_size = int(input_ids.size(0)) + local_seq_len = int(input_ids.size(1)) + else: + raise ValueError(f"Unsupported input_ids rank for pipeline runtime: {input_ids.dim()}.") + + if local_seq_len < 1: + raise ValueError("Pipeline tensor shape requires non-empty sequence.") + + tp_size = int(getattr(ps, "tp_size", 1) or 1) + cp_size = int(getattr(ps, "cp_size", 1) or 1) + # The model scatters THD activations into Megatron sequence-parallel + context-parallel form + # before the first layer, so PP-communicated hidden states carry padded_S / (CP * TP) per rank. + # The raw input_ids length is UNPADDED and not CP-divided; sizing the P2P buffer from it makes the + # receiver wait for CP*TP-too-many elements -> NCCL recv hang. Replicate thd.pack_nested_thd padding + # (each sequence padded to align = tp * (2*cp if cp>1 else 1)) then divide by CP*TP. + seq_lens = getattr(batch, "seq_lens", None) + if seq_lens is not None and cp_size * tp_size > 1: + sl = seq_lens if isinstance(seq_lens, torch.Tensor) else torch.as_tensor(seq_lens) + sl = sl.to(torch.int64).reshape(-1) + align = tp_size * (2 * cp_size if cp_size > 1 else 1) + padded = sl + (align - sl % align) % align + total_padded = int(padded.sum().item()) + local_seq_len = total_padded // (cp_size * tp_size) + elif tp_size > 1: + if local_seq_len % tp_size != 0: + raise ValueError( + f"Pipeline tensor sequence length {local_seq_len} is not divisible by TP={tp_size}." + ) + local_seq_len //= tp_size + + # Models with multi-head hyper-connections (e.g. DeepSeek V4) carry hc_mult + # parallel residual streams across pipeline stages. The inter-stage tensor folds + # hc_mult into the hidden dim ([B, S, hc_mult * H]); size the P2P buffer to match. + # hc_mult defaults to 1, so this is a no-op for every other model. + hc_mult = int(getattr(model_cfg, "hc_mult", 1) or 1) + return (local_seq_len, batch_size, int(model_cfg.hidden_size) * hc_mult) + + +def _last_loss_output(outputs: list[dict]) -> dict: + for output in reversed(outputs): + if output.get("loss") is not None: + return output + return {} + + +def _checkpoint_module(model: Any) -> torch.nn.Module: + if isinstance(model, torch.nn.Module): + return model + if isinstance(model, list | tuple): + return torch.nn.ModuleList(model) + raise TypeError( + f"Checkpoint model must be an nn.Module or sequence of modules, got {type(model).__name__}." + ) + + +def _pipeline_callbacks(forward_step: Callable, loss_fn: Callable | None): + def wrapped_forward_step(model, item): + batch, loss_context = split_loss_context(item) + if loss_context is None: + return forward_step(model, batch) + with use_loss_context(loss_context): + return forward_step(model, batch) + + if loss_fn is None: + return wrapped_forward_step, None + + def wrapped_loss_fn(output, item, loss_context=None): + batch, item_loss_context = split_loss_context(item) + loss_context = loss_context or item_loss_context or get_loss_context() + if loss_context is None: + return loss_fn(output, batch) + return loss_fn(output, batch, loss_context) + + return wrapped_forward_step, wrapped_loss_fn + + +class MegatronLiteRuntime(RuntimeBase): + """Megatron Lite default training backend (Megatron-style 5D parallel).""" + + def __init__(self, hf_path: str, cfg: MegatronLiteConfig | dict[str, Any]): + self._hf_path = hf_path + self._cfg = ( + cfg + if isinstance(cfg, MegatronLiteConfig) + else MegatronLiteConfig.from_dict(hf_path, cfg) + ) + + # ── build_model ── + + def build_model( + self, + hf_path: str | None = None, + cfg: MegatronLiteConfig | dict[str, Any] | None = None, + **kwargs, + ) -> ModelHandle: + if cfg is not None and isinstance(cfg, dict): + rt_cfg = MegatronLiteConfig.from_dict(hf_path or self._hf_path, cfg) + elif cfg is not None and isinstance(cfg, MegatronLiteConfig): + rt_cfg = cfg + else: + rt_cfg = self._cfg + + # ── init distributed ── + if not dist.is_initialized(): + dist.init_process_group("nccl", timeout=timedelta(minutes=10)) + torch.cuda.set_device(dist.get_rank() % torch.cuda.device_count()) + torch.cuda.manual_seed(42) + + # ── load model protocol module ── + proto = self._load_protocol(rt_cfg) + + _apply_attention_backend_env( + rt_cfg.attention_backend_override, tag=f"{rt_cfg.model_name}:{rt_cfg.impl}" + ) + + # ── escape hatch: model takes over ── + if hasattr(proto, "create_runtime"): + return proto.create_runtime(rt_cfg.hf_path, rt_cfg).build_model() + + # ── construct impl_cfg (parallel injected) ── + impl_cfg = _build_impl_cfg(proto, rt_cfg) + + # ── build model config ── + model_cfg = proto.build_model_config(rt_cfg.hf_path) + if callable(rt_cfg.model_config_hook): + model_cfg = rt_cfg.model_config_hook(model_cfg) + + # ── build model (model owns ps + optimizer + everything) ── + bundle = proto.build_model(model_cfg, impl_cfg=impl_cfg) + + # ── load HF weights (optional) ── + loaded_hf_weights = False + if rt_cfg.load_hf_weights and rt_cfg.hf_path and hasattr(proto, "load_hf_weights"): + for chunk in bundle.chunks: + proto.load_hf_weights(chunk, rt_cfg.hf_path, model_cfg, bundle.parallel_state) + loaded_hf_weights = True + + post_load_hook = bundle.extras.pop("post_model_load_hook", None) + if callable(post_load_hook): + post_load_updates = post_load_hook() + if post_load_updates is not None: + if not isinstance(post_load_updates, dict): + raise TypeError("post_model_load_hook must return a dict or None.") + if "optimizer" in post_load_updates: + bundle.optimizer = post_load_updates["optimizer"] + if "finalize_grads" in post_load_updates: + bundle.finalize_grads = post_load_updates["finalize_grads"] + extra_updates = post_load_updates.get("extras") + if extra_updates: + if not isinstance(extra_updates, dict): + raise TypeError("post_model_load_hook extras update must be a dict.") + bundle.extras.update(extra_updates) + + if loaded_hf_weights and bundle.optimizer is not None: + reload_model_params = getattr(bundle.optimizer, "reload_model_params", None) + if callable(reload_model_params): + reload_model_params() + + if bundle.forward_step is None: + raise ValueError("Megatron Lite model bundles must provide a typed forward_step.") + + p = rt_cfg.parallel + model = bundle.chunks[0] if len(bundle.chunks) == 1 else bundle.chunks + return ModelHandle( + model=model, + optimizer=bundle.optimizer, + lr_scheduler=None, + parallel_state=bundle.parallel_state, + config=rt_cfg, + _extras={ + "model_chunks": bundle.chunks, + "model_cfg": model_cfg, + "forward_step": bundle.forward_step, + "protocol": proto, + "finalize_grads": bundle.finalize_grads, + "world_size": dist.get_world_size(), + "cp_range": (p.cp, p.cp), + **bundle.extras, + }, + ) + + def _load_protocol(self, rt_cfg: MegatronLiteConfig): + """Load and return the model protocol module.""" + from megatron.lite.model.registry import TRAIN_RUNTIME_MODULES, resolve_runtime_model_name + + try: + runtime_key = resolve_runtime_model_name(rt_cfg.model_name, rt_cfg.impl) + except ValueError as exc: + raise ValueError( + f"No protocol registered for model={rt_cfg.model_name!r}, " + f"impl={rt_cfg.impl!r}. Register with register_model(...)." + ) from exc + + mod_path = TRAIN_RUNTIME_MODULES.get(runtime_key) + if mod_path is None: + raise ValueError(f"No protocol module for runtime key {runtime_key!r}") + + import importlib + + proto = importlib.import_module(mod_path) + + for fn_name in ("build_model_config", "build_model"): + if not callable(getattr(proto, fn_name, None)): + raise ValueError(f"Protocol module {mod_path} missing required function: {fn_name}") + if not hasattr(proto, "ImplConfig"): + raise ValueError(f"Protocol module {mod_path} missing ImplConfig class") + + return proto + + # ── Checkpoint ── + + def save_checkpoint(self, handle: ModelHandle, path: str, **kwargs) -> None: + from megatron.lite.primitive.ckpt import save_training_checkpoint + + step = kwargs.pop("step", None) + if step is None: + step = kwargs.pop("iteration", None) + if step is None: + step = kwargs.pop("global_step", 0) + use_dcp = bool(kwargs.pop("use_dcp", True)) + save_rng = bool(kwargs.pop("save_rng", True)) + get_placements, is_expert = _checkpoint_hooks(handle) + save_training_checkpoint( + _checkpoint_model(handle, use_dcp=use_dcp), + handle._optimizer, + int(step), + path, + _checkpoint_parallel_config(handle), + handle._parallel_state, + get_placements=kwargs.pop("get_placements", get_placements), + is_expert=kwargs.pop("is_expert", is_expert), + use_dcp=use_dcp, + save_rng=save_rng, + save_model=kwargs.pop("save_model", True), + save_optimizer=kwargs.pop("save_optimizer", True), + **kwargs, + ) + + def load_checkpoint(self, handle: ModelHandle, path: str, **kwargs) -> int: + from megatron.lite.primitive.ckpt import load_training_checkpoint + + use_dcp = bool(kwargs.pop("use_dcp", True)) + load_rng = bool(kwargs.pop("load_rng", True)) + update_legacy_format = bool( + kwargs.pop( + "load_parameter_state_update_legacy_format", + kwargs.pop("update_legacy_format", False), + ) + ) + get_placements, is_expert = _checkpoint_hooks(handle) + return load_training_checkpoint( + _checkpoint_model(handle, use_dcp=use_dcp), + handle._optimizer, + path, + _checkpoint_parallel_config(handle), + handle._parallel_state, + get_placements=kwargs.pop("get_placements", get_placements), + is_expert=kwargs.pop("is_expert", is_expert), + use_dcp=use_dcp, + load_rng=load_rng, + load_parameter_state_update_legacy_format=update_legacy_format, + load_model=kwargs.pop("load_model", True), + load_optimizer=kwargs.pop("load_optimizer", True), + **kwargs, + ) + + def export_weights(self, handle: ModelHandle, **kwargs) -> Iterator[tuple[str, torch.Tensor]]: + model_chunks = handle._extras.get("model_chunks", [handle._model]) + proto = handle._extras.get("protocol") + model_cfg = handle._extras.get("model_cfg") + ps = handle._parallel_state + + if proto and hasattr(proto, "export_hf_weights"): + yield from proto.export_hf_weights(model_chunks, model_cfg, ps, **kwargs) + else: + for chunk in model_chunks: + yield from chunk.named_parameters() + + # ── Memory ── + + def to( + self, + handle: ModelHandle, + device: str, + *, + model: bool = True, + optimizer: bool = True, + grad: bool = True, + ) -> None: + model_chunks = handle._extras.get("model_chunks", [handle._model]) + from megatron.lite.runtime.megatron_utils import ( + load_model_to_gpu, + load_optimizer, + offload_model_to_cpu, + offload_optimizer, + ) + + if device == "cpu": + if model: + offload_model_to_cpu(model_chunks) + if optimizer and handle._optimizer is not None: + offload_state = getattr(handle._optimizer, "offload_state_to_cpu", None) + if callable(offload_state): + offload_state() + else: + offload_optimizer(handle._optimizer) + elif device == "cuda": + if model: + load_model_to_gpu(model_chunks, load_grad=grad) + if optimizer and handle._optimizer is not None: + load_state = getattr(handle._optimizer, "load_state_to_device", None) + if callable(load_state): + load_state() + else: + load_optimizer(handle._optimizer) + + # ── Mode switching ── + + def train_mode(self, handle: ModelHandle): + return _TrainModeCtx(handle) + + def eval_mode(self, handle: ModelHandle): + return _EvalModeCtx(handle) + + # ── Training atoms ── + + def forward_backward( + self, + handle: ModelHandle, + data: Any, + loss_fn: Callable | None, + *, + num_microbatches: int = 1, + forward_only: bool = False, + ) -> ForwardResult: + from megatron.lite.primitive.train_step import run_microbatch_loop + + forward_step = handle._extras["forward_step"] + if num_microbatches < 1: + raise ValueError("num_microbatches must be >= 1") + + if hasattr(data, "__next__"): + data_iter = data + elif hasattr(data, "__iter__"): + data_iter = iter(data) + else: + data_iter = iter([data]) + + ps = handle._parallel_state + if ps.pp_size > 1: + from types import SimpleNamespace + + from megatron.lite.primitive.ckpt.hf_weights import unwrap_model + from megatron.lite.primitive.parallel.pipeline import forward_backward_pipelining + + first_item = next(data_iter) + first_batch, _loss_context = split_loss_context(first_item) + data_iter = chain([first_item], data_iter) + tensor_shape = _infer_pipeline_tensor_shape( + first_batch, handle._extras.get("model_cfg"), ps + ) + model_chunks = handle._extras.get("model_chunks", [handle._model]) + pipeline_chunks = [unwrap_model(chunk) for chunk in model_chunks] + pipeline_forward_step, pipeline_loss_fn = _pipeline_callbacks(forward_step, loss_fn) + outputs = forward_backward_pipelining( + pipeline_forward_step, + pipeline_chunks, + data_iter, + SimpleNamespace(num_microbatches=num_microbatches), + ps, + tensor_shape=tensor_shape, + pre_forward_hook=handle._extras.get("pre_forward_hook"), + loss_fn=pipeline_loss_fn, + forward_only=forward_only, + ) + out = _last_loss_output(outputs) + loss_obj = out.get("loss") if out else None + if isinstance(loss_obj, torch.Tensor): + loss_float = float(loss_obj.detach().item()) + elif loss_obj is not None: + loss_float = float(loss_obj) + loss_payload = torch.tensor( + [loss_obj is not None, loss_float if loss_obj is not None else 0.0], + device="cuda", + dtype=torch.float32, + ) + if ps.pp_group is not None and ps.pp_global_ranks is not None: + dist.broadcast(loss_payload, src=ps.pp_global_ranks[-1], group=ps.pp_group) + out = {"loss": loss_payload[1]} if bool(loss_payload[0].item()) else {} + else: + out = run_microbatch_loop( + handle._model, + data_iter, + num_microbatches, + forward_step, + optimizer=handle._optimizer if not forward_only else None, + dist_opt=not forward_only, + pre_forward_hook=handle._extras.get("pre_forward_hook"), + loss_fn=loss_fn, + forward_only=forward_only, + ) + + if not forward_only: + finalize_grads = handle._extras.get("finalize_grads") + if finalize_grads is not None: + finalize_grads() + + loss_tensor = out.get("loss") if out else None + metric_rows = out.get("_loss_fn_metrics", []) if out else [] + if ps.pp_size > 1: + metric_rows += [item["metrics"] for item in outputs if item.get("metrics")] + metrics: dict = {} + for row in metric_rows: + for key, value in row.items(): + metrics.setdefault(key, []).append(value) + + return ForwardResult( + model_output=ModelOutputs( + loss=loss_tensor, + vocab_parallel_logits=out.get("logits") if out else None, + log_probs=out.get("log_probs") if out else None, + routed_experts=out.get("routed_experts") if out else None, + ), + metrics=metrics, + ) + + def is_mp_src_rank_with_outputs(self, handle: ModelHandle) -> bool: + ps = handle._parallel_state + return ps.tp_rank == 0 and ps.cp_rank == 0 and ps.pp_rank == ps.pp_size - 1 + + def zero_grad(self, handle: ModelHandle) -> None: + for chunk in handle._extras.get("model_chunks", [handle._model]): + if hasattr(chunk, "zero_grad_buffer"): + chunk.zero_grad_buffer() + if handle._optimizer is not None: + handle._optimizer.zero_grad() + + def optimizer_step(self, handle: ModelHandle) -> tuple[bool, float, int | None]: + if handle._optimizer is None: + return True, 0.0, 0 + update_successful, grad_norm, num_zeros = handle._optimizer.step() + return update_successful, float(grad_norm), num_zeros + + def lr_scheduler_step(self, handle: ModelHandle) -> float | list[float]: + if handle._lr_scheduler is not None: + handle._lr_scheduler.step() + return handle._lr_scheduler.get_last_lr() + return 0.0 + + +# --------------------------------------------------------------------------- +# Context managers +# --------------------------------------------------------------------------- + + +class _TrainModeCtx: + def __init__(self, handle: ModelHandle): + self._handle = handle + + def __enter__(self): + for chunk in self._handle._extras.get("model_chunks", [self._handle._model]): + chunk.train() + return self + + def __exit__(self, *exc): + return False + + +class _EvalModeCtx: + def __init__(self, handle: ModelHandle): + self._handle = handle + self._prev_grad = torch.is_grad_enabled() + + def __enter__(self): + for chunk in self._handle._extras.get("model_chunks", [self._handle._model]): + chunk.eval() + torch.set_grad_enabled(False) + return self + + def __exit__(self, *exc): + torch.set_grad_enabled(self._prev_grad) + return False + + +def _checkpoint_parallel_config(handle: ModelHandle): + cfg = handle.config + if cfg is None: + return None + return getattr(cfg, "parallel", cfg) + + +def _checkpoint_model(handle: ModelHandle, *, use_dcp: bool): + model = handle._model + if not use_dcp or isinstance(model, torch.nn.Module): + return model + return torch.nn.ModuleList(handle._extras.get("model_chunks", model)) + + +def _checkpoint_hooks(handle: ModelHandle): + from megatron.lite.primitive.protocols import default_expert_classifier, default_placement_fn + + proto = handle._extras.get("protocol") + return ( + getattr(proto, "PLACEMENT_FN", default_placement_fn), + getattr(proto, "EXPERT_CLASSIFIER", default_expert_classifier), + ) diff --git a/experimental/lite/megatron/lite/runtime/contracts/__init__.py b/experimental/lite/megatron/lite/runtime/contracts/__init__.py new file mode 100644 index 00000000000..7087a5d70a9 --- /dev/null +++ b/experimental/lite/megatron/lite/runtime/contracts/__init__.py @@ -0,0 +1,68 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Lazy re-export hub for ``megatron.lite.runtime.contracts``. + +This preserves imports like ``from megatron.lite.runtime.contracts import X`` while +avoiding eager imports of heavyweight modules (for example ``torch`` from +``contracts.data``) during package import. +""" + +from __future__ import annotations + +import importlib +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from megatron.lite.runtime.backends.bridge.config import BridgeConfig + from megatron.lite.runtime.backends.mlite.config import DebugConfig, MegatronLiteConfig + from megatron.lite.runtime.contracts.config import ( + OptimizerConfig, + ParallelConfig, + RuntimeConfig, + ) + from megatron.lite.runtime.contracts.data import ( + Batch, + ForwardResult, + ModelOutputs, + PackedBatch, + TrainBatch, + ) + from megatron.lite.runtime.contracts.handle import ModelHandle + from megatron.lite.runtime.contracts.loss import LossContext + +__all__ = [ + "MegatronLiteConfig", + "Batch", + "BridgeConfig", + "DebugConfig", + "ForwardResult", + "LossContext", + "ModelHandle", + "ModelOutputs", + "OptimizerConfig", + "PackedBatch", + "ParallelConfig", + "RuntimeConfig", + "TrainBatch", +] + + +def __getattr__(name: str): + _lazy = { + "Batch": "megatron.lite.runtime.contracts.data", + "BridgeConfig": "megatron.lite.runtime.backends.bridge.config", + "DebugConfig": "megatron.lite.runtime.backends.mlite.config", + "MegatronLiteConfig": "megatron.lite.runtime.backends.mlite.config", + "ForwardResult": "megatron.lite.runtime.contracts.data", + "LossContext": "megatron.lite.runtime.contracts.loss", + "ModelHandle": "megatron.lite.runtime.contracts.handle", + "ModelOutputs": "megatron.lite.runtime.contracts.data", + "OptimizerConfig": "megatron.lite.runtime.contracts.config", + "PackedBatch": "megatron.lite.runtime.contracts.data", + "ParallelConfig": "megatron.lite.runtime.contracts.config", + "RuntimeConfig": "megatron.lite.runtime.contracts.config", + "TrainBatch": "megatron.lite.runtime.contracts.data", + } + if name in _lazy: + mod = importlib.import_module(_lazy[name]) + return getattr(mod, name) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/experimental/lite/megatron/lite/runtime/contracts/config.py b/experimental/lite/megatron/lite/runtime/contracts/config.py new file mode 100644 index 00000000000..74d3af85f97 --- /dev/null +++ b/experimental/lite/megatron/lite/runtime/contracts/config.py @@ -0,0 +1,88 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Shared runtime configuration for Megatron Lite.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from dataclasses import fields as dc_fields +from typing import TYPE_CHECKING, Any + + +def pick_fields(cls, src: dict[str, Any]) -> dict[str, Any]: + """Extract fields of dataclass *cls* that exist in *src*.""" + return {f.name: src[f.name] for f in dc_fields(cls) if f.name in src} + + +if TYPE_CHECKING: + from megatron.lite.runtime.backends.bridge.config import BridgeConfig + from megatron.lite.runtime.backends.mlite.config import MegatronLiteConfig + + +@dataclass +class ParallelConfig: + """Parallel dimensions used by Megatron Lite.""" + + tp: int = 1 + etp: int | None = None + ep: int = 1 + pp: int = 1 + vpp: int = 1 + cp: int = 1 + # Optional explicit mcore pipeline layout for advanced users (custom mode), + # e.g. "E|t*5|t*6|t,m,L" (MTP `m` must sit on the final/loss stage; standalone MTP + # is not supported yet). None -> auto-balanced from (num_layers, pp, mtp). + pp_layout: str | list | None = None + + +@dataclass +class OptimizerConfig: + """Optimizer + LR scheduler config. Aligned with VERL McoreOptimizerConfig. + + Stable VERL fields use VERL default values. + Compatibility aliases are lowered into backend-specific override dicts + by the adapter layer before consumption. + """ + + # --- stable VERL fields --- + optimizer: str = "adam" + lr: float = 1e-3 + min_lr: float = 0.0 + clip_grad: float = 1.0 + weight_decay: float = 0.01 + lr_warmup_steps_ratio: float = 0.0 + total_training_steps: int = -1 + lr_warmup_steps: int = -1 + lr_warmup_init: float = 0.0 + lr_decay_steps: int | None = None + lr_decay_style: str = "linear" + weight_decay_incr_style: str = "constant" + lr_wsd_decay_style: str = "exponential" + lr_wsd_decay_steps: int | None = None + use_checkpoint_opt_param_scheduler: bool = False + + # --- compatibility aliases --- + adam_beta1: float | None = None + adam_beta2: float | None = None + adam_eps: float | None = None + offload_fraction: float | None = None + use_precision_aware_optimizer: bool | None = None + decoupled_weight_decay: bool | None = None + + +@dataclass +class RuntimeConfig: + """Top-level runtime configuration. + + Attributes: + backend: Runtime backend name. Use ``"mlite"`` for Megatron Lite or + ``"bridge"`` for Megatron-Bridge. + hf_path: Path to HuggingFace model directory. Required for real runs. + backend_cfg: Backend config or a compatible dict. + """ + + backend: str = "mlite" + hf_path: str = "" + backend_cfg: MegatronLiteConfig | BridgeConfig | dict[str, Any] = field(default_factory=dict) + + +__all__ = ["OptimizerConfig", "ParallelConfig", "RuntimeConfig"] diff --git a/experimental/lite/megatron/lite/runtime/contracts/data.py b/experimental/lite/megatron/lite/runtime/contracts/data.py new file mode 100644 index 00000000000..d431a728bfd --- /dev/null +++ b/experimental/lite/megatron/lite/runtime/contracts/data.py @@ -0,0 +1,120 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Data contracts — forward_backward input/output types.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +import torch + + +class Batch: + """Protocol for data passed to Runtime.forward_backward. + + Sequences are packed without padding. ``sizes()`` gives per-sequence + lengths so the runtime can reconstruct ``cu_seqlens`` / ``position_ids`` + for THD attention. + + Subclass this to carry domain-specific fields (loss_mask, routed_experts, + etc.) while keeping the runtime interface uniform. + """ + + def __len__(self) -> int: + """Number of sequences in this batch.""" + raise NotImplementedError + + def sizes(self) -> torch.Tensor: + """Per-sequence token counts. Shape ``[num_seqs]``.""" + raise NotImplementedError + + +@dataclass(slots=True) +class PackedBatch(Batch): + """Variable-length packed batch — no padding. + + All token-level tensors are 1-D with length ``sum(seq_lens)``. + ``cu_seqlens`` and ``position_ids`` are derived automatically. + """ + + input_ids: torch.Tensor # [total_tokens] + labels: torch.Tensor # [total_tokens] + seq_lens: torch.Tensor # [num_seqs] + loss_mask: torch.Tensor | None = None # [total_tokens] + position_ids: torch.Tensor | None = None # [total_tokens], auto if None + routed_experts: torch.Tensor | None = None + extras: dict[str, Any] = field(default_factory=dict) + + def __len__(self) -> int: + return len(self.seq_lens) + + def sizes(self) -> torch.Tensor: + return self.seq_lens + + @property + def cu_seqlens(self) -> torch.Tensor: + """Cumulative sequence lengths for THD attention. Shape ``[num_seqs+1]``.""" + return torch.cat( + [ + torch.zeros(1, dtype=torch.int32, device=self.seq_lens.device), + self.seq_lens.cumsum(0).to(torch.int32), + ] + ) + + @property + def total_tokens(self) -> int: + return int(self.seq_lens.sum()) + + def make_position_ids(self) -> torch.Tensor: + """Generate per-token position_ids from seq_lens.""" + if self.position_ids is not None: + return self.position_ids + return torch.cat( + [torch.arange(s, device=self.seq_lens.device) for s in self.seq_lens.tolist()] + ) + + +@dataclass(slots=True) +class TrainBatch: + """Legacy fixed-shape batch (padded). Use PackedBatch for new code.""" + + input_ids: torch.Tensor + labels: torch.Tensor + loss_mask: torch.Tensor | None = None + position_ids: torch.Tensor | None = None + routed_experts: torch.Tensor | None = None + cp_size: int | None = None + extras: dict[str, Any] = field(default_factory=dict) + + +@dataclass(slots=True) +class ModelOutputs: + """Model forward output.""" + + loss: torch.Tensor | None = None + vocab_parallel_logits: torch.Tensor | None = None + log_probs: torch.Tensor | None = None + hidden_states: torch.Tensor | None = None + values: torch.Tensor | None = None + # MTP + mtp_logits: torch.Tensor | None = None + mtp_loss: torch.Tensor | None = None + # Router Replay: recorded routing decisions + routed_experts: torch.Tensor | None = None + + +@dataclass(slots=True) +class ForwardResult: + """Output of forward_backward.""" + + model_output: ModelOutputs = field(default_factory=ModelOutputs) + metrics: dict[str, Any] = field(default_factory=dict) + + +__all__ = [ + "Batch", + "ForwardResult", + "ModelOutputs", + "PackedBatch", + "TrainBatch", +] diff --git a/experimental/lite/megatron/lite/runtime/contracts/handle.py b/experimental/lite/megatron/lite/runtime/contracts/handle.py new file mode 100644 index 00000000000..85c4f120037 --- /dev/null +++ b/experimental/lite/megatron/lite/runtime/contracts/handle.py @@ -0,0 +1,65 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""ModelHandle — opaque handle returned by Runtime.build_model().""" + +from __future__ import annotations + +from typing import Any + + +class ModelHandle: + """Opaque handle returned by Runtime.build_model(). + + Only documented properties on this class are part of the public contract. + Internal helpers may still use ``_model`` / ``_optimizer`` / ``_extras`` + while the higher-level runtime API is being stabilized. + """ + + def __init__( + self, + *, + model: Any, + optimizer: Any = None, + lr_scheduler: Any = None, + parallel_state: Any = None, + config: Any = None, + _extras: dict[str, Any] | None = None, + ): + self._model = model + self._optimizer = optimizer + self._lr_scheduler = lr_scheduler + self._parallel_state = parallel_state + self._config = config + self._extras = _extras or {} + + @property + def dp_rank(self) -> int: + ps = self._parallel_state + if ps is None: + return 0 + return getattr(ps, "dp_rank", 0) + + @property + def dp_size(self) -> int: + ps = self._parallel_state + if ps is None: + return 1 + return getattr(ps, "dp_size", 1) + + @property + def dp_group(self): + ps = self._parallel_state + if ps is None: + return None + return getattr(ps, "dp_group", None) + + @property + def cp_range(self) -> tuple[int, int]: + return self._extras.get("cp_range", (1, 1)) + + @property + def config(self) -> Any: + """Backend config captured when this handle was built.""" + return self._config + + +__all__ = ["ModelHandle"] diff --git a/experimental/lite/megatron/lite/runtime/contracts/loss.py b/experimental/lite/megatron/lite/runtime/contracts/loss.py new file mode 100644 index 00000000000..47bcdb6114d --- /dev/null +++ b/experimental/lite/megatron/lite/runtime/contracts/loss.py @@ -0,0 +1,52 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Loss-side runtime context, kept separate from batch data contracts.""" + +from __future__ import annotations + +from collections.abc import Iterator +from contextlib import contextmanager +from contextvars import ContextVar +from dataclasses import dataclass +from typing import Any + + +@dataclass(frozen=True, slots=True) +class LossContext: + """Per-microbatch loss/output policy for model-owned loss helpers.""" + + temperature: float = 1.0 + calculate_entropy: bool = False + return_log_probs: bool = True + loss_scale: float = 1.0 + source_batch: Any | None = None + + +_CURRENT_LOSS_CONTEXT: ContextVar[LossContext | None] = ContextVar( + "megatron_lite_loss_context", default=None +) + + +def get_loss_context() -> LossContext | None: + return _CURRENT_LOSS_CONTEXT.get() + + +@contextmanager +def use_loss_context(loss_context: LossContext | None) -> Iterator[None]: + token = _CURRENT_LOSS_CONTEXT.set(loss_context) + try: + yield + finally: + _CURRENT_LOSS_CONTEXT.reset(token) + + +def split_loss_context(item): + if ( + isinstance(item, tuple) + and len(item) == 2 + and (item[1] is None or isinstance(item[1], LossContext)) + ): + return item + return item, None + + +__all__ = ["LossContext", "get_loss_context", "split_loss_context", "use_loss_context"] diff --git a/experimental/lite/megatron/lite/runtime/megatron_utils.py b/experimental/lite/megatron/lite/runtime/megatron_utils.py new file mode 100644 index 00000000000..54b5ec61843 --- /dev/null +++ b/experimental/lite/megatron/lite/runtime/megatron_utils.py @@ -0,0 +1,232 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Megatron-Core utilities aligned with VERL's megatron_utils. + +All functions here are ported from VERL and could theoretically be replaced +by ``from verl.utils.megatron_utils import ...`` if Megatron Lite ever depends on verl. + +Sources: + - verl/utils/megatron_utils.py (offload/load, register_megatron_training_hooks) + - verl/utils/megatron/optimizer.py (get_megatron_last_lr) +""" + +from __future__ import annotations + +import gc +from typing import Any + +import torch + +# ====================================================================== +# Rank utilities +# ====================================================================== + + +def is_mp_src_rank_with_outputs() -> bool: + """True on the rank that holds the final model output (loss). + + Only last PP stage, first TP rank, first CP rank has the output. + VERL: MegatronEngine.is_mp_src_rank_with_outputs + """ + from megatron.core import parallel_state as mpu + + return ( + mpu.get_tensor_model_parallel_rank() == 0 + and mpu.get_pipeline_model_parallel_rank() + == mpu.get_pipeline_model_parallel_world_size() - 1 + and mpu.get_context_parallel_rank() == 0 + ) + + +# ====================================================================== +# Training hooks — register_megatron_training_hooks +# ====================================================================== + + +def register_training_hooks(model_list: list, optimizer) -> None: + """Register megatron training callbacks on model config. + + Ref: megatron/training/training.py (core_v0.15.0rc7, L2039-L2057) + """ + from megatron.core.distributed import DistributedDataParallel as DDP + from megatron.core.distributed import finalize_model_grads + from megatron.core.utils import get_model_config + + for one_model in model_list: + config = get_model_config(one_model) + if optimizer is not None: + config.grad_scale_func = optimizer.scale_loss + config.finalize_model_grads_func = finalize_model_grads + + optimizer_config = getattr(optimizer, "config", None) + overlap_param_gather = getattr(optimizer_config, "overlap_param_gather", False) + overlap_grad_reduce = getattr(one_model.ddp_config, "overlap_grad_reduce", False) + align_grad_reduce = True + align_param_gather = getattr(one_model.ddp_config, "align_param_gather", False) + + if isinstance(model_list[0], DDP) and overlap_grad_reduce: + config.no_sync_func = [m.no_sync for m in model_list] + if len(model_list) == 1: + config.no_sync_func = config.no_sync_func[0] + if align_grad_reduce: + config.grad_sync_func = [m.start_grad_sync for m in model_list] + if len(model_list) == 1: + config.grad_sync_func = config.grad_sync_func[0] + if overlap_param_gather and align_param_gather: + config.param_sync_func = [m.start_param_sync for m in model_list] + if len(model_list) == 1: + config.param_sync_func = config.param_sync_func[0] + + +# ====================================================================== +# Model offload / load — offload_megatron_model_to_cpu / load_megatron_model_to_gpu +# ====================================================================== + + +def _is_megatron_ddp(model_chunk: Any) -> bool: + try: + from megatron.core.distributed import DistributedDataParallel as DDP + except Exception: + return False + + return ( + isinstance(model_chunk, DDP) + and hasattr(model_chunk, "buffers") + and hasattr(model_chunk, "expert_parallel_buffers") + and hasattr(model_chunk, "module") + ) + + +def offload_model_to_cpu(model_list: list) -> None: + """Offload DDP model to CPU via buffer-resize (zero-copy on GPU side).""" + for model_chunk in model_list: + if _is_megatron_ddp(model_chunk): + all_buffers = [model_chunk.buffers, model_chunk.expert_parallel_buffers] + for buffers in all_buffers: + for buffer in buffers: + if buffer.param_data.storage().size() > 0: + buffer.param_data.cpu_data = buffer.param_data.data.cpu().pin_memory() + buffer.param_data_size = buffer.param_data.storage().size() + buffer.param_data.storage().resize_(0) + + if buffer.grad_data.storage().size() > 0: + buffer.grad_data_size = buffer.grad_data.storage().size() + buffer.grad_data.storage().resize_(0) + + for param in model_chunk.module.parameters(): + if not param.requires_grad and param.device.type != "cpu": + param.data = param.data.to("cpu", non_blocking=True) + else: + model_chunk.to("cpu") + + +def load_model_to_gpu(model_list: list, load_grad: bool = True) -> None: + """Load DDP model back to GPU from pinned CPU copy.""" + for model_chunk in model_list: + if _is_megatron_ddp(model_chunk): + all_buffers = [model_chunk.buffers, model_chunk.expert_parallel_buffers] + for buffers in all_buffers: + for buffer in buffers: + if load_grad and hasattr(buffer, "grad_data_size"): + current_size = buffer.grad_data.storage().size() + if current_size == 0 or current_size == buffer.grad_data_size: + buffer.grad_data.storage().resize_(buffer.grad_data_size) + buffer.grad_data.zero_() + else: + buffer.grad_data.zero_() + + if buffer.param_data.storage().size() == 0: + buffer.param_data.storage().resize_(buffer.param_data_size) + buffer.param_data.copy_(buffer.param_data.cpu_data, non_blocking=True) + + for param in model_chunk.module.parameters(): + if not param.requires_grad and param.device.type == "cpu": + param.data = param.data.to("cuda", non_blocking=True) + else: + model_chunk.to("cuda") + + +# ====================================================================== +# Optimizer offload / load — offload_megatron_optimizer / load_megatron_optimizer +# ====================================================================== + + +def offload_optimizer(optimizer) -> None: + """Offload optimizer states to CPU.""" + from megatron.core.optimizer import ChainedOptimizer + + for _opt in _iter_opts(optimizer, ChainedOptimizer): + if _opt.optimizer is not None: + hdo = _opt.optimizer + if all( + hasattr(hdo, a) for a in ("sub_optimizers", "inner_param_to_orig_param", "state") + ): + for sub_opt in hdo.sub_optimizers: + for param, state in sub_opt.state.items(): + for k, v in state.items(): + if not isinstance(v, torch.Tensor): + continue + orig_param = hdo.inner_param_to_orig_param.get(param, param) + hdo.state[orig_param][k] = state[k] = v.to("cpu") + else: + for v in _opt.optimizer.state.values(): + if "exp_avg" in v: + v["exp_avg"] = v["exp_avg"].to("cpu", non_blocking=True) + if "exp_avg_sq" in v: + v["exp_avg_sq"] = v["exp_avg_sq"].to("cpu", non_blocking=True) + + gc.collect() + torch.cuda.empty_cache() + + +def load_optimizer(optimizer) -> None: + """Load optimizer states back to GPU.""" + from megatron.core.optimizer import ChainedOptimizer + + for _opt in _iter_opts(optimizer, ChainedOptimizer): + if _opt.optimizer is not None: + if hasattr(_opt.optimizer, "_move_new_state_to_right_device"): + _opt.optimizer._move_new_state_to_right_device() + else: + for v in _opt.optimizer.state.values(): + if "exp_avg" in v: + v["exp_avg"] = v["exp_avg"].to("cuda", non_blocking=True) + if "exp_avg_sq" in v: + v["exp_avg_sq"] = v["exp_avg_sq"].to("cuda", non_blocking=True) + + gc.collect() + torch.cuda.empty_cache() + + +def _iter_opts(optimizer, chained_cls): + if isinstance(optimizer, chained_cls): + return optimizer.chained_optimizers + return [optimizer] + + +# ====================================================================== +# Checkpoint helpers +# ====================================================================== + + +def build_sharded_state_dict( + model_list: list, optimizer: Any = None, lr_scheduler: Any = None +) -> dict[str, Any]: + """Build sharded state dict for model + optimizer + lr_scheduler. + + Uses ``model0.`` / ``model1.`` prefix for VPP (multiple model chunks). + """ + sharded_state_dict: dict[str, Any] = {} + + for i, model_chunk in enumerate(model_list): + prefix = f"model{i}." if len(model_list) > 1 else "model." + chunk_sd = model_chunk.sharded_state_dict(prefix=prefix) + sharded_state_dict.update(chunk_sd) + + if optimizer is not None: + opt_sd = optimizer.sharded_state_dict(model_sharded_state_dict=sharded_state_dict) + sharded_state_dict.update(opt_sd) + + if lr_scheduler is not None: + sharded_state_dict["lr_scheduler"] = lr_scheduler.state_dict() + + return sharded_state_dict diff --git a/experimental/lite/skills/README.md b/experimental/lite/skills/README.md new file mode 100644 index 00000000000..51a4468714d --- /dev/null +++ b/experimental/lite/skills/README.md @@ -0,0 +1,167 @@ +# Megatron Lite Skills + +This directory defines agent-agnostic skills for maintaining Megatron Lite. +A skill is an operational contract for agents: what context to load, what +invariants to protect, how to make a change, and what evidence to leave behind. + +These files are not Codex, Claude, or tool-specific skills. Do not add +agent-specific frontmatter, prompt wrappers, or runtime assumptions here. + +## Function Model + +Treat a skill as a function: + +- `Schema` is the signature and summary. +- `imports` are skills that must be loaded before execution, like Python imports. +- `calls` are every skill this function explicitly calls in the body. +- the body is Python-like pseudocode with bounded exits. + +An agent should be able to route work from the `Schema` alone. It should read +the body only when executing that skill. + +## File Layout + +Each skill file has three regions: + +1. Before `MLITE_SKILL_SCHEMA_BEGIN`: a short human-facing title or note. Agents + should not depend on this region for routing. +2. Between `MLITE_SKILL_SCHEMA_BEGIN` and `MLITE_SKILL_SCHEMA_END`: the compact + schema used for retrieval, routing, imports, and call planning. +3. After `MLITE_SKILL_SCHEMA_END`: the skill body. This must be Python-like + pseudocode, not free-form prose. + +The pseudocode is a contract, not executable Python. It should still be precise: +clear inputs, explicit outputs, explicit skill calls, and finite exits. + +Skills are organized like Python modules. A skill is a single `.md` file; +directories are namespaces, not skill bodies. Do not create `skill-name/README.md` +for individual skills. +Map schema names to paths by replacing `.` with `/` and `_` with `-`, then +adding `.md`: `model_compose.config_mapping` lives at +`model-compose/config-mapping.md`. + +## Loading Model + +Agents should load skills progressively: + +1. Read this file. +2. Read exactly one leaf skill for the current work type. +3. Read linked Megatron Lite docs or source files only when the leaf skill asks + for them. + +Keep the active context small. A skill should point to durable source files and +validation commands instead of copying long design background. + +## Defined Skills + +`basic`: +- `basic.constitution` +- `basic.find_reference` +- `basic.align_precision` +- `basic.align_e2e_precision` +- `basic.lint_skill` +- `basic.construct_proxy_task` +- `basic.review_threshold` + +`primitive`: +- `primitive.contract` +- `primitive.principle` +- `primitive.select_for_compose` +- `primitive.design` +- `primitive.validate` +- `primitive.fuse` +- `primitive.process_group` +- `primitive.parallel.tp` +- `primitive.parallel.ep` +- `primitive.parallel.pp` +- `primitive.parallel.cp` +- `primitive.optimizer.fsdp` +- `primitive.optimizer.distopt` +- `primitive.module.moe` +- `primitive.module.gqa` +- `primitive.module.thd` + +Primitive work has two reusable meta layers: `primitive.principle` defines what +must be true, and `primitive.select_for_compose` decides when a primitive belongs +in a model composition. + +`model_compose`: +- `model_compose.config_mapping` +- `model_compose.weight_mapping` +- `model_compose.build_model` +- `model_compose.qwen` + +`application`: +- `application.runtime` +- `application.verl` +- `application.bench` + +`perf`: +- `perf.measure` +- `perf.memory` +- `perf.fusion` +- `perf.optimize` + +`insight`: +- `insight.progressive_model_support` +- `insight.progressive_primitive_design` +- `insight.scope_control` + +## Skill Contract + +Each skill file contains a delimited schema block near the top. Agents may read +only this block for routing, then read the body only when executing the skill. + +````text + +```python +schema = Skill( + "basic.align_precision", kind="state_machine", purpose="align precision", + imports=["basic.constitution"], calls=["basic.constitution", "basic.find_reference"], + inputs=["task", "files", "reference", "budget"], + outputs=["patch", "evidence", "risks"], exits=["done", "blocked", "out_of_scope"], +) +``` + +```` + +Use the markers instead of fixed line counts. Keep the schema short enough to +scan without reading the body; prefer compact lists and short field names over +prose. + +After the schema, each skill body is Python-like pseudocode. The top-level +function should match the schema inputs and return one of the declared exits. +Use prose only inside short comments. + +Required sections: + +- one top-level function matching the schema name; +- explicit `return done(...)`, `return blocked(...)`, or + `return out_of_scope(...)`; +- explicit calls to skills as `namespace.skill(...)`. + +`imports` and `calls` are separate. `imports` says what an agent should preload. +`calls` says what the body invokes. If a loaded skill is directly invoked, list +it in `calls` too. + +Every loop must have a progress measure, a maximum attempt count, or a blocking +condition. A skill that can spin forever is invalid. + +## Skill Lint + +Every new or changed skill should pass `basic.lint_skill` before review. Lint is +structural: it checks that schema can be extracted, imports and calls resolve, +the file path matches the schema name, body calls match declared calls, exits +are explicit, and loops are bounded. Scenario dry-runs are optional but +preferred for complex skills; they use mocked inputs to verify the pseudocode +can reach a declared exit. + +## Boundaries + +- Skills describe how agents work; `docs/` describes Megatron Lite for users and + reviewers. +- Skills must not replace tests. They should name the right validation surface. +- Skills must not store task history, temporary job logs, or one-off debugging + transcripts. +- Skills should stay stable across agents. If a rule depends on one agent tool, + keep it outside this directory. diff --git a/experimental/lite/skills/application/bench.md b/experimental/lite/skills/application/bench.md new file mode 100644 index 00000000000..d8e5ba5c314 --- /dev/null +++ b/experimental/lite/skills/application/bench.md @@ -0,0 +1,31 @@ +# Bench Skill + +Run benchmark-style checks without weakening precision evidence. + +## Schema + + +```python +schema = Skill( + "application.bench", kind="state_machine", purpose="benchmark with controlled variables and evidence", + imports=["basic.constitution"], calls=["perf.measure"], + inputs=["task", "bench_config", "target", "budget"], + outputs=["bench", "evidence", "risks"], exits=["done", "blocked", "out_of_scope"], +) +``` + + +```python +def bench(task, bench_config, target, budget): + controlled = bench_config.variables.freeze_all_except(bench_config.axis) + if controlled.has_unknown_unfrozen_axes(): + return blocked("bench has uncontrolled variables", risks=controlled.unknown_axes) + + measurement = perf.measure(task, target=target, workload=bench_config.workload, budget=budget.measure) + if not measurement.done: + return blocked("bench measurement failed", evidence=measurement) + + evidence = record_evidence(task, run=measurement.run, comparison=measurement.metrics, environment=budget.env) + risks = ["benchmarks are not correctness proof", "uncontrolled variables can dominate"] + return done(bench=measurement.metrics, evidence=evidence, risks=risks) +``` diff --git a/experimental/lite/skills/application/miles.md b/experimental/lite/skills/application/miles.md new file mode 100644 index 00000000000..9193bed16d4 --- /dev/null +++ b/experimental/lite/skills/application/miles.md @@ -0,0 +1,15 @@ +# Miles Application Skill + +Use this when wiring Megatron Lite into radixark/miles examples. + +- Keep the external miles CLI backend as `--train-backend megatron`; the MLite + integration is activated by importing `miles_mlite.backend_patch` before miles + constructs `RayTrainGroup`. +- Add `experimental/lite`, `experimental/lite/examples`, this repository root, + and the miles checkout to `PYTHONPATH` for both the driver and Ray runtime + environment. +- Use `--model-name qwen3_moe` and `--megatron-to-hf-mode raw` for Qwen3 MoE + rollout resync. MLite `export_weights` already emits HF-format tensors. +- Use user-facing optimizer naming `dist_opt` in example CLI text. +- GPU validation must go through Slurm/container. Dry-run output is not evidence + of a passed SFT or GRPO path. diff --git a/experimental/lite/skills/application/runtime.md b/experimental/lite/skills/application/runtime.md new file mode 100644 index 00000000000..f3e956701df --- /dev/null +++ b/experimental/lite/skills/application/runtime.md @@ -0,0 +1,30 @@ +# Runtime Skill + +Validate the MLite runtime path end to end. + +## Schema + + +```python +schema = Skill( + "application.runtime", kind="state_machine", purpose="validate MLite runtime build/train/save/load", + imports=["basic.constitution"], calls=["basic.align_precision"], + inputs=["task", "runtime_config", "model", "budget"], + outputs=["runtime", "evidence", "risks"], exits=["done", "blocked", "out_of_scope"], +) +``` + + +```python +def runtime(task, runtime_config, model, budget): + if runtime_config.backend != "mlite": + return out_of_scope("not MLite runtime") + + run = execute_runtime_steps(["init", "build_model", "train_step", "save", "load"], runtime_config, model) + if not run.return_code == 0: + return blocked("runtime path failed", evidence=run) + + precision = basic.align_precision(task, target=model, variables=runtime_config.variables, budget=budget.precision) + evidence = record_evidence(task, run=run, comparison=precision, environment=budget.env) + return done(runtime=run, evidence=evidence, risks=["runtime success can hide model-local mismatch"]) +``` diff --git a/experimental/lite/skills/application/verl.md b/experimental/lite/skills/application/verl.md new file mode 100644 index 00000000000..e7257c2f139 --- /dev/null +++ b/experimental/lite/skills/application/verl.md @@ -0,0 +1,35 @@ +# VERL Skill + +Validate MLite through VERL SFT, RL, or GRPO workflows. + +## Schema + + +```python +schema = Skill( + "application.verl", kind="state_machine", purpose="validate VERL workflows with MLite", + imports=["basic.constitution"], calls=["basic.align_e2e_precision"], + inputs=["task", "verl_config", "model", "budget"], + outputs=["workflow", "evidence", "risks"], exits=["done", "blocked", "out_of_scope"], +) +``` + + +```python +def verl(task, verl_config, model, budget): + if verl_config.algorithm not in ["SFT", "RL", "GRPO"]: + return out_of_scope("not a VERL SFT/RL/GRPO workflow") + + modes = {"SFT": ["SFT"], "RL": ["RL"], "GRPO": ["RL"]}[verl_config.algorithm] + e2e = basic.align_e2e_precision(task, target=model, modes=modes, budget=budget.e2e) + if not e2e.done: + return blocked("VERL e2e precision failed", evidence=e2e) + + evidence = record_evidence(task, run=e2e.evidence.run, comparison=e2e.evidence, environment=budget.env) + risks = [ + "SFT loss curve can pass while a lower-level bug remains shared", + "RL reward variance can hide precision drift", + "rollout backend can become the reference accidentally", + ] + return done(workflow=verl_config.algorithm, evidence=evidence, risks=risks) +``` diff --git a/experimental/lite/skills/basic/align-e2e-precision.md b/experimental/lite/skills/basic/align-e2e-precision.md new file mode 100644 index 00000000000..c05c7414c50 --- /dev/null +++ b/experimental/lite/skills/basic/align-e2e-precision.md @@ -0,0 +1,59 @@ +# Align E2E Precision Skill + +Use pretrain, SFT, or RL as the strongest but highest-cost precision evidence. + +## Schema + + +```python +schema = Skill( + "basic.align_e2e_precision", kind="state_machine", purpose="validate precision through pretrain/SFT/RL", + imports=["basic.constitution"], calls=["basic.constitution", "basic.find_reference", "basic.review_threshold"], + inputs=["task", "target", "modes", "budget"], + outputs=["alignment", "evidence", "override", "risks"], exits=["done", "blocked", "out_of_scope"], +) +``` + + +```python +def align_e2e_precision(task, target, modes, budget): + if task.scope not in ["precision", "model", "runtime", "application"]: + return out_of_scope("not an end-to-end precision task") + if not budget.high_cost_approved: + return blocked("e2e precision requires high-cost approval") + + mode = choose_first_available(modes, order=["pretrain", "SFT", "RL"]) + if mode is None: + return blocked("no e2e mode selected") + + ref = basic.find_reference(task, layer=target.layer, candidates=task.references, budget=budget.reference) + if not ref.done: + return blocked("no usable e2e reference", evidence=ref) + + policy = basic.constitution(task, layer=target.layer, reference=ref.reference) + if not policy.done: + return blocked("e2e precision policy failed", evidence=policy) + + run = construct_e2e_run( + mode, + target, + reference=ref.reference, + freeze=["dataset", "tokenizer", "checkpoint", "schedule", "seed", "variables"], + compare=["loss_curve", "grad_norm", "reward_or_metric", "checkpoint_delta"], + ) + if run is None: + return blocked("cannot construct comparable e2e run") + + threshold = basic.review_threshold(task, ref.contract, requested_threshold=task.tolerance) + if not threshold.done: + return blocked("e2e precision threshold review failed", evidence=threshold) + compare_mode = threshold.compare_mode + + evidence = run_reference_and_target(run, compare_mode=compare_mode) + risks = ["high cost", "may preserve a shared bug", "can mask local mismatches"] + + if evidence.pass_: + return done(alignment="e2e_aligned", evidence=evidence, override=True, risks=risks) + + return done(alignment="e2e_mismatch", evidence=evidence, override=False, risks=risks) +``` diff --git a/experimental/lite/skills/basic/align-precision.md b/experimental/lite/skills/basic/align-precision.md new file mode 100644 index 00000000000..afbd29dccb3 --- /dev/null +++ b/experimental/lite/skills/basic/align-precision.md @@ -0,0 +1,99 @@ +# Align Precision Skill + +Align Megatron Lite behavior against a reference, or localize the first precision +break with controlled variables. + +## Schema + + +```python +schema = Skill( + "basic.align_precision", kind="state_machine", purpose="recursively align precision to reference", + imports=["basic.constitution"], + calls=[ + "basic.constitution", "basic.find_reference", "basic.construct_proxy_task", + "basic.review_threshold", "basic.align_precision", "basic.align_e2e_precision", + ], + inputs=["task", "target", "variables", "budget"], + outputs=["alignment", "evidence", "next"], exits=["done", "blocked", "out_of_scope"], +) +``` + + +```python +def align_precision(task, target, variables, budget): + if task.scope not in ["precision", "primitive", "model"]: + return out_of_scope("not a precision-alignment task") + if budget.depth > budget.max_depth or budget.attempts > budget.max_attempts: + return blocked("precision recursion budget exhausted") + + ref = basic.find_reference(task, layer=target.layer, candidates=task.references, budget=budget.reference) + if not ref.done: + return blocked("no usable reference", evidence=ref) + + policy = basic.constitution(task, layer=target.layer, reference=ref.reference) + if not policy.done: + return blocked("precision policy failed", evidence=policy) + + proxy_result = basic.construct_proxy_task( + task, + target=target, + reference=ref.reference, + variables=variables, + budget=budget.proxy, + ) + if not proxy_result.done: + return blocked("cannot construct minimal proxy task") + proxy_task = proxy_result.proxy + + controlled = variables.freeze_all_except(task.variable_under_test) + if controlled.has_unknown_unfrozen_axes(): + return blocked("unknown variables remain uncontrolled", risks=controlled.unknown_axes) + axes = controlled.axes(one_at_a_time=True) + + deterministic = run_same_setting(proxy_task, runs=3) + if not deterministic.bitwise_equal(): + proxy_task = remove_known_nondeterminism(proxy_task) + deterministic = run_same_setting(proxy_task, runs=3) + if not deterministic.bitwise_equal() and policy.validation.requires_bitwise: + return blocked("reference or target is not deterministic", evidence=deterministic) + + threshold = basic.review_threshold(task, ref.contract, requested_threshold=task.tolerance) + if not threshold.done: + return blocked("precision threshold review failed", evidence=threshold) + mode = threshold.compare_mode + + full = compare(proxy_task, target, ref.reference, mode=mode) + if full.pass_: + return done(alignment="aligned", evidence=[deterministic, full], next=[]) + + evidence = [deterministic, full] + for axis in axes: + experiment = vary_one_axis(proxy_task, axis=axis) + evidence.append(compare(experiment, target, ref.reference, mode=mode)) + if evidence[-1].first_failure: + break + + for unit in decompose(target, order=["layer", "submodule", "primitive"]): + child = basic.align_precision( + task.narrow_to(unit), + target=unit, + variables=variables.freeze_except(unit.related_axes), + budget=budget.child(increment=["depth", "attempts"]), + ) + evidence.append(child) + if child.alignment in ["aligned", "localized_mismatch"] or child.blocked: + break + + if budget.e2e.high_cost_approved: + e2e = basic.align_e2e_precision(task, target=target, modes=task.e2e_modes, budget=budget.e2e) + evidence.append(e2e) + if e2e.alignment == "e2e_aligned": + return done( + alignment="e2e_aligned_override", + evidence=evidence, + next=["local mismatch overridden by high-cost e2e evidence", *e2e.risks], + ) + + return done(alignment="localized_mismatch", evidence=evidence, next=owner_of_first_failure(evidence)) +``` diff --git a/experimental/lite/skills/basic/constitution.md b/experimental/lite/skills/basic/constitution.md new file mode 100644 index 00000000000..8d9c006a447 --- /dev/null +++ b/experimental/lite/skills/basic/constitution.md @@ -0,0 +1,45 @@ +# Basic Constitution Skill + +Global constraints for all Megatron Lite skills. + +## Schema + + +```python +schema = Skill( + "basic.constitution", kind="constitution", purpose="set global MLite constraints", + imports=[], calls=[], + inputs=["task", "layer", "reference"], + outputs=["constraints", "validation", "stop"], exits=["done", "blocked", "out_of_scope"], +) +``` + + +```python +def constitution(task, layer, reference): + if reference is None: + return blocked("no validation reference") + if task.requires_undefined_skill: + return out_of_scope("required procedural skill is undefined") + + constraints = [ + occam_razor("choose the smallest correct and reviewable design"), + modularity("primitives are replaceable unless explicitly fused"), + bounded_state_machine("every procedural skill has finite exits"), + ] + + validation = [ + reference_order(["Megatron", "HuggingFace", "Torch", "first principles"]), + require_bitwise_when_possible(reference), + minimal_primitive_check(scope=["single_gpu", "single_node"], reduce=["layers", "experts"]), + require_end_to_end_before_delivery(), + ] + + stop = [ + "missing reference", + "missing validation path", + "required procedural skill is undefined", + ] + + return done(constraints=constraints, validation=validation, stop=stop) +``` diff --git a/experimental/lite/skills/basic/construct-proxy-task.md b/experimental/lite/skills/basic/construct-proxy-task.md new file mode 100644 index 00000000000..3214f36d290 --- /dev/null +++ b/experimental/lite/skills/basic/construct-proxy-task.md @@ -0,0 +1,38 @@ +# Construct Proxy Task Skill + +Build the smallest task that can falsify a claim. + +## Schema + + +```python +schema = Skill( + "basic.construct_proxy_task", kind="state_machine", purpose="minimize validation while preserving claim", + imports=["basic.constitution"], calls=[], + inputs=["task", "target", "reference", "variables", "budget"], + outputs=["proxy", "controlled", "risks"], exits=["done", "blocked", "out_of_scope"], +) +``` + + +```python +def construct_proxy_task(task, target, reference, variables, budget): + if reference is None: + return blocked("proxy task needs a reference") + + proxy = minimize( + target, + start=["single_gpu", "single_node"], + reduce=["layers", "experts", "sequence", "hidden", "batch"], + preserve=["operation", "shape_rules", "dtype_rules", task.variable_under_test], + ) + if proxy is None or not proxy.still_tests(task.claim): + return blocked("cannot build minimal proxy that preserves claim") + + controlled = variables.freeze_all_except(task.variable_under_test) + if controlled.has_unknown_unfrozen_axes(): + return blocked("proxy has uncontrolled variables", risks=controlled.unknown_axes) + + risks = ["proxy may miss bugs that require full scale"] + return done(proxy=proxy, controlled=controlled, risks=risks) +``` diff --git a/experimental/lite/skills/basic/find-reference.md b/experimental/lite/skills/basic/find-reference.md new file mode 100644 index 00000000000..526f5b1defb --- /dev/null +++ b/experimental/lite/skills/basic/find-reference.md @@ -0,0 +1,51 @@ +# Find Reference Skill + +Find the strongest checkable reference for a Megatron Lite task. + +## Schema + + +```python +schema = Skill( + "basic.find_reference", kind="state_machine", purpose="select a checkable validation reference", + imports=[], calls=[], + inputs=["task", "layer", "candidates", "budget"], + outputs=["reference", "contract", "risks"], exits=["done", "blocked", "out_of_scope"], +) +``` + + +```python +def find_reference(task, layer, candidates, budget): + if task.scope not in ["primitive", "model", "runtime", "application", "precision"]: + return out_of_scope("unknown validation surface") + + ordered = [ + megatron_reference(task, layer), + huggingface_reference(task, layer), + torch_reference(task, layer), + first_principles_formula_or_distributed_invariant(task, layer), + *candidates, + ] + + risks = [] + for ref in ordered[:budget.max_candidates]: + if ref is None or not ref.exists(): + continue + + contract = extract_contract( + ref, + fields=["inputs", "outputs", "shape", "dtype", "seed", "variables", "tolerance"], + ) + if not contract.is_checkable(): + risks.append((ref, "reference exists but contract is incomplete")) + continue + if ref.requires_unavailable_assets(): + risks.append((ref, "reference requires unavailable assets")) + continue + + variables = freeze_variables(contract.variables, except_=task.variable_under_test) + return done(reference=ref, contract=contract.with_variables(variables), risks=risks) + + return blocked("no checkable reference", risks=risks) +``` diff --git a/experimental/lite/skills/basic/lint-skill.md b/experimental/lite/skills/basic/lint-skill.md new file mode 100644 index 00000000000..3700d05550d --- /dev/null +++ b/experimental/lite/skills/basic/lint-skill.md @@ -0,0 +1,58 @@ +# Lint Skill + +Validate a Megatron Lite skill file before review. + +## Schema + + +```python +schema = Skill( + "basic.lint_skill", kind="state_machine", purpose="validate skill structure and dry-runs", + imports=[], calls=[], + inputs=["skill_file", "registry", "scenarios", "budget"], + outputs=["lint", "dry_runs", "risks"], exits=["done", "blocked", "out_of_scope"], +) +``` + + +```python +def lint_skill(skill_file, registry, scenarios, budget): + if not skill_file.path.startswith("experimental/lite/skills/"): + return out_of_scope("not an MLite skill") + + schema = extract_schema_block(skill_file) + if schema is None: + return blocked("missing schema markers") + + spec = parse_skill_schema(schema) + expected_path = module_name_to_path(spec.name) + if skill_file.path != expected_path: + return blocked("schema name does not match file path", evidence=[spec.name, skill_file.path]) + + lint = [ + require_single_md_file(skill_file), + reject_readme_skill_body(skill_file), + require_python_like_body(skill_file), + require_top_level_function(skill_file, name=spec.name.split(".")[-1], inputs=spec.inputs), + require_declared_exits(skill_file, exits=spec.exits), + require_bounded_loops(skill_file), + require_resolved_imports(spec.imports, registry), + require_resolved_calls(spec.calls, registry), + ] + body_calls = extract_skill_calls(skill_file.body, registry=registry) + lint.extend([ + require_body_calls_declared(body_calls, declared=spec.calls), + require_declared_calls_used(spec.calls, body_calls, allow_recursive=True), + ]) + if any(check.fail for check in lint): + return blocked("skill lint failed", lint=lint) + + dry_runs = [] + for scenario in scenarios[:budget.max_scenarios]: + dry_runs.append(trace_pseudocode(skill_file, scenario, max_steps=budget.max_steps)) + if dry_runs[-1].exit not in spec.exits: + return blocked("scenario reached undeclared exit", dry_runs=dry_runs) + + risks = ["dry-runs are contract checks, not executable unit tests"] + return done(lint=lint, dry_runs=dry_runs, risks=risks) +``` diff --git a/experimental/lite/skills/basic/review-threshold.md b/experimental/lite/skills/basic/review-threshold.md new file mode 100644 index 00000000000..cd296ea3dd2 --- /dev/null +++ b/experimental/lite/skills/basic/review-threshold.md @@ -0,0 +1,33 @@ +# Review Threshold Skill + +Choose the comparison mode and require human review for non-bitwise thresholds. + +## Schema + + +```python +schema = Skill( + "basic.review_threshold", kind="state_machine", purpose="select bitwise or reviewed tolerance", + imports=["basic.constitution"], calls=[], + inputs=["task", "reference_contract", "requested_threshold"], + outputs=["compare_mode", "review", "risks"], exits=["done", "blocked", "out_of_scope"], +) +``` + + +```python +def review_threshold(task, reference_contract, requested_threshold): + if reference_contract is None: + return blocked("missing reference contract for threshold review") + + if reference_contract.bitwise_possible: + return done(compare_mode=bitwise(), review=False, risks=[]) + + threshold = requested_threshold or relative(0.01) + review = human_review_required( + reason="non-bitwise precision threshold; 1% relative is only a default", + threshold=threshold, + ) + risks = ["accepted threshold can hide real precision bugs"] + return done(compare_mode=threshold, review=review, risks=risks) +``` diff --git a/experimental/lite/skills/insight/progressive-model-support.md b/experimental/lite/skills/insight/progressive-model-support.md new file mode 100644 index 00000000000..2599564ea36 --- /dev/null +++ b/experimental/lite/skills/insight/progressive-model-support.md @@ -0,0 +1,201 @@ +# Progressive Model Support Skill + +Plan model support through either an HF-first path or a nearest-reference diff path. + +## Schema + + +```python +schema = Skill( + "insight.progressive_model_support", kind="state_machine", purpose="stage model support with HF and bridge/reference paths", + imports=["basic.constitution"], + calls=[ + "basic.find_reference", "basic.align_precision", "basic.align_e2e_precision", + "primitive.select_for_compose", + "model_compose.config_mapping", "model_compose.weight_mapping", "model_compose.build_model", + ], + inputs=["task", "model", "budget"], + outputs=["path", "stages", "evidence", "risks"], exits=["done", "blocked", "out_of_scope"], +) +``` + + +```python +def progressive_model_support(task, model, budget): + if model.is_pure_new or not model.has_bridge_or_nearest_reference(): + result = support_pure_new_model_from_hf(task, model, budget) + else: + result = support_model_from_bridge_reference(task, model, budget) + + if not result.done: + return blocked("progressive model support failed", evidence=result) + return done(path=result.path, stages=result.stages, evidence=result.evidence, risks=result.risks) + + +def support_pure_new_model_from_hf(task, model, budget): + stages = ["hf_config", "hf_weights", "PP", "EP", "CP_if_needed", "TP_if_needed", "e2e"] + evidence = [] + + config = model_compose.config_mapping(task, model.hf.config, model.lite_config, budget.config) + evidence.append(config) + if not config.done: + return blocked("pure-new HF path stopped at config mapping", evidence=evidence) + + weights = model_compose.weight_mapping(task, model.hf.checkpoint, model.lite_model, budget.weights) + evidence.append(weights) + if not weights.done: + return blocked("pure-new HF path stopped at weight mapping", evidence=evidence) + + primitive_steps = [ + step("PP", required=True, features=["pipeline_parallel"]), + step("EP", required=model.has_moe, features=["expert_parallel", "moe", "deepep"]), + step("CP", required=model.needs_long_context or model.uses_thd, features=["context_parallel", "thd"]), + step("TP", required=model.needs_tensor_parallel_for_memory_or_perf, features=["tensor_parallel"]), + ] + partial = model.empty_lite_model(config=config.mapping, weights=weights.mapping) + + for primitive_step in primitive_steps: + if not primitive_step.required: + continue + selection = primitive.select_for_compose( + task.narrow_to(primitive_step.name), + model_spec=model.spec.require(primitive_step.features), + candidates=model.primitive_candidates(primitive_step.features), + budget=budget.primitive_step(primitive_step.name), + ) + evidence.append(selection) + if not selection.done: + return blocked("pure-new primitive selection failed", evidence=evidence) + + partial = model_compose.build_model( + task.narrow_to(primitive_step.name), + model_spec=partial.spec.with_primitives(selection.selection), + primitives=partial.primitives + selection.selection, + budget=budget.build_step(primitive_step.name), + ) + evidence.append(partial) + if not partial.done: + return blocked("pure-new incremental model build failed", evidence=evidence) + + precision = compare_model_precision_ladder( + task.narrow_to(primitive_step.name), + target=partial.model, + reference=model.hf, + variables=model.variables.freeze_all_except(primitive_step.features), + budget=budget.precision_step(primitive_step.name), + ) + evidence.append(precision) + if not precision.done: + return blocked("pure-new precision ladder failed", evidence=evidence) + + e2e = basic.align_e2e_precision(task, target=partial.model, modes=task.e2e_modes, budget=budget.e2e) + evidence.append(e2e) + if not e2e.done: + return blocked("pure-new model stopped at e2e", evidence=evidence) + + risks = ["HF-first path can miss bugs shared by HF conversion and Lite compose"] + return done(path="hf_first_pure_new", stages=stages, evidence=evidence, risks=risks) + + +def support_model_from_bridge_reference(task, model, budget): + stages = ["nearest_reference", "diff_primitives", "bridge_precision", "hf_cross_check", "e2e"] + evidence = [] + + reference = basic.find_reference( + task, + layer="model", + candidates=[model.bridge_reference, *model.nearest_existing_models, "mbridge", "megatron-bridge"], + budget=budget.reference, + ) + evidence.append(reference) + if not reference.done: + return blocked("no nearest model reference for diff path", evidence=evidence) + + diff = model.diff_against(reference.reference) + selection = primitive.select_for_compose( + task.narrow_to(diff), + model_spec=diff.model_spec, + candidates=diff.primitive_candidates, + budget=budget.diff_primitives, + ) + evidence.append(selection) + if not selection.done: + return blocked("diff primitive selection failed", evidence=evidence) + + config = model_compose.config_mapping(task, model.hf.config, model.lite_config, budget.config) + weights = model_compose.weight_mapping(task, model.hf.checkpoint, model.lite_model, budget.weights) + evidence.extend([config, weights]) + if not config.done or not weights.done: + return blocked("HF config or weight mapping failed in reference path", evidence=evidence) + + candidate = model_compose.build_model( + task, + model_spec=model.spec.apply_diff(diff, config.mapping), + primitives=reference.reference.primitives + selection.selection, + budget=budget.build, + ) + evidence.append(candidate) + if not candidate.done: + return blocked("reference-diff model build failed", evidence=evidence) + + bridge_precision = compare_model_precision_ladder( + task.narrow_to("bridge_reference"), + target=candidate.model, + reference=reference.reference, + variables=model.variables.freeze_all_except(diff.changed_features), + budget=budget.bridge_precision, + ) + evidence.append(bridge_precision) + if not bridge_precision.done: + return blocked("reference-diff precision failed", evidence=evidence) + + hf_precision = compare_model_precision_ladder( + task.narrow_to("hf_cross_check"), + target=candidate.model, + reference=model.hf, + variables=model.variables.freeze_all_except(diff.changed_features), + budget=budget.hf_precision, + ) + evidence.append(hf_precision) + if not hf_precision.done: + return blocked("HF cross-check precision failed", evidence=evidence) + + e2e = basic.align_e2e_precision(task, target=candidate.model, modes=task.e2e_modes, budget=budget.e2e) + evidence.append(e2e) + if not e2e.done: + return blocked("reference-diff model stopped at e2e", evidence=evidence) + + risks = ["nearest-reference path can inherit reference bugs", "HF cross-check is still required"] + return done(path="nearest_reference_diff", stages=stages, evidence=evidence, risks=risks) + + +def compare_model_precision_ladder(task, target, reference, variables, budget): + forward = basic.align_precision( + task.with_phase("forward").with_reference(reference).with_metrics(["bitwise_when_possible", "cos_sim"]), + target=target.forward_proxy(), + variables=variables.freeze_all_except("forward_math"), + budget=budget.forward, + ) + if not forward.done: + return blocked("forward precision failed", evidence=forward) + + backward = basic.align_precision( + task.with_phase("backward_grad").with_reference(reference).with_metrics(["bitwise_when_possible", "grad_cos_sim"]), + target=target.backward_proxy(), + variables=variables.freeze_all_except("backward_grad"), + budget=budget.backward, + ) + if not backward.done: + return blocked("backward gradient precision failed", evidence=[forward, backward]) + + grad_norm = compare_grad_norm( + target, + reference, + mode=["bitwise_when_possible", "relative_threshold", "cos_sim_supporting_signal"], + tolerance=budget.grad_norm_tolerance, + ) + if not grad_norm.pass_: + return blocked("grad norm precision failed", evidence=[forward, backward, grad_norm]) + + return done(alignment="forward_backward_grad_norm_aligned", evidence=[forward, backward, grad_norm], risks=[]) +``` diff --git a/experimental/lite/skills/insight/progressive-primitive-design.md b/experimental/lite/skills/insight/progressive-primitive-design.md new file mode 100644 index 00000000000..4ef2f5857bd --- /dev/null +++ b/experimental/lite/skills/insight/progressive-primitive-design.md @@ -0,0 +1,39 @@ +# Progressive Primitive Design Skill + +Plan a new primitive from first principles to performance. + +## Schema + + +```python +schema = Skill( + "insight.progressive_primitive_design", kind="state_machine", purpose="stage new primitive design", + imports=["basic.constitution"], calls=["primitive.design", "primitive.validate", "perf.optimize"], + inputs=["task", "primitive", "budget"], + outputs=["stages", "evidence", "risks"], exits=["done", "blocked", "out_of_scope"], +) +``` + + +```python +def progressive_primitive_design(task, primitive, budget): + stages = ["principle", "single_device", "single_node", "distributed", "model_compose", "perf"] + evidence = [] + + design = primitive.design(task, primitive, primitive.requirements, budget.design) + evidence.append(design) + if not design.done: + return blocked("primitive design stopped at contract design", evidence=evidence) + + validation = primitive.validate(task, primitive=primitive, implementation=primitive.implementation, budget=budget.validation) + evidence.append(validation) + if not validation.done: + return blocked("primitive design stopped at validation", evidence=evidence) + + optimization = perf.optimize(task, target=primitive.implementation, constraints=primitive.perf_constraints, budget=budget.perf) + evidence.append(optimization) + if not optimization.done: + return blocked("primitive design stopped at performance", evidence=evidence) + + return done(stages=stages, evidence=evidence, risks=["performance stage must not rewrite correctness contract"]) +``` diff --git a/experimental/lite/skills/insight/scope-control.md b/experimental/lite/skills/insight/scope-control.md new file mode 100644 index 00000000000..f074c92e646 --- /dev/null +++ b/experimental/lite/skills/insight/scope-control.md @@ -0,0 +1,29 @@ +# Scope Control Skill + +Decide when to split work instead of expanding a skill or task. + +## Schema + + +```python +schema = Skill( + "insight.scope_control", kind="state_machine", purpose="prevent skill and task scope creep", + imports=["basic.constitution"], calls=["basic.lint_skill"], + inputs=["task", "change", "budget"], + outputs=["decision", "split", "risks"], exits=["done", "blocked", "out_of_scope"], +) +``` + + +```python +def scope_control(task, change, budget): + if change.touches_multiple_namespaces() or change.exceeds(budget.max_files): + return done(decision="split", split=propose_child_tasks(change), risks=["large review surface"]) + if change.adds_new_procedure_without_schema(): + return blocked("new procedure needs a skill schema") + + lint = basic.lint_skill(change.skill_file, registry=budget.registry, scenarios=[], budget=budget.lint) + if not lint.done: + return blocked("scope-control lint failed", evidence=lint) + return done(decision="continue", split=[], risks=[]) +``` diff --git a/experimental/lite/skills/model-compose/build-model.md b/experimental/lite/skills/model-compose/build-model.md new file mode 100644 index 00000000000..c921ddfcbc1 --- /dev/null +++ b/experimental/lite/skills/model-compose/build-model.md @@ -0,0 +1,35 @@ +# Build Model Skill + +Compose a Megatron Lite model from validated primitives. + +## Schema + + +```python +schema = Skill( + "model_compose.build_model", kind="state_machine", purpose="compose MLite model from primitives", + imports=["basic.constitution"], calls=["primitive.select_for_compose", "primitive.validate", "basic.align_precision"], + inputs=["task", "model_spec", "primitives", "budget"], + outputs=["model", "validation", "risks"], exits=["done", "blocked", "out_of_scope"], +) +``` + + +```python +def build_model(task, model_spec, primitives, budget): + selected = primitive.select_for_compose(task, model_spec=model_spec, candidates=primitives, budget=budget.selection) + if not selected.done: + return blocked("primitive selection failed before model compose", evidence=selected) + + for selected_primitive in selected.selection: + result = primitive.validate(task, primitive=selected_primitive, implementation=selected_primitive.impl, budget=budget.primitive) + if not result.done: + return blocked("primitive not validated before model compose", evidence=result) + + model = compose_layers(model_spec, selected.selection, boundary=["runtime", "model", "primitive"]) + precision = basic.align_precision(task, target=model, variables=model.variables, budget=budget.precision) + if not precision.done: + return blocked("model precision failed", evidence=precision) + + return done(model=model, validation=precision, risks=["composition can hide primitive boundary bugs"]) +``` diff --git a/experimental/lite/skills/model-compose/config-mapping.md b/experimental/lite/skills/model-compose/config-mapping.md new file mode 100644 index 00000000000..453695321c0 --- /dev/null +++ b/experimental/lite/skills/model-compose/config-mapping.md @@ -0,0 +1,34 @@ +# Config Mapping Skill + +Map external model configs into Megatron Lite configs. + +## Schema + + +```python +schema = Skill( + "model_compose.config_mapping", kind="state_machine", purpose="map model configs into MLite", + imports=["basic.constitution"], calls=["basic.find_reference"], + inputs=["task", "source_config", "target_config", "budget"], + outputs=["mapping", "evidence", "risks"], exits=["done", "blocked", "out_of_scope"], +) +``` + + +```python +def config_mapping(task, source_config, target_config, budget): + reference = basic.find_reference(task, layer="config", candidates=[source_config], budget=budget.reference) + if not reference.done: + return blocked("config reference not found", evidence=reference) + + mapping = map_fields( + source_config, + target_config, + required=["hidden_size", "num_layers", "num_heads", "dtype", "moe", "rope"], + ) + if mapping.has_missing_required_fields(): + return blocked("config mapping incomplete", evidence=mapping.missing) + + evidence = record_evidence(task, run=mapping.check, comparison=mapping.comparison, environment=budget.env) + return done(mapping=mapping, evidence=evidence, risks=["default mismatch", "alias mismatch"]) +``` diff --git a/experimental/lite/skills/model-compose/qwen.md b/experimental/lite/skills/model-compose/qwen.md new file mode 100644 index 00000000000..7dc6eae8fd5 --- /dev/null +++ b/experimental/lite/skills/model-compose/qwen.md @@ -0,0 +1,37 @@ +# Qwen Compose Skill + +Compose Qwen3 MoE and Qwen3.5 Megatron Lite models. + +## Schema + + +```python +schema = Skill( + "model_compose.qwen", kind="state_machine", purpose="compose Qwen3 MoE/Qwen3.5 MLite models", + imports=["basic.constitution"], calls=["model_compose.config_mapping", "model_compose.weight_mapping", "model_compose.build_model"], + inputs=["task", "hf_model", "lite_model", "budget"], + outputs=["model", "mapping", "evidence", "risks"], exits=["done", "blocked", "out_of_scope"], +) +``` + + +```python +def qwen(task, hf_model, lite_model, budget): + if hf_model.family not in ["qwen3_moe", "qwen3_5"]: + return out_of_scope("not a Qwen3 MoE/Qwen3.5 model") + + config = model_compose.config_mapping(task, hf_model.config, lite_model.config, budget.config) + if not config.done: + return blocked("Qwen config mapping failed", evidence=config) + + weights = model_compose.weight_mapping(task, hf_model.checkpoint, lite_model, budget.weights) + if not weights.done: + return blocked("Qwen weight mapping failed", evidence=weights) + + model = model_compose.build_model(task, lite_model.spec, lite_model.primitives, budget.build) + if not model.done: + return blocked("Qwen model build failed", evidence=model) + + risks = ["Qwen alias mismatch", "MoE router drift", "GQA head mapping drift"] + return done(model=model.model, mapping=[config.mapping, weights.mapping], evidence=[weights.evidence, model.validation], risks=risks) +``` diff --git a/experimental/lite/skills/model-compose/weight-mapping.md b/experimental/lite/skills/model-compose/weight-mapping.md new file mode 100644 index 00000000000..f5451539761 --- /dev/null +++ b/experimental/lite/skills/model-compose/weight-mapping.md @@ -0,0 +1,40 @@ +# Weight Mapping Skill + +Map checkpoint weights into Megatron Lite model state. + +## Schema + + +```python +schema = Skill( + "model_compose.weight_mapping", kind="state_machine", purpose="map checkpoint weights into MLite", + imports=["basic.constitution"], calls=["basic.find_reference", "basic.align_precision"], + inputs=["task", "source_checkpoint", "target_model", "budget"], + outputs=["mapping", "coverage", "evidence"], exits=["done", "blocked", "out_of_scope"], +) +``` + + +```python +def weight_mapping(task, source_checkpoint, target_model, budget): + candidates = [ + source_checkpoint, + "mbridge", + "megatron-bridge", + *budget.reference.extra_checkpoint_tools, + ] + reference = basic.find_reference(task, layer="checkpoint", candidates=candidates, budget=budget.reference) + if not reference.done: + return blocked("checkpoint reference not found", evidence=reference) + + mapping = map_weight_names_shapes_dtypes(source_checkpoint, target_model, reference_tools=["mbridge", "megatron-bridge"]) + coverage = mapping.coverage() + if not coverage.complete: + return blocked("weight mapping incomplete", evidence=coverage.missing) + + precision = basic.align_precision(task, target=target_model, variables=mapping.variables, budget=budget.precision) + if not precision.done: + return blocked("mapped model precision failed", evidence=precision) + + return done(mapping=mapping, coverage=coverage, evidence=precision) +``` diff --git a/experimental/lite/skills/perf/fusion.md b/experimental/lite/skills/perf/fusion.md new file mode 100644 index 00000000000..1be55e041ce --- /dev/null +++ b/experimental/lite/skills/perf/fusion.md @@ -0,0 +1,31 @@ +# Fusion Skill + +Evaluate fusion opportunities without breaking primitive boundaries. + +## Schema + + +```python +schema = Skill( + "perf.fusion", kind="state_machine", purpose="evaluate safe fusion opportunities", + imports=["basic.constitution"], calls=["primitive.fuse", "perf.measure"], + inputs=["task", "candidates", "target", "budget"], + outputs=["fusion", "evidence", "risks"], exits=["done", "blocked", "out_of_scope"], +) +``` + + +```python +def fusion(task, candidates, target, budget): + fused = propose_fusion(candidates, target) + decision = primitive.fuse(task, primitives=candidates, fused_design=fused, budget=budget.fuse) + if not decision.done: + return blocked("fusion decision failed", evidence=decision) + + measurement = perf.measure(task, target=fused, workload=budget.workload, budget=budget.measure) + if not measurement.done: + return blocked("fused target measurement failed", evidence=measurement) + + risks = [*decision.risks, "fusion must not replace precision validation"] + return done(fusion=fused, evidence=[decision, measurement], risks=risks) +``` diff --git a/experimental/lite/skills/perf/measure.md b/experimental/lite/skills/perf/measure.md new file mode 100644 index 00000000000..04a9a8d10cd --- /dev/null +++ b/experimental/lite/skills/perf/measure.md @@ -0,0 +1,30 @@ +# Measure Skill + +Measure runtime, memory, and quality metrics with a stable protocol. + +## Schema + + +```python +schema = Skill( + "perf.measure", kind="state_machine", purpose="measure performance without losing precision context", + imports=["basic.constitution"], calls=[], + inputs=["task", "target", "workload", "budget"], + outputs=["metrics", "run", "risks"], exits=["done", "blocked", "out_of_scope"], +) +``` + + +```python +def measure(task, target, workload, budget): + if workload is None: + return blocked("missing workload") + + run = execute_with_protocol(target, workload, warmup=budget.warmup, repeats=budget.repeats) + metrics = collect_metrics(run, fields=["tokens_per_sec", "step_time", "memory", "loss_or_reward"]) + if metrics.has_missing_fields(): + return blocked("performance evidence incomplete", evidence=metrics) + + risks = ["performance numbers without precision evidence are not sufficient"] + return done(metrics=metrics, run=run, risks=risks) +``` diff --git a/experimental/lite/skills/perf/memory.md b/experimental/lite/skills/perf/memory.md new file mode 100644 index 00000000000..d634f485c54 --- /dev/null +++ b/experimental/lite/skills/perf/memory.md @@ -0,0 +1,40 @@ +# Memory Skill + +Analyze memory across parameters, optimizer state, activations, and offload. + +## Schema + + +```python +schema = Skill( + "perf.memory", kind="state_machine", purpose="analyze MLite memory behavior", + imports=["basic.constitution"], calls=["perf.measure"], + inputs=["task", "target", "memory_config", "budget"], + outputs=["memory", "evidence", "risks"], exits=["done", "blocked", "out_of_scope"], +) +``` + + +```python +def memory(task, target, memory_config, budget): + measurement = perf.measure(task, target=target, workload=memory_config.workload, budget=budget.measure) + if not measurement.done: + return blocked("memory measurement failed", evidence=measurement) + + memory = split_memory(measurement.metrics.memory, buckets=["params", "grads", "optimizer", "activations", "offload"]) + model = estimate_memory( + target, + reference="https://developer.nvidia.cn/blog/explore-using-the-megatron-core-training-framework-to-improve-gpu-memory-efficiency-in-large-model-training/", + static_axes=["EP*PP changes expert/layer ownership", "FSDP changes param/grad/optimizer ownership"], + dynamic_axes=["TP*CP changes activation and attention working-set shape", "THD changes packed-token activation shape"], + ) + comparison = compare_estimate_to_measurement(model, measurement.metrics.memory) + if not comparison.within(memory_config.tolerance): + return blocked("memory estimate and measurement disagree", evidence=[model, measurement, comparison]) + + risks = [ + "offload can improve capacity while hiding transfer bottlenecks", + "static memory and dynamic peak must be inspected separately", + ] + return done(memory=memory, evidence=[measurement, model, comparison], risks=risks) +``` diff --git a/experimental/lite/skills/perf/optimize.md b/experimental/lite/skills/perf/optimize.md new file mode 100644 index 00000000000..df67b868e73 --- /dev/null +++ b/experimental/lite/skills/perf/optimize.md @@ -0,0 +1,61 @@ +# Optimize Skill + +Iterate on performance while preserving precision evidence. + +## Schema + + +```python +schema = Skill( + "perf.optimize", kind="state_machine", purpose="optimize performance under precision guardrails", + imports=["basic.constitution"], calls=["basic.align_precision", "perf.measure", "perf.fusion", "primitive.design"], + inputs=["task", "target", "candidates", "budget"], + outputs=["best", "evidence", "risks"], exits=["done", "blocked", "out_of_scope"], +) +``` + + +```python +def optimize(task, target, candidates, budget): + best = None + evidence = [] + queue = candidates[:budget.max_candidates] + + for attempt in range(budget.max_attempts): + if not queue: + break + candidate = queue.pop(0) + + experiment = run_experiment(candidate, workload=budget.workload, knobs=budget.tunable_knobs) + profile = perf.measure(task, target=candidate, workload=experiment.workload, budget=budget.measure) + evidence.append((candidate, experiment, profile)) + if not profile.done: + continue + + opportunities = analyze_profile( + profile.metrics, + spaces=["compute_communication_overlap", "kernel_fusion", "schedule_tuning", "new_or_split_primitive"], + ) + if opportunities.require_fusion: + fusion = perf.fusion(task, candidates=opportunities.fusion_candidates, target=candidate, budget=budget.fusion) + evidence.append(fusion) + if fusion.done: + queue.append(fusion.fusion) + if opportunities.require_primitive_design: + design = primitive.design(task, opportunities.primitive, opportunities.requirements, budget.primitive_design) + evidence.append(design) + if design.done: + queue.append(design.design) + + tuned = tune_parameters(candidate, opportunities.knobs) + precision = basic.align_precision(task, target=tuned, variables=tuned.variables, budget=budget.precision) + if not precision.done: + continue + measurement = perf.measure(task, target=tuned, workload=budget.workload, budget=budget.measure) + evidence.append((tuned, precision, measurement)) + best = choose_better(best, tuned, measurement.metrics) + + if best is None: + return blocked("no optimized candidate preserved precision", evidence=evidence) + return done(best=best, evidence=evidence, risks=["optimization search can overfit benchmark", "profiling must be repeated after every primitive change"]) +``` diff --git a/experimental/lite/skills/primitive/checkpoint/distckpt.md b/experimental/lite/skills/primitive/checkpoint/distckpt.md new file mode 100644 index 00000000000..fef29da385b --- /dev/null +++ b/experimental/lite/skills/primitive/checkpoint/distckpt.md @@ -0,0 +1,45 @@ +# Distopt Distributed Checkpoint Skill + +Define, implement, use, and validate Megatron Core distributed checkpointing for MLite distopt continuity. + +## Schema + + +```python +schema = Skill( + "primitive.checkpoint.distckpt", kind="primitive", purpose="checkpoint DistributedOptimizer state with mcore dist_checkpointing", + imports=["basic.constitution"], calls=["primitive.contract", "primitive.validate", "primitive.optimizer.distopt"], + inputs=["task", "implementation", "config", "reference", "budget"], + outputs=["principle", "implementation_contract", "usage_contract", "validation", "risks"], + exits=["done", "blocked", "out_of_scope"], +) +``` + + +```python +def distckpt(task, implementation, config, reference, budget): + contract = primitive.contract(implementation.distckpt, scope=task.scope, reference=reference) + if not contract.done: + return blocked("Distopt distckpt contract not satisfied", evidence=contract) + + principle = { + "semantics": "use mcore dist_checkpointing for DistributedOptimizer model and tensor optimizer state", + "invariants": ["save-load-continue matches uninterrupted training", "model sharded_state_dict keys are MLite-local"], + "reference": reference or "mcore DistributedOptimizer.sharded_state_dict roundtrip", + } + implementation_contract = { + "owned_files": ["primitive.ckpt.distckpt", "model protocol distopt compose sites"], + "state": ["model ShardedTensor metadata", "optimizer fp32 master params", "optimizer exp_avg", "optimizer exp_avg_sq", "optimizer step"], + "boundaries": ["do not attach sharded_state_dict to generic model classes", "do not change FSDP2 or mfsdp checkpoint paths"], + } + usage_contract = { + "choose_when": ["runtime use_dcp=True", "optimizer exposes sharded_state_dict", "model chunks expose sharded_state_dict"], + "avoid_when": ["use_dcp=False local checkpoint", "non-distopt optimizer", "cross-tool mcore-MLite checkpoint interchange"], + "compose_with": ["primitive.optimizer.distopt", "model-compose owned opt-in"], + } + validation = primitive.validate(task, primitive=implementation.distckpt, implementation=implementation, budget=budget) + risks = ["missing optimizer tensor state", "model key drift", "replica_id or shard offset mismatch"] + if not validation.done: + return blocked("Distopt distckpt validation failed", evidence=validation) + return done(principle=principle, implementation_contract=implementation_contract, usage_contract=usage_contract, validation=validation, risks=risks) +``` diff --git a/experimental/lite/skills/primitive/contract.md b/experimental/lite/skills/primitive/contract.md new file mode 100644 index 00000000000..7f270293f9f --- /dev/null +++ b/experimental/lite/skills/primitive/contract.md @@ -0,0 +1,61 @@ +# Primitive Contract Skill + +Define the required outputs for every MLite primitive skill. + +## Schema + + +```python +schema = Skill( + "primitive.contract", kind="constitution", purpose="define primitive skill outputs", + imports=["basic.constitution"], calls=[], + inputs=["primitive", "scope", "reference"], + outputs=["principle", "implementation_contract", "usage_contract", "validation", "risks"], + exits=["done", "blocked", "out_of_scope"], +) +``` + + +```python +def contract(primitive, scope, reference): + if reference is None: + return blocked("primitive needs a reference or first-principles invariant") + + principle = require([ + "math_or_parallel_semantics", + "invariants", + "shape_dtype_rank_rules", + "what_must_match_reference", + ]) + implementation_contract = require([ + "owned_files_or_modules", + "public_api", + "state_and_config", + "process_groups_or_device_placement", + "forward_backward_update_details", + "failure_modes", + ]) + usage_contract = require([ + "config_keys", + "minimal_example", + "valid_combinations", + "selection_rules", + "unsupported_combinations", + ]) + validation = require([ + "single_gpu_or_single_node_proxy", + "controlled_variables", + "precision_reference", + "composition_test", + "e2e_path_if_applicable", + ]) + risks = ["silent mismatch", "dtype drift", "hidden coupling", "wrong selection rule"] + + return done( + principle=principle, + implementation_contract=implementation_contract, + usage_contract=usage_contract, + validation=validation, + risks=risks, + ) +``` diff --git a/experimental/lite/skills/primitive/design.md b/experimental/lite/skills/primitive/design.md new file mode 100644 index 00000000000..8246a688b9e --- /dev/null +++ b/experimental/lite/skills/primitive/design.md @@ -0,0 +1,41 @@ +# Primitive Design Skill + +Design a replaceable MLite primitive before implementation. + +## Schema + + +```python +schema = Skill( + "primitive.design", kind="state_machine", purpose="design a modular primitive", + imports=["basic.constitution"], calls=["primitive.principle", "primitive.contract", "basic.find_reference"], + inputs=["task", "primitive", "requirements", "budget"], + outputs=["design", "reference", "risks"], exits=["done", "blocked", "out_of_scope"], +) +``` + + +```python +def design(task, primitive, requirements, budget): + ref = basic.find_reference(task, layer=primitive.layer, candidates=requirements.references, budget=budget.reference) + if not ref.done: + return blocked("no primitive reference", evidence=ref) + + principle = primitive.principle(primitive, reference=ref.reference, constraints=requirements.constraints) + if not principle.done: + return blocked("primitive principle failed", evidence=principle) + + contract = primitive.contract(primitive, scope=requirements.scope, reference=ref.reference) + if not contract.done: + return blocked("primitive contract failed", evidence=contract) + + design = { + "principle": principle.principle, + "implementation_details": define_owned_modules_and_dataflow(primitive), + "api": define_inputs_outputs_config_keys(primitive), + "composition": declare_valid_and_invalid_combinations(primitive), + "selection": decide_when_to_use_or_not_use(primitive), + "replaceability": require_no_hidden_dependency(primitive), + } + return done(design=design, reference=ref, risks=contract.risks) +``` diff --git a/experimental/lite/skills/primitive/fuse.md b/experimental/lite/skills/primitive/fuse.md new file mode 100644 index 00000000000..2c83f1b6238 --- /dev/null +++ b/experimental/lite/skills/primitive/fuse.md @@ -0,0 +1,40 @@ +# Primitive Fuse Skill + +Decide whether primitive coupling is allowed as an explicit fused primitive. + +## Schema + + +```python +schema = Skill( + "primitive.fuse", kind="state_machine", purpose="approve or reject primitive fusion", + imports=["basic.constitution"], calls=["primitive.validate", "basic.align_precision"], + inputs=["task", "primitives", "fused_design", "budget"], + outputs=["decision", "validation", "risks"], exits=["done", "blocked", "out_of_scope"], +) +``` + + +```python +def fuse(task, primitives, fused_design, budget): + if not fused_design.names_all_coupled_primitives(primitives): + return blocked("fusion hides primitive coupling") + if not fused_design.has_independent_fallback_or_reference(): + return blocked("fusion needs an unfused reference") + + structure = primitive.validate( + task, + primitive=fused_design.primitive, + implementation=fused_design.implementation, + budget=budget.validate, + ) + if not structure.done: + return blocked("fused primitive structure not validated", evidence=structure) + + precision = basic.align_precision(task, target=fused_design, variables=fused_design.variables, budget=budget.precision) + if not precision.done: + return blocked("fused primitive precision not validated", evidence=precision) + + risks = ["fusion can mask individual primitive bugs", "fusion reduces replaceability"] + return done(decision="approved_fused_primitive", validation=[structure, precision], risks=risks) +``` diff --git a/experimental/lite/skills/primitive/module/gqa.md b/experimental/lite/skills/primitive/module/gqa.md new file mode 100644 index 00000000000..53fdd65c767 --- /dev/null +++ b/experimental/lite/skills/primitive/module/gqa.md @@ -0,0 +1,46 @@ +# GQA Primitive Skill + +Define, implement, use, and validate grouped-query attention primitives. + +## Schema + + +```python +schema = Skill( + "primitive.module.gqa", kind="primitive", purpose="define and validate GQA module primitive", + imports=["basic.constitution"], calls=["primitive.contract", "primitive.validate"], + inputs=["task", "implementation", "config", "reference", "budget"], + outputs=["principle", "implementation_contract", "usage_contract", "validation", "risks"], + exits=["done", "blocked", "out_of_scope"], +) +``` + + +```python +def gqa(task, implementation, config, reference, budget): + contract = primitive.contract(implementation.gqa, scope=task.scope, reference=reference) + if not contract.done: + return blocked("GQA contract not satisfied", evidence=contract) + + principle = { + "semantics": "many query heads share fewer key/value heads", + "invariants": ["KV repeat mapping matches reference", "attention mask and rotary positions are unchanged"], + "reference": reference or "HuggingFace attention implementation", + } + implementation_contract = { + "details": ["qkv projection layout", "head mapping", "kv repeat", "attention call"], + "state": ["num_attention_heads", "num_key_value_heads", "head_dim"], + "boundaries": ["GQA owns head mapping; TP owns sharding of projection weights; THD owns packed sequence boundaries"], + } + usage_contract = { + "config": require_config_keys(config, ["num_attention_heads", "num_key_value_heads"]), + "choose_when": ["architecture declares grouped KV heads"], + "avoid_when": ["num_attention_heads not divisible by num_key_value_heads"], + "compose_with": ["TP projection sharding", "CP context sharding with explicit head/context mapping", "THD packed attention when use_thd=True"], + } + validation = primitive.validate(task, primitive=implementation.gqa, implementation=implementation, budget=budget) + risks = ["head repeat mismatch", "qkv layout bug", "attention mask drift"] + if not validation.done: + return blocked("GQA validation failed", evidence=validation) + return done(principle=principle, implementation_contract=implementation_contract, usage_contract=usage_contract, validation=validation, risks=risks) +``` diff --git a/experimental/lite/skills/primitive/module/moe.md b/experimental/lite/skills/primitive/module/moe.md new file mode 100644 index 00000000000..eb52ae650d8 --- /dev/null +++ b/experimental/lite/skills/primitive/module/moe.md @@ -0,0 +1,58 @@ +# MoE Primitive Skill + +Define, implement, use, and validate mixture-of-experts primitives. + +## Schema + + +```python +schema = Skill( + "primitive.module.moe", kind="primitive", purpose="define and validate MoE module primitive", + imports=["basic.constitution"], calls=["primitive.contract", "primitive.validate", "primitive.parallel.ep"], + inputs=["task", "implementation", "config", "reference", "budget"], + outputs=["principle", "implementation_contract", "usage_contract", "validation", "risks"], + exits=["done", "blocked", "out_of_scope"], +) +``` + + +```python +def moe(task, implementation, config, reference, budget): + contract = primitive.contract(implementation.moe, scope=task.scope, reference=reference) + if not contract.done: + return blocked("MoE contract not satisfied", evidence=contract) + + principle = { + "semantics": "router selects experts and combines weighted expert outputs", + "invariants": ["router logits/topk match reference", "dispatch/combine preserves token order"], + "reference": reference or "HuggingFace/Megatron MoE layer or first-principles weighted sum", + } + implementation_contract = { + "details": ["router", "topk", "token dispatcher", "DeepEP dispatcher", "expert MLP", "combine weights"], + "state": ["expert params", "router dtype", "capacity/drop policy", "DeepEP dispatch metadata"], + "boundaries": ["module owns routing math; EP owns distributed expert placement; DeepEP owns dispatch/combine transport"], + } + usage_contract = { + "config": require_config_keys(config, ["num_experts", "top_k", "expert_parallel_size", "use_deepep"]), + "choose_when": ["model architecture has sparse experts", "expert count justifies EP or grouped GEMM"], + "avoid_when": ["router tie behavior cannot be stabilized", "DeepEP metadata cannot be validated against all-to-all"], + "compose_with": ["primitive.parallel.ep", "DeepEP dispatcher when EP>1", "primitive.parallel.tp for expert MLP if explicit"], + } + ep_validation = None + if config.expert_parallel_size > 1: + ep_validation = primitive.parallel.ep(task, implementation=implementation, config=config, reference=reference, budget=budget.ep) + if not ep_validation.done: + return blocked("MoE EP composition failed", evidence=ep_validation) + + validation = primitive.validate(task, primitive=implementation.moe, implementation=implementation, budget=budget) + risks = ["router tie sensitivity", "expert capacity mismatch", "DeepEP metadata mismatch", "hidden state flattening bug"] + if not validation.done: + return blocked("MoE validation failed", evidence=validation) + return done( + principle=principle, + implementation_contract=implementation_contract, + usage_contract=usage_contract, + validation=[ep_validation, validation], + risks=risks, + ) +``` diff --git a/experimental/lite/skills/primitive/module/thd.md b/experimental/lite/skills/primitive/module/thd.md new file mode 100644 index 00000000000..762923c6242 --- /dev/null +++ b/experimental/lite/skills/primitive/module/thd.md @@ -0,0 +1,59 @@ +# THD Primitive Skill + +Define, implement, use, and validate packed THD variable-length attention. + +## Schema + + +```python +schema = Skill( + "primitive.module.thd", kind="primitive", purpose="define and validate THD packed-sequence primitive", + imports=["basic.constitution"], calls=["primitive.contract", "primitive.validate", "primitive.parallel.cp"], + inputs=["task", "implementation", "config", "reference", "budget"], + outputs=["principle", "implementation_contract", "usage_contract", "validation", "risks"], + exits=["done", "blocked", "out_of_scope"], +) +``` + + +```python +def thd(task, implementation, config, reference, budget): + contract = primitive.contract(implementation.thd, scope=task.scope, reference=reference) + if not contract.done: + return blocked("THD contract not satisfied", evidence=contract) + + principle = { + "semantics": "pack variable-length sequences as total_tokens x heads x dim for attention", + "invariants": ["cu_seqlens define sequence boundaries", "pack/unpack never crosses sequence boundaries"], + "reference": reference or "Megatron/Core TE THD packed-sequence attention contract", + } + implementation_contract = { + "details": ["PackedSeqParams", "PackedTHDBatch", "pack_nested_thd", "unpack_packed_thd_to_nested"], + "state": ["cu_seqlens", "cu_seqlens_padded", "max_seqlen", "qkv_format=thd"], + "boundaries": ["THD owns packing; CP owns zigzag rank partition when cp_size > 1"], + } + usage_contract = { + "config": require_config_keys(config, ["use_thd", "sequence_length", "context_parallel_size"]), + "choose_when": ["variable-length SFT/RL data", "padding waste dominates", "attention supports packed THD"], + "avoid_when": ["reference cannot expose cu_seqlens", "operator path lacks packed-sequence support"], + "compose_with": ["primitive.parallel.cp via zigzag THD slicing", "primitive.module.gqa attention path"], + } + + cp_validation = None + if config.context_parallel_size > 1 and not task.stack.contains("primitive.parallel.cp"): + cp_validation = primitive.parallel.cp(task.push("primitive.module.thd"), implementation=implementation, config=config, reference=reference, budget=budget.cp) + if not cp_validation.done: + return blocked("THD CP composition failed", evidence=cp_validation) + + validation = primitive.validate(task, primitive=implementation.thd, implementation=implementation, budget=budget) + risks = ["cu_seqlens mismatch", "padding alignment drift", "CP zigzag pack/unpack bug"] + if not validation.done: + return blocked("THD validation failed", evidence=validation) + return done( + principle=principle, + implementation_contract=implementation_contract, + usage_contract=usage_contract, + validation=[cp_validation, validation], + risks=risks, + ) +``` diff --git a/experimental/lite/skills/primitive/optimizer/distopt.md b/experimental/lite/skills/primitive/optimizer/distopt.md new file mode 100644 index 00000000000..25b3c41b687 --- /dev/null +++ b/experimental/lite/skills/primitive/optimizer/distopt.md @@ -0,0 +1,46 @@ +# Distributed Optimizer Skill + +Define, implement, use, and validate distributed optimizer sharding. + +## Schema + + +```python +schema = Skill( + "primitive.optimizer.distopt", kind="primitive", purpose="define and validate distributed optimizer primitive", + imports=["basic.constitution"], calls=["primitive.contract", "primitive.validate"], + inputs=["task", "implementation", "config", "reference", "budget"], + outputs=["principle", "implementation_contract", "usage_contract", "validation", "risks"], + exits=["done", "blocked", "out_of_scope"], +) +``` + + +```python +def distopt(task, implementation, config, reference, budget): + contract = primitive.contract(implementation.distopt, scope=task.scope, reference=reference) + if not contract.done: + return blocked("DistOpt contract not satisfied", evidence=contract) + + principle = { + "semantics": "shard optimizer state and updates across data-parallel ranks", + "invariants": ["global update equals unsharded optimizer", "state partition is deterministic"], + "reference": reference or "single-rank optimizer with same grads", + } + implementation_contract = { + "details": ["state partition", "grad shard ownership", "update and param sync"], + "state": ["momentum/variance shard", "master param ownership", "offload policy"], + "boundaries": ["optimizer primitive owns update state; data-parallel grad sync is a separate runtime contract"], + } + usage_contract = { + "config": require_config_keys(config, ["use_distributed_optimizer", "data_parallel_size"]), + "choose_when": ["optimizer state memory dominates", "replicated optimizer state is too expensive"], + "avoid_when": ["single-rank proxy debugging", "FSDP already owns optimizer sharding"], + "compose_with": ["data-parallel gradient path", "TP/EP/PP with clear DP group"], + } + validation = primitive.validate(task, primitive=implementation.distopt, implementation=implementation, budget=budget) + risks = ["state shard drift", "param sync mismatch", "offload update device mismatch"] + if not validation.done: + return blocked("DistOpt validation failed", evidence=validation) + return done(principle=principle, implementation_contract=implementation_contract, usage_contract=usage_contract, validation=validation, risks=risks) +``` diff --git a/experimental/lite/skills/primitive/optimizer/fsdp.md b/experimental/lite/skills/primitive/optimizer/fsdp.md new file mode 100644 index 00000000000..e3c85ebe39d --- /dev/null +++ b/experimental/lite/skills/primitive/optimizer/fsdp.md @@ -0,0 +1,46 @@ +# FSDP Skill + +Define, implement, use, and validate fully sharded data parallelism. + +## Schema + + +```python +schema = Skill( + "primitive.optimizer.fsdp", kind="primitive", purpose="define and validate FSDP optimizer primitive", + imports=["basic.constitution"], calls=["primitive.contract", "primitive.validate"], + inputs=["task", "implementation", "config", "reference", "budget"], + outputs=["principle", "implementation_contract", "usage_contract", "validation", "risks"], + exits=["done", "blocked", "out_of_scope"], +) +``` + + +```python +def fsdp(task, implementation, config, reference, budget): + contract = primitive.contract(implementation.fsdp, scope=task.scope, reference=reference) + if not contract.done: + return blocked("FSDP contract not satisfied", evidence=contract) + + principle = { + "semantics": "shard parameters, gradients, and optimizer state while materialized computation matches reference", + "invariants": ["materialized params equal reference", "optimizer update equals unsharded update"], + "reference": reference or "single-rank optimizer update with same params/grads", + } + implementation_contract = { + "details": ["wrap policy", "param shard", "all_gather before compute", "reduce_scatter grads"], + "state": ["optimizer state shard", "param offload", "optimizer offload", "update-state device"], + "boundaries": ["model modules expose params; optimizer primitive owns sharding/offload"], + } + usage_contract = { + "config": require_config_keys(config, ["fsdp_size", "param_offload", "optimizer_offload"]), + "choose_when": ["optimizer memory dominates", "param/optimizer offload is needed"], + "avoid_when": ["debugging model math; use unsharded optimizer first"], + "compose_with": ["TP/EP/PP through explicit param ownership", "DistOpt only with clear owner split"], + } + validation = primitive.validate(task, primitive=implementation.fsdp, implementation=implementation, budget=budget) + risks = ["offload device mismatch", "optimizer state drift", "materialization timing bug"] + if not validation.done: + return blocked("FSDP validation failed", evidence=validation) + return done(principle=principle, implementation_contract=implementation_contract, usage_contract=usage_contract, validation=validation, risks=risks) +``` diff --git a/experimental/lite/skills/primitive/parallel/cp.md b/experimental/lite/skills/primitive/parallel/cp.md new file mode 100644 index 00000000000..324568d1008 --- /dev/null +++ b/experimental/lite/skills/primitive/parallel/cp.md @@ -0,0 +1,52 @@ +# Context Parallel Skill + +Define, implement, use, and validate context parallelism. + +## Schema + + +```python +schema = Skill( + "primitive.parallel.cp", kind="primitive", purpose="define and validate CP primitive", + imports=["basic.constitution"], calls=["primitive.contract", "primitive.validate", "primitive.module.thd"], + inputs=["task", "implementation", "config", "reference", "budget"], + outputs=["principle", "implementation_contract", "usage_contract", "validation", "risks"], + exits=["done", "blocked", "out_of_scope"], +) +``` + + +```python +def cp(task, implementation, config, reference, budget): + contract = primitive.contract(implementation.cp, scope=task.scope, reference=reference) + if not contract.done: + return blocked("CP contract not satisfied", evidence=contract) + + principle = { + "semantics": "partition sequence context while preserving attention result", + "invariants": ["CP=1 equals unsharded attention", "position and mask mapping are identical"], + "reference": reference or "unsharded attention formula", + } + implementation_contract = { + "details": ["cp_group", "zigzag_split_for_cp", "zigzag_reconstruct_from_cp_parts", "position ids", "mask partition"], + "collectives": ["context gather/scatter or ring attention path"], + "autograd": ["no non-differentiable fallback collectives"], + } + usage_contract = { + "config": require_config_keys(config, ["context_parallel_size", "sequence_length"]), + "choose_when": ["long context exceeds local memory", "attention implementation supports CP"], + "avoid_when": ["reference cannot validate positions/masks", "model attention variant lacks CP support"], + "compose_with": ["TP only with explicit head/context ownership", "PP by stage-local context policy", "THD packed sequences via zigzag THD slicing"], + } + thd_validation = None + if config.use_thd and not task.stack.contains("primitive.module.thd"): + thd_validation = primitive.module.thd(task.push("primitive.parallel.cp"), implementation=implementation, config=config, reference=reference, budget=budget.thd) + if not thd_validation.done: + return blocked("CP THD composition failed", evidence=thd_validation) + + validation = primitive.validate(task, primitive=implementation.cp, implementation=implementation, budget=budget) + risks = ["zigzag reconstruction mismatch", "position mismatch", "mask mismatch", "non-differentiable gather fallback"] + if not validation.done: + return blocked("CP validation failed", evidence=validation) + return done(principle=principle, implementation_contract=implementation_contract, usage_contract=usage_contract, validation=[thd_validation, validation], risks=risks) +``` diff --git a/experimental/lite/skills/primitive/parallel/ep.md b/experimental/lite/skills/primitive/parallel/ep.md new file mode 100644 index 00000000000..e03eb6d2fac --- /dev/null +++ b/experimental/lite/skills/primitive/parallel/ep.md @@ -0,0 +1,46 @@ +# Expert Parallel Skill + +Define, implement, use, and validate expert parallelism. + +## Schema + + +```python +schema = Skill( + "primitive.parallel.ep", kind="primitive", purpose="define and validate EP primitive", + imports=["basic.constitution"], calls=["primitive.contract", "primitive.validate"], + inputs=["task", "implementation", "config", "reference", "budget"], + outputs=["principle", "implementation_contract", "usage_contract", "validation", "risks"], + exits=["done", "blocked", "out_of_scope"], +) +``` + + +```python +def ep(task, implementation, config, reference, budget): + contract = primitive.contract(implementation.ep, scope=task.scope, reference=reference) + if not contract.done: + return blocked("EP contract not satisfied", evidence=contract) + + principle = { + "semantics": "partition experts across ranks while router semantics stay global", + "invariants": ["EP=1 equals local MoE", "dispatch/combine preserve token order and weights"], + "reference": reference or "Megatron MoE expert parallel path", + } + implementation_contract = { + "details": ["ep_group", "expert placement", "token dispatcher", "combine weights"], + "collectives": ["all_to_all token exchange", "optional grouped GEMM locality"], + "state": ["expert ownership", "capacity/drop policy", "router dtype"], + } + usage_contract = { + "config": require_config_keys(config, ["expert_model_parallel_size", "num_experts"]), + "choose_when": ["num_experts exceeds local capacity", "MoE communication is acceptable"], + "avoid_when": ["router nondeterminism is unresolved", "expert count cannot map cleanly"], + "compose_with": ["TP only with explicit expert and tensor group nesting", "FSDP/DistOpt through optimizer owner"], + } + validation = primitive.validate(task, primitive=implementation.ep, implementation=implementation, budget=budget) + risks = ["router tie instability", "token drop mismatch", "all-to-all participant mismatch"] + if not validation.done: + return blocked("EP validation failed", evidence=validation) + return done(principle=principle, implementation_contract=implementation_contract, usage_contract=usage_contract, validation=validation, risks=risks) +``` diff --git a/experimental/lite/skills/primitive/parallel/pp.md b/experimental/lite/skills/primitive/parallel/pp.md new file mode 100644 index 00000000000..831da8578cd --- /dev/null +++ b/experimental/lite/skills/primitive/parallel/pp.md @@ -0,0 +1,46 @@ +# Pipeline Parallel Skill + +Define, implement, use, and validate pipeline parallelism. + +## Schema + + +```python +schema = Skill( + "primitive.parallel.pp", kind="primitive", purpose="define and validate PP primitive", + imports=["basic.constitution"], calls=["primitive.contract", "primitive.validate"], + inputs=["task", "implementation", "config", "reference", "budget"], + outputs=["principle", "implementation_contract", "usage_contract", "validation", "risks"], + exits=["done", "blocked", "out_of_scope"], +) +``` + + +```python +def pp(task, implementation, config, reference, budget): + contract = primitive.contract(implementation.pp, scope=task.scope, reference=reference) + if not contract.done: + return blocked("PP contract not satisfied", evidence=contract) + + principle = { + "semantics": "partition ordered layers into pipeline stages", + "invariants": ["stage concatenation equals full layer order", "microbatch schedule preserves gradients"], + "reference": reference or "Megatron pipeline schedule", + } + implementation_contract = { + "details": ["stage assignment", "send/recv activation tensors", "microbatch schedule"], + "state": ["layer ownership", "activation shape", "loss stage"], + "boundaries": ["first/last stage embeddings and heads", "MTP or auxiliary heads if present"], + } + usage_contract = { + "config": require_config_keys(config, ["pipeline_model_parallel_size", "num_layers"]), + "choose_when": ["model depth exceeds single-rank memory", "stageable layer stack"], + "avoid_when": ["tiny proxy unless testing PP itself", "uneven stage ownership without explicit plan"], + "compose_with": ["TP/EP inside each stage", "FSDP/DistOpt outside stage groups"], + } + validation = primitive.validate(task, primitive=implementation.pp, implementation=implementation, budget=budget) + risks = ["stage boundary off by one", "activation shape mismatch", "schedule deadlock"] + if not validation.done: + return blocked("PP validation failed", evidence=validation) + return done(principle=principle, implementation_contract=implementation_contract, usage_contract=usage_contract, validation=validation, risks=risks) +``` diff --git a/experimental/lite/skills/primitive/parallel/tp.md b/experimental/lite/skills/primitive/parallel/tp.md new file mode 100644 index 00000000000..4ffcad1358d --- /dev/null +++ b/experimental/lite/skills/primitive/parallel/tp.md @@ -0,0 +1,46 @@ +# Tensor Parallel Skill + +Define, implement, use, and validate tensor parallelism. + +## Schema + + +```python +schema = Skill( + "primitive.parallel.tp", kind="primitive", purpose="define and validate TP primitive", + imports=["basic.constitution"], calls=["primitive.contract", "primitive.validate"], + inputs=["task", "implementation", "config", "reference", "budget"], + outputs=["principle", "implementation_contract", "usage_contract", "validation", "risks"], + exits=["done", "blocked", "out_of_scope"], +) +``` + + +```python +def tp(task, implementation, config, reference, budget): + contract = primitive.contract(implementation.tp, scope=task.scope, reference=reference) + if not contract.done: + return blocked("TP contract not satisfied", evidence=contract) + + principle = { + "semantics": "split linear/vocab dimensions across tensor-parallel ranks", + "invariants": ["TP=1 equals unsharded reference", "row/column shard axes are explicit"], + "reference": reference or "Megatron tensor-parallel layers", + } + implementation_contract = { + "details": ["tp_group", "ColumnParallelLinear", "RowParallelLinear", "VocabParallelEmbedding"], + "collectives": ["all_gather output when needed", "reduce_scatter or all_reduce gradients"], + "state": ["sharded weight layout", "bias ownership", "vocab padding"], + } + usage_contract = { + "config": require_config_keys(config, ["tensor_model_parallel_size"]), + "choose_when": ["large matmul or vocab projection", "model has TP-compatible dimensions"], + "avoid_when": ["dimension not divisible and no padding contract", "debugging non-TP primitive"], + "compose_with": ["FSDP/DistOpt through optimizer owner", "PP", "EP with explicit group nesting", "CP/THD with explicit head/context ownership"], + } + validation = primitive.validate(task, primitive=implementation.tp, implementation=implementation, budget=budget) + risks = ["wrong shard axis", "missing collective", "dtype drift across shards"] + if not validation.done: + return blocked("TP validation failed", evidence=validation) + return done(principle=principle, implementation_contract=implementation_contract, usage_contract=usage_contract, validation=validation, risks=risks) +``` diff --git a/experimental/lite/skills/primitive/principle.md b/experimental/lite/skills/primitive/principle.md new file mode 100644 index 00000000000..a30753175dd --- /dev/null +++ b/experimental/lite/skills/primitive/principle.md @@ -0,0 +1,32 @@ +# Primitive Principle Skill + +State the principle and invariants of a primitive before implementation choices. + +## Schema + + +```python +schema = Skill( + "primitive.principle", kind="primitive", purpose="define primitive principle and invariants", + imports=["basic.constitution"], calls=[], + inputs=["primitive", "reference", "constraints"], + outputs=["principle", "invariants", "reference", "risks"], exits=["done", "blocked", "out_of_scope"], +) +``` + + +```python +def principle(primitive, reference, constraints): + if reference is None and not constraints.has_first_principles: + return blocked("primitive principle needs a reference or first-principles invariant") + + principle = state_math_or_parallel_semantics(primitive, reference=reference) + invariants = [ + define_shape_dtype_rank_rules(primitive), + define_forward_backward_update_equivalence(primitive), + define_bitwise_or_threshold_contract(primitive, reference), + define_single_gpu_or_single_node_proxy(primitive), + ] + risks = ["weak principle leads to implementation-specific tests", "missing invariant hides shared bugs"] + return done(principle=principle, invariants=invariants, reference=reference, risks=risks) +``` diff --git a/experimental/lite/skills/primitive/process-group.md b/experimental/lite/skills/primitive/process-group.md new file mode 100644 index 00000000000..fc9661b41d6 --- /dev/null +++ b/experimental/lite/skills/primitive/process-group.md @@ -0,0 +1,32 @@ +# Process Group Skill + +Validate process-group ownership and collective participation. + +## Schema + + +```python +schema = Skill( + "primitive.process_group", kind="state_machine", purpose="validate process groups and collectives", + imports=["basic.constitution"], calls=[], + inputs=["task", "groups", "collectives", "budget"], + outputs=["mapping", "validation", "risks"], exits=["done", "blocked", "out_of_scope"], +) +``` + + +```python +def process_group(task, groups, collectives, budget): + mapping = derive_rank_mapping(groups) + if mapping.has_overlap_without_owner(): + return blocked("rank belongs to overlapping groups without explicit owner") + if not collectives.have_same_participants(mapping): + return blocked("collective participants diverge", evidence=collectives.diff(mapping)) + + validation = [ + smoke_collective(groups, collectives, shape="tiny"), + vary_group_size_one_axis_at_a_time(groups, budget=budget), + ] + risks = ["NCCL hang from participant mismatch", "rank-order mismatch"] + return done(mapping=mapping, validation=validation, risks=risks) +``` diff --git a/experimental/lite/skills/primitive/select-for-compose.md b/experimental/lite/skills/primitive/select-for-compose.md new file mode 100644 index 00000000000..fa856d788d0 --- /dev/null +++ b/experimental/lite/skills/primitive/select-for-compose.md @@ -0,0 +1,44 @@ +# Primitive Select For Compose Skill + +Choose primitives for model composition without coupling their implementations. + +## Schema + + +```python +schema = Skill( + "primitive.select_for_compose", kind="state_machine", purpose="choose primitives for model compose", + imports=["basic.constitution"], calls=["primitive.principle"], + inputs=["task", "model_spec", "candidates", "budget"], + outputs=["selection", "rejected", "evidence", "risks"], exits=["done", "blocked", "out_of_scope"], +) +``` + + +```python +def select_for_compose(task, model_spec, candidates, budget): + selection = [] + rejected = [] + evidence = [] + + for candidate in candidates[:budget.max_candidates]: + principle = primitive.principle(candidate, reference=candidate.reference, constraints=model_spec.constraints) + evidence.append(principle) + if not principle.done: + rejected.append((candidate, "principle missing")) + continue + if not candidate.supports(model_spec.required_features): + rejected.append((candidate, "missing required model feature")) + continue + if candidate.creates_hidden_coupling(selection): + rejected.append((candidate, "hidden primitive coupling")) + continue + selection.append(candidate) + + selection = minimize_complexity(selection, prefer=["single_gpu_proxy", "single_node_proxy", "existing_reference"]) + if not covers_required_features(selection, model_spec.required_features): + return blocked("primitive selection does not cover model spec", evidence=[evidence, rejected]) + + risks = ["selection can overfit one model family", "unsupported combinations must stay explicit"] + return done(selection=selection, rejected=rejected, evidence=evidence, risks=risks) +``` diff --git a/experimental/lite/skills/primitive/validate.md b/experimental/lite/skills/primitive/validate.md new file mode 100644 index 00000000000..34e4b97b96a --- /dev/null +++ b/experimental/lite/skills/primitive/validate.md @@ -0,0 +1,38 @@ +# Primitive Validate Skill + +Validate one primitive before using it in a model. + +## Schema + + +```python +schema = Skill( + "primitive.validate", kind="state_machine", purpose="validate primitive correctness and precision", + imports=["basic.constitution"], calls=["basic.construct_proxy_task", "basic.align_precision"], + inputs=["task", "primitive", "implementation", "budget"], + outputs=["validation", "evidence", "risks"], exits=["done", "blocked", "out_of_scope"], +) +``` + + +```python +def validate(task, primitive, implementation, budget): + proxy = basic.construct_proxy_task( + task, target=primitive, reference=implementation.reference, variables=implementation.variables, budget=budget.proxy + ) + if not proxy.done: + return blocked("primitive proxy task not constructed", evidence=proxy) + + precision = basic.align_precision(task, target=primitive, variables=implementation.variables, budget=budget.precision) + if not precision.done: + return blocked("primitive precision not validated", evidence=precision) + + validation = [ + "static_contract", + "single_gpu_or_node_proxy", + "controlled_variable_precision", + "composition_with_adjacent_primitives", + "usage_example_runs", + ] + return done(validation=validation, evidence=[proxy, precision], risks=precision.next) +``` diff --git a/experimental/lite/tests/README.md b/experimental/lite/tests/README.md new file mode 100644 index 00000000000..6917dab6034 --- /dev/null +++ b/experimental/lite/tests/README.md @@ -0,0 +1,47 @@ +# Megatron Lite Validation + +`experimental/lite/tests` separates MLite validation into two layers: + +- Unit tests: CPU/single-process contract tests for primitive, model, runtime, checkpoint sentinels, config, and helper behavior. Pure helper tests stub optional imports when no Transformer Engine runtime path is exercised; tests that need the real package explicitly skip when unavailable. +- Smoke tests: real `torch.distributed` tests for TP/EP/PP/CP/FSDP2/offload/checkpoint/distopt and tiny Qwen lite forward/backward behavior. Smoke runs are capped at one node and at most 8 GPUs. + +Run unit coverage: + +```bash +PYTHONPATH="$(pwd):$(pwd)/experimental/lite" pytest experimental/lite/tests/unit +``` + +Run smoke coverage on one node: + +```bash +PYTHONPATH="$(pwd):$(pwd)/experimental/lite" MLITE_RUN_SMOKE=1 MLITE_SMOKE_NPROC=8 \ + experimental/lite/tests/run_primitive_validation.sh +``` + +The smoke suite is skipped by default in regular `pytest` runs. Enable it with `--mlite-smoke` or `MLITE_RUN_SMOKE=1`. + +Current matrix: + +| Surface | Unit | Smoke | +| --- | --- | --- | +| TP/EP/PP/CP/SP topology | `unit/primitive/test_parallel_unit.py`, `unit/primitive/test_parallel_dimensions_independent_unit.py` | `smoke/primitive/test_parallel_topologies_smoke.py` | +| TP linear/vocab primitives | `unit/primitive/test_parallel_dimensions_independent_unit.py` | Qwen model smoke exercises TP linear surfaces | +| EP token dispatch | `unit/primitive/test_parallel_dimensions_independent_unit.py` | Qwen model smoke exercises router, dispatcher, and experts | +| THD packing helpers | `unit/primitive/test_parallel_unit.py` | CP topology smoke exercises distributed CP groups | +| GQA/attention split contract | `unit/primitive/test_attention_moe_unit.py` | Qwen model smoke exercises attention forward/backward | +| MoE router/aux-loss contract | `unit/primitive/test_attention_moe_unit.py` | Qwen model smoke exercises router, dispatcher, and experts | +| LoRA adapter primitives | `unit/primitive/test_module_primitives_independent_unit.py` | Qwen model smoke can enable adapters in follow-up coverage | +| MTP/MRoPE/Gated Delta helper contracts | `unit/primitive/test_module_primitives_independent_unit.py`, `unit/primitive/test_ops_data_trainstep_unit.py` | Qwen3.5 MoE model smoke exercises MRoPE/Gated DeltaNet paths | +| Loss/logprob/math ops | `unit/primitive/test_ops_data_trainstep_unit.py` | Qwen model smoke exercises loss plumbing | +| Data/recompute/train-step primitives | `unit/primitive/test_ops_data_trainstep_unit.py` | model/runtime smoke exercises training loop integration | +| DDP + distributed optimizer | `unit/primitive/test_checkpoint_unit.py`, `unit/primitive/test_checkpoint_runtime.py` | `smoke/primitive/test_distopt_checkpoint_smoke.py` | +| FSDP2 config/wrap/offload | `unit/primitive/test_fsdp2_unit.py` | `smoke/primitive/test_fsdp2_offload_checkpoint_smoke.py` | +| FSDP2 save/load resume | `unit/primitive/test_checkpoint_unit.py`, `unit/primitive/test_checkpoint_runtime.py` | `smoke/primitive/test_fsdp2_offload_checkpoint_smoke.py` | +| Checkpoint restore vs direct training | `unit/primitive/test_checkpoint_unit.py`, `unit/primitive/test_checkpoint_runtime.py` | FSDP2 and distopt checkpoint smokes cover distributed restore paths | +| Runtime backend registry/config | `unit/primitive/test_runtime_config_unit.py`, `unit/runtime/test_runtime_backend_unit.py` | covered through checkpoint/model handles | +| Runtime env/offload controls | `unit/runtime/test_runtime_backend_unit.py` | `smoke/primitive/test_fsdp2_offload_checkpoint_smoke.py` | +| Optimizer update-state offload fraction | `unit/primitive/test_runtime_config_unit.py` and single-process CUDA coverage in `unit/primitive/test_fsdp2_offload_gpu.py` | multi-rank offloaded grad clipping is checked against the non-offloaded baseline in `smoke/primitive/test_fsdp2_offload_checkpoint_smoke.py` | +| Qwen3 MoE lite config/build/forward | `unit/model/test_qwen_config_unit.py` | `smoke/model/test_qwen_lite_forward_smoke.py` | +| Qwen3.5 MoE lite config/build/forward | `unit/model/test_qwen_config_unit.py` | `smoke/model/test_qwen_lite_forward_smoke.py` | + +Classic FSDP is not a separate MLite primitive in the current source tree; MLite's native sharded optimizer coverage is FSDP2 plus Megatron DDP/distopt. diff --git a/experimental/lite/tests/conftest.py b/experimental/lite/tests/conftest.py new file mode 100644 index 00000000000..ad048bdefbc --- /dev/null +++ b/experimental/lite/tests/conftest.py @@ -0,0 +1,104 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +from __future__ import annotations + +import os +import sys +import types +from pathlib import Path + +import pytest + +LITE_ROOT = Path(__file__).resolve().parents[1] +REPO_ROOT = Path(__file__).resolve().parents[3] +VERL_EXAMPLE_ROOT = LITE_ROOT / "examples" / "verl" +for root in (REPO_ROOT, LITE_ROOT, VERL_EXAMPLE_ROOT): + if str(root) not in sys.path: + sys.path.insert(0, str(root)) + + +def pytest_configure(config): + config.addinivalue_line("markers", "mlite: mark a test as Megatron Lite validation coverage") + config.addinivalue_line( + "markers", + "smoke: mark a Megatron Lite smoke test; skipped unless --mlite-smoke or MLITE_RUN_SMOKE=1 is set", + ) + config.addinivalue_line("markers", "gpu: mark a test as requiring CUDA") + config.addinivalue_line("markers", "distributed: mark a test as requiring torch.distributed") + + +def pytest_addoption(parser): + parser.addoption( + "--mlite-smoke", action="store_true", default=False, help="run Megatron Lite smoke tests" + ) + + +def pytest_collection_modifyitems(config, items): + run_smoke = config.getoption("--mlite-smoke") or os.getenv("MLITE_RUN_SMOKE") == "1" + if run_smoke: + return + skip_smoke = pytest.mark.skip(reason="set --mlite-smoke or MLITE_RUN_SMOKE=1 to run") + for item in items: + if "smoke" in item.keywords: + item.add_marker(skip_smoke) + + +@pytest.fixture +def transformer_engine_import_stub(monkeypatch): + def install() -> None: + try: + import transformer_engine.pytorch # noqa: F401 + + return + except (ModuleNotFoundError, OSError) as exc: + if isinstance(exc, ModuleNotFoundError) and exc.name not in { + "transformer_engine", + "transformer_engine.pytorch", + }: + raise + + class _UnavailableTE: + def __init__(self, *args, **kwargs): + raise RuntimeError("Transformer Engine is not installed in this test environment.") + + root = types.ModuleType("transformer_engine") + root.__version__ = "0.0.0" + pytorch = types.ModuleType("transformer_engine.pytorch") + pytorch.DotProductAttention = _UnavailableTE + pytorch.LayerNormLinear = _UnavailableTE + pytorch.Linear = _UnavailableTE + pytorch.RMSNorm = _UnavailableTE + permutation = types.ModuleType("transformer_engine.pytorch.permutation") + router = types.ModuleType("transformer_engine.pytorch.router") + cpp_extensions = types.ModuleType("transformer_engine.pytorch.cpp_extensions") + module = types.ModuleType("transformer_engine.pytorch.module") + module_base = types.ModuleType("transformer_engine.pytorch.module.base") + + def unavailable_kernel(*args, **kwargs): + raise RuntimeError("Transformer Engine fused kernel is not installed.") + + permutation.moe_permute = unavailable_kernel + permutation.moe_permute_and_pad_with_probs = unavailable_kernel + permutation.moe_permute_with_probs = unavailable_kernel + permutation.moe_unpermute = unavailable_kernel + router.fused_compute_score_for_moe_aux_loss = unavailable_kernel + router.fused_moe_aux_loss = unavailable_kernel + router.fused_topk_with_score_function = unavailable_kernel + cpp_extensions.general_gemm = lambda *args, **kwargs: None + module_base.get_workspace = lambda: None + module.base = module_base + pytorch.permutation = permutation + pytorch.router = router + pytorch.cpp_extensions = cpp_extensions + pytorch.module = module + root.pytorch = pytorch + monkeypatch.setitem(sys.modules, "transformer_engine", root) + monkeypatch.setitem(sys.modules, "transformer_engine.pytorch", pytorch) + monkeypatch.setitem(sys.modules, "transformer_engine.pytorch.permutation", permutation) + monkeypatch.setitem(sys.modules, "transformer_engine.pytorch.router", router) + monkeypatch.setitem( + sys.modules, "transformer_engine.pytorch.cpp_extensions", cpp_extensions + ) + monkeypatch.setitem(sys.modules, "transformer_engine.pytorch.module", module) + monkeypatch.setitem(sys.modules, "transformer_engine.pytorch.module.base", module_base) + + return install diff --git a/experimental/lite/tests/run_layering_contracts.sh b/experimental/lite/tests/run_layering_contracts.sh new file mode 100755 index 00000000000..967ef50b4d5 --- /dev/null +++ b/experimental/lite/tests/run_layering_contracts.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +cd "$REPO_ROOT" + +export PYTHONPATH="$REPO_ROOT:$REPO_ROOT/experimental/lite${PYTHONPATH:+:$PYTHONPATH}" +pytest -q experimental/lite/tests/unit/runtime/test_layering_contracts.py "$@" diff --git a/experimental/lite/tests/run_primitive_validation.sh b/experimental/lite/tests/run_primitive_validation.sh new file mode 100755 index 00000000000..d2884a2f1a7 --- /dev/null +++ b/experimental/lite/tests/run_primitive_validation.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +cd "${ROOT}" + +export PYTHONPATH="${ROOT}:${ROOT}/experimental/lite:${PYTHONPATH:-}" + +pytest experimental/lite/tests/unit "$@" + +if [[ "${MLITE_RUN_SMOKE:-0}" == "1" ]]; then + NPROC="${MLITE_SMOKE_NPROC:-${WORLD_SIZE:-1}}" + if (( NPROC < 1 || NPROC > 8 )); then + echo "MLITE smoke tests require 1 <= MLITE_SMOKE_NPROC <= 8, got ${NPROC}" >&2 + exit 2 + fi + torchrun --standalone --nproc_per_node="${NPROC}" \ + -m pytest --mlite-smoke experimental/lite/tests/smoke "$@" +fi diff --git a/experimental/lite/tests/smoke/model/test_glm5_lite_fsdp2_smoke.py b/experimental/lite/tests/smoke/model/test_glm5_lite_fsdp2_smoke.py new file mode 100644 index 00000000000..7aabf7b493d --- /dev/null +++ b/experimental/lite/tests/smoke/model/test_glm5_lite_fsdp2_smoke.py @@ -0,0 +1,115 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +from __future__ import annotations + +import os + +import pytest + +pytestmark = [pytest.mark.mlite, pytest.mark.smoke, pytest.mark.gpu, pytest.mark.distributed] + + +def _init_dist_or_skip(): + import torch + import torch.distributed as dist + + if not torch.cuda.is_available(): + pytest.skip("CUDA is required for GLM5 FSDP2 smoke.") + if "RANK" not in os.environ or "WORLD_SIZE" not in os.environ: + pytest.skip("Run with torchrun so FSDP2 ranks are available.") + if int(os.environ.get("WORLD_SIZE", "1")) < 2: + pytest.skip("GLM5 FSDP2 smoke requires at least 2 ranks.") + if int(os.environ.get("WORLD_SIZE", "1")) > 8: + pytest.skip("Megatron Lite smoke tests are capped at single-node 8 GPUs.") + + local_rank = int(os.environ.get("LOCAL_RANK", "0")) + torch.cuda.set_device(local_rank) + created_pg = False + if not dist.is_initialized(): + dist.init_process_group("nccl") + created_pg = True + yield torch.device("cuda", local_rank) + if created_pg and dist.is_initialized(): + dist.destroy_process_group() + + +@pytest.fixture(scope="module") +def cuda_dist(): + yield from _init_dist_or_skip() + + +def _tiny_config_kwargs(): + return dict( + num_hidden_layers=2, + hidden_size=128, + num_attention_heads=64, + num_key_value_heads=64, + head_dim=256, + vocab_size=32, + max_position_embeddings=512, + initializer_range=0.002, + q_lora_rank=16, + kv_lora_rank=512, + qk_head_dim=256, + qk_nope_head_dim=192, + qk_rope_head_dim=64, + v_head_dim=256, + index_head_dim=128, + index_n_heads=32, + index_topk=512, + intermediate_size=20, + moe_intermediate_size=6, + first_k_dense_replace=1, + n_routed_experts=3, + n_shared_experts=1, + num_experts_per_tok=3, + ) + + +def test_glm5_tiny_model_builds_and_steps_with_fsdp2_backend(cuda_dist): + import torch + + from megatron.lite.model.glm5.config import Glm5Config + from megatron.lite.model.glm5.lite import protocol + from megatron.lite.primitive.optimizers.fsdp2 import FSDP2Optimizer, fsdp2_available + from megatron.lite.runtime.contracts import OptimizerConfig, ParallelConfig + + if not fsdp2_available(): + pytest.skip("Installed PyTorch does not expose FSDP2 fully_shard.") + + device = cuda_dist + cfg = Glm5Config(**_tiny_config_kwargs()) + impl_cfg = protocol.ImplConfig( + parallel=ParallelConfig(), + optimizer="fsdp2", + optimizer_config=OptimizerConfig( + optimizer="adam", lr=1.0e-3, weight_decay=0.0, clip_grad=1.0, offload_fraction=0.0 + ), + deterministic=True, + ) + + torch.manual_seed(20260612) + torch.cuda.manual_seed_all(20260612) + bundle = protocol.build_model(cfg, impl_cfg=impl_cfg) + assert bundle.extras["optimizer_backend"] == "fsdp2" + assert bundle.optimizer is None + assert callable(bundle.extras["post_model_load_hook"]) + + model = bundle.chunks[0] + model.initialize_weights() + updates = bundle.extras["post_model_load_hook"]() + bundle.optimizer = updates["optimizer"] + assert isinstance(bundle.optimizer, FSDP2Optimizer) + + model.train() + batch, seq = 1, 512 + input_ids = torch.randint(0, cfg.vocab_size, (batch, seq), device=device) + bundle.optimizer.zero_grad() + output = model(input_ids=input_ids) + assert output["logits"].shape == (batch, seq, cfg.vocab_size) + loss = output["logits"].float().square().mean() + assert torch.isfinite(loss) + loss.backward() + + success, grad_norm, _ = bundle.optimizer.step() + assert success + assert torch.isfinite(torch.tensor(grad_norm, device=device)) diff --git a/experimental/lite/tests/smoke/model/test_qwen_lite_forward_smoke.py b/experimental/lite/tests/smoke/model/test_qwen_lite_forward_smoke.py new file mode 100644 index 00000000000..d5a03c32d20 --- /dev/null +++ b/experimental/lite/tests/smoke/model/test_qwen_lite_forward_smoke.py @@ -0,0 +1,164 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +from __future__ import annotations + +import os +from types import SimpleNamespace + +import pytest +import torch +import torch.distributed as dist + +pytestmark = [pytest.mark.mlite, pytest.mark.smoke, pytest.mark.gpu, pytest.mark.distributed] + + +def _qwen3_symbols(): + pytest.importorskip("transformer_engine.pytorch") + from megatron.lite.model.qwen3_moe.config import Qwen3MoEConfig + from megatron.lite.model.qwen3_moe.lite.model import Qwen3MoEModel + + return Qwen3MoEConfig, Qwen3MoEModel + + +def _qwen35_symbols(): + pytest.importorskip("transformer_engine.pytorch") + from megatron.lite.model.qwen3_5.config import Qwen35Config + from megatron.lite.model.qwen3_5.lite.model import Qwen35Model + + return Qwen35Config, Qwen35Model + + +@pytest.fixture(scope="module", autouse=True) +def _single_node_cuda_dist(): + if not torch.cuda.is_available(): + pytest.skip("CUDA is required for Qwen lite model smoke tests.") + if int(os.environ.get("WORLD_SIZE", "1")) > 8: + pytest.skip("Megatron Lite smoke tests are capped at single-node 8 GPUs.") + + os.environ.setdefault("RANK", "0") + os.environ.setdefault("WORLD_SIZE", "1") + os.environ.setdefault("LOCAL_RANK", "0") + os.environ.setdefault("MASTER_ADDR", "127.0.0.1") + os.environ.setdefault("MASTER_PORT", "29531") + + torch.cuda.set_device(int(os.environ["LOCAL_RANK"])) + created_pg = False + if not dist.is_initialized(): + dist.init_process_group(backend="nccl", init_method="env://") + created_pg = True + yield + if created_pg and dist.is_initialized(): + dist.destroy_process_group() + + +def _tiny_qwen3_config(): + Qwen3MoEConfig, _Qwen3MoEModel = _qwen3_symbols() + return Qwen3MoEConfig( + num_hidden_layers=1, + hidden_size=16, + num_attention_heads=4, + num_key_value_heads=2, + head_dim=4, + vocab_size=64, + num_experts=2, + num_experts_per_tok=1, + moe_intermediate_size=8, + max_position_embeddings=16, + layer_types=["full_attention"], + ) + + +def _tiny_qwen35_config(): + Qwen35Config, _Qwen35Model = _qwen35_symbols() + return Qwen35Config( + num_hidden_layers=1, + hidden_size=16, + num_attention_heads=4, + num_key_value_heads=2, + head_dim=4, + vocab_size=64, + num_experts=2, + num_experts_per_tok=1, + moe_intermediate_size=8, + shared_expert_intermediate_size=8, + linear_num_key_heads=2, + linear_key_head_dim=4, + linear_num_value_heads=2, + linear_value_head_dim=4, + linear_conv_kernel_dim=2, + max_position_embeddings=16, + partial_rotary_factor=1.0, + mrope_section=[1, 1, 0], + layer_types=["linear_attention"], + ) + + +def _parallel_state(): + from megatron.lite.primitive.parallel import init_parallel + from megatron.lite.runtime.contracts.config import ParallelConfig + + return init_parallel(ParallelConfig(tp=1, etp=1, ep=1, pp=1, cp=1)) + + +def _token_batch(vocab_size: int): + torch.manual_seed(9876 + dist.get_rank()) + input_ids = torch.randint(0, vocab_size, (2, 4), device="cuda") + labels = torch.randint(0, vocab_size, (2, 4), device="cuda") + return input_ids, labels + + +def _assert_loss_and_backward(output: dict, model: torch.nn.Module): + loss = output["loss"] + assert loss.ndim == 0 + assert torch.isfinite(loss) + loss.backward() + grad_params = [ + param for param in model.parameters() if param.requires_grad and param.grad is not None + ] + assert grad_params + assert all( + torch.isfinite(param.grad.detach().float()).all() for param in grad_params + ) + + +def test_qwen3_moe_lite_tiny_forward_backward_smoke(): + _Qwen3MoEConfig, Qwen3MoEModel = _qwen3_symbols() + config = _tiny_qwen3_config() + model = ( + Qwen3MoEModel(config, _parallel_state(), use_deepep=False) + .cuda() + .to(torch.bfloat16) + ) + input_ids, labels = _token_batch(config.vocab_size) + + output = model(input_ids=input_ids, labels=labels, return_log_probs=True) + + assert output["hidden_states"].shape[-1] == config.hidden_size + assert output["log_probs"].shape == labels.shape + _assert_loss_and_backward(output, model) + + +def test_qwen35_lite_tiny_forward_backward_smoke(): + _Qwen35Config, Qwen35Model = _qwen35_symbols() + config = _tiny_qwen35_config() + train_config = SimpleNamespace( + tp=1, + ep=1, + etp=1, + pp=1, + cp=1, + vpp=None, + use_deepep=False, + fp8=False, + recompute_modules=[], + deterministic=True, + ) + model = ( + Qwen35Model(config, train_config, _parallel_state()).cuda().to(torch.bfloat16) + ) + input_ids, labels = _token_batch(config.vocab_size) + + output = model(input_ids=input_ids, labels=labels) + + assert output["hidden_states"].shape[-1] == config.hidden_size + assert output["log_probs"].shape == labels.shape + _assert_loss_and_backward(output, model) diff --git a/experimental/lite/tests/smoke/primitive/test_auto_pipeline_layout_smoke.py b/experimental/lite/tests/smoke/primitive/test_auto_pipeline_layout_smoke.py new file mode 100644 index 00000000000..64a4cb52ea3 --- /dev/null +++ b/experimental/lite/tests/smoke/primitive/test_auto_pipeline_layout_smoke.py @@ -0,0 +1,454 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Pipeline-layout smoke: real models build through the layout and train on PP>1. + +Builds {qwen3_5, qwen3_moe, kimi_k2, glm5, deepseek_v4} with ``num_hidden_layers=9`` +on pp=4 (not divisible) and asserts the layout drives a real model, not just the +layout math: each builds, trains a finite-loss step over the uneven split, and HF +export reconstructs every decoder layer. This exercises the auto layout mode on real +models; the custom ``pp_layout`` string mode and the exact per-stage splits (incl. +the real 43/61/78 counts) are pinned in the deterministic unit tests. + +Topology follows the save/load/export smoke's validated dist_opt capability: +tp2/ep2 for TP-capable models, tp1/ep2 for glm5 / deepseek_v4 (native lite TP=1). +CP=1: cp>1 on these tiny proxy seqs breaks TE attention backend selection +(orthogonal to layout; covered by the dedicated CP smokes). + +Run: torchrun --nproc_per_node=8 -m pytest --mlite-smoke, selecting per-env subsets +with -k (qwen3_5 on the qwen3.5 site; the rest on the DSA overlay). Models also +gate themselves with importorskip. +""" +from __future__ import annotations + +import os +import re +from datetime import timedelta + +import pytest +import torch +import torch.distributed as dist +from megatron.lite.primitive.ckpt.hf_weights import unwrap_model +from megatron.lite.primitive.deterministic import set_deterministic +from megatron.lite.runtime.backends.mlite.runtime import MegatronLiteRuntime +from megatron.lite.runtime.contracts.config import OptimizerConfig, ParallelConfig +from megatron.lite.runtime.contracts.data import PackedBatch + +pytestmark = [pytest.mark.mlite, pytest.mark.smoke, pytest.mark.gpu, pytest.mark.distributed] + +# 9 decoders over pp=4 is non-divisible; 9 is small enough to stay fast yet leaves no +# empty stage even for the MTP model (deepseek_v4 adds a slot on the last stage). +_NUM_LAYERS = 9 +_PP = 4 + +# GLM5 / DeepSeek-V4 native lite support TP=1 only (matches the save/load smoke). +_TP1_ONLY = {"glm5", "deepseek_v4"} + + +def _require_te() -> None: + te = pytest.importorskip( + "transformer_engine.pytorch", + reason="auto pipeline-layout smoke requires real Transformer Engine.", + ) + assert hasattr(te, "Linear"), "smoke requires real Transformer Engine Linear." + + +def _optimizer_config() -> OptimizerConfig: + return OptimizerConfig( + optimizer="adam", + lr=1.0e-3, + weight_decay=0.0, + adam_beta1=0.9, + adam_beta2=0.95, + adam_eps=1.0e-8, + clip_grad=1.0, + offload_fraction=0.0, + ) + + +def _random_packed_batch(vocab_size: int) -> PackedBatch: + return PackedBatch( + input_ids=torch.randint(0, vocab_size, (2048,), device="cuda"), + labels=torch.randint(0, vocab_size, (2048,), device="cuda"), + seq_lens=torch.full((1,), 2048, dtype=torch.int64, device="cuda"), + ) + + +# ────────────────────────────────────────────────────────────────────────── +# Tiny non-divisible model configs (mirror the save/load smoke, num_hidden_layers=9). +# ────────────────────────────────────────────────────────────────────────── +def _qwen3_5(): + pytest.importorskip("fla", reason="qwen3_5 needs the FLA / GatedDeltaNet stack.") + _require_te() + from megatron.lite.model.qwen3_5.config import Qwen35Config + from megatron.lite.model.qwen3_5.lite import protocol + + cfg = Qwen35Config( + num_hidden_layers=_NUM_LAYERS, + hidden_size=16, + num_attention_heads=4, + num_key_value_heads=2, + head_dim=4, + vocab_size=64, + num_experts=4, + num_experts_per_tok=2, + moe_intermediate_size=8, + shared_expert_intermediate_size=8, + linear_num_key_heads=2, + linear_key_head_dim=4, + linear_num_value_heads=2, + linear_value_head_dim=4, + linear_conv_kernel_dim=4, + layer_types=(["full_attention", "linear_attention"] * ((_NUM_LAYERS + 1) // 2))[ + :_NUM_LAYERS + ], + partial_rotary_factor=1.0, + max_position_embeddings=4096, + ) + return cfg, protocol + + +def _qwen3_moe(): + _require_te() + from megatron.lite.model.qwen3_moe.config import Qwen3MoEConfig + from megatron.lite.model.qwen3_moe.lite import protocol + + cfg = Qwen3MoEConfig( + num_hidden_layers=_NUM_LAYERS, + hidden_size=16, + num_attention_heads=4, + num_key_value_heads=2, + head_dim=4, + vocab_size=64, + num_experts=4, + num_experts_per_tok=1, + moe_intermediate_size=8, + max_position_embeddings=4096, + layer_types=["full_attention"] * _NUM_LAYERS, + ) + return cfg, protocol + + +def _kimi_k2(): + _require_te() + from megatron.lite.model.kimi_k2.config import KimiK2Config + from megatron.lite.model.kimi_k2.lite import protocol + + cfg = KimiK2Config( + num_hidden_layers=_NUM_LAYERS, + hidden_size=64, + num_attention_heads=4, + num_key_value_heads=4, + vocab_size=128, + intermediate_size=96, + moe_intermediate_size=16, + n_routed_experts=4, + n_shared_experts=1, + num_experts_per_tok=2, + n_group=2, + topk_group=1, + first_k_dense_replace=1, + q_lora_rank=16, + kv_lora_rank=12, + qk_nope_head_dim=8, + qk_rope_head_dim=8, + v_head_dim=8, + max_position_embeddings=4096, + rope_theta=10000.0, + rope_scaling={ + "type": "yarn", + "factor": 1.0, + "original_max_position_embeddings": 4096, + "beta_fast": 1.0, + "beta_slow": 1.0, + "mscale": 1.0, + "mscale_all_dim": 1.0, + }, + ) + return cfg, protocol + + +def _glm5(): + pytest.importorskip("cudnn", reason="glm5 fused DSA needs the cudnn DSA stack.") + _require_te() + from megatron.lite.model.glm5.config import Glm5Config + from megatron.lite.model.glm5.lite import protocol + + cfg = Glm5Config( + num_hidden_layers=_NUM_LAYERS, + hidden_size=128, + num_attention_heads=64, + num_key_value_heads=64, + head_dim=256, + vocab_size=32, + max_position_embeddings=4096, + initializer_range=0.002, + q_lora_rank=16, + kv_lora_rank=512, + qk_head_dim=256, + qk_nope_head_dim=192, + qk_rope_head_dim=64, + v_head_dim=256, + index_head_dim=128, + index_n_heads=32, + index_topk=512, + intermediate_size=20, + moe_intermediate_size=6, + first_k_dense_replace=1, + n_routed_experts=4, + n_shared_experts=1, + num_experts_per_tok=2, + ) + return cfg, protocol + + +def _deepseek_v4(): + pytest.importorskip("cudnn", reason="deepseek_v4 fused DSA needs the cudnn DSA stack.") + _require_te() + from megatron.lite.model.deepseek_v4.config import DeepseekV4Config + from megatron.lite.model.deepseek_v4.lite import protocol + + cfg = DeepseekV4Config( + vocab_size=64, + hidden_size=128, + moe_intermediate_size=16, + num_hidden_layers=_NUM_LAYERS, + num_attention_heads=8, + num_key_value_heads=1, + head_dim=64, + qk_rope_head_dim=16, + q_lora_rank=32, + o_lora_rank=32, + o_groups=2, + n_routed_experts=4, + n_shared_experts=1, + num_experts_per_tok=2, + routed_scaling_factor=1.5, + max_position_embeddings=4096, + compress_ratios=[4, 4], + sliding_window=128, + num_hash_layers=2, + hc_mult=2, + index_head_dim=64, + index_n_heads=8, + index_topk=512, + # Real MTP: an extra nextn layer on the last stage exercises the + # MTP-aware branch of the auto layout balancing. + num_nextn_predict_layers=1, + rms_norm_eps=1e-6, + ) + return cfg, protocol + + +MODELS = { + "qwen3_5": _qwen3_5, + "qwen3_moe": _qwen3_moe, + "kimi_k2": _kimi_k2, + "glm5": _glm5, + "deepseek_v4": _deepseek_v4, +} + + +# ────────────────────────────────────────────────────────────────────────── +# Distributed harness (mirrors the save/load/export smoke). +# ────────────────────────────────────────────────────────────────────────── +@pytest.fixture(scope="module", autouse=True) +def _single_node_cuda_dist(): + if not torch.cuda.is_available(): + pytest.skip("CUDA is required for auto pipeline-layout smoke.") + if int(os.environ.get("WORLD_SIZE", "1")) > 8: + pytest.skip("Megatron Lite smoke tests are capped at single-node 8 GPUs.") + + os.environ.setdefault("CUBLAS_WORKSPACE_CONFIG", ":4096:8") + os.environ.setdefault("NVTE_ALLOW_NONDETERMINISTIC_ALGO", "0") + os.environ.setdefault("RANK", "0") + os.environ.setdefault("WORLD_SIZE", "1") + os.environ.setdefault("LOCAL_RANK", "0") + os.environ.setdefault("MASTER_ADDR", "127.0.0.1") + os.environ.setdefault("MASTER_PORT", "29577") + + torch.cuda.set_device(int(os.environ["LOCAL_RANK"])) + created_pg = False + if not dist.is_initialized(): + timeout_s = int(os.environ.get("MLITE_DIST_TIMEOUT_S", "180")) + dist.init_process_group( + backend="nccl", init_method="env://", timeout=timedelta(seconds=timeout_s) + ) + created_pg = True + yield + try: + from megatron.core import parallel_state as mpu + + if mpu.is_initialized(): + mpu.destroy_model_parallel() + finally: + if created_pg and dist.is_initialized(): + dist.destroy_process_group() + + +# Process groups lite's init_parallel creates per build, recorded so the autouse +# teardown can free them (mcore's destroy_model_parallel frees only mcore's groups). +_BUILT_PARALLEL_STATES: list = [] +_PS_GROUP_ATTRS = ( + "tp_group", "ep_group", "etp_group", "cp_group", "pp_group", "pp_cpu_group", + "dp_group", "dp_cp_group", "tp_ep_group", "ep_dp_group", +) + + +@pytest.fixture(autouse=True) +def _reset_parallel_state_between_tests(): + yield + import gc + + from megatron.core import parallel_state as mpu + + if mpu.is_initialized(): + mpu.destroy_model_parallel() + # Finalize the just-built model first so DDP releases its group references, then + # explicitly destroy the process groups lite's init_parallel created for this + # test. Without this, lite's ~10 fresh NCCL/gloo groups per build leak across the + # ~6 builds in one torchrun process and make a later PP P2P / export collective + # fail nondeterministically (mcore's destroy_model_parallel frees only its own). + gc.collect() + for ps in _BUILT_PARALLEL_STATES: + for attr in _PS_GROUP_ATTRS: + group = getattr(ps, attr, None) + if group is not None: + try: + dist.destroy_process_group(group) + except Exception: + pass + _BUILT_PARALLEL_STATES.clear() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + + +def _proxy_topology(model_name: str, pp_layout=None) -> ParallelConfig: + # PP is held at 4 (the layout axis under test); the rest follow the save/load + # smoke's validated dist_opt capability. CP=1 — see module docstring. + # pp_layout=None -> auto mode; a string -> custom mode (advanced explicit layout). + if model_name in _TP1_ONLY: # glm5 / deepseek_v4: native lite is TP=1 only. + return ParallelConfig(tp=1, ep=2, etp=1, pp=_PP, cp=1, pp_layout=pp_layout) + return ParallelConfig(tp=2, ep=2, etp=1, pp=_PP, cp=1, pp_layout=pp_layout) + + +def _build_handle(model_name: str, *, seed: int, pp_layout=None): + from types import SimpleNamespace + + from megatron.lite.runtime.contracts.handle import ModelHandle + + cfg, protocol = MODELS[model_name]() + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + + parallel = _proxy_topology(model_name, pp_layout=pp_layout) + impl_cfg = protocol.ImplConfig( + parallel=parallel, + optimizer="dist_opt", + optimizer_config=_optimizer_config(), + use_deepep=False, + deterministic=True, + ) + bundle = protocol.build_model(cfg, impl_cfg=impl_cfg) + chunks = bundle.chunks + extras = dict(bundle.extras) + extras.update( + { + "model_chunks": chunks, + "forward_step": bundle.forward_step, + "finalize_grads": bundle.finalize_grads, + "protocol": protocol, + } + ) + handle = ModelHandle( + model=chunks, + optimizer=bundle.optimizer, + parallel_state=bundle.parallel_state, + config=SimpleNamespace(parallel=parallel), + _extras=extras, + ) + _BUILT_PARALLEL_STATES.append(bundle.parallel_state) + return handle, cfg + + +def _local_layer_indices(handle) -> list[int]: + chunk = unwrap_model(handle._extras["model_chunks"][0]) + return list(chunk.layer_indices) + + +# Decoder layer ids in exported HF names: matches both `model.layers.N.` (HF-rooted, +# most models) and bare `layers.N.` (deepseek_v4-flash release naming). +_HF_LAYER_RE = re.compile(r"(?:^|\.)layers\.(\d+)\.") + + +def _hf_decoder_layer_indices(names) -> set[int]: + """Decoder layer ids referenced by exported HF weight names (``...layers.N...``).""" + return {int(m.group(1)) for n in names if (m := _HF_LAYER_RE.search(n))} + + +@pytest.mark.parametrize("model_name", list(MODELS)) +def test_uneven_pp_builds_trains_and_exports(model_name): + """A non-divisible layer count builds an uneven PP split, trains a step, and + exports every layer. Build + train + export share one model build (one heavy + distributed build per test keeps the builds in one torchrun process bounded). + + - build: the non-divisible count produces a valid balanced uneven split (not + "not divisible"); this stage owns a sorted, in-range, contiguous decoder run. + - train: one real step over the uneven pipeline yields a finite loss (proves PP + P2P across stages of unequal depth actually runs the built model). + - export: HF export all-gathers the per-stage shards (keyed on global layer + names) back to a complete state — every decoder layer 0..N-1 reconstructed, + every tensor finite. Regression guard for the deepseek_v4 local-vs-global + layer-key bug; a dropped/duplicated stage would silently corrupt exports. + """ + if dist.get_world_size() != 8: + pytest.skip("auto pipeline-layout proxy smoke requires exactly 8 GPUs.") + + set_deterministic(2026) + + handle, cfg = _build_handle(model_name, seed=4242) + ps = handle._parallel_state + assert ps.pp_size == _PP + assert cfg.num_hidden_layers == _NUM_LAYERS + + local = _local_layer_indices(handle) + assert local == sorted(local) and len(set(local)) == len(local), local + assert local and all(0 <= i < _NUM_LAYERS for i in local), local + assert local == list(range(local[0], local[0] + len(local))), local + + runtime = MegatronLiteRuntime.__new__(MegatronLiteRuntime) + batch = _random_packed_batch(cfg.vocab_size) + runtime.zero_grad(handle) + result = runtime.forward_backward(handle, iter([batch]), None, num_microbatches=1) + runtime.optimizer_step(handle) + loss = result.model_output.loss + assert loss is not None and torch.isfinite(loss).all(), ( + f"{model_name}: non-finite loss {loss} on uneven PP layout" + ) + + # Export across the uneven PP split (every rank materializes the full HF state). + protocol = handle._extras["protocol"] + chunks = handle._extras["model_chunks"] + exported = list(protocol.export_hf_weights(chunks, cfg, ps)) + present = _hf_decoder_layer_indices([name for name, _ in exported]) + expected = set(range(cfg.num_hidden_layers)) + assert expected.issubset(present), ( + f"{model_name}: uneven-PP export dropped decoder layers " + f"{sorted(expected - present)} (have {sorted(present)}); PP gather lost a stage" + ) + for name, tensor in exported: + assert torch.isfinite(tensor.float()).all(), ( + f"{model_name}: non-finite exported tensor {name} from uneven PP gather" + ) + if dist.get_rank() == 0: + print( + "NON_SKIP_UNEVEN_PP_BUILD_TRAIN_EXPORT " + f"model={model_name} num_layers={cfg.num_hidden_layers} " + f"split={local} exported_decoder_layers={len(present & expected)}" + ) + + +# Custom-string mode (``ParallelConfig.pp_layout``) is covered by the deterministic +# unit tests (test_parallel_unit.py: the explicit string is honored verbatim, differs +# from the auto split, and a >pp-stage string raises). It shares the build path with +# the auto matrix above (only ``ps.pp_layout`` differs), so it is not given a separate +# GPU build here: keeping this smoke to one heavy distributed build per model avoids a +# trailing build tripping NCCL P2P comm-init timeouts from process-group churn. diff --git a/experimental/lite/tests/smoke/primitive/test_distopt_checkpoint_smoke.py b/experimental/lite/tests/smoke/primitive/test_distopt_checkpoint_smoke.py new file mode 100644 index 00000000000..00561ef825b --- /dev/null +++ b/experimental/lite/tests/smoke/primitive/test_distopt_checkpoint_smoke.py @@ -0,0 +1,315 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +from __future__ import annotations + +import os +from types import SimpleNamespace + +import pytest +import torch +import torch.distributed as dist +import torch.nn as nn +from torch.distributed.tensor import Replicate, Shard + +from megatron.core.dist_checkpointing import load_plain_tensors +from megatron.core.dist_checkpointing.dict_utils import diff +from megatron.core.distributed import DistributedDataParallel, DistributedDataParallelConfig +from megatron.core.optimizer import OptimizerConfig, get_megatron_optimizer +from megatron.core.transformer import TransformerConfig +from megatron.lite.primitive.ckpt import attach_model_sharded_state_dict +from megatron.lite.primitive.optimizers.megatron_wrap import build_dist_opt_stack +from megatron.lite.primitive.parallel import ParallelState, init_parallel +from megatron.lite.runtime.backends.mlite.runtime import MegatronLiteRuntime +from megatron.lite.runtime.contracts.config import OptimizerConfig as LiteOptimizerConfig +from megatron.lite.runtime.contracts.config import ParallelConfig +from megatron.lite.runtime.contracts.handle import ModelHandle + +pytestmark = [pytest.mark.mlite, pytest.mark.smoke, pytest.mark.gpu, pytest.mark.distributed] + + +class TinyDense(nn.Module): + def __init__(self): + super().__init__() + self.fc1 = nn.Linear(8, 8, bias=False) + self.fc2 = nn.Linear(8, 4, bias=False) + + def forward(self, x): + return self.fc2(torch.relu(self.fc1(x))) + + +class TinyTopologyAwareState(nn.Module): + dense_shape = (8, 8) + expert_shape = (8, 4) + + def __init__(self, ps: ParallelState): + super().__init__() + self.dense_weight = nn.Parameter( + _local_shard(_global_tensor(self.dense_shape, 1.0), 0, ps.tp_rank, ps.tp_size) + .cuda() + .bfloat16() + ) + self.experts_weight = nn.Parameter( + _local_shard(_global_tensor(self.expert_shape, 101.0), 0, ps.etp_rank, ps.etp_size) + .cuda() + .bfloat16() + ) + + def forward(self, x): + return x + + +@pytest.fixture(scope="module", autouse=True) +def _single_node_cuda_dist_opt(): + if not torch.cuda.is_available(): + pytest.skip("CUDA is required for dist_opt smoke tests.") + if int(os.environ.get("WORLD_SIZE", "1")) > 8: + pytest.skip("Megatron Lite smoke tests are capped at single-node 8 GPUs.") + + os.environ.setdefault("RANK", "0") + os.environ.setdefault("WORLD_SIZE", "1") + os.environ.setdefault("LOCAL_RANK", "0") + os.environ.setdefault("MASTER_ADDR", "127.0.0.1") + os.environ.setdefault("MASTER_PORT", "29541") + + torch.cuda.set_device(int(os.environ["LOCAL_RANK"])) + created_pg = False + if not dist.is_initialized(): + dist.init_process_group(backend="nccl", init_method="env://") + created_pg = True + + from megatron.core import parallel_state as mpu + + created_mpu = False + if not mpu.is_initialized(): + mpu.initialize_model_parallel( + tensor_model_parallel_size=1, + pipeline_model_parallel_size=1, + virtual_pipeline_model_parallel_size=None, + context_parallel_size=1, + expert_model_parallel_size=1, + expert_tensor_parallel_size=1, + ) + created_mpu = True + yield + if created_mpu: + mpu.destroy_model_parallel() + if created_pg and dist.is_initialized(): + dist.destroy_process_group() + + +def _global_tensor(shape: tuple[int, ...], offset: float) -> torch.Tensor: + return torch.arange(offset, offset + int(torch.tensor(shape).prod().item())).reshape(shape) + + +def _local_shard(tensor: torch.Tensor, dim: int, rank: int, size: int) -> torch.Tensor: + if size <= 1: + return tensor.clone() + chunks = torch.chunk(tensor, size, dim=dim) + return chunks[rank].contiguous().clone() + + +def _topology_placements(name: str) -> list: + if _is_expert_param(name): + return [Replicate(), Replicate(), Replicate(), Shard(0)] + return [Replicate(), Replicate(), Replicate(), Shard(0)] + + +def _is_expert_param(name: str) -> bool: + return "experts" in name + + +def _build_sharded_model_and_dist_opt(parallel: ParallelConfig): + ps = init_parallel(parallel) + model = TinyTopologyAwareState(ps) + model_cfg = SimpleNamespace( + num_hidden_layers=1, + hidden_size=8, + num_attention_heads=2, + num_experts=2, + moe_intermediate_size=16, + add_bias_linear=False, + ) + engine_cfg = SimpleNamespace( + model_name="tiny_topology_state", + parallel=parallel, + optimizer=LiteOptimizerConfig(optimizer="adam", lr=1.0e-3, weight_decay=0.0), + deterministic=False, + ) + wrapped_chunks, optimizer = build_dist_opt_stack( + [model], model_cfg=model_cfg, engine_cfg=engine_cfg, ps=ps, is_expert=_is_expert_param + ) + _seed_optimizer_state(optimizer) + attach_model_sharded_state_dict( + wrapped_chunks, ps, get_placements=_topology_placements, is_expert=_is_expert_param + ) + return wrapped_chunks, optimizer, ps + + +def _seed_optimizer_state(optimizer) -> None: + for inner_optimizer in _inner_optimizers(optimizer): + for group in inner_optimizer.param_groups: + for param in group["params"]: + state = inner_optimizer.state[param] + base = param.detach().float().abs() + state["exp_avg"] = (base + 0.125).to(dtype=param.dtype) + state["exp_avg_sq"] = (base + 0.25).to(dtype=param.dtype) + reload_model_params = getattr(optimizer, "reload_model_params", None) + if callable(reload_model_params): + reload_model_params() + + +def _inner_optimizers(optimizer): + chained = getattr(optimizer, "chained_optimizers", None) + if chained is not None: + for chained_optimizer in chained: + yield from _inner_optimizers(chained_optimizer) + return + try: + yield optimizer.optimizer + except AttributeError: + yield optimizer + + +def _dist_opt_handle(wrapped_chunks, optimizer, ps: ParallelState, parallel: ParallelConfig): + return ModelHandle( + model=wrapped_chunks, + optimizer=optimizer, + parallel_state=ps, + config=SimpleNamespace(parallel=parallel), + _extras={ + "model_chunks": wrapped_chunks, + "protocol": SimpleNamespace( + PLACEMENT_FN=_topology_placements, EXPERT_CLASSIFIER=_is_expert_param + ), + }, + ) + + +def _build_model_and_dist_opt(): + torch.manual_seed(2468) + model = TinyDense().bfloat16().cuda() + ddp_config = DistributedDataParallelConfig(use_distributed_optimizer=True) + wrapped = DistributedDataParallel( + TransformerConfig(num_attention_heads=1, num_layers=1), ddp_config, model + ) + optimizer = get_megatron_optimizer( + OptimizerConfig(optimizer="adam", lr=1.0e-3, bf16=True, use_distributed_optimizer=True), + [wrapped], + ) + attach_model_sharded_state_dict([wrapped], _single_node_parallel_state()) + return wrapped, optimizer + + +def _single_node_parallel_state() -> ParallelState: + rank = torch.distributed.get_rank() + world = torch.distributed.get_world_size() + return ParallelState(dp_size=world, dp_rank=rank, dp_cp_size=world, dp_cp_rank=rank) + + +def _shared_tmp_path(tmp_path) -> str: + payload = [str(tmp_path) if torch.distributed.get_rank() == 0 else None] + torch.distributed.broadcast_object_list(payload, src=0) + return payload[0] + + +def _train_step(model, optimizer, x: torch.Tensor): + output = model(x) + loss = output.float().square().mean() + loss.backward() + optimizer.step() + optimizer.zero_grad() + if hasattr(model, "zero_grad_buffer"): + model.zero_grad_buffer() + return loss.detach() + + +def _local_named_params(model) -> dict[str, torch.Tensor]: + return {name: param.detach().cpu().float().clone() for name, param in model.named_parameters()} + + +def _assert_model_close(lhs, rhs): + lhs_params = _local_named_params(lhs) + rhs_params = _local_named_params(rhs) + assert lhs_params.keys() == rhs_params.keys() + for name in lhs_params: + torch.testing.assert_close(lhs_params[name], rhs_params[name], atol=0.0, rtol=0.0) + + +def test_dist_opt_checkpoint_load_matches_uninterrupted_training_single_node(tmp_path): + model_for_ckpt, optimizer_for_ckpt = _build_model_and_dist_opt() + direct_model, direct_optimizer = _build_model_and_dist_opt() + loaded_model, loaded_optimizer = _build_model_and_dist_opt() + runtime = MegatronLiteRuntime.__new__(MegatronLiteRuntime) + + torch.manual_seed(1357) + x0 = torch.randn(4, 8, device="cuda", dtype=torch.bfloat16) + x1 = torch.randn(4, 8, device="cuda", dtype=torch.bfloat16) + + _train_step(model_for_ckpt, optimizer_for_ckpt, x0) + _train_step(direct_model, direct_optimizer, x0) + checkpoint_dir = _shared_tmp_path(tmp_path) + + runtime.save_checkpoint( + ModelHandle( + model=model_for_ckpt, + optimizer=optimizer_for_ckpt, + _extras={"model_chunks": [model_for_ckpt]}, + ), + checkpoint_dir, + step=1, + ) + assert ( + runtime.load_checkpoint( + ModelHandle( + model=loaded_model, + optimizer=loaded_optimizer, + _extras={"model_chunks": [loaded_model]}, + ), + checkpoint_dir, + ) + == 1 + ) + + _train_step(direct_model, direct_optimizer, x1) + _train_step(loaded_model, loaded_optimizer, x1) + _assert_model_close(direct_model, loaded_model) + + +def test_dist_opt_checkpoint_reshards_from_pp_ep_to_tp_pp_ep_etp(tmp_path): + if torch.distributed.get_world_size() < 8: + pytest.skip("TP2/PP2/EP2/ETP2 dist_opt reshard smoke requires 8 GPUs.") + + runtime = MegatronLiteRuntime.__new__(MegatronLiteRuntime) + checkpoint_root = _shared_tmp_path(tmp_path) + source_dir = os.path.join(checkpoint_root, "source") + reserialized_dir = os.path.join(checkpoint_root, "reserialized") + source_parallel = ParallelConfig(tp=1, ep=2, etp=1, pp=2, cp=1) + target_parallel = ParallelConfig(tp=2, ep=2, etp=2, pp=2, cp=1) + + source_chunks, source_optimizer, source_ps = _build_sharded_model_and_dist_opt(source_parallel) + runtime.save_checkpoint( + _dist_opt_handle(source_chunks, source_optimizer, source_ps, source_parallel), + source_dir, + step=3, + save_rng=False, + ) + + target_chunks, target_optimizer, target_ps = _build_sharded_model_and_dist_opt(target_parallel) + assert ( + runtime.load_checkpoint( + _dist_opt_handle(target_chunks, target_optimizer, target_ps, target_parallel), + source_dir, + load_rng=False, + ) + == 3 + ) + runtime.save_checkpoint( + _dist_opt_handle(target_chunks, target_optimizer, target_ps, target_parallel), + reserialized_dir, + step=3, + save_rng=False, + ) + + plain_source = load_plain_tensors(os.path.join(source_dir, "step_3")) + plain_reserialized = load_plain_tensors(os.path.join(reserialized_dir, "step_3")) + diffs = diff(plain_source, plain_reserialized) + assert not any(map(bool, diffs)), diffs diff --git a/experimental/lite/tests/smoke/primitive/test_fsdp2_offload_checkpoint_smoke.py b/experimental/lite/tests/smoke/primitive/test_fsdp2_offload_checkpoint_smoke.py new file mode 100644 index 00000000000..fbaebad32b5 --- /dev/null +++ b/experimental/lite/tests/smoke/primitive/test_fsdp2_offload_checkpoint_smoke.py @@ -0,0 +1,267 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +from __future__ import annotations + +import os +from types import SimpleNamespace + +import pytest +import torch +import torch.distributed as dist +import torch.nn as nn + +from megatron.lite.primitive.optimizers.fsdp2 import ( + FSDP2Config, + build_fsdp2_adamw, + build_fsdp2_device_mesh, + fsdp2_available, + wrap_fsdp2, +) +from megatron.lite.primitive.optimizers.fsdp2.adamw import iter_torch_optimizers, to_local_tensor +from megatron.lite.primitive.parallel.state import ParallelState +from megatron.lite.runtime.backends.mlite.runtime import MegatronLiteRuntime +from megatron.lite.runtime.contracts.config import ParallelConfig +from megatron.lite.runtime.contracts.handle import ModelHandle + +pytestmark = [pytest.mark.mlite, pytest.mark.smoke, pytest.mark.gpu, pytest.mark.distributed] + + +class TinyUnit(nn.Module): + def __init__(self): + super().__init__() + self.linear = nn.Linear(8, 8) + self.act = nn.GELU() + + def forward(self, x): + return self.act(self.linear(x)) + + +class TinyModel(nn.Module): + def __init__(self): + super().__init__() + self.unit0 = TinyUnit() + self.unit1 = TinyUnit() + self.out = nn.Linear(8, 4) + + def forward(self, x): + return self.out(self.unit1(self.unit0(x))) + + +@pytest.fixture(scope="module", autouse=True) +def _single_node_cuda_dist(): + if not torch.cuda.is_available(): + pytest.skip("CUDA is required for FSDP2 smoke tests.") + if not fsdp2_available(): + pytest.skip("Installed PyTorch does not expose FSDP2 fully_shard.") + if int(os.environ.get("WORLD_SIZE", "1")) > 8: + pytest.skip("Megatron Lite smoke tests are capped at single-node 8 GPUs.") + + os.environ.setdefault("RANK", "0") + os.environ.setdefault("WORLD_SIZE", "1") + os.environ.setdefault("LOCAL_RANK", "0") + os.environ.setdefault("MASTER_ADDR", "127.0.0.1") + os.environ.setdefault("MASTER_PORT", "29521") + + torch.cuda.set_device(int(os.environ["LOCAL_RANK"])) + created_pg = False + if not dist.is_initialized(): + dist.init_process_group(backend="nccl", init_method="env://") + created_pg = True + yield + if created_pg and dist.is_initialized(): + dist.destroy_process_group() + + +def _parallel_state() -> ParallelState: + rank = dist.get_rank() + world_size = dist.get_world_size() + return ParallelState( + dp_group=dist.group.WORLD, + dp_cp_group=dist.group.WORLD, + dp_size=world_size, + dp_cp_size=world_size, + dp_rank=rank, + dp_cp_rank=rank, + ) + + +def _shared_tmp_path(tmp_path) -> str: + payload = [str(tmp_path) if dist.get_rank() == 0 else None] + dist.broadcast_object_list(payload, src=0) + return payload[0] + + +def _checkpoint_config(): + return SimpleNamespace(parallel=ParallelConfig()) + + +def _build_fsdp2_model(dtype: torch.dtype = torch.bfloat16) -> tuple[nn.Module, ParallelState]: + torch.manual_seed(1234) + model = TinyModel().cuda().to(dtype=dtype) + ps = _parallel_state() + config = FSDP2Config(unit_modules=(TinyUnit,), reshard_after_forward=True) + mesh = build_fsdp2_device_mesh(ps, config) + return wrap_fsdp2(model, ps, config, mesh=mesh), ps + + +def _build_optimizer(model: nn.Module, ps: ParallelState, *, offload_fraction: float): + return build_fsdp2_adamw( + [model], + SimpleNamespace( + optimizer="adam", + lr=1.0e-3, + weight_decay=0.0, + adam_beta1=0.9, + adam_beta2=0.95, + adam_eps=1.0e-8, + clip_grad=1.0, + offload_fraction=offload_fraction, + ), + ps, + use_fp32_master=True, + ) + + +def _local_param_devices(model: nn.Module) -> set[str]: + return {to_local_tensor(param.detach()).device.type for param in model.parameters()} + + +def _optimizer_state_devices(optimizer) -> set[str]: + devices: set[str] = set() + for child in iter_torch_optimizers(optimizer.optimizer): + for param_state in getattr(child, "state", {}).values(): + if not isinstance(param_state, dict): + continue + for value in param_state.values(): + if isinstance(value, torch.Tensor): + devices.add(to_local_tensor(value).device.type) + return devices + + +def _local_named_params(model: nn.Module) -> dict[str, torch.Tensor]: + return { + name: to_local_tensor(param.detach()).cpu().clone() + for name, param in model.named_parameters() + } + + +def _train_step(model: nn.Module, optimizer, x: torch.Tensor, target: torch.Tensor): + optimizer.zero_grad() + loss = torch.nn.functional.mse_loss(model(x).float(), target.float()) + loss.backward() + success, grad_norm, _ = optimizer.step() + assert success + assert torch.isfinite(torch.tensor(grad_norm)) + return loss.detach(), float(grad_norm) + + +def _assert_grad_norm_exact(lhs: float, rhs: float) -> None: + assert lhs == rhs + + +def _assert_local_params_close(lhs: nn.Module, rhs: nn.Module): + lhs_params = _local_named_params(lhs) + rhs_params = _local_named_params(rhs) + assert lhs_params.keys() == rhs_params.keys() + for name in lhs_params: + torch.testing.assert_close(lhs_params[name], rhs_params[name], atol=0.0, rtol=0.0) + + +def test_fsdp2_runtime_model_and_optimizer_offload_roundtrip_single_node(): + model, ps = _build_fsdp2_model() + optimizer = _build_optimizer(model, ps, offload_fraction=0.0) + handle = ModelHandle( + model=model, optimizer=optimizer, parallel_state=ps, _extras={"model_chunks": [model]} + ) + runtime = MegatronLiteRuntime.__new__(MegatronLiteRuntime) + + assert _local_param_devices(model) == {"cuda"} + assert _optimizer_state_devices(optimizer) == {"cuda"} + + runtime.to(handle, "cpu", model=True, optimizer=True, grad=True) + assert _local_param_devices(model) == {"cpu"} + assert _optimizer_state_devices(optimizer) == {"cpu"} + + runtime.to(handle, "cuda", model=True, optimizer=True, grad=True) + assert _local_param_devices(model) == {"cuda"} + assert _optimizer_state_devices(optimizer) == {"cuda"} + + +def test_fsdp2_offload_fraction_matches_non_offloaded_grad_clip_single_node(): + if dist.get_world_size() < 2: + pytest.skip( + "multi-rank FSDP2 grad clipping equivalence requires WORLD_SIZE > 1." + ) + + baseline_model, baseline_ps = _build_fsdp2_model() + baseline_optimizer = _build_optimizer( + baseline_model, baseline_ps, offload_fraction=0.0 + ) + offload_model, offload_ps = _build_fsdp2_model() + offload_optimizer = _build_optimizer( + offload_model, offload_ps, offload_fraction=1.0 + ) + + assert _optimizer_state_devices(baseline_optimizer) == {"cuda"} + assert _optimizer_state_devices(offload_optimizer) == {"cpu"} + + torch.manual_seed(4321) + x = torch.randn(4, 8, device="cuda", dtype=torch.bfloat16) + target = torch.randn(4, 4, device="cuda", dtype=torch.bfloat16) + _loss, baseline_grad_norm = _train_step( + baseline_model, baseline_optimizer, x, target + ) + _loss, offload_grad_norm = _train_step(offload_model, offload_optimizer, x, target) + + _assert_grad_norm_exact(offload_grad_norm, baseline_grad_norm) + _assert_local_params_close(offload_model, baseline_model) + assert _local_param_devices(offload_model) == {"cuda"} + assert _optimizer_state_devices(offload_optimizer) == {"cpu"} + + +def test_fsdp2_checkpoint_load_matches_uninterrupted_training_single_node(tmp_path): + model_for_ckpt, ps = _build_fsdp2_model() + optimizer_for_ckpt = _build_optimizer(model_for_ckpt, ps, offload_fraction=0.0) + direct_model, direct_ps = _build_fsdp2_model() + direct_optimizer = _build_optimizer(direct_model, direct_ps, offload_fraction=0.0) + loaded_model, loaded_ps = _build_fsdp2_model() + loaded_optimizer = _build_optimizer(loaded_model, loaded_ps, offload_fraction=0.0) + runtime = MegatronLiteRuntime.__new__(MegatronLiteRuntime) + + torch.manual_seed(4321) + x0 = torch.randn(4, 8, device="cuda", dtype=torch.bfloat16) + y0 = torch.randn(4, 4, device="cuda", dtype=torch.bfloat16) + x1 = torch.randn(4, 8, device="cuda", dtype=torch.bfloat16) + y1 = torch.randn(4, 4, device="cuda", dtype=torch.bfloat16) + + _train_step(model_for_ckpt, optimizer_for_ckpt, x0, y0) + _train_step(direct_model, direct_optimizer, x0, y0) + checkpoint_dir = _shared_tmp_path(tmp_path) + + runtime.save_checkpoint( + ModelHandle( + model=model_for_ckpt, + optimizer=optimizer_for_ckpt, + parallel_state=ps, + config=_checkpoint_config(), + _extras={"model_chunks": [model_for_ckpt]}, + ), + checkpoint_dir, + step=1, + ) + assert ( + runtime.load_checkpoint( + ModelHandle( + model=loaded_model, + optimizer=loaded_optimizer, + parallel_state=loaded_ps, + config=_checkpoint_config(), + _extras={"model_chunks": [loaded_model]}, + ), + checkpoint_dir, + ) + == 1 + ) + + _train_step(direct_model, direct_optimizer, x1, y1) + _train_step(loaded_model, loaded_optimizer, x1, y1) + _assert_local_params_close(direct_model, loaded_model) diff --git a/experimental/lite/tests/smoke/primitive/test_parallel_topologies_smoke.py b/experimental/lite/tests/smoke/primitive/test_parallel_topologies_smoke.py new file mode 100644 index 00000000000..1a1c0f3d224 --- /dev/null +++ b/experimental/lite/tests/smoke/primitive/test_parallel_topologies_smoke.py @@ -0,0 +1,148 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +from __future__ import annotations + +import os +from types import SimpleNamespace + +import pytest +import torch +import torch.distributed as dist + +from megatron.lite.primitive.parallel import ( + PackedSeqParams, + contiguous_to_zigzag_chunks, + init_parallel, + zigzag_split_for_cp, + zigzag_to_contiguous_chunks, +) + +pytestmark = [pytest.mark.mlite, pytest.mark.smoke, pytest.mark.gpu, pytest.mark.distributed] + + +@pytest.fixture(scope="module", autouse=True) +def _single_node_cuda_dist(): + if not torch.cuda.is_available(): + pytest.skip("CUDA is required for distributed primitive smoke tests.") + if int(os.environ.get("WORLD_SIZE", "1")) > 8: + pytest.skip("Megatron Lite smoke tests are capped at single-node 8 GPUs.") + + os.environ.setdefault("RANK", "0") + os.environ.setdefault("WORLD_SIZE", "1") + os.environ.setdefault("LOCAL_RANK", "0") + os.environ.setdefault("MASTER_ADDR", "127.0.0.1") + os.environ.setdefault("MASTER_PORT", "29511") + + torch.cuda.set_device(int(os.environ["LOCAL_RANK"])) + created_pg = False + if not dist.is_initialized(): + dist.init_process_group(backend="nccl", init_method="env://") + created_pg = True + yield + if created_pg and dist.is_initialized(): + dist.destroy_process_group() + + +def _assert_group_size(group, expected: int): + if expected == 1: + return + assert group is not None + assert dist.get_world_size(group) == expected + + +def _topologies(world_size: int): + yield "dp_only", SimpleNamespace(tp=1, ep=1, etp=1, cp=1, pp=1) + if world_size >= 2: + yield "tp2", SimpleNamespace(tp=2, ep=1, etp=1, cp=1, pp=1) + yield "cp2", SimpleNamespace(tp=1, ep=1, etp=1, cp=2, pp=1) + yield "pp2", SimpleNamespace(tp=1, ep=1, etp=1, cp=1, pp=2) + yield "ep2", SimpleNamespace(tp=1, ep=2, etp=1, cp=1, pp=1) + yield "etp2", SimpleNamespace(tp=1, ep=1, etp=2, cp=1, pp=1) + if world_size >= 4: + yield "tp2_ep2_pp2", SimpleNamespace(tp=2, ep=2, etp=1, cp=1, pp=2) + if world_size >= 8: + yield "tp2_ep2_cp2_pp2", SimpleNamespace(tp=2, ep=2, etp=1, cp=2, pp=2) + yield "tp2_ep2_etp2_pp2", SimpleNamespace(tp=2, ep=2, etp=2, cp=1, pp=2) + + +def test_parallel_state_builds_expected_primitive_groups(): + world_size = dist.get_world_size() + for name, cfg in _topologies(world_size): + if world_size % (cfg.tp * cfg.cp * cfg.pp) != 0: + continue + if world_size % (cfg.etp * cfg.ep * cfg.pp) != 0: + continue + + ps = init_parallel(cfg) + dense_dp = world_size // (cfg.tp * cfg.cp * cfg.pp) + expert_dp = world_size // (cfg.etp * cfg.ep * cfg.pp) + + assert ps.tp_size == cfg.tp + assert ps.cp_size == cfg.cp + assert ps.pp_size == cfg.pp + assert ps.ep_size == cfg.ep + assert ps.dp_size == dense_dp + assert ps.expert_dp_size == expert_dp + assert 0 <= ps.tp_rank < cfg.tp + assert 0 <= ps.cp_rank < cfg.cp + assert 0 <= ps.pp_rank < cfg.pp + assert 0 <= ps.ep_rank < cfg.ep + _assert_group_size(ps.tp_group, cfg.tp) + _assert_group_size(ps.cp_group, cfg.cp) + _assert_group_size(ps.pp_group, cfg.pp) + _assert_group_size(ps.ep_group, cfg.ep) + _assert_group_size(ps.dp_group, dense_dp) + _assert_group_size(ps.dp_cp_group, dense_dp * cfg.cp) + _assert_group_size(ps.ep_dp_group, expert_dp) + # tp_ep follows Megatron Core's expert tensor + expert model group, + # not the dense-layer TP group. + _assert_group_size(ps.tp_ep_group, cfg.etp * cfg.ep) + _assert_group_size(ps.etp_group, cfg.etp) + if cfg.pp > 1: + assert ps.pp_next_rank in ps.pp_global_ranks + assert ps.pp_prev_rank in ps.pp_global_ranks + + +def test_cp_zigzag_contiguous_chunk_swap_roundtrip(): + if dist.get_world_size() < 2: + pytest.skip("CP chunk swap smoke requires at least 2 ranks.") + + ps = init_parallel(SimpleNamespace(tp=1, ep=1, etp=1, cp=2, pp=1)) + full = torch.arange(8, device="cuda", dtype=torch.float32).reshape(1, 8, 1) + 100 * ps.dp_rank + local_zigzag = zigzag_split_for_cp(full, ps.cp_rank, cp_size=2, seq_dim=1) + + local_contiguous = zigzag_to_contiguous_chunks(local_zigzag, ps.cp_group, seq_dim=1) + expected = full[:, ps.cp_rank * 4 : (ps.cp_rank + 1) * 4, :] + assert torch.equal(local_contiguous, expected) + + restored = contiguous_to_zigzag_chunks(local_contiguous, ps.cp_group, seq_dim=1) + assert torch.equal(restored, local_zigzag) + + +def test_gdn_rejects_thd_context_parallel_until_validated(): + if dist.get_world_size() < 2: + pytest.skip("GDN THD+CP guard smoke requires at least 2 ranks.") + + pytest.importorskip("transformer_engine.pytorch") + from megatron.lite.primitive.modules.gated_delta_net import GatedDeltaNet + + ps = init_parallel(SimpleNamespace(tp=1, ep=1, etp=1, cp=2, pp=1)) + gdn = ( + GatedDeltaNet( + hidden_size=16, + linear_num_key_heads=2, + linear_key_head_dim=4, + linear_num_value_heads=2, + linear_value_head_dim=4, + linear_conv_kernel_dim=2, + rms_norm_eps=1e-6, + ps=ps, + ) + .cuda() + .to(torch.bfloat16) + ) + cu_seqlens = torch.tensor([0, 8], dtype=torch.int32, device="cuda") + packed_seq_params = PackedSeqParams.from_cu_seqlens(cu_seqlens, max_seqlen=8) + local_hidden = torch.randn(4, 1, 16, device="cuda", dtype=torch.bfloat16) + + with pytest.raises(NotImplementedError, match="all-gather CP"): + gdn(local_hidden, packed_seq_params=packed_seq_params) diff --git a/experimental/lite/tests/smoke/primitive/test_qwen35_hf_numeric_roundtrip_smoke.py b/experimental/lite/tests/smoke/primitive/test_qwen35_hf_numeric_roundtrip_smoke.py new file mode 100644 index 00000000000..34a6bd67d86 --- /dev/null +++ b/experimental/lite/tests/smoke/primitive/test_qwen35_hf_numeric_roundtrip_smoke.py @@ -0,0 +1,224 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Qwen3.5 HF<->lite checkpoint NUMERIC round-trip + ground-truth smoke. + +The existing export unit tests (tests/unit/model/test_qwen35_export.py) and the +save/load/export coverage smoke only assert NAMES / DTYPE / FINITENESS, never +numeric fidelity — so a transform that is self-consistent on the round-trip but +wrong vs real HF would pass silently. This smoke closes that blind spot: + + test_qwen35_save_load_numeric_roundtrip + build A(seed1) -> save_hf -> build B(seed2) -> load_hf(B) -> assert every + param allclose(A, B). Catches ASYMMETRIC load/export bugs (load != inverse + of export). Vocab embed/head rows >= vocab_size are unused TP-padding (random + in A, zeroed on reload) and are excluded; the real model has vocab%128==0 so + no padding exists. + + test_qwen35_real_hf_load_export_matches_original [opt-in: QWEN35_HF_DIR] + GROUND TRUTH against real Qwen3.5 safetensors: load real HF -> export -> + compare to the ORIGINAL safetensors tensor-by-tensor. Catches HF-file-level + mismatches that native round-trip cannot see. It does NOT prove the loaded + native layout is semantically correct if load/export are mutually inverse but + both wrong vs forward semantics. Needs an 80GB GPU; loads only the first 8 + decoder layers to bound memory/time. + +Run single GPU: + torchrun --nproc_per_node=1 -m pytest tests/smoke/primitive/test_qwen35_hf_numeric_roundtrip_smoke.py +""" +from __future__ import annotations + +import os + +import pytest +import torch +import torch.distributed as dist + +pytestmark = [pytest.mark.mlite, pytest.mark.smoke, pytest.mark.gpu] + + +def _qwen35_tiny_cfg(): + pytest.importorskip("fla", reason="qwen3_5 needs the FLA / GatedDeltaNet stack.") + pytest.importorskip( + "transformer_engine.pytorch", reason="qwen3_5 smoke needs real Transformer Engine." + ) + from megatron.lite.model.qwen3_5.config import Qwen35Config + + return Qwen35Config( + num_hidden_layers=2, + hidden_size=16, + num_attention_heads=4, + num_key_value_heads=2, + head_dim=4, + vocab_size=64, + num_experts=4, + num_experts_per_tok=2, + moe_intermediate_size=8, + shared_expert_intermediate_size=8, + linear_num_key_heads=2, + linear_key_head_dim=4, + linear_num_value_heads=2, + linear_value_head_dim=4, + linear_conv_kernel_dim=4, + layer_types=["full_attention", "linear_attention"], + partial_rotary_factor=1.0, + max_position_embeddings=4096, + ) + + +def _build(cfg, seed): + from megatron.lite.model.qwen3_5.lite import protocol + from megatron.lite.runtime.contracts.config import ParallelConfig + + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + parallel = ParallelConfig(tp=1, pp=1, cp=1, ep=1, etp=1, vpp=1) + impl = protocol.ImplConfig( + parallel=parallel, optimizer="dist_opt", use_deepep=False, deterministic=True + ) + bundle = protocol.build_model(cfg, impl_cfg=impl) + return bundle, protocol + + +def _named(chunks): + out = {} + for i, chunk in enumerate(chunks): + for name, param in chunk.named_parameters(): + out[f"{i}.{name}"] = param.detach().float().cpu().clone() + return out + + +@pytest.fixture(scope="module", autouse=True) +def _single_rank_dist(): + if not torch.cuda.is_available(): + pytest.skip("CUDA required for qwen3.5 numeric round-trip smoke.") + created = False + if not dist.is_initialized(): + for k, v in { + "MASTER_ADDR": "127.0.0.1", + "MASTER_PORT": "29597", + "RANK": "0", + "WORLD_SIZE": "1", + "LOCAL_RANK": "0", + }.items(): + os.environ.setdefault(k, v) + local_rank = int(os.environ.get("LOCAL_RANK", "0")) + torch.cuda.set_device(local_rank) + dist.init_process_group("nccl") + created = True + yield + if created and dist.is_initialized(): + dist.destroy_process_group() + + +def test_qwen35_save_load_numeric_roundtrip(tmp_path): + if dist.get_world_size() != 1: + pytest.skip("numeric round-trip smoke runs single-rank (tp1).") + cfg = _qwen35_tiny_cfg() + a, proto = _build(cfg, seed=1) + b, _ = _build(cfg, seed=2) + + before = _named(a.chunks) + b_before = _named(b.chunks) + assert sum(1 for k in before if not torch.equal(before[k], b_before[k])) > 0 + + out_dir = str(tmp_path / "hf") + os.makedirs(out_dir, exist_ok=True) + proto.save_hf_weights(a.chunks, out_dir, cfg, a.parallel_state) + proto.load_hf_weights(b.chunks[0], out_dir, cfg, b.parallel_state) + + pa, pb = _named(a.chunks), _named(b.chunks) + vocab = cfg.vocab_size + mism = [] + for k in pa: + ta, tb = pa[k], pb[k] + if k.endswith("embed.embedding.weight") or k.endswith("head.col.linear.weight"): + assert ta.shape[0] >= vocab + ta, tb = ta[:vocab], tb[:vocab] # exclude unused TP-padding rows + # atol/rtol=1e-3 are bf16-cast round-trip tolerances. + if not torch.allclose(ta, tb, atol=1e-3, rtol=1e-3): + mism.append(f"{k} (max_abs_diff={(ta - tb).abs().max().item()})") + assert not mism, "HF save->load not numeric round-trip:\n" + "\n".join(mism) + + +@pytest.mark.skipif( + not os.environ.get("QWEN35_HF_DIR"), + reason="set QWEN35_HF_DIR to a real Qwen3.5 HF checkpoint to run the ground-truth check.", +) +def test_qwen35_real_hf_load_export_matches_original(tmp_path): + if dist.get_world_size() != 1: + pytest.skip("ground-truth smoke runs single-rank (tp1).") + import json + + from safetensors import safe_open + + from megatron.lite.model.qwen3_5.config import Qwen35Config + from megatron.lite.model.qwen3_5.lite import protocol + from megatron.lite.runtime.contracts.config import ParallelConfig + + model_dir = os.environ["QWEN35_HF_DIR"] + cfg = Qwen35Config.from_hf(model_dir) + n = min(8, cfg.num_hidden_layers) + cfg.num_hidden_layers = n + cfg.layer_types = cfg.layer_types[:n] + cfg.num_nextn_predict_layers = 0 + # Ensure both attention branches are actually exercised in the truncated model. + assert "full_attention" in cfg.layer_types + assert "linear_attention" in cfg.layer_types + + parallel = ParallelConfig(tp=1, pp=1, cp=1, ep=1, etp=1, vpp=1) + impl = protocol.ImplConfig( + parallel=parallel, optimizer="dist_opt", use_deepep=False, deterministic=True + ) + bundle = protocol.build_model(cfg, impl_cfg=impl) + protocol.load_hf_weights(bundle.chunks[0], model_dir, cfg, bundle.parallel_state) + + out_dir = str(tmp_path / "hf_export") + os.makedirs(out_dir, exist_ok=True) + protocol.save_hf_weights(bundle.chunks, out_dir, cfg, bundle.parallel_state) + + orig_map = json.load(open(os.path.join(model_dir, "model.safetensors.index.json")))[ + "weight_map" + ] + + def _orig(key): + with safe_open(os.path.join(model_dir, orig_map[key]), framework="pt") as fh: + return fh.get_tensor(key).float() + + exported = {} + for fn in os.listdir(out_dir): + if fn.endswith(".safetensors"): + with safe_open(os.path.join(out_dir, fn), framework="pt") as fh: + for key in fh.keys(): + exported[key] = fh.get_tensor(key).float() + + # Derive the expected comparison set from the ORIGINAL weight map (not from + # `exported`): otherwise a tensor the exporter silently drops would never be + # compared and a missing-export bug would pass undetected. + # + # Scope to what this truncated text-only build actually produces: the first + # `n` decoder layers (prefix `model.language_model.layers.{i}.`) plus the + # global embed/norm/head tensors. The original checkpoint also carries the + # MTP head (`mtp.*`, disabled here via num_nextn_predict_layers=0) and the + # vision tower (`model.visual.*`, not built on the text path); both are + # intentionally NOT built/exported, so they are excluded from expected_keys. + global_keys = ( + "model.language_model.embed_tokens.weight", + "model.language_model.norm.weight", + "lm_head.weight", + ) + layer_prefixes = tuple(f"model.language_model.layers.{i}." for i in range(n)) + expected_keys = { + k for k in orig_map if k.startswith(layer_prefixes) or k in global_keys + } + assert expected_keys, "no comparable tensors found in original weight map." + + missing = expected_keys - set(exported) + assert not missing, "export missing tensors: " + ", ".join(sorted(missing)) + + mism = [] + for k in sorted(expected_keys): + o, e = _orig(k), exported[k] + if o.shape != e.shape: + mism.append(f"{k} shape {tuple(o.shape)} != {tuple(e.shape)}") + elif not torch.allclose(o, e, atol=2e-2, rtol=2e-2): # bf16-cast export tolerance + mism.append(f"{k} max_abs_diff={(o - e).abs().max().item()}") + assert not mism, "lite export does not match original HF safetensors:\n" + "\n".join(mism) diff --git a/experimental/lite/tests/smoke/primitive/test_qwen3_moe_distopt_checkpoint_smoke.py b/experimental/lite/tests/smoke/primitive/test_qwen3_moe_distopt_checkpoint_smoke.py new file mode 100644 index 00000000000..7cf98b11df0 --- /dev/null +++ b/experimental/lite/tests/smoke/primitive/test_qwen3_moe_distopt_checkpoint_smoke.py @@ -0,0 +1,207 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +from __future__ import annotations + +import os +from types import SimpleNamespace +from typing import Any + +import pytest +import torch +import torch.distributed as dist + +from megatron.lite.primitive.deterministic import set_deterministic +from megatron.lite.runtime.backends.mlite.runtime import MegatronLiteRuntime +from megatron.lite.runtime.contracts.config import OptimizerConfig, ParallelConfig +from megatron.lite.runtime.contracts.data import PackedBatch +from megatron.lite.runtime.contracts.handle import ModelHandle + +pytestmark = [pytest.mark.mlite, pytest.mark.smoke, pytest.mark.gpu, pytest.mark.distributed] + + +def _qwen3_moe_symbols(): + te = pytest.importorskip( + "transformer_engine.pytorch", + reason="Qwen3MoE dist_opt checkpoint smoke requires real Transformer Engine.", + ) + assert hasattr(te, "Linear"), "Qwen3MoE smoke requires real Transformer Engine Linear." + from megatron.lite.model.qwen3_moe.config import Qwen3MoEConfig + from megatron.lite.model.qwen3_moe.lite import protocol + + return Qwen3MoEConfig, protocol + + +@pytest.fixture(scope="module", autouse=True) +def _single_node_cuda_dist(): + if not torch.cuda.is_available(): + pytest.skip("CUDA is required for Qwen3MoE dist_opt checkpoint smoke tests.") + if int(os.environ.get("WORLD_SIZE", "1")) > 8: + pytest.skip("Megatron Lite smoke tests are capped at single-node 8 GPUs.") + + os.environ.setdefault("CUBLAS_WORKSPACE_CONFIG", ":4096:8") + os.environ.setdefault("NVTE_ALLOW_NONDETERMINISTIC_ALGO", "0") + os.environ.setdefault("RANK", "0") + os.environ.setdefault("WORLD_SIZE", "1") + os.environ.setdefault("LOCAL_RANK", "0") + os.environ.setdefault("MASTER_ADDR", "127.0.0.1") + os.environ.setdefault("MASTER_PORT", "29541") + + torch.cuda.set_device(int(os.environ["LOCAL_RANK"])) + created_pg = False + if not dist.is_initialized(): + dist.init_process_group(backend="nccl", init_method="env://") + created_pg = True + yield + try: + from megatron.core import parallel_state as mpu + + if mpu.is_initialized(): + mpu.destroy_model_parallel() + finally: + if created_pg and dist.is_initialized(): + dist.destroy_process_group() + + +def _topology() -> ParallelConfig: + return ParallelConfig(tp=2, ep=2, etp=1, pp=2, cp=1) + + +def _tiny_qwen3_moe_config(): + Qwen3MoEConfig, _protocol = _qwen3_moe_symbols() + return Qwen3MoEConfig( + num_hidden_layers=2, + hidden_size=16, + num_attention_heads=4, + num_key_value_heads=2, + head_dim=4, + vocab_size=64, + num_experts=4, + num_experts_per_tok=1, + moe_intermediate_size=8, + max_position_embeddings=16, + layer_types=["full_attention", "full_attention"], + ) + + +def _build_handle(model_seed: int) -> ModelHandle: + _Qwen3MoEConfig, protocol = _qwen3_moe_symbols() + torch.manual_seed(model_seed) + torch.cuda.manual_seed_all(model_seed) + + parallel = _topology() + model_cfg = _tiny_qwen3_moe_config() + impl_cfg = protocol.ImplConfig( + parallel=parallel, + optimizer="dist_opt", + optimizer_config=OptimizerConfig( + optimizer="adam", lr=1.0e-3, weight_decay=0.0, clip_grad=1.0 + ), + use_deepep=False, + deterministic=True, + ) + bundle = protocol.build_model(model_cfg, impl_cfg=impl_cfg) + extras = dict(bundle.extras) + extras.update( + { + "model_chunks": bundle.chunks, + "forward_step": bundle.forward_step, + "finalize_grads": bundle.finalize_grads, + "protocol": protocol, + } + ) + return ModelHandle( + model=bundle.chunks, + optimizer=bundle.optimizer, + parallel_state=bundle.parallel_state, + config=SimpleNamespace(parallel=parallel), + _extras=extras, + ) + + +def _shared_tmp_path(tmp_path) -> str: + payload = [str(tmp_path) if dist.get_rank() == 0 else None] + dist.broadcast_object_list(payload, src=0) + return payload[0] + + +def _random_batch(vocab_size: int) -> PackedBatch: + # Raw THD PackedBatch: 1-D packed tokens + true seq lengths (protocol packs). + return PackedBatch( + input_ids=torch.randint(0, vocab_size, (8,), device="cuda"), + labels=torch.randint(0, vocab_size, (8,), device="cuda"), + seq_lens=torch.full((2,), 4, dtype=torch.int64, device="cuda"), + ) + + +def _clone_batch(batch: PackedBatch) -> PackedBatch: + return PackedBatch( + input_ids=batch.input_ids.detach().clone(), + labels=batch.labels.detach().clone(), + seq_lens=batch.seq_lens.detach().clone(), + ) + + +def _assert_batch_equal(actual: PackedBatch, expected: PackedBatch) -> None: + assert torch.equal(actual.input_ids, expected.input_ids) + assert torch.equal(actual.labels, expected.labels) + + +def _train_step(runtime: MegatronLiteRuntime, handle: ModelHandle, batch: dict[str, Any]) -> None: + runtime.zero_grad(handle) + runtime.forward_backward(handle, iter([batch]), None, num_microbatches=1) + runtime.optimizer_step(handle) + runtime.zero_grad(handle) + + +def _local_named_params(handle: ModelHandle) -> dict[str, torch.Tensor]: + params: dict[str, torch.Tensor] = {} + for chunk_idx, chunk in enumerate(handle._extras["model_chunks"]): + for name, param in chunk.named_parameters(): + params[f"{chunk_idx}.{name}"] = param.detach().cpu().float().clone() + return params + + +def _assert_params_bitwise_equal(lhs: ModelHandle, rhs: ModelHandle) -> None: + lhs_params = _local_named_params(lhs) + rhs_params = _local_named_params(rhs) + assert lhs_params.keys() == rhs_params.keys() + for name in lhs_params: + torch.testing.assert_close(lhs_params[name], rhs_params[name], atol=0.0, rtol=0.0) + + +def test_qwen3_moe_dist_opt_checkpoint_restores_rng_and_continues_bitwise_tp2_pp2_ep2(tmp_path): + if dist.get_world_size() != 8: + pytest.skip("Qwen3MoE tp2/pp2/ep2 dist_opt checkpoint smoke requires exactly 8 GPUs.") + + set_deterministic(2026) + model_cfg = _tiny_qwen3_moe_config() + runtime = MegatronLiteRuntime.__new__(MegatronLiteRuntime) + model_for_ckpt = _build_handle(model_seed=4242) + direct_model = _build_handle(model_seed=4242) + loaded_model = _build_handle(model_seed=4242) + + torch.manual_seed(1357 + dist.get_rank()) + torch.cuda.manual_seed_all(1357 + dist.get_rank()) + step0_batch = _random_batch(model_cfg.vocab_size) + + cpu_rng_before_step0 = torch.get_rng_state() + cuda_rng_before_step0 = torch.cuda.get_rng_state() + _train_step(runtime, model_for_ckpt, step0_batch) + + torch.set_rng_state(cpu_rng_before_step0) + torch.cuda.set_rng_state(cuda_rng_before_step0) + _train_step(runtime, direct_model, _clone_batch(step0_batch)) + _assert_params_bitwise_equal(model_for_ckpt, direct_model) + + checkpoint_dir = _shared_tmp_path(tmp_path) + runtime.save_checkpoint(model_for_ckpt, checkpoint_dir, step=1) + + direct_step1_batch = _random_batch(model_cfg.vocab_size) + expected_step1_batch = _clone_batch(direct_step1_batch) + _train_step(runtime, direct_model, direct_step1_batch) + + assert runtime.load_checkpoint(loaded_model, checkpoint_dir) == 1 + loaded_step1_batch = _random_batch(model_cfg.vocab_size) + _assert_batch_equal(loaded_step1_batch, expected_step1_batch) + _train_step(runtime, loaded_model, loaded_step1_batch) + + _assert_params_bitwise_equal(direct_model, loaded_model) diff --git a/experimental/lite/tests/smoke/primitive/test_save_load_export_smoke.py b/experimental/lite/tests/smoke/primitive/test_save_load_export_smoke.py new file mode 100644 index 00000000000..5f366bee946 --- /dev/null +++ b/experimental/lite/tests/smoke/primitive/test_save_load_export_smoke.py @@ -0,0 +1,720 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Save / load / export coverage smoke across the supported models and backends. + +Matrix: {qwen3_5, qwen3_moe, kimi_k2, glm5, deepseek_v4} x {dist_opt, fsdp2}. + +Each (model, backend) case does a faithful round-trip: + build -> train one real step -> save checkpoint -> build fresh -> load -> + assert parameters restored bit-exactly -> export HF weights (bf16) -> + reload exported safetensors and assert dtype/finiteness. + +dist_opt cases use the Megatron distributed-checkpoint path on a +tp2/ep2/pp2 topology (exactly 8 GPUs). fsdp2 cases use torch DCP on a +pure-DP mesh. Both checkpoint paths are exercised through the unified +MegatronLiteRuntime so the matrix guards the runtime entry points. + +Run with torchrun --nproc_per_node=8 -m pytest, selecting per-env subsets +with -k (qwen3_5 needs the qwen3.5 canary site; the four deepseek/qwen3_moe +models need the DSA overlay). Models gate themselves with importorskip so a +wrong-env invocation skips rather than errors. +""" +from __future__ import annotations + +import os +from datetime import timedelta +from types import SimpleNamespace + +import pytest +import torch +import torch.distributed as dist + +from megatron.lite.primitive.deterministic import set_deterministic +from megatron.lite.runtime.backends.mlite.runtime import MegatronLiteRuntime +from megatron.lite.runtime.contracts.config import OptimizerConfig, ParallelConfig +from megatron.lite.runtime.contracts.data import PackedBatch +from megatron.lite.runtime.contracts.handle import ModelHandle + +pytestmark = [pytest.mark.mlite, pytest.mark.smoke, pytest.mark.gpu, pytest.mark.distributed] + + +# ────────────────────────────────────────────────────────────────────────── +# Model registry: name -> builder returning (tiny_config, protocol_module). +# Each builder importorskips its env-specific deps so a wrong-env run skips. +# ────────────────────────────────────────────────────────────────────────── +def _require_te() -> None: + te = pytest.importorskip( + "transformer_engine.pytorch", + reason="save/load/export smoke requires real Transformer Engine.", + ) + assert hasattr(te, "Linear"), "smoke requires real Transformer Engine Linear." + + +def _qwen3_5(): + pytest.importorskip("fla", reason="qwen3_5 needs the FLA / GatedDeltaNet stack.") + _require_te() + from megatron.lite.model.qwen3_5.config import Qwen35Config + from megatron.lite.model.qwen3_5.lite import protocol + + cfg = Qwen35Config( + num_hidden_layers=2, + hidden_size=16, + num_attention_heads=4, + num_key_value_heads=2, + head_dim=4, + vocab_size=64, + num_experts=4, + num_experts_per_tok=2, + moe_intermediate_size=8, + shared_expert_intermediate_size=8, + linear_num_key_heads=2, + linear_key_head_dim=4, + linear_num_value_heads=2, + linear_value_head_dim=4, + linear_conv_kernel_dim=4, + # Mix full + linear attention so the distckpt linear_attn TP-shard + # path (the in_proj/conv1d/layer_norm sharding fixes) is covered. + layer_types=["full_attention", "linear_attention"], + partial_rotary_factor=1.0, + max_position_embeddings=4096, + ) + return cfg, protocol + + +def _qwen3_moe(): + _require_te() + from megatron.lite.model.qwen3_moe.config import Qwen3MoEConfig + from megatron.lite.model.qwen3_moe.lite import protocol + + cfg = Qwen3MoEConfig( + num_hidden_layers=2, + hidden_size=16, + num_attention_heads=4, + num_key_value_heads=2, + head_dim=4, + vocab_size=64, + num_experts=4, + num_experts_per_tok=1, + moe_intermediate_size=8, + max_position_embeddings=4096, + layer_types=["full_attention", "full_attention"], + ) + return cfg, protocol + + +def _kimi_k2(): + _require_te() + from megatron.lite.model.kimi_k2.config import KimiK2Config + from megatron.lite.model.kimi_k2.lite import protocol + + cfg = KimiK2Config( + num_hidden_layers=2, + hidden_size=64, + num_attention_heads=4, + num_key_value_heads=4, + vocab_size=128, + intermediate_size=96, + moe_intermediate_size=16, + n_routed_experts=4, + n_shared_experts=1, + num_experts_per_tok=2, + n_group=2, + topk_group=1, + first_k_dense_replace=1, + q_lora_rank=16, + kv_lora_rank=12, + qk_nope_head_dim=8, + qk_rope_head_dim=8, + v_head_dim=8, + max_position_embeddings=4096, + rope_theta=10000.0, + rope_scaling={ + "type": "yarn", + "factor": 1.0, + "original_max_position_embeddings": 4096, + "beta_fast": 1.0, + "beta_slow": 1.0, + "mscale": 1.0, + "mscale_all_dim": 1.0, + }, + ) + return cfg, protocol + + +def _glm5(): + pytest.importorskip("cudnn", reason="glm5 fused DSA needs the cudnn DSA stack.") + _require_te() + from megatron.lite.model.glm5.config import Glm5Config + from megatron.lite.model.glm5.lite import protocol + + cfg = Glm5Config( + num_hidden_layers=2, + hidden_size=128, + num_attention_heads=64, + num_key_value_heads=64, + head_dim=256, + vocab_size=32, + max_position_embeddings=4096, + initializer_range=0.002, + q_lora_rank=16, + kv_lora_rank=512, + qk_head_dim=256, + qk_nope_head_dim=192, + qk_rope_head_dim=64, + v_head_dim=256, + index_head_dim=128, + index_n_heads=32, + index_topk=512, + intermediate_size=20, + moe_intermediate_size=6, + first_k_dense_replace=1, + n_routed_experts=4, + n_shared_experts=1, + num_experts_per_tok=2, + ) + return cfg, protocol + + +def _deepseek_v4(): + pytest.importorskip("cudnn", reason="deepseek_v4 fused DSA needs the cudnn DSA stack.") + _require_te() + from megatron.lite.model.deepseek_v4.config import DeepseekV4Config + from megatron.lite.model.deepseek_v4.lite import protocol + + cfg = DeepseekV4Config( + vocab_size=64, + hidden_size=128, + moe_intermediate_size=16, + num_hidden_layers=2, + num_attention_heads=8, + num_key_value_heads=1, + head_dim=64, + qk_rope_head_dim=16, + q_lora_rank=32, + o_lora_rank=32, + o_groups=2, + n_routed_experts=4, + n_shared_experts=1, + num_experts_per_tok=2, + routed_scaling_factor=1.5, + max_position_embeddings=4096, + compress_ratios=[4, 4], + sliding_window=128, + num_hash_layers=2, + hc_mult=2, + index_head_dim=64, + index_n_heads=8, + index_topk=512, + # DeepSeek-V4 really has MTP; its ImplConfig defaults mtp_enable=True and + # requires >=1 nextn layer, so give it one (exercises MTP weight IO too). + num_nextn_predict_layers=1, + rms_norm_eps=1e-6, + ) + return cfg, protocol + + +MODELS = { + "qwen3_5": _qwen3_5, + "qwen3_moe": _qwen3_moe, + "kimi_k2": _kimi_k2, + "glm5": _glm5, + "deepseek_v4": _deepseek_v4, +} + +BACKENDS = ("dist_opt", "fsdp2") + + +# ────────────────────────────────────────────────────────────────────────── +# Distributed harness +# ────────────────────────────────────────────────────────────────────────── +@pytest.fixture(scope="module", autouse=True) +def _single_node_cuda_dist(): + if not torch.cuda.is_available(): + pytest.skip("CUDA is required for save/load/export smoke.") + if int(os.environ.get("WORLD_SIZE", "1")) > 8: + pytest.skip("Megatron Lite smoke tests are capped at single-node 8 GPUs.") + + os.environ.setdefault("CUBLAS_WORKSPACE_CONFIG", ":4096:8") + os.environ.setdefault("NVTE_ALLOW_NONDETERMINISTIC_ALGO", "0") + os.environ.setdefault("RANK", "0") + os.environ.setdefault("WORLD_SIZE", "1") + os.environ.setdefault("LOCAL_RANK", "0") + os.environ.setdefault("MASTER_ADDR", "127.0.0.1") + os.environ.setdefault("MASTER_PORT", "29555") + + torch.cuda.set_device(int(os.environ["LOCAL_RANK"])) + + # Diagnostic: dump all thread stacks and force-exit after N seconds so a + # hang self-reports fast (keeps each run well under the time budget) instead + # of stalling on NCCL's slow watchdog/abort. Opt-in via MLITE_HANG_DUMP_S. + hang_dump_s = os.environ.get("MLITE_HANG_DUMP_S") + if hang_dump_s: + import faulthandler + + faulthandler.dump_traceback_later(int(hang_dump_s), exit=True) + + created_pg = False + if not dist.is_initialized(): + # Short timeout so a desynced collective fails fast instead of stalling + # the whole job on the multi-minute NCCL watchdog default. + timeout_s = int(os.environ.get("MLITE_DIST_TIMEOUT_S", "180")) + dist.init_process_group( + backend="nccl", init_method="env://", timeout=timedelta(seconds=timeout_s) + ) + created_pg = True + yield + try: + from megatron.core import parallel_state as mpu + + if mpu.is_initialized(): + mpu.destroy_model_parallel() + finally: + if created_pg and dist.is_initialized(): + dist.destroy_process_group() + + +@pytest.fixture(autouse=True) +def _reset_parallel_state_between_tests(): + """Tear down Megatron model-parallel groups after each case. + + The matrix builds models with different topologies (tp2/ep2/pp2 for + dist_opt, pure-DP for fsdp2). Leaving a prior case's mpu groups initialized + desyncs the next case's collectives, so reset between tests. + """ + yield + from megatron.core import parallel_state as mpu + + if mpu.is_initialized(): + mpu.destroy_model_parallel() + + +# GLM5 / DeepSeek-V4 native lite support TP=ETP=VPP=1 only (EP/PP/CP wired +# through primitives), so their dist_opt topology shards via ep/pp/cp instead. +_TP1_ONLY = {"glm5", "deepseek_v4"} + + +def _topology(model_name: str, backend: str) -> ParallelConfig: + if backend == "fsdp2": + # fsdp2 + pp2: FSDP2 shards over dp(=4) within each of 2 pipeline stages, + # so save/load is exercised with pipeline parallelism (not just pure DP). + return ParallelConfig(tp=1, ep=1, etp=1, pp=2, cp=1) + # Diagnostic hook: MLITE_FORCE_TOPO="tp,ep,etp,pp,cp" overrides any model's + # topology to isolate which parallel dim triggers a hang. + forced = os.environ.get("MLITE_FORCE_TOPO") + if forced: + tp, ep, etp, pp, cp = (int(x) for x in forced.split(",")) + return ParallelConfig(tp=tp, ep=ep, etp=etp, pp=pp, cp=cp) + # Diagnostic hook: force the tp1/pp2 topology on any model to isolate + # whether a tp1+pp2 pipeline-P2P bug is generic (not DSA-specific). + if model_name in _TP1_ONLY or os.environ.get("MLITE_FORCE_TP1"): + # tp1 x pp2 x cp1 x dp4 = 8 ranks; ep2 within the expert space. + # CP is intentionally 1: save/load fidelity does not need the DSA CP path + # (covered by the dedicated CP smokes), and CP+tiny-seq risks fused-DSA hangs. + return ParallelConfig(tp=1, ep=2, etp=1, pp=2, cp=1) + # tp2 x pp2 x cp1 x dp2 = 8 ranks; ep2 within the expert space. + return ParallelConfig(tp=2, ep=2, etp=1, pp=2, cp=1) + + +def _optimizer_config(offload_fraction: float = 0.0) -> OptimizerConfig: + return OptimizerConfig( + optimizer="adam", + lr=1.0e-3, + weight_decay=0.0, + adam_beta1=0.9, + adam_beta2=0.95, + adam_eps=1.0e-8, + clip_grad=1.0, + offload_fraction=offload_fraction, + ) + + +def _build_handle( + model_name: str, + backend: str, + *, + seed: int, + topology: ParallelConfig | None = None, + offload_fraction: float = 0.0, +): + cfg, protocol = MODELS[model_name]() + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + + parallel = topology or _topology(model_name, backend) + impl_cfg = protocol.ImplConfig( + parallel=parallel, + optimizer=backend, + optimizer_config=_optimizer_config(offload_fraction), + use_deepep=False, + deterministic=True, + ) + bundle = protocol.build_model(cfg, impl_cfg=impl_cfg) + chunks = bundle.chunks + + if bundle.extras.get("optimizer_backend") == "fsdp2": + for chunk in chunks: + if hasattr(chunk, "initialize_weights"): + chunk.initialize_weights() + optimizer = bundle.extras["post_model_load_hook"]()["optimizer"] + else: + optimizer = bundle.optimizer + + extras = dict(bundle.extras) + extras.update( + { + "model_chunks": chunks, + "forward_step": bundle.forward_step, + "finalize_grads": bundle.finalize_grads, + "protocol": protocol, + } + ) + handle = ModelHandle( + model=chunks, + optimizer=optimizer, + parallel_state=bundle.parallel_state, + config=SimpleNamespace(parallel=parallel), + _extras=extras, + ) + return handle, cfg, protocol + + +def _shared_tmp_path(tmp_path, suffix: str) -> str: + payload = [os.path.join(str(tmp_path), suffix) if dist.get_rank() == 0 else None] + dist.broadcast_object_list(payload, src=0) + path = payload[0] + if dist.get_rank() == 0: + os.makedirs(path, exist_ok=True) + dist.barrier() + return path + + +def _random_packed_batch(vocab_size: int) -> PackedBatch: + return PackedBatch( + input_ids=torch.randint(0, vocab_size, (2048,), device="cuda"), + labels=torch.randint(0, vocab_size, (2048,), device="cuda"), + seq_lens=torch.full((1,), 2048, dtype=torch.int64, device="cuda"), + ) + + +def _train_step(handle: ModelHandle, backend: str, cfg) -> None: + # Unified path for both backends: the runtime routes pp>1 through the + # pipeline schedule regardless of optimizer backend, so fsdp2 also exercises + # pipeline parallelism here (not just pure DP). + runtime = MegatronLiteRuntime.__new__(MegatronLiteRuntime) + batch = _random_packed_batch(cfg.vocab_size) + runtime.zero_grad(handle) + runtime.forward_backward(handle, iter([batch]), None, num_microbatches=1) + runtime.optimizer_step(handle) + runtime.zero_grad(handle) + + +def _local_named_params(handle: ModelHandle) -> dict[str, torch.Tensor]: + from megatron.lite.primitive.optimizers.fsdp2.adamw import to_local_tensor + + params: dict[str, torch.Tensor] = {} + for chunk_idx, chunk in enumerate(handle._extras["model_chunks"]): + for name, param in chunk.named_parameters(): + params[f"{chunk_idx}.{name}"] = to_local_tensor(param.detach()).cpu().float().clone() + return params + + +def _assert_params_bitwise_equal(lhs: ModelHandle, rhs: ModelHandle) -> None: + lhs_params = _local_named_params(lhs) + rhs_params = _local_named_params(rhs) + assert lhs_params.keys() == rhs_params.keys() + assert lhs_params, "expected at least one local parameter to compare." + mismatches = [] + for name in lhs_params: + if not torch.equal(lhs_params[name], rhs_params[name]): + diff = (lhs_params[name] - rhs_params[name]).abs().max().item() + mismatches.append(f"{name} (max_abs_diff={diff})") + assert not mismatches, "save/load not bitwise; mismatched params:\n" + "\n".join(mismatches) + + +def _is_valid_hf_export_key(key: str, model_name: str) -> bool: + """Whether an exported key matches the model's real HF release naming. + + Most models use the ``model.``-rooted HF convention. DeepSeek-V4-Flash ships + a bare layout (``embed.weight`` / ``head.weight`` / ``norm.weight`` / + ``layers.N.*`` / ``mtp.N.*`` / ``hc_head_*``), so allow that for deepseek_v4. + """ + if model_name == "deepseek_v4": + return key.startswith(("layers.", "mtp.")) or key in ( + "embed.weight", + "head.weight", + "norm.weight", + "hc_head_base", + "hc_head_fn", + "hc_head_scale", + ) + return key.startswith("model.") or key in ("lm_head.weight",) + + +def _export_and_reload(handle: ModelHandle, cfg, protocol, out_dir: str, model_name: str) -> None: + """Export HF weights (bf16) and assert the reloaded shards are valid. + + Prefer the model's ``save_hf_weights`` wrapper; some models only expose the + ``export_hf_weights`` generator, so fall back to gathering it and writing + the safetensors ourselves (rank 0). Every supported model must offer one of + the two — otherwise it has no export path at all, which is a hard failure. + """ + chunks = handle._extras["model_chunks"] + ps = handle._parallel_state + + if hasattr(protocol, "save_hf_weights"): + protocol.save_hf_weights(chunks, out_dir, cfg, ps) + elif hasattr(protocol, "export_hf_weights"): + from megatron.lite.primitive.ckpt.hf_weights import save_safetensors + + weights = dict(protocol.export_hf_weights(chunks, cfg, ps, rank0_only=True)) + if dist.get_rank() == 0 and weights: + save_safetensors(weights, out_dir) + else: + raise AssertionError(f"{protocol.__name__} exposes no HF export path.") + dist.barrier() + + if dist.get_rank() != 0: + dist.barrier() + return + + from safetensors import safe_open + + shards = [f for f in os.listdir(out_dir) if f.endswith(".safetensors")] + assert shards, f"no safetensors exported to {out_dir}" + keys: set[str] = set() + # The shared exporter casts EVERY floating tensor to bf16 (``_cast_export_tensor`` + # with export_dtype=bf16) and leaves integers untouched. So every float weight + # — including the router correction bias — must be bf16, while integer auxiliary + # buffers (e.g. DS4 hash-routing ``tid2eid`` index tables; casting them would + # corrupt the indices) keep their native integral dtype. Excluding such buffers + # from the export would be a de-scope; we keep them and check their dtype. + for shard in shards: + with safe_open(os.path.join(out_dir, shard), framework="pt") as fh: + for key in fh.keys(): + tensor = fh.get_tensor(key) + if tensor.dtype.is_floating_point: + assert tensor.dtype == torch.bfloat16, ( + f"{key} exported as {tensor.dtype}, want bf16" + ) + else: + assert tensor.dtype in (torch.int64, torch.int32, torch.bool), ( + f"{key} exported as unexpected non-float dtype {tensor.dtype}" + ) + assert torch.isfinite(tensor.float()).all(), f"{key} has non-finite values" + assert _is_valid_hf_export_key(key, model_name), ( + f"unexpected non-HF export key: {key}" + ) + keys.add(key) + assert keys, "exported zero tensors" + # PP-gather completeness: the rank-0 export must carry EVERY decoder layer's + # weights (all pipeline stages gathered), not just the first stage's — guards + # against a pp-blind export silently dropping later stages' layers. + num_layers = int(getattr(cfg, "num_hidden_layers")) + + def _has_layer(i: int) -> bool: + # Match both the ``model.``-rooted convention (``model.layers.{i}.``) and + # the bare DeepSeek-V4-Flash layout (``layers.{i}.``), which has no prefix. + prefix = f"layers.{i}." + return any(k.startswith(prefix) or f".{prefix}" in k for k in keys) + + missing = [i for i in range(num_layers) if not _has_layer(i)] + assert not missing, ( + f"export missing decoder layers {missing} (PP gather incomplete); " + f"sample keys: {sorted(keys)[:6]}" + ) + dist.barrier() + + +@pytest.mark.parametrize("backend", BACKENDS) +@pytest.mark.parametrize("model_name", list(MODELS)) +def test_save_load_roundtrip(model_name, backend, tmp_path): + """Checkpoint save -> fresh build -> load restores parameters bit-exactly. + + Covers all 5 models x {dist_opt + distckpt, fsdp2 + dcp} (10 combos) — the + primary regression guard for the runtime checkpoint entry points. + """ + if dist.get_world_size() != 8: + pytest.skip("save/load proxy smoke requires exactly 8 GPUs.") + + set_deterministic(2026) + + saved, cfg, _protocol = _build_handle(model_name, backend, seed=4242) + _train_step(saved, backend, cfg) + + ckpt_dir = _shared_tmp_path(tmp_path, "ckpt") + runtime = MegatronLiteRuntime.__new__(MegatronLiteRuntime) + runtime.save_checkpoint(saved, ckpt_dir, step=1) + + loaded, _cfg2, _proto2 = _build_handle(model_name, backend, seed=9999) + assert runtime.load_checkpoint(loaded, ckpt_dir) == 1 + _assert_params_bitwise_equal(saved, loaded) + + +@pytest.mark.parametrize("model_name", list(MODELS)) +def test_export_hf_bf16_reload(model_name, tmp_path): + """Export HF weights (bf16) and reload the safetensors shards. + + Export output is backend-agnostic, so this exercises the canonical export + path: a dist_opt model whose TP/EP/PP shards are gathered to full HF + tensors. NOTE: exporting directly from a live fsdp2 (DTensor-sharded) model + is NOT covered here — save_hf_weights' gather is not DTensor-aware and + deadlocks; tracked as a known gap. + """ + if dist.get_world_size() != 8: + pytest.skip("export proxy smoke requires exactly 8 GPUs.") + + set_deterministic(2026) + + handle, cfg, protocol = _build_handle(model_name, "dist_opt", seed=4242) + _train_step(handle, "dist_opt", cfg) + + export_dir = _shared_tmp_path(tmp_path, "hf_export") + _export_and_reload(handle, cfg, protocol, export_dir, model_name) + + +# ────────────────────────────────────────────────────────────────────────── +# Runtime offload / onload roundtrip (RL Best tier: runtime.to(cpu/cuda)) +# +# Exercises the real runtime.to() used to reclaim GPU between train and rollout: +# param + optimizer state move CPU<->GPU as a whole (NOT offload_fraction). +# 3 delivery models x 2 optimizers on the 8-GPU proxy2 topology. +# ────────────────────────────────────────────────────────────────────────── + +# qwen3_5 offload is validated separately (its GatedDeltaNet linear-attention +# path and run env differ); the others run here. +DELIVERY_MODELS = ("deepseek_v4", "glm5", "kimi_k2") + + +def _offload_topology(model_name: str) -> ParallelConfig: + # proxy2 = 8-GPU pp2/ep2/cp2. CSA/DSA (glm5, ds4) are TP=1 only, so they + # fill 8 ranks with dp2 (tp1·cp2·pp2·dp2=8); TP-capable MoE models use tp2 + # (tp2·cp2·pp2·dp1=8). + forced = os.environ.get("MLITE_FORCE_TOPO") + if forced: + tp, ep, etp, pp, cp = (int(x) for x in forced.split(",")) + return ParallelConfig(tp=tp, ep=ep, etp=etp, pp=pp, cp=cp) + if model_name in _TP1_ONLY: # glm5, deepseek_v4: CSA/DSA are TP=1 only + return ParallelConfig(tp=1, ep=2, etp=1, pp=2, cp=2) + return ParallelConfig(tp=2, ep=2, etp=1, pp=2, cp=2) + + +def _iter_opt_state_tensors(handle: ModelHandle, backend: str): + """Yield (key, tensor) over optimizer state for either backend — the same + tensors runtime.to() offloads, so device / value can be inspected.""" + opt = handle._optimizer + if backend == "fsdp2": + from megatron.lite.primitive.optimizers.fsdp2.adamw import iter_torch_optimizers + + for ci, child in enumerate(iter_torch_optimizers(opt.optimizer)): + for pi, st in enumerate(getattr(child, "state", {}).values()): + if isinstance(st, dict): + for k, v in st.items(): + if isinstance(v, torch.Tensor): + yield f"{ci}.{pi}.{k}", v + else: + from megatron.core.optimizer import ChainedOptimizer + + opts = opt.chained_optimizers if isinstance(opt, ChainedOptimizer) else [opt] + for oi, sub in enumerate(opts): + inner = getattr(sub, "optimizer", None) + if inner is None: + continue + for pi, st in enumerate(inner.state.values()): + for k, v in st.items(): + if isinstance(v, torch.Tensor): + yield f"{oi}.{pi}.{k}", v + + +def _opt_state_devices(handle: ModelHandle, backend: str) -> set[str]: + from megatron.lite.primitive.optimizers.fsdp2.adamw import to_local_tensor + + return {to_local_tensor(v).device.type for _, v in _iter_opt_state_tensors(handle, backend)} + + +def _opt_state_snapshot(handle: ModelHandle, backend: str) -> dict[str, torch.Tensor]: + from megatron.lite.primitive.optimizers.fsdp2.adamw import to_local_tensor + + return { + k: to_local_tensor(v.detach()).cpu().float().clone() + for k, v in _iter_opt_state_tensors(handle, backend) + } + + +def _local_param_devices(handle: ModelHandle) -> set[str]: + from megatron.lite.primitive.optimizers.fsdp2.adamw import to_local_tensor + + return { + to_local_tensor(p.detach()).device.type + for chunk in handle._extras["model_chunks"] + for p in chunk.parameters() + } + + +def _assert_named_bitwise_equal(lhs: dict, rhs: dict, label: str) -> None: + assert lhs.keys() == rhs.keys(), f"{label} keys differ across offload roundtrip." + mismatches = [] + for name in lhs: + if not torch.equal(lhs[name], rhs[name]): + diff = (lhs[name] - rhs[name]).abs().max().item() + mismatches.append(f"{name} (max_abs_diff={diff})") + assert not mismatches, f"{label} not bitwise after offload/onload:\n" + "\n".join(mismatches) + + +@pytest.mark.parametrize("backend", BACKENDS) +@pytest.mark.parametrize("model_name", DELIVERY_MODELS) +def test_offload_onload_roundtrip(model_name, backend, tmp_path): + """runtime.to(cpu) -> to(cuda) restores params + optimizer state exactly and + training continues — the RL train<->rollout GPU-reclaim path.""" + if dist.get_world_size() != 8: + pytest.skip("offload/onload proxy smoke requires exactly 8 GPUs.") + + set_deterministic(2026) + handle, cfg, _ = _build_handle( + model_name, backend, seed=4242, topology=_offload_topology(model_name) + ) + _train_step(handle, backend, cfg) # populate optimizer (exp_avg) state + + runtime = MegatronLiteRuntime.__new__(MegatronLiteRuntime) + params_before = _local_named_params(handle) + opt_before = _opt_state_snapshot(handle, backend) + assert opt_before, "expected optimizer state after a train step." + assert _opt_state_devices(handle, backend) == {"cuda"} + + runtime.to(handle, "cpu", model=True, optimizer=True, grad=True) + assert _opt_state_devices(handle, backend) == {"cpu"}, "optimizer state not offloaded to CPU." + if backend == "fsdp2": + # fsdp2 moves params to CPU directly; dist_opt instead frees the GPU + # buffer storage (params keep a 0-size cuda handle), so assert only fsdp2. + assert _local_param_devices(handle) == {"cpu"}, "params not offloaded to CPU." + + runtime.to(handle, "cuda", model=True, optimizer=True, grad=True) + assert _opt_state_devices(handle, backend) == {"cuda"}, "optimizer state not back on GPU." + assert _local_param_devices(handle) == {"cuda"}, "params not back on GPU." + + _assert_named_bitwise_equal(params_before, _local_named_params(handle), "param") + _assert_named_bitwise_equal(opt_before, _opt_state_snapshot(handle, backend), "optimizer-state") + + _train_step(handle, backend, cfg) # continues training after onload + + +@pytest.mark.parametrize("backend", BACKENDS) +@pytest.mark.parametrize("model_name", ["kimi_k2"]) +def test_offload_fraction_keeps_optimizer_state_on_cpu(model_name, backend, tmp_path): + """offload_fraction>0 keeps the optimizer update state on CPU and still + trains. One delivery model is enough to guard the real-model wiring + (generic TinyModel coverage already exists).""" + if dist.get_world_size() != 8: + pytest.skip("offload_fraction proxy smoke requires exactly 8 GPUs.") + + set_deterministic(2026) + handle, cfg, _ = _build_handle( + model_name, + backend, + seed=4242, + topology=_offload_topology(model_name), + offload_fraction=1.0, + ) + _train_step(handle, backend, cfg) + assert "cpu" in _opt_state_devices(handle, backend), ( + "offload_fraction=1.0 should keep optimizer update state on CPU." + ) + _train_step(handle, backend, cfg) # trains with the offloaded update state diff --git a/experimental/lite/tests/unit/examples/test_bench_example.py b/experimental/lite/tests/unit/examples/test_bench_example.py new file mode 100644 index 00000000000..9a9c00a9bbb --- /dev/null +++ b/experimental/lite/tests/unit/examples/test_bench_example.py @@ -0,0 +1,310 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Unit tests for the benchmark example.""" + +from __future__ import annotations + +import json +import sys +from contextlib import nullcontext +from pathlib import Path + +import torch + +from megatron.lite.runtime.contracts.config import ParallelConfig +from megatron.lite.runtime.contracts.data import ForwardResult, ModelOutputs +from megatron.lite.runtime.contracts.handle import ModelHandle + +_LITE_ROOT = str(Path(__file__).resolve().parents[3]) +sys.path = [path for path in sys.path if path != _LITE_ROOT] +sys.path.insert(0, _LITE_ROOT) + + +def test_bench_builds_mlite_runtime_config_with_model_hook(): + from examples.bench.bench import BenchCliConfig, build_runtime_config + from megatron.lite.model.qwen3_5.config import Qwen35Config + from megatron.lite.runtime.backends.mlite.config import MegatronLiteConfig + + cfg = BenchCliConfig( + backend="mlite", + hf_path="/tmp/hf", + model_name="qwen3_5", + use_thd=True, + truncate_layers=2, + disable_mtp=True, + ) + + runtime_cfg = build_runtime_config(cfg) + + assert runtime_cfg.backend == "mlite" + assert isinstance(runtime_cfg.backend_cfg, MegatronLiteConfig) + assert runtime_cfg.backend_cfg.impl_cfg["use_thd"] is True + assert callable(runtime_cfg.backend_cfg.model_config_hook) + + model_cfg = runtime_cfg.backend_cfg.model_config_hook(Qwen35Config()) + assert model_cfg.num_hidden_layers == 2 + assert len(model_cfg.layer_types) == 2 + assert model_cfg.num_nextn_predict_layers == 0 + + +def test_bench_mlite_deterministic_mounts_native_vision_not_mbridge(monkeypatch): + from examples.bench.bench import BenchCliConfig, build_runtime_config + + monkeypatch.setenv("MEGATRON_LITE_DETERMINISTIC", "1") + + runtime_cfg = build_runtime_config( + BenchCliConfig(backend="mlite", hf_path="/tmp/hf", model_name="qwen3_5") + ) + + impl_cfg = runtime_cfg.backend_cfg.impl_cfg + assert impl_cfg["mount_vision_model"] is True + assert ("mount_" + "mbridge_vision_model") not in impl_cfg + + +def test_bench_builds_bridge_dry_run_plan_without_bridge_import(): + from examples.bench.bench import BenchCliConfig, build_dry_run_plan + + plan = build_dry_run_plan( + BenchCliConfig( + backend="bridge", + hf_path="/tmp/hf", + model_name="qwen3_5", + truncate_layers=2, + override_transformer_json='{"attention_backend": "unfused"}', + dry_run=True, + ) + ) + + assert plan["dry_run"] is True + assert plan["runtime"]["backend"] == "bridge" + backend_cfg = plan["runtime"]["backend_cfg"] + assert backend_cfg["model_name"] == "qwen3_5" + assert backend_cfg["override_transformer_config"] == {"attention_backend": "unfused"} + assert backend_cfg["bridge_post_init"].startswith(" None: + pass + + def forward_backward(self, handle, data, loss_fn, *, num_microbatches: int = 1): + self.loss += 1 + return ForwardResult(model_output=ModelOutputs(loss=torch.tensor(float(self.loss)))) + + def optimizer_step(self, handle): + return True, 3.5, 0 + + def lr_scheduler_step(self, handle): + return 0.0 + + +def test_pretrain_session_runs_with_fake_runtime_on_cpu(): + from examples.bench.session import PretrainSessionConfig, run_pretrain_session + + handle = ModelHandle( + model=object(), + optimizer=object(), + parallel_state=None, + config=type( + "Cfg", (), {"model_name": "fake", "impl": "lite", "parallel": ParallelConfig()} + )(), + _extras={"optimizer_backend": "fake"}, + ) + + result = run_pretrain_session( + _FakeRuntime(), + handle, + PretrainSessionConfig(steps=3, warmup=1, device="cpu", seq_len=4), + data_iter=iter([{}, {}, {}]), + ) + + assert result.backend == "mlite" + assert result.seq_len == 4 + assert result.num_microbatches == 1 + assert len(result.step_traces) == 2 + assert [trace.loss for trace in result.step_traces] == [2.0, 3.0] + assert result.step_traces[0].grad_norm == 3.5 + + +def test_bench_main_writes_dry_run_output_json(tmp_path): + from examples.bench.bench import main + + output_path = tmp_path / "dry_run.json" + + artifact = main( + [ + "--backend", + "mlite", + "--hf-path", + "/tmp/hf", + "--model-name", + "qwen3_5", + "--truncate-layers", + "2", + "--disable-mtp", + "--dry-run", + "--output-json", + str(output_path), + ] + ) + + assert output_path.exists() + assert json.loads(output_path.read_text()) == artifact + + +def test_bench_main_writes_output_json_only_on_rank_zero(tmp_path, monkeypatch): + from examples.bench.bench import main + + output_path = tmp_path / "rank_one.json" + monkeypatch.setenv("RANK", "1") + + artifact = main( + [ + "--backend", + "mlite", + "--hf-path", + "/tmp/hf", + "--model-name", + "qwen3_5", + "--dry-run", + "--output-json", + str(output_path), + ] + ) + + assert artifact["dry_run"] is True + assert not output_path.exists() + + +def test_result_artifact_summary_and_trace_compare(tmp_path): + from examples.bench.results import compare_step_traces, load_result_artifact, result_summary + + baseline = { + "summary": { + "backend": "mlite", + "avg_step_ms": 10.0, + "tok_per_s": 3200.0, + "steps_measured": 2, + }, + "result": { + "step_traces": [ + {"step": 0, "loss": 1.0, "grad_norm": 2.0, "step_ms": 10.0}, + {"step": 1, "loss": 1.5, "grad_norm": 2.5, "step_ms": 10.0}, + ] + }, + } + candidate = { + "summary": { + "backend": "bridge", + "avg_step_ms": 11.0, + "tok_per_s": 2900.0, + "steps_measured": 2, + }, + "result": { + "step_traces": [ + {"step": 0, "loss": 1.00001, "grad_norm": 2.00001, "step_ms": 11.0}, + {"step": 1, "loss": 1.49999, "grad_norm": 2.49999, "step_ms": 11.0}, + ] + }, + } + baseline_path = tmp_path / "mlite.json" + baseline_path.write_text(json.dumps(baseline), encoding="utf-8") + + loaded = load_result_artifact(baseline_path) + + assert result_summary(loaded)["backend"] == "mlite" + assert compare_step_traces(baseline, candidate, atol=1e-3, rtol=0.0)["passed"] is True + + +def test_result_trace_compare_reports_metric_level_failures(): + from examples.bench.results import compare_step_traces + + baseline = { + "result": {"step_traces": [{"step": 0, "loss": 1.0, "grad_norm": 2.0, "step_ms": 10.0}]} + } + candidate = { + "result": {"step_traces": [{"step": 0, "loss": 1.00001, "grad_norm": 3.0, "step_ms": 10.0}]} + } + + comparison = compare_step_traces(baseline, candidate, atol=1e-3, rtol=0.0) + + assert comparison["passed"] is False + assert comparison["loss_passed"] is True + assert comparison["grad_norm_passed"] is False + + +def test_correctness_compare_requires_bitwise_fields(): + from examples.bench.results import compare_correctness_artifacts + + baseline = { + "eval_logits": {"sha256": "a", "shape": [1], "dtype": "torch.bfloat16"}, + "steps": [ + { + "loss": {"value": 1.0, "float_hex": (1.0).hex()}, + "logits": {"sha256": "b"}, + "grad_fingerprint": {"sha256": "c", "tensor_count": 1}, + "grad_norm": {"value": 2.0, "float_hex": (2.0).hex()}, + "update_successful": True, + "num_zeros": 0, + "post_step_weights": {"sha256": "d", "tensor_count": 1}, + } + ], + } + candidate = json.loads(json.dumps(baseline)) + + assert compare_correctness_artifacts(baseline, candidate)["passed"] is True + + candidate["steps"][0]["grad_norm"] = {"value": 2.5, "float_hex": (2.5).hex()} + comparison = compare_correctness_artifacts(baseline, candidate) + + assert comparison["passed"] is False + assert comparison["max_grad_norm_abs"] == 0.5 diff --git a/experimental/lite/tests/unit/model/test_deepseek_v4_hf_load_local_to_global.py b/experimental/lite/tests/unit/model/test_deepseek_v4_hf_load_local_to_global.py new file mode 100644 index 00000000000..40895e435bd --- /dev/null +++ b/experimental/lite/tests/unit/model/test_deepseek_v4_hf_load_local_to_global.py @@ -0,0 +1,78 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""DeepSeek-V4 HF load must lift LOCAL pipeline layer indices to GLOBAL. + +Under PP the model's ``self.layers`` ModuleDict is keyed by LOCAL pipeline +position, so a non-first stage's native ``state_dict`` keys carry local indices +(``layers.0`` ...). The HF release is keyed by GLOBAL layer index, so -- exactly +like the exporter -- ``load_hf_weights`` must map local->global via +``to_global_layer_name(name, layer_map)`` before resolving HF names. Without it a +non-first stage reads the wrong global layer's weights. + +This is a CPU unit test: a minimal stand-in stage (no GPU/TE) whose layers are +keyed locally but carry ``layer_indices`` = the global ids it owns, plus a tiny +on-disk safetensors keyed by GLOBAL names. ``load_hf_weights`` must copy each +local layer the GLOBAL layer's tensor; pre-fix it resolves local names, finds +nothing, and leaves the params untouched. +""" +from __future__ import annotations + +from types import SimpleNamespace + +import pytest +import torch +import torch.nn as nn + +pytestmark = pytest.mark.mlite + + +class _LayerNorm(nn.Module): + def __init__(self, dim: int): + super().__init__() + self.weight = nn.Parameter(torch.zeros(dim)) + + +class _Block(nn.Module): + def __init__(self, dim: int): + super().__init__() + self.input_layernorm = _LayerNorm(dim) + + +class _Stage(nn.Module): + """A non-first PP stage: layers keyed by LOCAL position, ``layer_indices`` + gives the GLOBAL ids it owns (e.g. [4, 5] for the 3rd stage of pp).""" + + def __init__(self, global_ids: list[int], dim: int): + super().__init__() + self.layer_indices = list(global_ids) + self.layers = nn.ModuleDict({str(i): _Block(dim) for i in range(len(global_ids))}) + + +def test_ds4_load_hf_resolves_local_pp_layer_to_global(tmp_path): + from safetensors.torch import save_file + + from megatron.lite.model.deepseek_v4.config import DeepseekV4Config + from megatron.lite.model.deepseek_v4.lite import checkpoint as ckpt + + dim = 4 + global_ids = [4, 5] # this stage owns global layers 4 and 5, keyed local 0 and 1 + model = _Stage(global_ids, dim) + cfg = DeepseekV4Config(num_hidden_layers=8, n_routed_experts=8) + ps = SimpleNamespace(tp_size=1, etp_size=1, ep_size=1, ep_rank=0) + + # Real-release layout is keyed by GLOBAL layer index; input_layernorm maps to + # the bare V4-Flash ``attn_norm.weight``. + save_file( + {f"layers.{g}.attn_norm.weight": torch.full((dim,), float(g)) for g in global_ids}, + str(tmp_path / "model.safetensors"), + ) + + ckpt.load_hf_weights(model, str(tmp_path), cfg, ps) + + # local layer 0 -> global 4 -> filled with 4.0; local 1 -> global 5 -> 5.0. + # Pre-fix, load resolved layers.0/layers.1 (local), found nothing, left zeros. + torch.testing.assert_close( + model.layers["0"].input_layernorm.weight.detach(), torch.full((dim,), 4.0) + ) + torch.testing.assert_close( + model.layers["1"].input_layernorm.weight.detach(), torch.full((dim,), 5.0) + ) diff --git a/experimental/lite/tests/unit/model/test_glm5_lite_cp_smoke.py b/experimental/lite/tests/unit/model/test_glm5_lite_cp_smoke.py new file mode 100644 index 00000000000..8e769d1110b --- /dev/null +++ b/experimental/lite/tests/unit/model/test_glm5_lite_cp_smoke.py @@ -0,0 +1,463 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +from __future__ import annotations + +import pytest + + +def _make_train_config(ps): + from types import SimpleNamespace + + return SimpleNamespace( + tp=ps.tp_size, + ep=ps.ep_size, + etp=ps.etp_size, + pp=ps.pp_size, + cp=ps.cp_size, + vpp=None, + use_deepep=False, + fp8=False, + recompute_modules=[], + deterministic=True, + ) + + +def _make_glm5_model(cfg, ps, **kwargs): + from megatron.lite.model.glm5.lite.model import Glm5Model + + return Glm5Model(cfg, _make_train_config(ps), ps, **kwargs) + + +def _init_dist_or_skip(): + import os + + import torch + import torch.distributed as dist + + if not torch.cuda.is_available(): + pytest.skip("CUDA is required for GLM5 CP smoke.") + if "RANK" not in os.environ or "WORLD_SIZE" not in os.environ: + pytest.skip("Run with torchrun so CP ranks are available.") + + local_rank = int(os.environ.get("LOCAL_RANK", "0")) + torch.cuda.set_device(local_rank) + if not dist.is_initialized(): + dist.init_process_group("nccl") + if dist.get_world_size() < 2: + pytest.skip("GLM5 CP smoke requires at least 2 ranks.") + return torch.device("cuda", local_rank) + + +def _tiny_config_kwargs(): + return dict( + num_hidden_layers=2, + hidden_size=128, + num_attention_heads=64, + num_key_value_heads=64, + head_dim=256, + vocab_size=32, + max_position_embeddings=512, + initializer_range=0.002, + q_lora_rank=16, + kv_lora_rank=512, + qk_head_dim=256, + qk_nope_head_dim=192, + qk_rope_head_dim=64, + v_head_dim=256, + index_head_dim=128, + index_n_heads=32, + index_topk=512, + intermediate_size=20, + moe_intermediate_size=6, + first_k_dense_replace=1, + n_routed_experts=3, + n_shared_experts=1, + num_experts_per_tok=3, + ) + + +def _tiny_hf_parity_config_kwargs(): + kwargs = _tiny_config_kwargs() + kwargs.update(index_topk=512, max_position_embeddings=512) + return kwargs + + +def _fused_dsa_seq_len(world: int) -> int: + seq = 512 + if seq % (2 * world) != 0: + pytest.skip(f"GLM5 fused DSA CP smoke requires seq={seq} divisible by 2*world={2 * world}.") + return seq + + +def _to_hf_deepseek_v3_config(cfg): + from transformers.models.deepseek_v3.configuration_deepseek_v3 import DeepseekV3Config + + return DeepseekV3Config( + hidden_size=cfg.hidden_size, + intermediate_size=cfg.intermediate_size, + moe_intermediate_size=cfg.moe_intermediate_size, + num_hidden_layers=cfg.num_hidden_layers, + num_attention_heads=cfg.num_attention_heads, + num_key_value_heads=cfg.num_key_value_heads, + vocab_size=cfg.vocab_size, + n_shared_experts=cfg.n_shared_experts, + n_routed_experts=cfg.n_routed_experts, + routed_scaling_factor=cfg.routed_scaling_factor, + kv_lora_rank=cfg.kv_lora_rank, + q_lora_rank=cfg.q_lora_rank, + qk_rope_head_dim=cfg.qk_rope_head_dim, + v_head_dim=cfg.v_head_dim, + qk_nope_head_dim=cfg.qk_nope_head_dim, + n_group=cfg.n_group, + topk_group=cfg.topk_group, + num_experts_per_tok=cfg.num_experts_per_tok, + first_k_dense_replace=cfg.first_k_dense_replace, + norm_topk_prob=cfg.norm_topk_prob, + max_position_embeddings=cfg.max_position_embeddings, + rms_norm_eps=cfg.rms_norm_eps, + tie_word_embeddings=False, + rope_theta=cfg.rope_theta, + rope_scaling=None, + rope_interleave=cfg.rope_interleave, + attention_bias=False, + attention_dropout=0.0, + use_cache=False, + ) + + +def _capture_tensor_output(outputs): + return outputs[0].detach() if isinstance(outputs, tuple) else outputs.detach() + + +def _distributed_diff_stats(actual, expected) -> tuple[float, float]: + import torch + import torch.distributed as dist + + diff = (actual.float() - expected.float()).abs() + max_abs = diff.max() + scale = torch.maximum(actual.float().abs().max(), expected.float().abs().max()).clamp_min(1e-6) + stats = torch.stack([max_abs, scale]) + if dist.is_initialized(): + dist.all_reduce(stats, op=dist.ReduceOp.MAX) + return float(stats[0].item()), float((stats[0] / stats[1]).item()) + + +def _hf_state_dict_for_glm5_loader(model): + return { + name: tensor.detach().cpu().contiguous().clone() + for name, tensor in model.state_dict().items() + } + + +def _make_dsa(*, cp_size: int = 1, cp_rank: int = 0, cp_group=None): + from megatron.lite.primitive.modules.attention import DynamicSparseAttention + + return DynamicSparseAttention( + hidden_size=128, + num_attention_heads=64, + q_lora_rank=16, + kv_lora_rank=512, + qk_nope_head_dim=192, + qk_rope_head_dim=64, + v_head_dim=256, + index_n_heads=32, + index_head_dim=128, + index_topk=512, + rms_norm_eps=1e-5, + cp_size=cp_size, + cp_rank=cp_rank, + cp_group=cp_group, + ) + + +@pytest.mark.gpu +def test_glm5_dsa_cp2_matches_full_sequence_reference_forward_and_grad(): + import torch + import torch.distributed as dist + + from megatron.lite.primitive.modules.attention import build_rope_cache + from megatron.lite.primitive.parallel.cp import zigzag_position_ids_for_cp, zigzag_slice_for_cp + from megatron.lite.primitive.parallel.state import ParallelState + + device = _init_dist_or_skip() + world = dist.get_world_size() + rank = dist.get_rank() + ps = ParallelState(cp_group=dist.group.WORLD, cp_size=world, cp_rank=rank) + + torch.manual_seed(2026) + cp_attn = _make_dsa(cp_size=world, cp_rank=rank, cp_group=ps.cp_group).to( + device=device, dtype=torch.bfloat16 + ) + torch.manual_seed(2026) + ref_attn = _make_dsa().to(device=device, dtype=torch.bfloat16) + + batch, seq = 1, _fused_dsa_seq_len(world) + torch.manual_seed(99) + full_x = torch.randn(batch, seq, 128, device=device, dtype=torch.bfloat16) + local_x = zigzag_slice_for_cp(full_x, rank, world, seq_dim=1).detach().requires_grad_(True) + ref_x = full_x.detach().clone().requires_grad_(True) + + cos, sin = build_rope_cache( + dim=64, max_position_embeddings=seq, rope_theta=1_000_000.0, device=device + ) + local_pos = zigzag_position_ids_for_cp(seq, rank, world, device).expand(batch, -1) + full_pos = torch.arange(seq, device=device, dtype=torch.long).unsqueeze(0).expand(batch, -1) + + cp_out = cp_attn(local_x, cos=cos, sin=sin, position_ids=local_pos) + ref_out = ref_attn(ref_x, cos=cos, sin=sin, position_ids=full_pos) + expected = zigzag_slice_for_cp(ref_out, rank, world, seq_dim=1) + torch.testing.assert_close(cp_out, expected, atol=3e-2, rtol=3e-2) + + cp_out.float().sum().backward() + ref_out.float().sum().backward() + expected_grad = zigzag_slice_for_cp(ref_x.grad, rank, world, seq_dim=1) + assert local_x.grad is not None + torch.testing.assert_close(local_x.grad, expected_grad, atol=8e-2, rtol=8e-2) + + +@pytest.mark.gpu +def test_glm5_tiny_model_cp2_matches_full_sequence_reference_forward(): + import torch + import torch.distributed as dist + + from megatron.lite.model.glm5.config import Glm5Config + from megatron.lite.primitive.parallel.cp import zigzag_slice_for_cp + from megatron.lite.primitive.parallel.state import ParallelState + + device = _init_dist_or_skip() + world = dist.get_world_size() + rank = dist.get_rank() + cfg = Glm5Config(**_tiny_config_kwargs()) + cfg.mlp_layer_types = ["dense", "dense"] + ps = ParallelState(cp_group=dist.group.WORLD, cp_size=world, cp_rank=rank) + + torch.manual_seed(777) + cp_model = _make_glm5_model(cfg, ps=ps).to(device=device, dtype=torch.bfloat16) + torch.manual_seed(777) + ref_model = _make_glm5_model(cfg, ps=ParallelState()).to( + device=device, dtype=torch.bfloat16 + ) + cp_model.eval() + ref_model.eval() + + batch, seq = 1, _fused_dsa_seq_len(world) + torch.manual_seed(100) + full_hidden = torch.randn(batch, seq, cfg.hidden_size, device=device, dtype=torch.bfloat16) + local_hidden = zigzag_slice_for_cp(full_hidden, rank, world, seq_dim=1).contiguous() + + with torch.no_grad(): + cp_hidden = cp_model(hidden_states=local_hidden)["hidden_states"] + ref_hidden = ref_model(hidden_states=full_hidden)["hidden_states"] + expected = zigzag_slice_for_cp(ref_hidden, rank, world, seq_dim=1) + + torch.testing.assert_close(cp_hidden, expected, atol=1e-1, rtol=1e-1) + + +@pytest.mark.gpu +def test_glm5_tiny_model_cp2_forward_backward_smoke(): + import torch + import torch.distributed as dist + + from megatron.lite.model.glm5.config import Glm5Config + from megatron.lite.primitive.parallel.cp import zigzag_slice_for_cp + from megatron.lite.primitive.parallel.state import ParallelState + + device = _init_dist_or_skip() + world = dist.get_world_size() + rank = dist.get_rank() + cfg = Glm5Config(**_tiny_config_kwargs()) + cfg.mlp_layer_types = ["dense", "dense"] + ps = ParallelState(cp_group=dist.group.WORLD, cp_size=world, cp_rank=rank) + + torch.manual_seed(1234) + model = _make_glm5_model(cfg, ps=ps).to(device=device, dtype=torch.bfloat16) + + batch, seq = 1, _fused_dsa_seq_len(world) + torch.manual_seed(55) + full_ids = torch.randint(0, cfg.vocab_size, (batch, seq), device=device) + input_ids = zigzag_slice_for_cp(full_ids, rank, world, seq_dim=1).contiguous() + + output = model(input_ids=input_ids) + assert output["hidden_states"].shape == (batch, seq // world, cfg.hidden_size) + assert torch.isfinite(output["hidden_states"].float()).all() + loss = output["hidden_states"].float().square().mean() + assert loss.ndim == 0 + assert torch.isfinite(loss) + loss.backward() + + grad_norm = torch.zeros((), device=device) + for param in model.parameters(): + if param.grad is not None: + grad_norm = grad_norm + param.grad.detach().float().norm() + assert torch.isfinite(grad_norm) + + +@pytest.mark.gpu +def test_glm5_packed_thd_variable_sequence_cp2_forward_backward_smoke(): + import torch + import torch.distributed as dist + + from megatron.lite.model.glm5.config import Glm5Config + from megatron.lite.primitive.parallel.state import ParallelState + from megatron.lite.primitive.parallel.thd import pack_nested_thd, unpack_packed_thd_to_nested + + device = _init_dist_or_skip() + world = dist.get_world_size() + rank = dist.get_rank() + cfg_kwargs = _tiny_config_kwargs() + cfg_kwargs.update(max_position_embeddings=64, num_nextn_predict_layers=1) + cfg = Glm5Config(**cfg_kwargs) + cfg.mlp_layer_types = ["dense", "dense"] + ps = ParallelState(cp_group=dist.group.WORLD, cp_size=world, cp_rank=rank) + + torch.manual_seed(20260614) + model = _make_glm5_model(cfg, ps=ps, mtp_enable=True, mtp_enable_train=True).to( + device=device, dtype=torch.bfloat16 + ) + model.train() + + lengths = [16, 20, 24] + ids = torch.nested.as_nested_tensor( + [ + torch.randint(0, cfg.vocab_size, (length,), device=device, dtype=torch.long) + for length in lengths + ], + layout=torch.jagged, + ) + labels = torch.nested.as_nested_tensor( + [ + torch.randint(0, cfg.vocab_size, (length,), device=device, dtype=torch.long) + for length in lengths + ], + layout=torch.jagged, + ) + loss_mask = torch.nested.as_nested_tensor( + [torch.ones(length, device=device, dtype=torch.float32) for length in lengths], + layout=torch.jagged, + ) + packed = pack_nested_thd( + ids, + cp_size=world, + cp_rank=rank, + cp_group=ps.cp_group, + labels=labels, + loss_mask=loss_mask, + ) + + out = model( + input_ids=packed.input_ids, + labels=packed.labels, + loss_mask=packed.loss_mask, + position_ids=packed.position_ids, + packed_seq_params=packed.packed_seq_params, + ) + assert torch.isfinite(out["loss"]) + assert "mtp_loss" in out + out["loss"].backward() + + grad_norm = torch.zeros((), device=device) + for param in model.parameters(): + if param.grad is not None: + grad_norm = grad_norm + param.grad.detach().float().norm() + assert torch.isfinite(grad_norm) + + nested_log_probs = unpack_packed_thd_to_nested(out["log_probs"], packed) + assert nested_log_probs.offsets().numel() == len(lengths) + 1 + assert [int(x) for x in nested_log_probs.offsets().diff().cpu()] == lengths + + if rank == 0: + print( + "NON_SKIP_GLM5_THD_CP_SMOKE_PASSED " + f"world_size={world} lengths={lengths} " + f"loss={float(out['loss'].detach().item()):.6e} " + f"grad_norm={float(grad_norm.detach().item()):.6e}" + ) + + +@pytest.mark.gpu +def test_glm5_tiny_model_cp2_matches_hf_reference_logits(tmp_path): + import torch + import torch.distributed as dist + from transformers.models.deepseek_v3.modeling_deepseek_v3 import DeepseekV3ForCausalLM + + from megatron.lite.model.glm5.config import Glm5Config + from megatron.lite.model.glm5.lite.checkpoint import load_hf_weights + from megatron.lite.primitive.ckpt.hf_weights import save_safetensors + from megatron.lite.primitive.parallel.cp import zigzag_slice_for_cp + from megatron.lite.primitive.parallel.state import ParallelState + + device = _init_dist_or_skip() + world = dist.get_world_size() + rank = dist.get_rank() + cfg = Glm5Config(**_tiny_hf_parity_config_kwargs()) + + torch.manual_seed(20260611) + hf_ref = DeepseekV3ForCausalLM(_to_hf_deepseek_v3_config(cfg)).to( + device=device, dtype=torch.bfloat16 + ) + hf_ref.eval() + rank_tmp_path = tmp_path / f"rank{rank}" + save_safetensors(_hf_state_dict_for_glm5_loader(hf_ref), str(rank_tmp_path)) + + ps = ParallelState(cp_group=dist.group.WORLD, cp_size=world, cp_rank=rank) + native = _make_glm5_model(cfg, ps=ps).to(device=device, dtype=torch.bfloat16) + native.eval() + load_hf_weights(native, str(rank_tmp_path), cfg, ps) + + batch, seq = 1, _fused_dsa_seq_len(world) + torch.manual_seed(311) + full_ids = torch.randint(0, cfg.vocab_size, (batch, seq), device=device) + local_ids = zigzag_slice_for_cp(full_ids, rank, world, seq_dim=1).contiguous() + + hf_layer_outputs = [] + native_layer_outputs = [] + hooks = [] + for layer in hf_ref.model.layers: + hooks.append( + layer.register_forward_hook( + lambda _module, _inputs, outputs: hf_layer_outputs.append( + _capture_tensor_output(outputs) + ) + ) + ) + for layer in native.layers: + hooks.append( + layer.register_forward_hook( + lambda _module, _inputs, outputs: native_layer_outputs.append( + _capture_tensor_output(outputs) + ) + ) + ) + + with torch.no_grad(): + if rank == 0: + print( + "glm5_hf_native_parity reference=transformers.DeepseekV3ForCausalLM " + "mode=dense_mla_reference_with_full_topk_dsa" + ) + hf_logits = hf_ref(full_ids).logits + native_logits = native(input_ids=local_ids)["logits"] + + for hook in hooks: + hook.remove() + + assert len(hf_layer_outputs) == cfg.num_hidden_layers + assert len(native_layer_outputs) == cfg.num_hidden_layers + for layer_idx, (actual, full_expected) in enumerate( + zip(native_layer_outputs, hf_layer_outputs, strict=True) + ): + expected = zigzag_slice_for_cp(full_expected, rank, world, seq_dim=1).contiguous() + max_abs, max_rel = _distributed_diff_stats(actual, expected) + if rank == 0: + print( + f"glm5_hf_native_parity layer={layer_idx} " + f"max_abs_diff={max_abs:.6e} max_rel_diff={max_rel:.6e}" + ) + torch.testing.assert_close(actual.float(), expected.float(), atol=1.5e-1, rtol=1.5e-1) + + expected = zigzag_slice_for_cp(hf_logits, rank, world, seq_dim=1).contiguous() + max_abs, max_rel = _distributed_diff_stats(native_logits, expected) + if rank == 0: + print( + "glm5_hf_native_parity logits " f"max_abs_diff={max_abs:.6e} max_rel_diff={max_rel:.6e}" + ) + torch.testing.assert_close(native_logits.float(), expected.float(), atol=1.5e-1, rtol=1.5e-1) diff --git a/experimental/lite/tests/unit/model/test_glm5_lite_static.py b/experimental/lite/tests/unit/model/test_glm5_lite_static.py new file mode 100644 index 00000000000..8eab6ed6b27 --- /dev/null +++ b/experimental/lite/tests/unit/model/test_glm5_lite_static.py @@ -0,0 +1,646 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Static and CPU smoke tests for native GLM-5 lite.""" + +from __future__ import annotations + +from pathlib import Path + + +def _make_train_config(ps): + from types import SimpleNamespace + + return SimpleNamespace( + tp=ps.tp_size, + ep=ps.ep_size, + etp=ps.etp_size, + pp=ps.pp_size, + cp=ps.cp_size, + vpp=None, + use_deepep=False, + fp8=False, + recompute_modules=[], + deterministic=True, + ) + + +def _make_glm5_model(cfg, ps=None, **kwargs): + from megatron.lite.model.glm5.lite.model import Glm5Model + from megatron.lite.primitive.parallel import ParallelState + + ps = ParallelState() if ps is None else ps + return Glm5Model(cfg, _make_train_config(ps), ps, **kwargs) + + +def _tiny_config_kwargs(): + return dict( + num_hidden_layers=2, + hidden_size=16, + num_attention_heads=2, + num_key_value_heads=2, + head_dim=4, + vocab_size=32, + max_position_embeddings=16, + q_lora_rank=8, + kv_lora_rank=4, + qk_head_dim=8, + qk_nope_head_dim=4, + qk_rope_head_dim=4, + v_head_dim=4, + index_head_dim=8, + index_n_heads=2, + index_topk=2, + intermediate_size=20, + moe_intermediate_size=6, + first_k_dense_replace=1, + n_routed_experts=3, + n_shared_experts=1, + num_experts_per_tok=2, + ) + + +def test_glm5_registry_resolves_lite(): + from megatron.lite.model.registry import ( + get_train_runtime_module, + resolve_model_type_from_hf, + resolve_runtime_model_name, + ) + + runtime_name = resolve_runtime_model_name("glm5", "lite") + assert runtime_name == "glm5" + module = get_train_runtime_module(runtime_name) + assert module.__name__ == "megatron.lite.model.glm5.lite.protocol" + assert resolve_model_type_from_hf({"model_type": "glm_moe_dsa"}) == "glm5" + + +def test_glm5_config_reads_hf_architecture_fields(): + from megatron.lite.model.glm5.config import Glm5Config + + cfg = Glm5Config._from_hf_dict( + { + "model_type": "glm_moe_dsa", + "hidden_size": 6144, + "num_hidden_layers": 78, + "num_attention_heads": 64, + "num_key_value_heads": 64, + "q_lora_rank": 2048, + "kv_lora_rank": 512, + "qk_head_dim": 256, + "qk_nope_head_dim": 192, + "qk_rope_head_dim": 64, + "v_head_dim": 256, + "index_head_dim": 128, + "index_n_heads": 32, + "index_topk": 2048, + "first_k_dense_replace": 3, + "n_routed_experts": 256, + "n_shared_experts": 1, + "num_experts_per_tok": 8, + "vocab_size": 154880, + "rope_parameters": {"rope_theta": 1000000, "rope_type": "default"}, + } + ) + + assert cfg.q_lora_rank == 2048 + assert cfg.kv_lora_rank == 512 + assert cfg.index_topk == 2048 + assert cfg.num_nextn_predict_layers == 1 + assert cfg.rope_theta == 1_000_000.0 + assert cfg.is_moe_layer(2) is False + assert cfg.is_moe_layer(3) is True + + +def test_glm5_config_ignores_null_hf_optional_fields(): + from megatron.lite.model.glm5.config import Glm5Config + + cfg = Glm5Config._from_hf_dict( + { + "model_type": "glm_moe_dsa", + "indexer_rope_first": None, + "indexer_use_hadamard": None, + "mlp_layer_types": None, + } + ) + + assert cfg.indexer_rope_first is True + assert cfg.indexer_use_hadamard is False + assert cfg.mlp_layer_types is None + + +def test_glm5_config_preserves_mtp_aliases_and_layer_types(): + from megatron.lite.model.glm5.config import Glm5Config + + cfg = Glm5Config._from_hf_dict( + { + **_tiny_config_kwargs(), + "num_nextn_predict": 1, + "mtp_loss_scaling_factor": 0.2, + "mlp_layer_types": ["dense", "sparse", "sparse"], + } + ) + + assert cfg.num_nextn_predict_layers == 1 + assert cfg.mtp_loss_scaling_factor == 0.2 + assert cfg.is_moe_layer(2) is True + + +def test_glm5_lite_does_not_import_wrappers_or_sibling_models(): + root = Path(__file__).resolve().parents[3] / "megatron" / "lite" / "model" / "glm5" / "lite" + for path in root.glob("*.py"): + text = path.read_text() + assert "megatron.lite.model.qwen" not in text + assert "mbridge" not in text + assert "MCore" not in text + assert "megatron.core" not in text + + +def test_glm5_lite_uses_shared_mla_and_dsa_primitive(): + root = Path(__file__).resolve().parents[3] / "megatron" / "lite" + model_text = (root / "model" / "glm5" / "lite" / "model.py").read_text() + primitive_text = (root / "primitive" / "modules" / "attention" / "dsa.py").read_text() + kernel_text = (root / "primitive" / "kernels" / "dsa_kernels.py").read_text() + + assert "DynamicSparseAttention" in model_text + assert ( + "from megatron.lite.primitive.modules.attention.mla import MultiLatentAttention" + in primitive_text + ) + assert "class DynamicSparseAttention" in primitive_text + assert "class MultiLatentAttention" not in primitive_text + assert "class DSAIndexer" in primitive_text + assert "megatron.core" not in primitive_text + assert "dsa_kernels.fused_indexer_sparse_attn" in primitive_text + assert "dsa_kernels.dsa_sparse_attn" in primitive_text + assert "dsa_kernels.indexer_topk" in primitive_text + assert "value_dim" in kernel_text + assert "from cudnn.deepseek_sparse_attention import DSA" in kernel_text + assert "from cudnn import DSA" in kernel_text + assert "cudnn.deepseek_sparse_attention.indexer_forward._interface_sm90" in kernel_text + assert "cudnn.deepseek_sparse_attention.indexer_forward._interface" in kernel_text + assert "torch.cuda.get_device_capability(device)" in kernel_text + assert "torch.topk" not in primitive_text + assert "torch.softmax" not in primitive_text + assert "torch.matmul" not in primitive_text + + +def test_glm5_dsa_kernel_routes_indexer_forward_by_sm(monkeypatch): + from megatron.lite.primitive.kernels import dsa_kernels + + sm90_entry = object() + sm100_entry = object() + + monkeypatch.setattr(dsa_kernels, "_load_indexer_fwd_sm90", lambda: sm90_entry) + monkeypatch.setattr(dsa_kernels, "_load_indexer_fwd_sm100", lambda: sm100_entry) + + monkeypatch.setattr(dsa_kernels.torch.cuda, "get_device_capability", lambda device: (9, 0)) + assert dsa_kernels._select_indexer_forward(None) is sm90_entry + + monkeypatch.setattr(dsa_kernels.torch.cuda, "get_device_capability", lambda device: (10, 0)) + assert dsa_kernels._select_indexer_forward(None) is sm100_entry + + monkeypatch.setattr(dsa_kernels.torch.cuda, "get_device_capability", lambda device: (8, 0)) + assert dsa_kernels._select_indexer_forward(None) is None + + +def test_glm5_dsa_training_forward_uses_fused_kernel(monkeypatch): + import pytest + import torch + + from megatron.lite.primitive.modules.attention import dsa + from megatron.lite.primitive.modules.attention import DynamicSparseAttention, build_rope_cache + + if not torch.cuda.is_available(): + pytest.skip("GLM-5 native attention requires CUDA (Transformer Engine RMSNorm)") + device = torch.device("cuda") + + calls = {} + + def fake_fused_indexer_sparse_attn( + query, + kv_full, + attn_sink, + window_idxs, + q_indexer, + k_indexer, + weights, + indexer_topk, + ratio, + softmax_scale, + indexer_softmax_scale=1.0, + loss_coeff=0.0, + sparse_loss=False, + kv_offset=0, + calculate_per_token_loss=False, + value_dim=None, + ): + del attn_sink, q_indexer, k_indexer, weights, softmax_scale, indexer_softmax_scale + calls["training"] = { + "query_shape": tuple(query.shape), + "kv_shape": tuple(kv_full.shape), + "window_shape": tuple(window_idxs.shape), + "indexer_topk": indexer_topk, + "ratio": ratio, + "loss_coeff": loss_coeff, + "sparse_loss": sparse_loss, + "kv_offset": kv_offset, + "calculate_per_token_loss": calculate_per_token_loss, + "value_dim": value_dim, + } + return query.new_zeros( + query.shape[0], query.shape[1], query.shape[2] * value_dim + ), torch.zeros((), device=query.device, dtype=torch.float32) + + monkeypatch.setattr( + dsa._dsa_kernels, "fused_indexer_sparse_attn", fake_fused_indexer_sparse_attn + ) + + attn = DynamicSparseAttention( + hidden_size=16, + num_attention_heads=2, + q_lora_rank=8, + kv_lora_rank=4, + qk_nope_head_dim=4, + qk_rope_head_dim=4, + v_head_dim=4, + index_n_heads=2, + index_head_dim=8, + index_topk=2, + rms_norm_eps=1e-5, + ) + attn.to(device=device, dtype=torch.bfloat16) + attn.train() + x = torch.randn(1, 4, 16, device=device, dtype=torch.bfloat16) + cos, sin = build_rope_cache( + dim=4, max_position_embeddings=4, rope_theta=1_000_000.0, device=device + ) + position_ids = torch.arange(4, device=device).unsqueeze(0) + + out = attn(x, cos=cos, sin=sin, position_ids=position_ids) + + assert out.shape == (1, 4, 16) + assert calls["training"] == { + "query_shape": (4, 1, 2, 8), + "kv_shape": (4, 1, 8), + "window_shape": (1, 4, 0), + "indexer_topk": 2, + "ratio": 1, + "loss_coeff": 0.0, + "sparse_loss": False, + "kv_offset": 0, + "calculate_per_token_loss": False, + "value_dim": 4, + } + + +def test_glm5_dsa_eval_forward_uses_fused_sparse_attention(monkeypatch): + import pytest + import torch + + from megatron.lite.primitive.modules.attention import dsa + from megatron.lite.primitive.modules.attention import DynamicSparseAttention, build_rope_cache + + if not torch.cuda.is_available(): + pytest.skip("GLM-5 native attention requires CUDA (Transformer Engine RMSNorm)") + device = torch.device("cuda") + + calls = {} + + def fake_indexer_topk(q_indexer, k_indexer, weights, topk, ratio, indexer_softmax_scale=1.0): + del q_indexer, k_indexer, weights, indexer_softmax_scale + calls["indexer"] = {"topk": topk, "ratio": ratio} + idx = torch.zeros((1, 4, topk), dtype=torch.int32, device=device) + return idx, torch.full((1, 4), topk, dtype=torch.int32, device=device) + + def fake_dsa_sparse_attn( + query, + kv_full, + attn_sink, + topk_idxs, + softmax_scale, + topk_length=None, + indexer_topk=0, + value_dim=None, + ): + del kv_full, attn_sink, topk_idxs, softmax_scale, indexer_topk + calls["sparse"] = {"topk_length_is_set": topk_length is not None, "value_dim": value_dim} + return query.new_zeros(query.shape[0], query.shape[1], query.shape[2] * value_dim) + + monkeypatch.setattr(dsa._dsa_kernels, "indexer_topk", fake_indexer_topk) + monkeypatch.setattr(dsa._dsa_kernels, "dsa_sparse_attn", fake_dsa_sparse_attn) + + attn = DynamicSparseAttention( + hidden_size=16, + num_attention_heads=2, + q_lora_rank=8, + kv_lora_rank=4, + qk_nope_head_dim=4, + qk_rope_head_dim=4, + v_head_dim=4, + index_n_heads=2, + index_head_dim=8, + index_topk=2, + rms_norm_eps=1e-5, + ) + attn.to(device=device, dtype=torch.bfloat16) + attn.eval() + x = torch.randn(1, 4, 16, device=device, dtype=torch.bfloat16) + cos, sin = build_rope_cache( + dim=4, max_position_embeddings=4, rope_theta=1_000_000.0, device=device + ) + position_ids = torch.arange(4, device=device).unsqueeze(0) + + with torch.no_grad(): + out = attn(x, cos=cos, sin=sin, position_ids=position_ids) + + assert out.shape == (1, 4, 16) + assert calls["indexer"] == {"topk": 2, "ratio": 1} + assert calls["sparse"] == {"topk_length_is_set": True, "value_dim": 4} + + +def test_glm5_lite_model_exports_native_state_names(): + from megatron.lite.model.glm5.config import Glm5Config + + model = _make_glm5_model(Glm5Config(**_tiny_config_kwargs())) + keys = set(model.state_dict()) + + assert "embed.embedding.weight" in keys + assert "layers.0.self_attention.self_attention.q_a_proj.weight" in keys + assert "layers.0.mlp.gate_up.linear.weight" in keys + assert "layers.1.moe.router.gate.weight" in keys + assert "layers.1.moe.experts.fc1.weight0" in keys + assert "layers.1.moe.shared_expert.gate_up.linear.weight" in keys + assert "head.col.linear.weight" in keys + + +def test_glm5_checkpoint_exports_and_saves_hf_style_weights(tmp_path): + import torch + from safetensors import safe_open + + from megatron.lite.model.glm5.config import Glm5Config + from megatron.lite.model.glm5.lite.checkpoint import ( + export_hf_weights, + save_hf_weights, + ) + from megatron.lite.primitive.parallel import ParallelState + + cfg = Glm5Config(**_tiny_config_kwargs()) + ps = ParallelState() + model = _make_glm5_model(cfg, ps=ps) + model.layers[1].moe.router.expert_bias.copy_(torch.tensor([0.25, -0.5, 1.0])) + + exported = dict(export_hf_weights(model, cfg, ps)) + state = model.state_dict() + + assert torch.equal( + exported["model.layers.1.mlp.experts.2.gate_proj.weight"], + state["layers.1.moe.experts.fc1.weight2"][: cfg.moe_intermediate_size].detach().cpu(), + ) + assert torch.equal( + exported["model.layers.1.mlp.gate.e_score_correction_bias"], + state["layers.1.moe.router.expert_bias"].detach().cpu(), + ) + assert "model.layers.1.mlp.experts.gate_up_proj" not in exported + + hf_dir = tmp_path / "hf" + save_hf_weights(model, str(hf_dir), cfg, ps) + with safe_open(str(hf_dir / "model.safetensors"), framework="pt", device="cpu") as handle: + assert torch.equal( + handle.get_tensor("model.layers.1.mlp.experts.2.down_proj.weight"), + state["layers.1.moe.experts.fc2.weight2"].detach().cpu(), + ) + assert torch.equal( + handle.get_tensor("model.layers.1.mlp.gate.e_score_correction_bias"), + state["layers.1.moe.router.expert_bias"].detach().cpu(), + ) + + loaded = _make_glm5_model(cfg, ps=ps) + from megatron.lite.model.glm5.lite.checkpoint import load_hf_weights + + load_hf_weights(loaded, str(hf_dir), cfg, ps) + assert torch.equal( + loaded.state_dict()["layers.1.moe.router.expert_bias"].detach().cpu(), + state["layers.1.moe.router.expert_bias"].detach().cpu(), + ) + + hf_bf16_dir = tmp_path / "hf_bf16" + save_hf_weights(model, str(hf_bf16_dir), cfg, ps, export_dtype=torch.bfloat16) + with safe_open(str(hf_bf16_dir / "model.safetensors"), framework="pt", device="cpu") as handle: + floating_dtypes = { + handle.get_tensor(key).dtype + for key in handle.keys() + if handle.get_tensor(key).is_floating_point() + } + assert floating_dtypes == {torch.bfloat16} + + loaded_bf16 = _make_glm5_model(cfg, ps=ps) + load_hf_weights(loaded_bf16, str(hf_bf16_dir), cfg, ps) + assert torch.equal( + loaded_bf16.state_dict()["layers.1.moe.experts.fc1.weight2"][ + cfg.moe_intermediate_size : + ] + .detach() + .cpu(), + state["layers.1.moe.experts.fc1.weight2"][cfg.moe_intermediate_size :] + .detach() + .cpu() + .to(torch.bfloat16) + .to(torch.float32), + ) + + +def test_glm5_checkpoint_exports_and_loads_mtp_layers(tmp_path): + import torch + + from megatron.lite.model.glm5.config import Glm5Config + from megatron.lite.model.glm5.lite.checkpoint import export_hf_weights, load_hf_weights + from megatron.lite.primitive.ckpt.hf_weights import save_safetensors + from megatron.lite.primitive.parallel import ParallelState + + cfg = Glm5Config(**_tiny_config_kwargs(), num_nextn_predict_layers=1) + ps = ParallelState() + model = _make_glm5_model(cfg, ps=ps, mtp_enable=True) + state = model.state_dict() + + assert "mtp.layers.0.eh_proj.linear.weight" in state + assert "mtp.layers.0.transformer_layer.input_layernorm.weight" in state + + exported = dict(export_hf_weights(model, cfg, ps)) + assert "model.layers.2.eh_proj.weight" in exported + assert "model.layers.2.enorm.weight" in exported + assert "model.layers.2.hnorm.weight" in exported + assert "model.layers.2.shared_head.norm.weight" in exported + assert "model.layers.2.input_layernorm.weight" in exported + assert "model.layers.2.mlp.gate.weight" in exported + + save_safetensors(exported, str(tmp_path)) + loaded = _make_glm5_model(cfg, ps=ps, mtp_enable=True) + load_hf_weights(loaded, str(tmp_path), cfg, ps) + assert torch.equal( + loaded.state_dict()["mtp.layers.0.eh_proj.linear.weight"], + state["mtp.layers.0.eh_proj.linear.weight"], + ) + + +def test_glm5_router_modules_use_current_names_and_bias_buffers(): + import torch + + from megatron.lite.model.glm5.config import Glm5Config + from megatron.lite.model.glm5.lite.model import Glm5SigmoidTopKRouter + + model = _make_glm5_model( + Glm5Config(**_tiny_config_kwargs(), num_nextn_predict_layers=1), mtp_enable=True + ) + routers = [module for module in model.modules() if isinstance(module, Glm5SigmoidTopKRouter)] + assert len(routers) == 2 + for router in routers: + assert hasattr(router, "gate") + assert hasattr(router, "expert_bias") + assert torch.isfinite(router.gate.weight).all() + assert torch.equal( + router.expert_bias, torch.zeros_like(router.expert_bias) + ) + + +def test_glm5_protocol_allows_cp_only_parallel_scope(): + import pytest + + from megatron.lite.model.glm5.lite.protocol import _validate_parallel_scope + from megatron.lite.runtime.contracts import ParallelConfig + + # CP-only as well as PP/VPP/EP are supported and must validate cleanly. + _validate_parallel_scope(ParallelConfig(tp=1, ep=1, etp=1, cp=2, pp=1, vpp=1)) + _validate_parallel_scope(ParallelConfig(tp=1, ep=1, etp=1, cp=1, pp=2, vpp=2)) + # GLM-5 native DSA attention rejects tensor / expert-tensor parallelism. + with pytest.raises(NotImplementedError): + _validate_parallel_scope(ParallelConfig(tp=2, ep=1, etp=1, cp=1, pp=1, vpp=1)) + with pytest.raises(NotImplementedError): + _validate_parallel_scope(ParallelConfig(tp=1, ep=1, etp=2, cp=1, pp=1, vpp=1)) + + +def test_glm5_impl_config_accepts_runtime_mtp_fields(): + from megatron.lite.model.glm5.config import Glm5Config + from megatron.lite.model.glm5.lite.protocol import ImplConfig + + cfg = Glm5Config(**_tiny_config_kwargs(), num_nextn_predict_layers=1) + + assert ImplConfig(mtp_enable=False, mtp_enable_train=False).mtp_enable is False + assert ImplConfig(mtp_enable=True, mtp_enable_train=True).mtp_enable_train is True + assert cfg.num_nextn_predict_layers == 1 + + +def test_glm5_protocol_uses_mlite_optimizer_api(): + from megatron.lite.model.glm5.lite.protocol import ImplConfig + + protocol_path = ( + Path(__file__).resolve().parents[3] + / "megatron" + / "lite" + / "model" + / "glm5" + / "lite" + / "protocol.py" + ) + protocol_text = protocol_path.read_text() + + assert ImplConfig().optimizer == "dist_opt" + assert "build_dist_opt_training_optimizer" in protocol_text + + +def test_glm5_lite_tiny_forward_backward(monkeypatch): + import pytest + import torch + + from megatron.lite.model.glm5.config import Glm5Config + from megatron.lite.primitive.modules.attention import dsa + + if not torch.cuda.is_available(): + pytest.skip("GLM-5 native model requires CUDA (Transformer Engine RMSNorm)") + device = torch.device("cuda") + + def fake_fused_indexer_sparse_attn( + query, + kv_full, + attn_sink, + window_idxs, + q_indexer, + k_indexer, + weights, + indexer_topk, + ratio, + softmax_scale, + indexer_softmax_scale=1.0, + loss_coeff=0.0, + sparse_loss=False, + kv_offset=0, + calculate_per_token_loss=False, + value_dim=None, + ): + del ( + kv_full, + attn_sink, + window_idxs, + q_indexer, + k_indexer, + weights, + indexer_topk, + ratio, + softmax_scale, + indexer_softmax_scale, + loss_coeff, + sparse_loss, + kv_offset, + calculate_per_token_loss, + ) + return query.new_zeros( + query.shape[0], query.shape[1], query.shape[2] * value_dim + ), torch.zeros((), device=query.device, dtype=torch.float32) + + monkeypatch.setattr( + dsa._dsa_kernels, "fused_indexer_sparse_attn", fake_fused_indexer_sparse_attn + ) + + torch.manual_seed(1234) + model = _make_glm5_model(Glm5Config(**_tiny_config_kwargs())).to( + device=device, dtype=torch.bfloat16 + ) + input_ids = torch.randint(0, model.config.vocab_size, (2, 5), device=device) + labels = torch.randint(0, model.config.vocab_size, (2, 5), device=device) + + output = model(input_ids=input_ids, labels=labels) + + # hidden_states stays in (seq, batch, hidden) layout; logits are transposed + # back to (batch, seq, *) inside forward. + assert output["hidden_states"].shape == (5, 2, model.config.hidden_size) + assert output["loss"].ndim == 0 + output["loss"].backward() + grad_norm = sum( + param.grad.detach().float().norm() for param in model.parameters() if param.grad is not None + ) + assert torch.isfinite(grad_norm) + + mtp_model = _make_glm5_model( + Glm5Config(**_tiny_config_kwargs(), num_nextn_predict_layers=1), + mtp_enable=True, + mtp_enable_train=True, + ).to(device=device, dtype=torch.bfloat16) + # Inference path (no labels) exposes the per-MTP-head logits. + mtp_infer = mtp_model(input_ids=input_ids) + assert len(mtp_infer["mtp_hidden_states"]) == 1 + assert len(mtp_infer["mtp_logits"]) == 1 + assert mtp_infer["mtp_hidden_states"][0].shape == (5, 2, mtp_model.config.hidden_size) + assert mtp_infer["mtp_logits"][0].shape == (2, 5, mtp_model.config.vocab_size) + + # Training path (with labels) returns the MTP loss instead of logits. + mtp_output = mtp_model( + input_ids=input_ids, labels=labels, loss_mask=torch.ones_like(labels, dtype=torch.float32) + ) + + assert len(mtp_output["mtp_hidden_states"]) == 1 + assert mtp_output["mtp_hidden_states"][0].shape == (5, 2, mtp_model.config.hidden_size) + assert mtp_output["mtp_loss"].ndim == 0 + mtp_output["loss"].backward() + mtp_grad_norm = sum( + param.grad.detach().float().norm() + for param in mtp_model.parameters() + if param.grad is not None + ) + assert torch.isfinite(mtp_grad_norm) diff --git a/experimental/lite/tests/unit/model/test_kimi_k2_int4_dequant.py b/experimental/lite/tests/unit/model/test_kimi_k2_int4_dequant.py new file mode 100644 index 00000000000..ee814a14484 --- /dev/null +++ b/experimental/lite/tests/unit/model/test_kimi_k2_int4_dequant.py @@ -0,0 +1,141 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Bit-exact parity for the Kimi K2 INT4 load-path dequant. + +Kimi ships INT4-packed release weights (eight offset-binary INT4 values per +int32 slot + per-group scales). The MLite load path dequantizes them in +``kimi_k2/lite/checkpoint.py`` without any ``transformers`` / external-bridge +dependency. This test pins that dequant to be bit-exact against the reference +algorithm in ``megatron.bridge`` (``conversion/quantization_utils.py``: +``quantize_to_int4`` / ``dequantize_int4``), which the load path was written to +match. The reference is vendored here (CPU-only, no GPU) so the guard runs in +unit CI and does not import megatron.bridge. +""" + +from __future__ import annotations + +import torch + + +# --------------------------------------------------------------------------- +# Reference INT4 quant/dequant, copied verbatim from megatron.bridge +# conversion/quantization_utils.py (quantize_to_int4 / dequantize_int4). These +# define the "ground truth" packed layout + group-scale semantics that the +# MLite load path must reproduce exactly. +# --------------------------------------------------------------------------- +def _bridge_quantize_to_int4( + weight: torch.Tensor, group_size: int = 32, scale_dtype: torch.dtype = torch.bfloat16 +): + out_features, in_features = weight.shape + weight_shape = torch.tensor([out_features, in_features], dtype=torch.int32) + + w = weight.float() + num_groups = (in_features + group_size - 1) // group_size + w_grouped = w.view(out_features, num_groups, -1) + + group_max = w_grouped.abs().amax(dim=-1) + scale = (group_max / 7.0).clamp(min=1e-10) + + scale_expanded = scale.unsqueeze(-1).expand_as(w_grouped) + w_q = (w_grouped / scale_expanded).round().clamp(-8, 7) + + w_q = w_q.view(out_features, -1)[:, :in_features] + w_q = (w_q + 8).to(torch.uint8) + + assert in_features % 8 == 0 + w_q_grouped = w_q.view(out_features, in_features // 8, 8).to(torch.int32) + packed = torch.zeros(out_features, in_features // 8, dtype=torch.int32) + for i in range(8): + packed |= (w_q_grouped[:, :, i] & 0xF) << (i * 4) + + return packed, scale.to(scale_dtype), weight_shape + + +def _bridge_dequantize_int4( + weight_packed: torch.Tensor, weight_scale: torch.Tensor, weight_shape: torch.Tensor +) -> torch.Tensor: + local_out, local_packed_in = weight_packed.shape + local_in = local_packed_in * 8 + + shifts = torch.arange(8) * 4 + unpacked = ((weight_packed.unsqueeze(-1) >> shifts) & 0xF).float() + unpacked = unpacked.reshape(local_out, local_in) - 8 + + scale = weight_scale.float() + if scale.ndim == 1: + scale = scale.view(local_out, scale.numel() // local_out) + else: + scale = scale.view(local_out, -1) + elements_per_group = local_in // scale.shape[1] + scale_expanded = scale.repeat_interleave(elements_per_group, dim=1)[:, :local_in] + + return (unpacked * scale_expanded).to(torch.bfloat16) + + +class _StubReader: + """Minimal ``SafeTensorReader`` stand-in for the dequant helpers. + + The load helpers only call ``.get_tensor(name)`` and consult ``.index`` (via + ``_has``); a dict + key set is enough. + """ + + def __init__(self, tensors: dict[str, torch.Tensor]): + self._tensors = dict(tensors) + self.index = set(self._tensors) + + def get_tensor(self, name: str) -> torch.Tensor: + return self._tensors[name] + + +def test_kimi_int4_dequant_bit_exact_to_bridge(): + from megatron.lite.model.kimi_k2.lite.checkpoint import _dequant_int4_weight + + torch.manual_seed(2026) + # out_features arbitrary; in_features divisible by both 8 (pack factor) and 32 + # (group_size) so the packed/group layout is exercised with >1 group per row. + weight = torch.randn(40, 128, dtype=torch.bfloat16) + + packed, scale, shape = _bridge_quantize_to_int4(weight) + reference = _bridge_dequantize_int4(packed, scale, shape) + + reader = _StubReader( + {"w_packed": packed, "w_scale": scale, "w_shape": shape} + ) + got = _dequant_int4_weight(reader, "w") + + assert got.dtype == torch.bfloat16 + assert got.shape == reference.shape + assert torch.equal(got, reference), ( + "MLite INT4 dequant diverged from the megatron.bridge reference: " + f"max|diff|={(got.float() - reference.float()).abs().max().item()}" + ) + + +def test_kimi_get_passes_through_bf16_when_unquantized(): + """``_get`` must load a plain bf16 tensor unchanged (no scale, no packing). + + MLite export/save emits bf16, so reloading an exported (or already bf16) + checkpoint must hit the no-dequant path — the third load case alongside + FP8 (``*_scale_inv``) and INT4 (``*_packed``). + """ + from megatron.lite.model.kimi_k2.lite.checkpoint import _get + + torch.manual_seed(7) + weight = torch.randn(16, 24, dtype=torch.bfloat16) + reader = _StubReader({"layer.weight": weight}) + + got = _get(reader, "layer.weight") + assert got.dtype == torch.bfloat16 + assert torch.equal(got, weight) + + +def test_kimi_get_dispatches_to_int4_when_packed_present(): + """When only ``*_packed`` exists (no plain tensor), ``_get`` dequantizes INT4.""" + from megatron.lite.model.kimi_k2.lite.checkpoint import _get + + torch.manual_seed(11) + weight = torch.randn(8, 64, dtype=torch.bfloat16) + packed, scale, shape = _bridge_quantize_to_int4(weight) + reader = _StubReader({"w_packed": packed, "w_scale": scale, "w_shape": shape}) + + got = _get(reader, "w") + assert torch.equal(got, _bridge_dequantize_int4(packed, scale, shape)) diff --git a/experimental/lite/tests/unit/model/test_kimi_k2_lite_cp_smoke.py b/experimental/lite/tests/unit/model/test_kimi_k2_lite_cp_smoke.py new file mode 100644 index 00000000000..0b48e570c63 --- /dev/null +++ b/experimental/lite/tests/unit/model/test_kimi_k2_lite_cp_smoke.py @@ -0,0 +1,535 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + + +def _init_dist_or_skip(): + import os + + import torch + import torch.distributed as dist + + if not torch.cuda.is_available(): + pytest.skip("CUDA is required for Kimi K2 CP smoke.") + if "RANK" not in os.environ or "WORLD_SIZE" not in os.environ: + pytest.skip("Run with torchrun so CP ranks are available.") + + local_rank = int(os.environ.get("LOCAL_RANK", "0")) + torch.cuda.set_device(local_rank) + if not dist.is_initialized(): + dist.init_process_group("nccl") + if dist.get_world_size() < 2: + pytest.skip("Kimi K2 CP smoke requires at least 2 ranks.") + return torch.device("cuda", local_rank) + + +def _tiny_config(): + from megatron.lite.model.kimi_k2.config import KimiK2Config + + return KimiK2Config( + num_hidden_layers=2, + hidden_size=64, + num_attention_heads=4, + num_key_value_heads=4, + vocab_size=128, + intermediate_size=96, + moe_intermediate_size=16, + n_routed_experts=4, + n_shared_experts=1, + num_experts_per_tok=2, + n_group=2, + topk_group=1, + first_k_dense_replace=1, + q_lora_rank=16, + kv_lora_rank=12, + qk_nope_head_dim=8, + qk_rope_head_dim=8, + v_head_dim=8, + max_position_embeddings=128, + rope_theta=10000.0, + rope_scaling={ + "type": "yarn", + "factor": 1.0, + "original_max_position_embeddings": 128, + "beta_fast": 1.0, + "beta_slow": 1.0, + "mscale": 1.0, + "mscale_all_dim": 1.0, + }, + ) + + +def _tiny_hf_parity_config(): + from megatron.lite.model.kimi_k2.config import KimiK2Config + + return KimiK2Config( + num_hidden_layers=2, + hidden_size=64, + num_attention_heads=4, + num_key_value_heads=4, + vocab_size=128, + intermediate_size=96, + moe_intermediate_size=16, + n_routed_experts=4, + n_shared_experts=1, + num_experts_per_tok=2, + n_group=2, + topk_group=1, + first_k_dense_replace=1, + q_lora_rank=16, + kv_lora_rank=12, + qk_nope_head_dim=8, + qk_rope_head_dim=8, + v_head_dim=8, + max_position_embeddings=128, + rope_theta=10000.0, + rope_scaling={ + "type": "yarn", + "factor": 1.0, + "original_max_position_embeddings": 128, + "beta_fast": 1.0, + "beta_slow": 1.0, + "mscale": 1.0, + "mscale_all_dim": 1.0, + }, + ) + + +def _train_cfg(cp: int): + return SimpleNamespace( + tp=1, + ep=1, + etp=1, + pp=1, + cp=cp, + vpp=None, + use_deepep=False, + fp8=False, + recompute_modules={}, + deterministic=False, + ) + + +def _to_hf_deepseek_v3_config(cfg): + from transformers.models.deepseek_v3.configuration_deepseek_v3 import DeepseekV3Config + + rope_scaling = dict(cfg.rope_scaling) + rope_scaling.setdefault("rope_type", rope_scaling.get("type", "yarn")) + return DeepseekV3Config( + hidden_size=cfg.hidden_size, + intermediate_size=cfg.intermediate_size, + moe_intermediate_size=cfg.moe_intermediate_size, + num_hidden_layers=cfg.num_hidden_layers, + num_attention_heads=cfg.num_attention_heads, + num_key_value_heads=cfg.num_key_value_heads, + vocab_size=cfg.vocab_size, + n_shared_experts=cfg.n_shared_experts, + n_routed_experts=cfg.n_routed_experts, + routed_scaling_factor=cfg.routed_scaling_factor, + kv_lora_rank=cfg.kv_lora_rank, + q_lora_rank=cfg.q_lora_rank, + qk_rope_head_dim=cfg.qk_rope_head_dim, + v_head_dim=cfg.v_head_dim, + qk_nope_head_dim=cfg.qk_nope_head_dim, + n_group=cfg.n_group, + topk_group=cfg.topk_group, + num_experts_per_tok=cfg.num_experts_per_tok, + first_k_dense_replace=cfg.first_k_dense_replace, + norm_topk_prob=cfg.norm_topk_prob, + max_position_embeddings=cfg.max_position_embeddings, + rms_norm_eps=cfg.rms_norm_eps, + tie_word_embeddings=cfg.tie_word_embeddings, + rope_theta=cfg.rope_theta, + rope_scaling=rope_scaling, + rope_interleave=True, + attention_bias=False, + attention_dropout=0.0, + use_cache=False, + ) + + +def _capture_tensor_output(outputs): + return outputs[0].detach() if isinstance(outputs, tuple) else outputs.detach() + + +def _distributed_diff_stats(actual, expected) -> tuple[float, float]: + import torch + import torch.distributed as dist + + diff = (actual.float() - expected.float()).abs() + max_abs = diff.max() + scale = torch.maximum(actual.float().abs().max(), expected.float().abs().max()).clamp_min(1e-6) + stats = torch.stack([max_abs, scale]) + if dist.is_initialized(): + dist.all_reduce(stats, op=dist.ReduceOp.MAX) + return float(stats[0].item()), float((stats[0] / stats[1]).item()) + + +def _hf_state_dict_for_kimi_loader(model): + state = { + name: tensor.detach().cpu().contiguous().clone() + for name, tensor in model.state_dict().items() + } + for name, gate_up in list(state.items()): + if not name.endswith(".mlp.experts.gate_up_proj"): + continue + prefix = name.removesuffix(".gate_up_proj") + down = state.get(f"{prefix}.down_proj") + gate, up = gate_up.chunk(2, dim=1) + for expert_idx in range(gate_up.size(0)): + state[f"{prefix}.{expert_idx}.gate_proj.weight"] = gate[expert_idx].contiguous().clone() + state[f"{prefix}.{expert_idx}.up_proj.weight"] = up[expert_idx].contiguous().clone() + if down is not None: + state[f"{prefix}.{expert_idx}.down_proj.weight"] = ( + down[expert_idx].contiguous().clone() + ) + return state + + +def test_kimi_k2_mla_cp2_matches_full_sequence_reference_forward_and_grad(): + import torch + import torch.distributed as dist + + device = _init_dist_or_skip() + from megatron.lite.primitive.modules.attention import MultiLatentAttention + from megatron.lite.primitive.parallel import ParallelState + from megatron.lite.primitive.parallel.cp import zigzag_slice_for_cp + from megatron.lite.primitive.parallel.state import init_parallel + from megatron.lite.runtime.contracts import ParallelConfig + + world = dist.get_world_size() + rank = dist.get_rank() + cfg = _tiny_config() + ps = init_parallel(ParallelConfig(tp=1, ep=1, etp=1, cp=world, pp=1)) + + kwargs = dict( + hidden_size=cfg.hidden_size, + num_attention_heads=cfg.num_attention_heads, + q_lora_rank=cfg.q_lora_rank, + kv_lora_rank=cfg.kv_lora_rank, + qk_nope_head_dim=cfg.qk_nope_head_dim, + qk_rope_head_dim=cfg.qk_rope_head_dim, + v_head_dim=cfg.v_head_dim, + rms_norm_eps=cfg.rms_norm_eps, + rope_theta=cfg.rope_theta, + rope_scaling=cfg.rope_scaling, + use_thd=False, + ) + torch.manual_seed(20260531) + cp_layer = MultiLatentAttention(ps=ps, **kwargs).to(device=device, dtype=torch.bfloat16) + torch.manual_seed(20260531) + ref_layer = MultiLatentAttention(ps=ParallelState(), **kwargs).to( + device=device, + dtype=torch.bfloat16, + ) + + seq, batch = 8 * world, 1 + torch.manual_seed(123) + full_x = torch.randn(seq, batch, cfg.hidden_size, device=device, dtype=torch.bfloat16) + local_x = zigzag_slice_for_cp(full_x, rank, world, seq_dim=0).detach().requires_grad_(True) + ref_x = full_x.detach().clone().requires_grad_(True) + + cp_out = cp_layer(local_x) + ref_out = ref_layer(ref_x) + expected = zigzag_slice_for_cp(ref_out, rank, world, seq_dim=0) + torch.testing.assert_close(cp_out, expected, atol=5e-2, rtol=5e-2) + + cp_out.float().sum().backward() + expected.float().sum().backward() + expected_grad = zigzag_slice_for_cp(ref_x.grad, rank, world, seq_dim=0) + assert local_x.grad is not None + torch.testing.assert_close(local_x.grad, expected_grad, atol=1e-1, rtol=1e-1) + + +def test_kimi_k2_tiny_model_cp2_matches_full_sequence_reference_forward(): + import torch + import torch.distributed as dist + + device = _init_dist_or_skip() + from megatron.lite.model.kimi_k2.lite.model import KimiK2Model + from megatron.lite.primitive.parallel import ParallelState + from megatron.lite.primitive.parallel.cp import zigzag_slice_for_cp + from megatron.lite.primitive.parallel.state import init_parallel + from megatron.lite.runtime.contracts import ParallelConfig + + world = dist.get_world_size() + rank = dist.get_rank() + cfg = _tiny_config() + ps = init_parallel(ParallelConfig(tp=1, ep=1, etp=1, cp=world, pp=1)) + + torch.manual_seed(777) + cp_model = KimiK2Model(cfg, _train_cfg(world), ps, use_thd=False).to( + device=device, + dtype=torch.bfloat16, + ) + torch.manual_seed(777) + ref_model = KimiK2Model(cfg, _train_cfg(1), ParallelState(), use_thd=False).to( + device=device, + dtype=torch.bfloat16, + ) + cp_model.eval() + ref_model.eval() + + batch, seq = 1, 8 * world + torch.manual_seed(100) + full_ids = torch.randint(0, cfg.vocab_size, (batch, seq), device=device) + full_labels = torch.randint(0, cfg.vocab_size, (batch, seq), device=device) + input_ids = zigzag_slice_for_cp(full_ids, rank, world, seq_dim=1).contiguous() + labels = zigzag_slice_for_cp(full_labels, rank, world, seq_dim=1).contiguous() + + with torch.no_grad(): + cp_out = cp_model(input_ids=input_ids, labels=labels) + ref_out = ref_model(input_ids=full_ids, labels=full_labels) + + expected_hidden = zigzag_slice_for_cp(ref_out["hidden_states"], rank, world, seq_dim=0) + expected_log_probs = zigzag_slice_for_cp(ref_out["log_probs"], rank, world, seq_dim=1) + torch.testing.assert_close(cp_out["hidden_states"], expected_hidden, atol=1e-1, rtol=1e-1) + torch.testing.assert_close(cp_out["log_probs"], expected_log_probs, atol=1e-1, rtol=1e-1) + + +def test_kimi_k2_tiny_model_cp2_matches_hf_reference_logits(tmp_path): + import torch + import torch.distributed as dist + from transformers.models.deepseek_v3.modeling_deepseek_v3 import DeepseekV3ForCausalLM + + device = _init_dist_or_skip() + from megatron.lite.model.kimi_k2.lite.checkpoint import load_hf_weights + from megatron.lite.model.kimi_k2.lite.model import KimiK2Model + from megatron.lite.primitive.ckpt.hf_weights import save_safetensors + from megatron.lite.primitive.parallel.cp import zigzag_slice_for_cp + from megatron.lite.primitive.parallel.state import init_parallel + from megatron.lite.runtime.contracts import ParallelConfig + + world = dist.get_world_size() + rank = dist.get_rank() + cfg = _tiny_hf_parity_config() + + torch.manual_seed(20260610) + hf_ref = DeepseekV3ForCausalLM(_to_hf_deepseek_v3_config(cfg)).to( + device=device, + dtype=torch.bfloat16, + ) + hf_ref.eval() + rank_tmp_path = tmp_path / f"rank{rank}" + save_safetensors( + _hf_state_dict_for_kimi_loader(hf_ref), + str(rank_tmp_path), + ) + + ps = init_parallel(ParallelConfig(tp=1, ep=1, etp=1, cp=world, pp=1)) + native = KimiK2Model(cfg, _train_cfg(world), ps, use_thd=False).to( + device=device, + dtype=torch.bfloat16, + ) + native.eval() + load_hf_weights(native, str(rank_tmp_path), cfg, ps) + + batch, seq = 1, 4 * world + torch.manual_seed(310) + full_ids = torch.randint(0, cfg.vocab_size, (batch, seq), device=device) + local_ids = zigzag_slice_for_cp(full_ids, rank, world, seq_dim=1).contiguous() + + hf_layer_outputs = [] + native_layer_outputs = [] + hooks = [] + for layer in hf_ref.model.layers: + hooks.append( + layer.register_forward_hook( + lambda _module, _inputs, outputs: hf_layer_outputs.append( + _capture_tensor_output(outputs) + ) + ) + ) + for layer in native.layers: + hooks.append( + layer.register_forward_hook( + lambda _module, _inputs, outputs: native_layer_outputs.append( + _capture_tensor_output(outputs) + ) + ) + ) + + with torch.no_grad(): + if rank == 0: + print( + "kimi_k2_hf_native_parity reference=transformers.DeepseekV3ForCausalLM " + "rope=DeepseekV3RotaryEmbedding+apply_rotary_pos_emb_interleave" + ) + hf_logits = hf_ref(full_ids).logits + native_logits = native(input_ids=local_ids)["logits"] + + for hook in hooks: + hook.remove() + + assert len(hf_layer_outputs) == cfg.num_hidden_layers + assert len(native_layer_outputs) == cfg.num_hidden_layers + for layer_idx, (actual, full_expected) in enumerate( + zip(native_layer_outputs, hf_layer_outputs, strict=True) + ): + expected = zigzag_slice_for_cp(full_expected, rank, world, seq_dim=1) + expected = expected.transpose(0, 1).contiguous() + max_abs, max_rel = _distributed_diff_stats(actual, expected) + if rank == 0: + print( + f"kimi_k2_hf_native_parity layer={layer_idx} " + f"max_abs_diff={max_abs:.6e} max_rel_diff={max_rel:.6e}" + ) + torch.testing.assert_close( + actual.float(), + expected.float(), + atol=1.5e-1, + rtol=1.5e-1, + ) + + expected = zigzag_slice_for_cp(hf_logits, rank, world, seq_dim=1).contiguous() + max_abs, max_rel = _distributed_diff_stats(native_logits, expected) + if rank == 0: + print( + "kimi_k2_hf_native_parity logits " + f"max_abs_diff={max_abs:.6e} max_rel_diff={max_rel:.6e}" + ) + torch.testing.assert_close( + native_logits.float(), + expected.float(), + atol=1.5e-1, + rtol=1.5e-1, + ) + + +def test_kimi_k2_packed_thd_variable_sequence_cp2_smoke(): + import torch + import torch.distributed as dist + + device = _init_dist_or_skip() + from megatron.lite.model.kimi_k2.lite.model import KimiK2Model + from megatron.lite.primitive.parallel.state import init_parallel + from megatron.lite.primitive.parallel.thd import pack_nested_thd, unpack_packed_thd_to_nested + from megatron.lite.runtime.contracts import ParallelConfig + + world = dist.get_world_size() + rank = dist.get_rank() + cfg = _tiny_config() + ps = init_parallel(ParallelConfig(tp=1, ep=1, etp=1, cp=world, pp=1)) + + torch.manual_seed(20260614) + model = KimiK2Model(cfg, _train_cfg(world), ps, use_thd=True).to( + device=device, + dtype=torch.bfloat16, + ) + model.train() + + lengths = [5, 7, 9] + ids = torch.nested.as_nested_tensor( + [ + torch.randint(0, cfg.vocab_size, (length,), device=device, dtype=torch.long) + for length in lengths + ], + layout=torch.jagged, + ) + labels = torch.nested.as_nested_tensor( + [ + torch.randint(0, cfg.vocab_size, (length,), device=device, dtype=torch.long) + for length in lengths + ], + layout=torch.jagged, + ) + loss_mask = torch.nested.as_nested_tensor( + [torch.ones(length, device=device, dtype=torch.float32) for length in lengths], + layout=torch.jagged, + ) + packed = pack_nested_thd( + ids, + cp_size=world, + cp_rank=rank, + cp_group=ps.cp_group, + labels=labels, + loss_mask=loss_mask, + ) + + out = model( + input_ids=packed.input_ids, + labels=packed.labels, + loss_mask=packed.loss_mask, + position_ids=packed.position_ids, + packed_seq_params=packed.packed_seq_params, + ) + assert torch.isfinite(out["loss"]) + out["loss"].backward() + nested_log_probs = unpack_packed_thd_to_nested(out["log_probs"], packed) + assert nested_log_probs.offsets().numel() == len(lengths) + 1 + assert [int(x) for x in nested_log_probs.offsets().diff().cpu()] == lengths + + if rank == 0: + print( + "NON_SKIP_KIMI_K2_THD_CP_SMOKE_PASSED " + f"world_size={world} lengths={lengths} " + f"loss={float(out['loss'].detach().item()):.6e}" + ) + + +def test_kimi_k2_tiny_model_fsdp2_optimizer_step_smoke(): + import torch + import torch.distributed as dist + + device = _init_dist_or_skip() + from megatron.lite.model.kimi_k2.lite.protocol import ImplConfig, build_model + from megatron.lite.primitive.optimizers.fsdp2 import FSDP2Optimizer, fsdp2_available + from megatron.lite.runtime.contracts import OptimizerConfig, ParallelConfig + + if not fsdp2_available(): + pytest.skip("Installed PyTorch does not expose FSDP2 fully_shard.") + + rank = dist.get_rank() + world = dist.get_world_size() + cfg = _tiny_config() + impl_cfg = ImplConfig( + parallel=ParallelConfig(tp=1, ep=1, etp=1, cp=1, pp=1), + optimizer="fsdp2", + optimizer_config=OptimizerConfig( + optimizer="adam", + lr=1.0e-4, + weight_decay=0.0, + clip_grad=1.0, + adam_beta1=0.9, + adam_beta2=0.95, + adam_eps=1.0e-8, + ), + ) + + torch.manual_seed(20260612) + bundle = build_model(cfg, impl_cfg=impl_cfg) + assert bundle.optimizer is None + assert bundle.extras["optimizer_backend"] == "fsdp2" + post_model_load_hook = bundle.extras["post_model_load_hook"] + assert post_model_load_hook is not None + updates = post_model_load_hook() + optimizer = updates["optimizer"] + assert isinstance(optimizer, FSDP2Optimizer) + assert optimizer.name == "fsdp2" + + model = bundle.chunks[0] + model.train() + optimizer.zero_grad() + + batch, seq = 1, 8 + torch.manual_seed(20260613 + rank) + input_ids = torch.randint(0, cfg.vocab_size, (batch, seq), device=device) + labels = torch.randint(0, cfg.vocab_size, (batch, seq), device=device) + + out = model(input_ids=input_ids, labels=labels) + loss = out["loss"] + assert torch.isfinite(loss) + loss.backward() + success, grad_norm, num_zeros = optimizer.step() + assert success + assert num_zeros == 0 + assert torch.isfinite(torch.tensor(grad_norm)) + if rank == 0: + print( + "kimi_k2_fsdp2_smoke optimizer=fsdp2 " + f"world_size={world} loss={float(loss.detach().item()):.6e} " + f"grad_norm={grad_norm:.6e}" + ) diff --git a/experimental/lite/tests/unit/model/test_kimi_k2_lite_static.py b/experimental/lite/tests/unit/model/test_kimi_k2_lite_static.py new file mode 100644 index 00000000000..67c9e1da08e --- /dev/null +++ b/experimental/lite/tests/unit/model/test_kimi_k2_lite_static.py @@ -0,0 +1,340 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Static smoke coverage for Kimi K2 lite native implementation.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + + +def test_kimi_k2_lite_registry_resolves(): + from megatron.lite.model.registry import ( + get_train_runtime_module, + resolve_model_type_from_hf, + resolve_runtime_model_name, + ) + + runtime_name = resolve_runtime_model_name("kimi_k2", "lite") + assert runtime_name == "kimi_k2" + assert resolve_model_type_from_hf({"model_type": "kimi_k2"}) == "kimi_k2" + assert resolve_model_type_from_hf({"model_type": "deepseek_v3"}) == "kimi_k2" + module = get_train_runtime_module(runtime_name) + assert module.__name__ == "megatron.lite.model.kimi_k2.lite.protocol" + + +def test_kimi_k2_config_reads_hf_fields(): + from megatron.lite.model.kimi_k2.config import KimiK2Config + + cfg = KimiK2Config._from_hf_dict( + { + "model_type": "kimi_k2", + "num_hidden_layers": 3, + "hidden_size": 32, + "num_attention_heads": 4, + "num_key_value_heads": 4, + "vocab_size": 128, + "intermediate_size": 64, + "moe_intermediate_size": 16, + "n_routed_experts": 8, + "n_shared_experts": 2, + "num_experts_per_tok": 2, + "n_group": 4, + "topk_group": 2, + "topk_method": "noaux_tc", + "norm_topk_prob": True, + "scoring_func": "sigmoid", + "seq_aux": True, + "first_k_dense_replace": 1, + "q_lora_rank": 12, + "kv_lora_rank": 10, + "qk_nope_head_dim": 6, + "qk_rope_head_dim": 2, + "v_head_dim": 8, + "rope_scaling": {"type": "yarn", "factor": 32.0}, + } + ) + + assert cfg.num_experts == 8 + assert cfg.n_group == 4 + assert cfg.topk_group == 2 + assert cfg.shared_expert_intermediate_size == 32 + assert cfg.q_head_dim == 8 + assert not cfg.is_moe_layer(0) + assert cfg.is_moe_layer(1) + + +def test_kimi_k2_lite_does_not_import_wrappers_or_sibling_models(): + root = Path(__file__).resolve().parents[3] / "megatron" / "lite" / "model" / "kimi_k2" / "lite" + forbidden = ( + "megatron.lite.model.qwen3_5", + "megatron.lite.model.qwen3_moe", + "bridge_model", + "hybrid", + "build_mcore_context", + "from mbridge", + "import mbridge", + ) + for path in root.glob("*.py"): + text = path.read_text() + for token in forbidden: + assert token not in text + + +def test_kimi_k2_lite_implementation_files_stay_small(): + root = Path(__file__).resolve().parents[3] / "megatron" / "lite" / "model" / "kimi_k2" / "lite" + for name in ("model.py", "protocol.py", "checkpoint.py"): + line_count = len((root / name).read_text().splitlines()) + assert line_count < 1000, f"{name} has {line_count} lines" + + +def test_kimi_k2_lite_uses_shared_mla_primitive(): + lite_root = Path(__file__).resolve().parents[3] / "megatron" / "lite" + model_root = lite_root / "model" / "kimi_k2" / "lite" + primitive_mla = lite_root / "primitive" / "modules" / "attention" / "mla.py" + model_text = (model_root / "model.py").read_text() + mla_text = primitive_mla.read_text() + + assert ( + "from megatron.lite.primitive.modules.attention import MultiLatentAttention" in model_text + ) + assert not (model_root / "mla.py").exists() + assert "class MultiLatentAttention" not in model_text + assert "class MultiLatentAttention" in mla_text + assert "megatron.core" not in mla_text + assert "KimiK2SigmoidTopKRouter" in model_text + + +def test_kimi_k2_lite_optimizer_names_are_current(): + root = Path(__file__).resolve().parents[3] / "megatron" / "lite" / "model" / "kimi_k2" / "lite" + protocol_text = (root / "protocol.py").read_text() + + assert 'optimizer: str | None = "dist_opt"' in protocol_text + assert 'impl_cfg.optimizer == "dist_opt"' in protocol_text + assert 'impl_cfg.optimizer == "fsdp2"' in protocol_text + for forbidden in ("m" + "c_full", 'optimizer == "m' + 'c"'): + assert forbidden not in protocol_text + + +def test_kimi_k2_dist_opt_deterministic_default_is_enabled(): + from megatron.lite.model.kimi_k2.lite.protocol import ImplConfig + + cfg = ImplConfig(optimizer="dist_opt", deterministic=True, mtp_enable=True) + + assert cfg.optimizer == "dist_opt" + assert cfg.deterministic is True + assert cfg.mtp_enable is True + + +def test_kimi_k2_impl_config_accepts_runtime_mtp_fields(): + from megatron.lite.model.kimi_k2.config import KimiK2Config + from megatron.lite.model.kimi_k2.lite.protocol import ImplConfig + + cfg = KimiK2Config( + num_hidden_layers=1, + hidden_size=8, + num_attention_heads=2, + num_key_value_heads=2, + vocab_size=16, + intermediate_size=12, + moe_intermediate_size=4, + n_routed_experts=2, + n_shared_experts=1, + num_experts_per_tok=1, + n_group=1, + topk_group=1, + first_k_dense_replace=1, + q_lora_rank=4, + kv_lora_rank=4, + qk_nope_head_dim=2, + qk_rope_head_dim=2, + v_head_dim=2, + num_nextn_predict_layers=1, + ) + + assert ImplConfig(mtp_enable=False, mtp_enable_train=False).mtp_enable is False + assert ImplConfig(mtp_enable=True, mtp_enable_train=True).mtp_enable_train is True + assert cfg.num_nextn_predict_layers == 1 + + +def test_kimi_k2_mtp_and_pp_layout_rules_are_explicit(): + from megatron.lite.primitive.parallel import ParallelState, build_pipeline_chunk_layout + + model_text = ( + Path(__file__).resolve().parents[3] + / "megatron" + / "lite" + / "model" + / "kimi_k2" + / "lite" + / "model.py" + ).read_text() + + rank0 = ParallelState(pp_size=2, pp_rank=0, pp_is_first=True, pp_is_last=False) + rank1 = ParallelState(pp_size=2, pp_rank=1, pp_is_first=False, pp_is_last=True) + + assert build_pipeline_chunk_layout(4, rank0).layer_indices == [0, 1] + assert build_pipeline_chunk_layout(4, rank1).layer_indices == [2, 3] + # MTP is built on the stage the layout designates (layout.has_mtp), not a + # hard-coded head rank. + assert ( + "if mtp_enable and config.num_nextn_predict_layers > 0 and layout.has_mtp" + in model_text + ) + assert "self.mtp_embed = mtp_embedding" in model_text + + # pp-only: VPP / interleaving raises rather than silently mis-splitting. + with pytest.raises(NotImplementedError): + build_pipeline_chunk_layout(4, rank1, vpp=2, vpp_chunk_id=1) + + # Non-divisible counts auto-balance (no "not divisible" error): 3/pp2 -> [2, 1]. + assert build_pipeline_chunk_layout(3, rank0).layer_indices == [0, 1] + assert build_pipeline_chunk_layout(3, rank1).layer_indices == [2] + + +def test_kimi_k2_checkpoint_exports_hf_names(): + torch = pytest.importorskip("torch") + + from megatron.lite.model.kimi_k2.config import KimiK2Config + from megatron.lite.model.kimi_k2.lite import protocol + from megatron.lite.model.kimi_k2.lite.checkpoint import ( + KimiK2WeightSpec, + export_hf_weights, + save_hf_weights, + ) + + cfg = KimiK2Config( + num_hidden_layers=2, + hidden_size=8, + num_attention_heads=2, + num_key_value_heads=2, + vocab_size=16, + intermediate_size=12, + moe_intermediate_size=4, + n_routed_experts=2, + n_shared_experts=1, + num_experts_per_tok=1, + n_group=1, + topk_group=1, + first_k_dense_replace=1, + q_lora_rank=4, + kv_lora_rank=4, + qk_nope_head_dim=2, + qk_rope_head_dim=2, + v_head_dim=2, + num_nextn_predict_layers=1, + ) + spec = KimiK2WeightSpec(cfg) + + assert callable(export_hf_weights) + assert callable(save_hf_weights) + assert hasattr(protocol, "export_hf_weights") + assert spec.tp_spec("layers.1.self_attention.linear_proj.linear.weight") == (1, 0) + assert spec.tp_spec("layers.1.moe.experts.fc1.weight0") == (0, 1) + assert spec.expert_global_id("layers.1.moe.experts.fc2.weight1") == 1 + + gate_up = torch.arange(8 * 8, dtype=torch.float32).view(8, 8) + exported = dict(spec.native_to_hf("layers.1.moe.experts.fc1.weight0", gate_up)) + assert set(exported) == { + "model.layers.1.mlp.experts.0.gate_proj.weight", + "model.layers.1.mlp.experts.0.up_proj.weight", + } + torch.testing.assert_close( + exported["model.layers.1.mlp.experts.0.gate_proj.weight"], + gate_up[:4], + ) + torch.testing.assert_close( + exported["model.layers.1.mlp.experts.0.up_proj.weight"], + gate_up[4:], + ) + + bias = torch.arange(2, dtype=torch.float32) + assert spec.native_to_hf("layers.1.moe.router.expert_bias", bias) == [ + ("model.layers.1.mlp.gate.e_score_correction_bias", bias) + ] + assert spec.native_to_hf("mtp.layers.0.enorm.weight", bias) == [ + ("model.layers.2.enorm.weight", bias) + ] + assert spec.native_to_hf("mtp.layers.0.eh_proj.linear.weight", gate_up) == [ + ("model.layers.2.eh_proj.weight", gate_up) + ] + assert spec.native_to_hf("mtp.layers.0.final_layernorm.weight", bias) == [ + ("model.layers.2.shared_head.norm.weight", bias) + ] + mtp_export = dict( + spec.native_to_hf("mtp.layers.0.transformer_layer.mlp.gate_up.linear.weight", gate_up) + ) + assert set(mtp_export) == { + "model.layers.2.mlp.gate_proj.weight", + "model.layers.2.mlp.up_proj.weight", + } + + +def test_kimi_k2_fp8_checkpoint_dequant_cpu_path(): + torch = pytest.importorskip("torch") + if not hasattr(torch, "float8_e4m3fn"): + pytest.skip("torch float8_e4m3fn is required for this smoke.") + + from megatron.lite.model.kimi_k2.lite.checkpoint import _dequant_fp8_weight + + class Reader: + index = {"w_scale_inv": "fake.safetensors"} + + @staticmethod + def get_tensor(name): + assert name == "w_scale_inv" + return torch.full((1, 1), 2.0, dtype=torch.float32) + + weight = torch.tensor([[1.0, -2.0], [3.0, -4.0]], dtype=torch.float32).to(torch.float8_e4m3fn) + out = _dequant_fp8_weight(Reader(), "w", weight) + + torch.testing.assert_close(out, weight.float() * 2.0) + + +def test_kimi_k2_int4_checkpoint_dequant_cpu_path(): + torch = pytest.importorskip("torch") + + from megatron.lite.model.kimi_k2.lite.checkpoint import _get + + values = torch.tensor([[-8, -7, -1, 0, 1, 2, 6, 7, -8, 7]], dtype=torch.int8) + unsigned = (values + 8).to(torch.int32) + packed = torch.zeros((1, 2), dtype=torch.int32) + for offset in range(8): + packed[:, 0] |= unsigned[:, offset] << (4 * offset) + for offset in range(2): + packed[:, 1] |= unsigned[:, 8 + offset] << (4 * offset) + + class Reader: + index = { + "w_packed": "fake.safetensors", + "w_scale": "fake.safetensors", + "w_shape": "fake.safetensors", + } + + @staticmethod + def get_tensor(name): + return { + "w_packed": packed, + "w_scale": torch.tensor([[0.5, 2.0]], dtype=torch.float32), + "w_shape": torch.tensor([1, 10], dtype=torch.int64), + }[name] + + out = _get(Reader(), "w") + expected = torch.cat([values[:, :5].float() * 0.5, values[:, 5:].float() * 2.0], dim=1) + + torch.testing.assert_close(out, expected) + + +def test_kimi_k2_real_checkpoint_prefix_helpers(): + from megatron.lite.model.kimi_k2.lite.checkpoint import _lm_head_name, _text_prefix + + class Reader: + index = { + "language_model.model.embed_tokens.weight": "fake.safetensors", + "language_model.lm_head.weight": "fake.safetensors", + } + + prefix = _text_prefix(Reader()) + + assert prefix == "language_model.model" + assert _lm_head_name(Reader(), prefix) == "language_model.lm_head.weight" diff --git a/experimental/lite/tests/unit/model/test_qwen35_export.py b/experimental/lite/tests/unit/model/test_qwen35_export.py new file mode 100644 index 00000000000..b34f3df3bd1 --- /dev/null +++ b/experimental/lite/tests/unit/model/test_qwen35_export.py @@ -0,0 +1,725 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +from types import SimpleNamespace + +import torch +import torch.nn as nn + +from megatron.lite.model.qwen3_5.config import Qwen35Config +from megatron.lite.model.qwen3_5.lite.checkpoint import ( + Qwen35WeightSpec, + _merge_full_attn_qkvg, + _merge_gate_up_tp_shards, + _merge_linear_attn_conv1d_tp_shards, + _merge_linear_attn_in_proj_tp_shards, + export_hf_weights, +) +from megatron.lite.model.qwen3_5.lite.protocol import ImplConfig +from megatron.lite.model.registry import TRAIN_RUNTIME_MODULES, resolve_runtime_model_name + + +def _tiny_config() -> Qwen35Config: + return Qwen35Config( + num_hidden_layers=1, + hidden_size=8, + num_attention_heads=2, + num_key_value_heads=1, + head_dim=2, + vocab_size=16, + num_experts=4, + num_experts_per_tok=2, + moe_intermediate_size=4, + shared_expert_intermediate_size=4, + linear_num_key_heads=2, + linear_key_head_dim=2, + linear_num_value_heads=2, + linear_value_head_dim=2, + linear_conv_kernel_dim=2, + layer_types=["full_attention"], + partial_rotary_factor=1.0, + ) + + +def _single_rank_parallel_state() -> SimpleNamespace: + return SimpleNamespace( + pp_size=1, tp_size=1, tp_group=None, ep_size=1, ep_group=None, etp_size=1, etp_group=None + ) + + +def test_qwen35_protocol_registers_vllm_export_entrypoint() -> None: + key = resolve_runtime_model_name("qwen3_5", "lite") + module = __import__(TRAIN_RUNTIME_MODULES[key], fromlist=["export_hf_weights"]) + + assert key == "qwen3_5" + assert callable(module.export_hf_weights) + + +def test_qwen35_gdn_cp_mode_is_configurable() -> None: + assert ImplConfig().gdn_cp_mode == "replicated" + assert ImplConfig(gdn_cp_mode="sharded").gdn_cp_mode == "sharded" + + +def test_qwen35_export_uses_hf_checkpoint_names_without_module_prefix() -> None: + class TinyQwen35Module(nn.Module): + def __init__(self) -> None: + super().__init__() + self.embed = nn.Module() + self.embed.embedding = nn.Embedding(16, 8) + self.norm = nn.LayerNorm(8) + self.head = nn.Module() + self.head.col = nn.Module() + self.head.col.linear = nn.Linear(8, 16, bias=False) + + cfg = _tiny_config() + model = TinyQwen35Module() + + exported = dict(export_hf_weights(model, cfg, _single_rank_parallel_state())) + + assert set(exported) == { + "model.language_model.embed_tokens.weight", + "model.language_model.norm.weight", + "lm_head.weight", + } + assert all(not name.startswith("module.") for name in exported) + assert all(not name.startswith(("embed.", "norm.", "head.")) for name in exported) + + +def test_qwen35_export_dtype_cast_is_opt_in() -> None: + class TinyQwen35Module(nn.Module): + def __init__(self, config: Qwen35Config) -> None: + super().__init__() + self.norm = nn.LayerNorm(8) + self.layers = nn.ModuleList([nn.Module()]) + self.layers[0].moe = nn.Module() + self.layers[0].moe.experts = nn.Module() + self.layers[0].moe.experts.fc1 = nn.Module() + + rows = config.moe_intermediate_size * 2 + for expert_idx in range(config.num_experts): + tensor = torch.arange(rows * config.hidden_size, dtype=torch.float32).reshape( + rows, config.hidden_size + ) + tensor = tensor + expert_idx * 1000 + self.layers[0].moe.experts.fc1.register_parameter( + f"weight{expert_idx}", nn.Parameter(tensor) + ) + + cfg = _tiny_config() + model = TinyQwen35Module(cfg) + + default_export = dict(export_hf_weights(model, cfg, _single_rank_parallel_state())) + bf16_export = dict( + export_hf_weights(model, cfg, _single_rank_parallel_state(), export_dtype="bfloat16") + ) + + assert default_export["model.language_model.norm.weight"].dtype == torch.float32 + assert bf16_export["model.language_model.norm.weight"].dtype == torch.bfloat16 + assert ( + default_export["model.language_model.layers.0.mlp.experts.gate_up_proj"].dtype + == torch.float32 + ) + assert ( + bf16_export["model.language_model.layers.0.mlp.experts.gate_up_proj"].dtype + == torch.bfloat16 + ) + + +def test_qwen35_export_preserves_runtime_parameter_dtype_by_default() -> None: + class TinyQwen35Module(nn.Module): + def __init__(self, config: Qwen35Config) -> None: + super().__init__() + self.norm = nn.LayerNorm(8).to(torch.bfloat16) + self.layers = nn.ModuleList([nn.Module()]) + self.layers[0].moe = nn.Module() + self.layers[0].moe.experts = nn.Module() + self.layers[0].moe.experts.fc1 = nn.Module() + + rows = config.moe_intermediate_size * 2 + for expert_idx in range(config.num_experts): + tensor = torch.arange(rows * config.hidden_size, dtype=torch.bfloat16).reshape( + rows, config.hidden_size + ) + tensor = tensor + expert_idx * 1000 + self.layers[0].moe.experts.fc1.register_parameter( + f"weight{expert_idx}", nn.Parameter(tensor) + ) + + cfg = _tiny_config() + model = TinyQwen35Module(cfg) + + exported = dict(export_hf_weights(model, cfg, _single_rank_parallel_state())) + + assert exported["model.language_model.norm.weight"].dtype == torch.bfloat16 + assert ( + exported["model.language_model.layers.0.mlp.experts.gate_up_proj"].dtype == torch.bfloat16 + ) + + +def test_qwen35_export_batches_ep_expert_gather(monkeypatch) -> None: + class TinyQwen35Module(nn.Module): + def __init__(self, config: Qwen35Config) -> None: + super().__init__() + self.layers = nn.ModuleList([nn.Module()]) + self.layers[0].moe = nn.Module() + self.layers[0].moe.experts = nn.Module() + self.layers[0].moe.experts.fc1 = nn.Module() + + rows = config.moe_intermediate_size * 2 + for local_idx in range(config.num_experts // 2): + tensor = torch.arange(rows * config.hidden_size, dtype=torch.bfloat16).reshape( + rows, config.hidden_size + ) + tensor = tensor + local_idx * 1000 + self.layers[0].moe.experts.fc1.register_parameter( + f"weight{local_idx}", nn.Parameter(tensor) + ) + + cfg = _tiny_config() + cfg.num_experts = 16 + model = TinyQwen35Module(cfg) + ps = SimpleNamespace( + pp_size=1, + tp_size=1, + tp_group=None, + ep_size=2, + ep_group=object(), + etp_size=1, + etp_group=None, + ) + gather_calls = [] + + def fake_all_gather(outputs, tensor, group=None): + del group + gather_calls.append(tensor.clone()) + outputs[0].copy_(tensor) + outputs[1].copy_(tensor + 2000) + + monkeypatch.setattr("megatron.lite.primitive.ckpt.hf_weights.dist.all_gather", fake_all_gather) + + exported = dict(export_hf_weights(model, cfg, ps)) + + assert len(gather_calls) == 2 + assert all(call.shape[0] <= 4 for call in gather_calls) + local_tensors = [ + getattr(model.layers[0].moe.experts.fc1, f"weight{i}").detach() + for i in range(cfg.num_experts // ps.ep_size) + ] + expected = torch.stack(local_tensors + [tensor + 2000 for tensor in local_tensors]) + assert torch.equal(exported["model.language_model.layers.0.mlp.experts.gate_up_proj"], expected) + + +def test_qwen35_export_uses_packed_expert_group_names(monkeypatch) -> None: + class TinyQwen35Module(nn.Module): + def __init__(self, config: Qwen35Config) -> None: + super().__init__() + self.layers = nn.ModuleList([nn.Module()]) + self.layers[0].moe = nn.Module() + self.layers[0].moe.experts = nn.Module() + self.layers[0].moe.experts.fc1 = nn.Module() + + rows = config.moe_intermediate_size * 2 + for expert_idx in range(config.num_experts): + tensor = torch.arange(rows * config.hidden_size, dtype=torch.bfloat16).reshape( + rows, config.hidden_size + ) + tensor = tensor + expert_idx * 1000 + self.layers[0].moe.experts.fc1.register_parameter( + f"weight{expert_idx}", nn.Parameter(tensor) + ) + + cfg = _tiny_config() + seen_native_names = [] + original = Qwen35WeightSpec.native_to_hf + + def spy_native_to_hf(self, native_name, tensor): + seen_native_names.append(native_name) + return original(self, native_name, tensor) + + monkeypatch.setattr(Qwen35WeightSpec, "native_to_hf", spy_native_to_hf) + + exported = dict(export_hf_weights(TinyQwen35Module(cfg), cfg, _single_rank_parallel_state())) + + assert seen_native_names == ["layers.0.moe.experts.fc1.packed"] + assert set(exported) == {"model.language_model.layers.0.mlp.experts.gate_up_proj"} + + +def test_qwen35_export_rank0_only_still_participates_in_ep_gather(monkeypatch) -> None: + class TinyQwen35Module(nn.Module): + def __init__(self, config: Qwen35Config) -> None: + super().__init__() + self.layers = nn.ModuleList([nn.Module()]) + self.layers[0].moe = nn.Module() + self.layers[0].moe.experts = nn.Module() + self.layers[0].moe.experts.fc1 = nn.Module() + + rows = config.moe_intermediate_size * 2 + for local_idx in range(config.num_experts // 2): + tensor = torch.zeros(rows, config.hidden_size, dtype=torch.bfloat16) + local_idx + self.layers[0].moe.experts.fc1.register_parameter( + f"weight{local_idx}", nn.Parameter(tensor) + ) + + cfg = _tiny_config() + ps = SimpleNamespace( + pp_size=1, + tp_size=1, + tp_group=None, + ep_size=2, + ep_group=object(), + etp_size=1, + etp_group=None, + ) + gather_calls = [] + + def fake_all_gather(outputs, tensor, group=None): + del group + gather_calls.append(tensor.clone()) + outputs[0].copy_(tensor) + outputs[1].copy_(tensor + 2) + + monkeypatch.setattr("megatron.lite.primitive.ckpt.hf_weights.dist.is_initialized", lambda: True) + monkeypatch.setattr("megatron.lite.primitive.ckpt.hf_weights.dist.get_rank", lambda: 1) + monkeypatch.setattr("megatron.lite.primitive.ckpt.hf_weights.dist.all_gather", fake_all_gather) + + exported = list(export_hf_weights(TinyQwen35Module(cfg), cfg, ps, rank0_only=True)) + + assert exported == [] + assert len(gather_calls) == 1 + + +def test_qwen35_export_maps_top_level_and_layer_norm_names() -> None: + cfg = _tiny_config() + spec = Qwen35WeightSpec(cfg) + tensor = torch.arange(cfg.hidden_size) + + cases = { + "embed.embedding.weight": "model.language_model.embed_tokens.weight", + "norm.weight": "model.language_model.norm.weight", + "head.col.linear.weight": "lm_head.weight", + "layers.0.full_attn.qkv.linear.layer_norm_weight": ( + "model.language_model.layers.0.input_layernorm.weight" + ), + "layers.0.mlp_norm.weight": "model.language_model.layers.0.post_attention_layernorm.weight", + } + + for native_name, hf_name in cases.items(): + exported = dict(spec.native_to_hf(native_name, tensor)) + assert set(exported) == {hf_name} + assert torch.equal(exported[hf_name], tensor) + + +def test_qwen35_export_unpacks_full_attention_q_gate() -> None: + cfg = _tiny_config() + spec = Qwen35WeightSpec(cfg) + hidden = cfg.hidden_size + q_gate = torch.arange(cfg.num_attention_heads * 2 * cfg.head_dim * hidden).reshape(-1, hidden) + key = torch.arange( + q_gate.numel(), q_gate.numel() + cfg.num_key_value_heads * cfg.head_dim * hidden + ).reshape(-1, hidden) + value = torch.arange( + key[-1, -1] + 1, key[-1, -1] + 1 + cfg.num_key_value_heads * cfg.head_dim * hidden + ).reshape(-1, hidden) + + packed = _merge_full_attn_qkvg(q_gate, key, value, cfg=cfg) + exported = dict(spec.native_to_hf("layers.0.full_attn.qkv.linear.weight", packed)) + + assert set(exported) == { + "model.language_model.layers.0.self_attn.q_proj.weight", + "model.language_model.layers.0.self_attn.k_proj.weight", + "model.language_model.layers.0.self_attn.v_proj.weight", + } + assert torch.equal(exported["model.language_model.layers.0.self_attn.q_proj.weight"], q_gate) + assert torch.equal(exported["model.language_model.layers.0.self_attn.k_proj.weight"], key) + assert torch.equal(exported["model.language_model.layers.0.self_attn.v_proj.weight"], value) + + +def test_qwen35_export_maps_linear_attention_to_hf_checkpoint_names() -> None: + cfg = _tiny_config() + spec = Qwen35WeightSpec(cfg) + qk_dim = cfg.linear_num_key_heads * cfg.linear_key_head_dim + v_dim = cfg.linear_num_value_heads * cfg.linear_value_head_dim + rows = qk_dim * 2 + v_dim * 2 + cfg.linear_num_value_heads * 2 + tensor = torch.arange(rows * cfg.hidden_size).reshape(rows, cfg.hidden_size) + + exported = dict(spec.native_to_hf("layers.0.linear_attn.in_proj.linear.weight", tensor)) + + assert set(exported) == { + "model.language_model.layers.0.linear_attn.in_proj_qkv.weight", + "model.language_model.layers.0.linear_attn.in_proj_z.weight", + "model.language_model.layers.0.linear_attn.in_proj_b.weight", + "model.language_model.layers.0.linear_attn.in_proj_a.weight", + } + assert ( + exported["model.language_model.layers.0.linear_attn.in_proj_qkv.weight"].shape[0] + == qk_dim * 2 + v_dim + ) + assert exported["model.language_model.layers.0.linear_attn.in_proj_z.weight"].shape[0] == v_dim + assert ( + exported["model.language_model.layers.0.linear_attn.in_proj_b.weight"].shape[0] + == cfg.linear_num_value_heads + ) + assert ( + exported["model.language_model.layers.0.linear_attn.in_proj_a.weight"].shape[0] + == cfg.linear_num_value_heads + ) + + +def test_qwen35_export_reorders_linear_attention_tp_shards_before_hf_split() -> None: + cfg = _tiny_config() + qk_dim = cfg.linear_num_key_heads * cfg.linear_key_head_dim + v_dim = cfg.linear_num_value_heads * cfg.linear_value_head_dim + hidden = cfg.hidden_size + parts = [ + torch.arange(0, qk_dim * hidden).reshape(qk_dim, hidden), + torch.arange(100, 100 + qk_dim * hidden).reshape(qk_dim, hidden), + torch.arange(200, 200 + v_dim * hidden).reshape(v_dim, hidden), + torch.arange(300, 300 + v_dim * hidden).reshape(v_dim, hidden), + torch.arange(400, 400 + cfg.linear_num_value_heads * hidden).reshape( + cfg.linear_num_value_heads, hidden + ), + torch.arange(500, 500 + cfg.linear_num_value_heads * hidden).reshape( + cfg.linear_num_value_heads, hidden + ), + ] + full = torch.cat(parts, dim=0) + shards = [torch.cat([part.chunk(2, dim=0)[rank] for part in parts], dim=0) for rank in range(2)] + + merged = _merge_linear_attn_in_proj_tp_shards(shards, cfg=cfg) + + assert torch.equal(merged, full) + + +def test_qwen35_export_reorders_linear_attention_conv1d_tp_shards() -> None: + cfg = _tiny_config() + qk_dim = cfg.linear_num_key_heads * cfg.linear_key_head_dim + v_dim = cfg.linear_num_value_heads * cfg.linear_value_head_dim + trailing = (1, cfg.linear_conv_kernel_dim) + parts = [ + torch.arange(0, qk_dim * trailing[0] * trailing[1], dtype=torch.float32).reshape( + qk_dim, *trailing + ), + torch.arange(100, 100 + qk_dim * trailing[0] * trailing[1], dtype=torch.float32).reshape( + qk_dim, *trailing + ), + torch.arange(200, 200 + v_dim * trailing[0] * trailing[1], dtype=torch.float32).reshape( + v_dim, *trailing + ), + ] + full = torch.cat(parts, dim=0) + shards = [torch.cat([part.chunk(2, dim=0)[rank] for part in parts], dim=0) for rank in range(2)] + + merged = _merge_linear_attn_conv1d_tp_shards(shards, cfg=cfg) + + assert torch.equal(merged, full) + + +def test_qwen35_export_uses_mbridge_conv1d_tp_gather(monkeypatch) -> None: + class TinyQwen35Module(nn.Module): + def __init__(self, local_shard: torch.Tensor) -> None: + super().__init__() + self.layers = nn.ModuleList([nn.Module()]) + self.layers[0].linear_attn = nn.Module() + self.layers[0].linear_attn.conv1d = nn.Module() + self.layers[0].linear_attn.conv1d.register_parameter( + "weight", nn.Parameter(local_shard.clone()) + ) + + cfg = _tiny_config() + qk_dim = cfg.linear_num_key_heads * cfg.linear_key_head_dim + v_dim = cfg.linear_num_value_heads * cfg.linear_value_head_dim + trailing = (1, cfg.linear_conv_kernel_dim) + parts = [ + torch.arange(0, qk_dim * trailing[0] * trailing[1], dtype=torch.float32).reshape( + qk_dim, *trailing + ), + torch.arange(100, 100 + qk_dim * trailing[0] * trailing[1], dtype=torch.float32).reshape( + qk_dim, *trailing + ), + torch.arange(200, 200 + v_dim * trailing[0] * trailing[1], dtype=torch.float32).reshape( + v_dim, *trailing + ), + ] + full = torch.cat(parts, dim=0) + shards = [torch.cat([part.chunk(2, dim=0)[rank] for part in parts], dim=0) for rank in range(2)] + ps = SimpleNamespace( + pp_size=1, + tp_size=2, + tp_group=object(), + ep_size=1, + ep_group=None, + etp_size=1, + etp_group=None, + ) + gather_calls = [] + + def fake_all_gather(outputs, tensor, group=None): + assert group is ps.tp_group + gather_calls.append(tensor.clone()) + outputs[0].copy_(shards[0]) + outputs[1].copy_(shards[1]) + + monkeypatch.setattr( + "megatron.lite.model.qwen3_5.lite.checkpoint.dist.all_gather", fake_all_gather + ) + + exported = dict(export_hf_weights(TinyQwen35Module(shards[0]), cfg, ps)) + + assert len(gather_calls) == 1 + assert torch.equal(gather_calls[0], shards[0]) + assert torch.equal(exported["model.language_model.layers.0.linear_attn.conv1d.weight"], full) + + +def test_qwen35_export_reorders_shared_expert_gate_up_tp_shards() -> None: + gate = torch.arange(0, 32).reshape(4, 8) + up = torch.arange(100, 132).reshape(4, 8) + full = torch.cat([gate, up], dim=0) + shards = [ + torch.cat([gate.chunk(2, dim=0)[rank], up.chunk(2, dim=0)[rank]], dim=0) + for rank in range(2) + ] + + merged = _merge_gate_up_tp_shards(shards) + + assert torch.equal(merged, full) + + +def test_qwen35_export_restores_zero_centered_linear_attention_norm() -> None: + cfg = _tiny_config() + spec = Qwen35WeightSpec(cfg) + tensor = torch.tensor([-0.5, 0.0, 0.5]) + + exported = dict(spec.native_to_hf("layers.0.linear_attn.norm.weight", tensor)) + + assert set(exported) == {"model.language_model.layers.0.linear_attn.norm.weight"} + assert torch.equal( + exported["model.language_model.layers.0.linear_attn.norm.weight"], tensor + 1 + ) + + +def test_qwen35_export_maps_shared_expert_to_hf_checkpoint_names() -> None: + cfg = _tiny_config() + spec = Qwen35WeightSpec(cfg) + tensor = torch.arange(cfg.shared_expert_intermediate_size * 2 * cfg.hidden_size).reshape( + -1, cfg.hidden_size + ) + + exported = dict(spec.native_to_hf("layers.0.moe.shared_expert.gate_up.linear.weight", tensor)) + + assert set(exported) == { + "model.language_model.layers.0.mlp.shared_expert.gate_proj.weight", + "model.language_model.layers.0.mlp.shared_expert.up_proj.weight", + } + gate, up = tensor.chunk(2, dim=0) + assert torch.equal( + exported["model.language_model.layers.0.mlp.shared_expert.gate_proj.weight"], gate + ) + assert torch.equal( + exported["model.language_model.layers.0.mlp.shared_expert.up_proj.weight"], up + ) + + +def test_qwen35_export_packs_base_expert_fc1_to_hf_gate_up_proj() -> None: + cfg = _tiny_config() + spec = Qwen35WeightSpec(cfg) + base = torch.arange(cfg.moe_intermediate_size * 2 * cfg.hidden_size).reshape( + -1, cfg.hidden_size + ) + + exported = {} + expert_tensors = [] + for expert_idx in range(cfg.num_experts): + tensor = base + expert_idx * 1000 + expert_tensors.append(tensor) + exported.update( + dict(spec.native_to_hf(f"layers.0.moe.experts.fc1.weight{expert_idx}", tensor)) + ) + + assert set(exported) == {"model.language_model.layers.0.mlp.experts.gate_up_proj"} + assert torch.equal( + exported["model.language_model.layers.0.mlp.experts.gate_up_proj"], + torch.stack(expert_tensors, dim=0), + ) + + +def test_qwen35_export_matches_mbridge_qwen35_moe_packed_expert_contract() -> None: + cfg = _tiny_config() + spec = Qwen35WeightSpec(cfg) + rows = cfg.moe_intermediate_size * 2 + fc1_tensors = [ + torch.arange(rows * cfg.hidden_size, dtype=torch.bfloat16).reshape(rows, cfg.hidden_size) + + expert_idx * 1000 + for expert_idx in range(cfg.num_experts) + ] + fc2_tensors = [ + torch.arange(cfg.hidden_size * cfg.moe_intermediate_size, dtype=torch.bfloat16).reshape( + cfg.hidden_size, cfg.moe_intermediate_size + ) + + expert_idx * 1000 + for expert_idx in range(cfg.num_experts) + ] + + fc1_exported = {} + fc2_exported = {} + for expert_idx, (fc1, fc2) in enumerate(zip(fc1_tensors, fc2_tensors, strict=True)): + fc1_exported.update( + dict(spec.native_to_hf(f"layers.0.moe.experts.fc1.weight{expert_idx}", fc1)) + ) + fc2_exported.update( + dict(spec.native_to_hf(f"layers.0.moe.experts.fc2.weight{expert_idx}", fc2)) + ) + + assert set(fc1_exported) == {"model.language_model.layers.0.mlp.experts.gate_up_proj"} + assert set(fc2_exported) == {"model.language_model.layers.0.mlp.experts.down_proj"} + assert torch.equal( + fc1_exported["model.language_model.layers.0.mlp.experts.gate_up_proj"], + torch.stack(fc1_tensors, dim=0), + ) + assert torch.equal( + fc2_exported["model.language_model.layers.0.mlp.experts.down_proj"], + torch.stack(fc2_tensors, dim=0), + ) + + +def test_qwen35_export_vllm_target_uses_runtime_prefix_and_packed_expert_names() -> None: + cfg = _tiny_config() + spec = Qwen35WeightSpec(cfg, target="vllm") + dense = torch.arange(cfg.hidden_size) + + exported_embed = dict(spec.native_to_hf("embed.embedding.weight", dense)) + exported_norm = dict(spec.native_to_hf("norm.weight", dense)) + exported_head = dict(spec.native_to_hf("head.col.linear.weight", dense)) + exported_mlp_norm = dict(spec.native_to_hf("layers.0.mlp_norm.weight", dense)) + assert set(exported_embed) == {"language_model.model.embed_tokens.weight"} + assert set(exported_norm) == {"language_model.model.norm.weight"} + assert set(exported_head) == {"language_model.lm_head.weight"} + assert set(exported_mlp_norm) == { + "language_model.model.layers.0.post_attention_layernorm.weight" + } + assert torch.equal(exported_embed["language_model.model.embed_tokens.weight"], dense) + assert torch.equal(exported_norm["language_model.model.norm.weight"], dense) + assert torch.equal(exported_head["language_model.lm_head.weight"], dense) + assert torch.equal( + exported_mlp_norm["language_model.model.layers.0.post_attention_layernorm.weight"], dense + ) + + fc1_tensors = [ + torch.arange(cfg.moe_intermediate_size * 2 * cfg.hidden_size, dtype=torch.bfloat16).reshape( + -1, cfg.hidden_size + ) + + expert_idx * 1000 + for expert_idx in range(cfg.num_experts) + ] + fc2_tensors = [ + torch.arange(cfg.hidden_size * cfg.moe_intermediate_size, dtype=torch.bfloat16).reshape( + cfg.hidden_size, cfg.moe_intermediate_size + ) + + expert_idx * 2000 + for expert_idx in range(cfg.num_experts) + ] + + fc1_exported = {} + fc2_exported = {} + for expert_idx, (fc1, fc2) in enumerate(zip(fc1_tensors, fc2_tensors, strict=True)): + fc1_exported.update( + dict(spec.native_to_hf(f"layers.0.moe.experts.fc1.weight{expert_idx}", fc1)) + ) + fc2_exported.update( + dict(spec.native_to_hf(f"layers.0.moe.experts.fc2.weight{expert_idx}", fc2)) + ) + + assert set(fc1_exported) == {"language_model.model.layers.0.mlp.experts.gate_up_proj"} + assert set(fc2_exported) == {"language_model.model.layers.0.mlp.experts.down_proj"} + assert torch.equal( + fc1_exported["language_model.model.layers.0.mlp.experts.gate_up_proj"], + torch.stack(fc1_tensors, dim=0), + ) + assert torch.equal( + fc2_exported["language_model.model.layers.0.mlp.experts.down_proj"], + torch.stack(fc2_tensors, dim=0), + ) + + +def test_qwen35_export_vllm_target_packs_experts_with_runtime_prefix() -> None: + class TinyQwen35Module(nn.Module): + def __init__(self, config: Qwen35Config) -> None: + super().__init__() + self.layers = nn.ModuleList([nn.Module()]) + self.layers[0].moe = nn.Module() + self.layers[0].moe.experts = nn.Module() + self.layers[0].moe.experts.fc1 = nn.Module() + self.layers[0].moe.experts.fc2 = nn.Module() + + rows = config.moe_intermediate_size * 2 + for expert_idx in range(config.num_experts): + fc1 = torch.arange(rows * config.hidden_size, dtype=torch.bfloat16).reshape( + rows, config.hidden_size + ) + fc1 = fc1 + expert_idx * 1000 + fc2 = torch.arange( + config.hidden_size * config.moe_intermediate_size, dtype=torch.bfloat16 + ).reshape(config.hidden_size, config.moe_intermediate_size) + fc2 = fc2 + expert_idx * 2000 + self.layers[0].moe.experts.fc1.register_parameter( + f"weight{expert_idx}", nn.Parameter(fc1) + ) + self.layers[0].moe.experts.fc2.register_parameter( + f"weight{expert_idx}", nn.Parameter(fc2) + ) + + cfg = _tiny_config() + model = TinyQwen35Module(cfg) + + exported = dict(export_hf_weights(model, cfg, _single_rank_parallel_state(), target="vllm")) + + assert "model.language_model.layers.0.mlp.experts.gate_up_proj" not in exported + assert "model.language_model.layers.0.mlp.experts.down_proj" not in exported + assert "language_model.model.layers.0.mlp.experts.0.gate_proj.weight" not in exported + assert "language_model.model.layers.0.mlp.experts.0.down_proj.weight" not in exported + assert set(exported) == { + "language_model.model.layers.0.mlp.experts.gate_up_proj", + "language_model.model.layers.0.mlp.experts.down_proj", + } + + expected_fc1 = [] + expected_fc2 = [] + for expert_idx in range(cfg.num_experts): + fc1 = getattr(model.layers[0].moe.experts.fc1, f"weight{expert_idx}").detach() + fc2 = getattr(model.layers[0].moe.experts.fc2, f"weight{expert_idx}").detach() + expected_fc1.append(fc1) + expected_fc2.append(fc2) + + assert torch.equal( + exported["language_model.model.layers.0.mlp.experts.gate_up_proj"], + torch.stack(expected_fc1, dim=0), + ) + assert torch.equal( + exported["language_model.model.layers.0.mlp.experts.down_proj"], + torch.stack(expected_fc2, dim=0), + ) + + +def test_qwen35_export_packs_base_expert_fc2_and_expert_metadata() -> None: + cfg = _tiny_config() + spec = Qwen35WeightSpec(cfg) + base = torch.arange(cfg.hidden_size * cfg.moe_intermediate_size).reshape( + cfg.hidden_size, cfg.moe_intermediate_size + ) + native_name = "layers.0.moe.experts.fc2.weight2" + + exported = {} + expert_tensors = [] + for expert_idx in range(cfg.num_experts): + tensor = base + expert_idx * 1000 + expert_tensors.append(tensor) + exported.update( + dict(spec.native_to_hf(f"layers.0.moe.experts.fc2.weight{expert_idx}", tensor)) + ) + + assert set(exported) == {"model.language_model.layers.0.mlp.experts.down_proj"} + assert torch.equal( + exported["model.language_model.layers.0.mlp.experts.down_proj"], + torch.stack(expert_tensors, dim=0), + ) + assert spec.is_expert(native_name) + assert spec.expert_global_id(native_name) == 2 + assert spec.expert_local_name(native_name, 0) == "layers.0.moe.experts.fc2.weight0" + assert spec.tp_spec(native_name) == (1, 1) diff --git a/experimental/lite/tests/unit/model/test_qwen_config_unit.py b/experimental/lite/tests/unit/model/test_qwen_config_unit.py new file mode 100644 index 00000000000..339b1a21e0b --- /dev/null +++ b/experimental/lite/tests/unit/model/test_qwen_config_unit.py @@ -0,0 +1,146 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +from __future__ import annotations + +import ast +from pathlib import Path + +import pytest + +from megatron.lite.model.qwen3_5.config import Qwen35Config +from megatron.lite.model.qwen3_moe.config import Qwen3MoEConfig +from megatron.lite.model.registry import resolve_model_type_from_hf, resolve_runtime_model_name + +pytestmark = pytest.mark.mlite + +LITE_ROOT = Path(__file__).resolve().parents[3] + + +def _tiny_qwen3_hf_dict() -> dict: + return { + "model_type": "qwen3_moe", + "hidden_size": 16, + "num_attention_heads": 4, + "num_key_value_heads": 2, + "num_hidden_layers": 1, + "vocab_size": 64, + "num_experts": 2, + "num_experts_per_tok": 1, + "moe_intermediate_size": 8, + "rope_parameters": {"rope_theta": 12345.0}, + } + + +def _tiny_qwen35_text_config() -> dict: + return { + "hidden_size": 16, + "num_attention_heads": 4, + "num_key_value_heads": 2, + "head_dim": 4, + "num_hidden_layers": 2, + "vocab_size": 64, + "num_experts": 2, + "num_experts_per_tok": 1, + "moe_intermediate_size": 8, + "shared_expert_intermediate_size": 8, + "linear_num_key_heads": 2, + "linear_key_head_dim": 4, + "linear_num_value_heads": 2, + "linear_value_head_dim": 4, + "linear_conv_kernel_dim": 2, + "num_nextn_predict_layers": 1, + "layer_types": ["linear_attention", "full_attention", "full_attention"], + "rope_parameters": {"partial_rotary_factor": 1.0, "mrope_section": [1, 1, 0]}, + } + + +def test_registry_resolves_qwen_lite_model_names(): + assert resolve_model_type_from_hf({"model_type": "qwen3_moe"}) == "qwen3" + assert resolve_model_type_from_hf({"model_type": "qwen3_5_moe"}) == "qwen3_5" + assert resolve_runtime_model_name("qwen3", "lite") == "qwen3" + assert resolve_runtime_model_name("qwen3_moe", "lite") == "qwen3_moe" + assert resolve_runtime_model_name("qwen3_5", "lite") == "qwen3_5" + + +def test_qwen3_config_from_hf_dict_derives_head_dim_and_rope_theta(): + cfg = Qwen3MoEConfig._from_hf_dict(_tiny_qwen3_hf_dict()) + + assert cfg.hidden_size == 16 + assert cfg.head_dim == 4 + assert cfg.layer_types == ["full_attention"] + assert cfg.rope_theta == 12345.0 + + +def test_qwen3_config_rejects_invalid_expert_topk(): + hf = _tiny_qwen3_hf_dict() + hf["num_experts_per_tok"] = 3 + + with pytest.raises(ValueError, match="num_experts_per_tok"): + Qwen3MoEConfig._from_hf_dict(hf) + + +def test_qwen35_config_from_text_config_splits_mtp_layer_types(): + cfg = Qwen35Config._from_hf_dict( + {"model_type": "qwen3_5_moe", "text_config": _tiny_qwen35_text_config()} + ) + + assert cfg.layer_types == ["linear_attention", "full_attention"] + assert cfg.mtp_layer_types == ["full_attention"] + assert cfg.rotary_dim == 4 + assert cfg.mrope_section == [1, 1, 0] + + +def test_qwen_lite_protocols_build_configs_from_hf_dicts(transformer_engine_import_stub): + transformer_engine_import_stub() + + from megatron.lite.model.qwen3_5.lite import protocol as qwen35_protocol + from megatron.lite.model.qwen3_moe.lite import protocol as qwen3_protocol + + qwen3_cfg = qwen3_protocol.build_model_config(_tiny_qwen3_hf_dict(), vocab_size=128) + qwen35_cfg = qwen35_protocol.build_model_config( + {"model_type": "qwen3_5_moe", "text_config": _tiny_qwen35_text_config()}, vocab_size=128 + ) + + assert qwen3_cfg.vocab_size == 128 + assert qwen35_cfg.vocab_size == 128 + assert qwen35_cfg.layer_type_at(0) == "linear_attention" + assert qwen35_cfg.layer_type_at(1) == "full_attention" + + +def test_qwen_lite_protocols_reexport_checkpoint_hook_names(): + protocol_paths = [ + LITE_ROOT / "megatron/lite/model/qwen3_moe/lite/protocol.py", + LITE_ROOT / "megatron/lite/model/qwen3_5/lite/protocol.py", + ] + + for path in protocol_paths: + tree = ast.parse(path.read_text()) + exported = _string_list_assignment(tree, "__all__") + checkpoint_imports = _checkpoint_import_names(tree) + + assert "EXPERT_CLASSIFIER" in exported + assert "PLACEMENT_FN" in exported + assert "EXPERT_CLASSIFIER" in checkpoint_imports + assert "PLACEMENT_FN" in checkpoint_imports + + +def _string_list_assignment(tree: ast.Module, name: str) -> set[str]: + for node in tree.body: + if not isinstance(node, ast.Assign): + continue + if not any(isinstance(target, ast.Name) and target.id == name for target in node.targets): + continue + if not isinstance(node.value, (ast.List, ast.Tuple)): + return set() + return {item.value for item in node.value.elts if isinstance(item, ast.Constant)} + return set() + + +def _checkpoint_import_names(tree: ast.Module) -> set[str]: + names: set[str] = set() + for node in tree.body: + if not isinstance(node, ast.ImportFrom): + continue + if node.module is None or not node.module.endswith(".lite.checkpoint"): + continue + names.update(alias.name for alias in node.names) + return names diff --git a/experimental/lite/tests/unit/primitive/test_attention_moe_unit.py b/experimental/lite/tests/unit/primitive/test_attention_moe_unit.py new file mode 100644 index 00000000000..3945e32c283 --- /dev/null +++ b/experimental/lite/tests/unit/primitive/test_attention_moe_unit.py @@ -0,0 +1,149 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +from __future__ import annotations + +from types import SimpleNamespace + +import pytest +import torch + +pytestmark = pytest.mark.mlite + + +@pytest.fixture(autouse=True) +def _te_import_stub(transformer_engine_import_stub): + transformer_engine_import_stub() + + +def _split_grouped_qkvg(): + from megatron.lite.primitive.modules import split_grouped_qkvg + + return split_grouped_qkvg + + +def _moe_aux_scaler(): + from megatron.lite.primitive.modules.moe import MoEAuxLossAutoScaler + + return MoEAuxLossAutoScaler + + +def _router_and_parallel_state(monkeypatch): + from megatron.lite.primitive.modules.router import TopKRouter + from megatron.lite.primitive.parallel import ParallelState + + del monkeypatch + return TopKRouter, ParallelState + + +def _router_config(): + return SimpleNamespace( + hidden_size=4, num_experts=4, num_experts_per_tok=2, router_aux_loss_coef=0.1 + ) + + +def _walk_grad_fn_names(tensor: torch.Tensor) -> set[str]: + names: set[str] = set() + stack = [tensor.grad_fn] + while stack: + fn = stack.pop() + if fn is None: + continue + names.add(type(fn).__name__) + stack.extend(parent for parent, _idx in fn.next_functions) + return names + + +def test_gqa_split_grouped_qkvg_preserves_q_gate_kv_order(): + split_grouped_qkvg = _split_grouped_qkvg() + qkv = torch.arange(24).reshape(1, 24) + + query, gate, key, value = split_grouped_qkvg(qkv, num_heads=4, num_kv_heads=2, head_dim=2) + + assert query.shape == (1, 4, 2) + assert gate.shape == (1, 4, 2) + assert key.shape == (1, 2, 2) + assert value.shape == (1, 2, 2) + assert torch.equal(query, torch.tensor([[[0, 1], [2, 3], [12, 13], [14, 15]]])) + assert torch.equal(gate, torch.tensor([[[4, 5], [6, 7], [16, 17], [18, 19]]])) + assert torch.equal(key, torch.tensor([[[8, 9], [20, 21]]])) + assert torch.equal(value, torch.tensor([[[10, 11], [22, 23]]])) + + +def test_moe_aux_loss_auto_scaler_threads_scaled_aux_gradient(): + MoEAuxLossAutoScaler = _moe_aux_scaler() + MoEAuxLossAutoScaler.set_loss_scale(torch.tensor([0.25])) + output = torch.randn(3, requires_grad=True) + aux_loss = torch.tensor(2.0, requires_grad=True) + + scaled_output = MoEAuxLossAutoScaler.apply(output * 2.0, aux_loss) + scaled_output.sum().backward() + + torch.testing.assert_close(output.grad, torch.full_like(output, 2.0)) + torch.testing.assert_close(aux_loss.grad, torch.tensor(0.25)) + MoEAuxLossAutoScaler.main_loss_backward_scale = None + + +def test_topk_router_returns_finite_scores_and_valid_expert_indices(monkeypatch): + TopKRouter, ParallelState = _router_and_parallel_state(monkeypatch) + config = _router_config() + router = TopKRouter(config, ParallelState(), compute_aux_loss=False) + hidden = torch.randn(5, 4) + + scores, indices = router(hidden) + + assert scores.shape == (5, 2) + assert indices.shape == (5, 2) + assert scores.dtype == hidden.dtype + assert torch.isfinite(scores).all() + assert indices.min().item() >= 0 + assert indices.max().item() < config.num_experts + + +def test_topk_router_scores_are_normalized_and_deterministic_in_eval(monkeypatch): + TopKRouter, ParallelState = _router_and_parallel_state(monkeypatch) + config = _router_config() + router = TopKRouter(config, ParallelState(), compute_aux_loss=False) + hidden = torch.randn(5, config.hidden_size) + + router.eval() + scores_1, indices_1 = router(hidden) + scores_2, indices_2 = router(hidden) + + torch.testing.assert_close(scores_1.sum(dim=-1), torch.ones(hidden.size(0))) + torch.testing.assert_close(scores_1, scores_2, atol=0, rtol=0) + assert torch.equal(indices_1, indices_2) + + +def test_topk_router_does_not_attach_aux_scaler_in_eval(monkeypatch): + TopKRouter, ParallelState = _router_and_parallel_state(monkeypatch) + config = _router_config() + router = TopKRouter(config, ParallelState(), compute_aux_loss=True) + hidden = torch.randn(5, config.hidden_size) + + router.eval() + scores, _indices = router(hidden) + + assert not any("MoEAuxLoss" in name for name in _walk_grad_fn_names(scores)) + + +def test_topk_router_aux_loss_contributes_gate_gradient(monkeypatch): + TopKRouter, ParallelState = _router_and_parallel_state(monkeypatch) + config = _router_config() + router = TopKRouter(config, ParallelState(), compute_aux_loss=True) + hidden = torch.randn(8, config.hidden_size) + + router.train() + scores, _indices = router(hidden) + scores.sum().backward() + grad_with_aux = router.gate.weight.grad.detach().clone() + + router.zero_grad() + saved_coeff = router.aux_loss_coeff + router.aux_loss_coeff = 0.0 + scores_no_aux, _indices = router(hidden) + scores_no_aux.sum().backward() + grad_no_aux = router.gate.weight.grad.detach().clone() + router.aux_loss_coeff = saved_coeff + + assert torch.isfinite(grad_with_aux).all() + assert torch.isfinite(grad_no_aux).all() + assert (grad_with_aux - grad_no_aux).abs().sum().item() > 0.0 diff --git a/experimental/lite/tests/unit/primitive/test_checkpoint_runtime.py b/experimental/lite/tests/unit/primitive/test_checkpoint_runtime.py new file mode 100644 index 00000000000..c2377b89f62 --- /dev/null +++ b/experimental/lite/tests/unit/primitive/test_checkpoint_runtime.py @@ -0,0 +1,291 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +from __future__ import annotations + +import copy +import random +from types import SimpleNamespace +from unittest.mock import patch + +import numpy as np +import torch +import torch.nn as nn + +from megatron.lite.primitive.ckpt import save_training_checkpoint +from megatron.lite.runtime.backends.mlite.runtime import MegatronLiteRuntime +from megatron.lite.runtime.contracts.config import ParallelConfig +from megatron.lite.runtime.contracts.handle import ModelHandle + + +class TinyMLP(nn.Module): + def __init__(self): + super().__init__() + self.layers = nn.Sequential(nn.Linear(4, 8), nn.GELU(), nn.Linear(8, 2)) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.layers(x) + + +def _step(model: nn.Module, optimizer: torch.optim.Optimizer, x: torch.Tensor, y: torch.Tensor): + optimizer.zero_grad(set_to_none=True) + loss = torch.nn.functional.mse_loss(model(x), y) + loss.backward() + optimizer.step() + return loss.detach() + + +def _clone_model_and_optimizer(model: nn.Module): + clone = copy.deepcopy(model) + optimizer = torch.optim.AdamW(clone.parameters(), lr=1.0e-3, weight_decay=0.0) + return clone, optimizer + + +def _assert_model_close(lhs: nn.Module, rhs: nn.Module): + for (lhs_name, lhs_param), (rhs_name, rhs_param) in zip( + lhs.named_parameters(), rhs.named_parameters(), strict=True + ): + assert lhs_name == rhs_name + torch.testing.assert_close(lhs_param, rhs_param, atol=0.0, rtol=0.0) + + +def test_runtime_local_checkpoint_load_matches_uninterrupted_training(tmp_path): + torch.manual_seed(2029) + base = TinyMLP() + ckpt_model, ckpt_optimizer = _clone_model_and_optimizer(base) + direct_model, direct_optimizer = _clone_model_and_optimizer(base) + loaded_model, loaded_optimizer = _clone_model_and_optimizer(base) + x0, y0 = torch.randn(3, 4), torch.randn(3, 2) + x1, y1 = torch.randn(3, 4), torch.randn(3, 2) + + _step(ckpt_model, ckpt_optimizer, x0, y0) + _step(direct_model, direct_optimizer, x0, y0) + + runtime = MegatronLiteRuntime.__new__(MegatronLiteRuntime) + runtime.save_checkpoint( + ModelHandle(model=ckpt_model, optimizer=ckpt_optimizer), + str(tmp_path), + step=1, + use_dcp=False, + ) + + assert ( + runtime.load_checkpoint( + ModelHandle(model=loaded_model, optimizer=loaded_optimizer), + str(tmp_path), + use_dcp=False, + ) + == 1 + ) + + _step(direct_model, direct_optimizer, x1, y1) + _step(loaded_model, loaded_optimizer, x1, y1) + + _assert_model_close(direct_model, loaded_model) + + +class DistOptLike: + """Small optimizer wrapper with the same checkpoint contract as dist_opt.""" + + def __init__(self, optimizer: torch.optim.Optimizer): + self.optimizer = optimizer + self.load_calls = 0 + self.parameter_save_calls = 0 + self.parameter_load_calls = 0 + self.update_legacy_format = None + + def zero_grad(self): + self.optimizer.zero_grad(set_to_none=True) + + def step(self): + self.optimizer.step() + return True, 0.0, 0 + + def state_dict(self): + state = self.optimizer.state_dict() + state["dist_opt_like_marker"] = {"load_calls": self.load_calls} + return state + + def load_state_dict(self, state): + marker = state.pop("dist_opt_like_marker") + self.load_calls = int(marker["load_calls"]) + 1 + self.optimizer.load_state_dict(state) + + def save_parameter_state(self, filename: str): + self.parameter_save_calls += 1 + torch.save({"parameter_save_calls": self.parameter_save_calls}, filename) + + def load_parameter_state(self, filename: str, *, update_legacy_format: bool = False): + state = torch.load(filename, weights_only=False) + self.parameter_load_calls = int(state["parameter_save_calls"]) + self.update_legacy_format = update_legacy_format + + +def test_runtime_local_checkpoint_uses_optimizer_parameter_state_contract(tmp_path): + torch.manual_seed(2030) + model = TinyMLP() + optimizer = DistOptLike(torch.optim.AdamW(model.parameters(), lr=1.0e-3)) + x, y = torch.randn(3, 4), torch.randn(3, 2) + optimizer.zero_grad() + torch.nn.functional.mse_loss(model(x), y).backward() + optimizer.step() + + runtime = MegatronLiteRuntime.__new__(MegatronLiteRuntime) + runtime.save_checkpoint( + ModelHandle(model=model, optimizer=optimizer), str(tmp_path), step=7, use_dcp=False + ) + + loaded_model = TinyMLP() + loaded_optimizer = DistOptLike(torch.optim.AdamW(loaded_model.parameters(), lr=1.0e-3)) + + assert ( + runtime.load_checkpoint( + ModelHandle(model=loaded_model, optimizer=loaded_optimizer), + str(tmp_path), + update_legacy_format=True, + use_dcp=False, + ) + == 7 + ) + assert loaded_optimizer.load_calls == 1 + assert loaded_optimizer.parameter_load_calls == 1 + assert loaded_optimizer.update_legacy_format is True + assert (tmp_path / "training_state.optimizer_parameter_state.pt").exists() + _assert_model_close(model, loaded_model) + + +def test_runtime_local_checkpoint_restores_rng_state(tmp_path): + model = TinyMLP() + runtime = MegatronLiteRuntime.__new__(MegatronLiteRuntime) + + random.seed(2031) + np.random.seed(2031) + torch.manual_seed(2031) + + runtime.save_checkpoint( + ModelHandle(model=model, optimizer=None), str(tmp_path), step=9, use_dcp=False + ) + + expected_python = random.random() + expected_numpy = np.random.random(4) + expected_torch = torch.rand(4) + + random.seed(9999) + np.random.seed(9999) + torch.manual_seed(9999) + + assert ( + runtime.load_checkpoint( + ModelHandle(model=model, optimizer=None), str(tmp_path), use_dcp=False + ) + == 9 + ) + assert random.random() == expected_python + np.testing.assert_allclose(np.random.random(4), expected_numpy, atol=0.0, rtol=0.0) + torch.testing.assert_close(torch.rand(4), expected_torch, atol=0.0, rtol=0.0) + + +def test_runtime_local_checkpoint_uses_rank_specific_files_when_distributed(tmp_path): + model = TinyMLP() + runtime = MegatronLiteRuntime.__new__(MegatronLiteRuntime) + + with ( + patch("megatron.lite.primitive.ckpt.dcp.dist.is_available", return_value=True), + patch("megatron.lite.primitive.ckpt.dcp.dist.is_initialized", return_value=True), + patch("megatron.lite.primitive.ckpt.dcp.dist.get_rank", return_value=3), + ): + runtime.save_checkpoint( + ModelHandle(model=model, optimizer=None), str(tmp_path), step=11, use_dcp=False + ) + assert (tmp_path / "training_state_rank_00003.pt").exists() + assert not (tmp_path / "training_state.pt").exists() + assert ( + runtime.load_checkpoint( + ModelHandle(model=model, optimizer=None), str(tmp_path), use_dcp=False + ) + == 11 + ) + + +def test_primitive_local_checkpoint_keeps_optimizer_checkpoints_local(tmp_path): + model = TinyMLP() + optimizer = torch.optim.AdamW(model.parameters(), lr=1.0e-3) + + with patch("megatron.lite.primitive.ckpt.dcp.dcp.save") as dcp_save_mock: + save_training_checkpoint(model, optimizer, 12, str(tmp_path), use_dcp=False) + + dcp_save_mock.assert_not_called() + assert (tmp_path / "training_state.pt").exists() + + +def test_primitive_explicit_dcp_saves_optimizer_rank_sidecar(tmp_path): + model = TinyMLP() + optimizer = torch.optim.AdamW(model.parameters(), lr=1.0e-3) + parallel = ParallelConfig(tp=1, ep=1, pp=1, cp=1) + + with ( + patch("megatron.lite.primitive.ckpt.dcp._build_meshes", return_value=(None, None)), + patch( + "megatron.lite.primitive.ckpt.dcp.DTensor.from_local", + side_effect=lambda tensor, *args, **kwargs: tensor, + ), + patch("megatron.lite.primitive.ckpt.dcp.dcp.save") as dcp_save_mock, + ): + save_training_checkpoint( + model, optimizer, 12, str(tmp_path), parallel, object(), use_dcp=True + ) + + dcp_save_mock.assert_called_once() + assert (tmp_path / "step_12" / "optimizer_rank_0.pt").exists() + + +def test_runtime_dcp_checkpoint_threads_parallel_config_and_protocol_hooks(tmp_path): + model = TinyMLP() + parallel = ParallelConfig(tp=2, ep=1, pp=1, cp=1) + ps = object() + + def placement_fn(name: str): + return ["placement", name] + + def expert_classifier(name: str): + return name.endswith("expert") + + proto = SimpleNamespace(PLACEMENT_FN=placement_fn, EXPERT_CLASSIFIER=expert_classifier) + handle = ModelHandle( + model=[model], + optimizer=None, + parallel_state=ps, + config=SimpleNamespace(parallel=parallel), + _extras={"model_chunks": [model], "protocol": proto}, + ) + runtime = MegatronLiteRuntime.__new__(MegatronLiteRuntime) + + with patch("megatron.lite.primitive.ckpt.save_training_checkpoint") as save_mock: + runtime.save_checkpoint(handle, str(tmp_path), global_step=13, use_dcp=True) + + save_args = save_mock.call_args.args + save_kwargs = save_mock.call_args.kwargs + assert isinstance(save_args[0], nn.ModuleList) + assert save_args[0][0] is model + assert save_args[2] == 13 + assert save_args[3] == str(tmp_path) + assert save_args[4] is parallel + assert save_args[5] is ps + assert save_kwargs["get_placements"] is placement_fn + assert save_kwargs["is_expert"] is expert_classifier + assert save_kwargs["use_dcp"] is True + assert save_kwargs["save_rng"] is True + + with patch( + "megatron.lite.primitive.ckpt.load_training_checkpoint", return_value=13 + ) as load_mock: + assert runtime.load_checkpoint(handle, str(tmp_path), use_dcp=True) == 13 + + load_args = load_mock.call_args.args + load_kwargs = load_mock.call_args.kwargs + assert isinstance(load_args[0], nn.ModuleList) + assert load_args[0][0] is model + assert load_args[3] is parallel + assert load_args[4] is ps + assert load_kwargs["get_placements"] is placement_fn + assert load_kwargs["is_expert"] is expert_classifier + assert load_kwargs["use_dcp"] is True + assert load_kwargs["load_rng"] is True diff --git a/experimental/lite/tests/unit/primitive/test_checkpoint_unit.py b/experimental/lite/tests/unit/primitive/test_checkpoint_unit.py new file mode 100644 index 00000000000..9f0c816bc56 --- /dev/null +++ b/experimental/lite/tests/unit/primitive/test_checkpoint_unit.py @@ -0,0 +1,138 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +from __future__ import annotations + +import copy + +import pytest +import torch +import torch.nn as nn + +from megatron.lite.runtime.backends.mlite.runtime import MegatronLiteRuntime +from megatron.lite.runtime.contracts.handle import ModelHandle + +pytestmark = pytest.mark.mlite + + +class TinyMLP(nn.Module): + def __init__(self): + super().__init__() + self.layers = nn.Sequential(nn.Linear(4, 8), nn.GELU(), nn.Linear(8, 2)) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.layers(x) + + +def _step(model: nn.Module, optimizer: torch.optim.Optimizer, x: torch.Tensor, y: torch.Tensor): + optimizer.zero_grad(set_to_none=True) + loss = torch.nn.functional.mse_loss(model(x), y) + loss.backward() + optimizer.step() + return loss.detach() + + +def _clone_model_and_optimizer(model: nn.Module): + clone = copy.deepcopy(model) + optimizer = torch.optim.AdamW(clone.parameters(), lr=1.0e-3, weight_decay=0.0) + return clone, optimizer + + +def _assert_model_close(lhs: nn.Module, rhs: nn.Module): + for (lhs_name, lhs_param), (rhs_name, rhs_param) in zip( + lhs.named_parameters(), rhs.named_parameters(), strict=True + ): + assert lhs_name == rhs_name + torch.testing.assert_close(lhs_param, rhs_param, atol=0.0, rtol=0.0) + + +def test_runtime_checkpoint_load_matches_uninterrupted_training(tmp_path): + torch.manual_seed(2029) + base = TinyMLP() + ckpt_model, ckpt_optimizer = _clone_model_and_optimizer(base) + direct_model, direct_optimizer = _clone_model_and_optimizer(base) + loaded_model, loaded_optimizer = _clone_model_and_optimizer(base) + x0, y0 = torch.randn(3, 4), torch.randn(3, 2) + x1, y1 = torch.randn(3, 4), torch.randn(3, 2) + + _step(ckpt_model, ckpt_optimizer, x0, y0) + _step(direct_model, direct_optimizer, x0, y0) + + runtime = MegatronLiteRuntime.__new__(MegatronLiteRuntime) + ckpt_handle = ModelHandle( + model=ckpt_model, optimizer=ckpt_optimizer, _extras={"model_chunks": [ckpt_model]} + ) + runtime.save_checkpoint(ckpt_handle, str(tmp_path), step=1, use_dcp=False) + + loaded_handle = ModelHandle( + model=loaded_model, optimizer=loaded_optimizer, _extras={"model_chunks": [loaded_model]} + ) + assert runtime.load_checkpoint(loaded_handle, str(tmp_path), use_dcp=False) == 1 + + _step(direct_model, direct_optimizer, x1, y1) + _step(loaded_model, loaded_optimizer, x1, y1) + + _assert_model_close(direct_model, loaded_model) + + +class DistOptLike: + """Small optimizer wrapper with the same checkpoint contract as dist_opt.""" + + def __init__(self, optimizer: torch.optim.Optimizer): + self.optimizer = optimizer + self.load_calls = 0 + self.parameter_save_calls = 0 + self.parameter_load_calls = 0 + + def zero_grad(self): + self.optimizer.zero_grad(set_to_none=True) + + def step(self): + self.optimizer.step() + return True, 0.0, 0 + + def state_dict(self): + state = self.optimizer.state_dict() + state["dist_opt_like_marker"] = {"load_calls": self.load_calls} + return state + + def load_state_dict(self, state): + marker = state.pop("dist_opt_like_marker") + self.load_calls = int(marker["load_calls"]) + 1 + self.optimizer.load_state_dict(state) + + def save_parameter_state(self, filename: str): + self.parameter_save_calls += 1 + torch.save({"parameter_save_calls": self.parameter_save_calls}, filename) + + def load_parameter_state(self, filename: str, *, update_legacy_format: bool = False): + state = torch.load(filename) + self.parameter_load_calls = int(state["parameter_save_calls"]) + + +def test_runtime_checkpoint_uses_optimizer_state_dict_contract(tmp_path): + torch.manual_seed(2030) + model = TinyMLP() + optimizer = DistOptLike(torch.optim.AdamW(model.parameters(), lr=1.0e-3)) + x, y = torch.randn(3, 4), torch.randn(3, 2) + optimizer.zero_grad() + torch.nn.functional.mse_loss(model(x), y).backward() + optimizer.step() + + runtime = MegatronLiteRuntime.__new__(MegatronLiteRuntime) + runtime.save_checkpoint( + ModelHandle(model=model, optimizer=optimizer, _extras={"model_chunks": [model]}), + str(tmp_path), + step=7, + use_dcp=False, + ) + + loaded_model = TinyMLP() + loaded_optimizer = DistOptLike(torch.optim.AdamW(loaded_model.parameters(), lr=1.0e-3)) + loaded_handle = ModelHandle( + model=loaded_model, optimizer=loaded_optimizer, _extras={"model_chunks": [loaded_model]} + ) + + assert runtime.load_checkpoint(loaded_handle, str(tmp_path), use_dcp=False) == 7 + assert loaded_optimizer.load_calls == 1 + assert loaded_optimizer.parameter_load_calls == 1 + assert (tmp_path / "training_state.optimizer_parameter_state.pt").exists() + _assert_model_close(model, loaded_model) diff --git a/experimental/lite/tests/unit/primitive/test_dist_opt_validation.py b/experimental/lite/tests/unit/primitive/test_dist_opt_validation.py new file mode 100644 index 00000000000..ffe9b112020 --- /dev/null +++ b/experimental/lite/tests/unit/primitive/test_dist_opt_validation.py @@ -0,0 +1,29 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +from __future__ import annotations + +import pytest + +from megatron.lite.primitive.optimizers.megatron_wrap import ( + validate_dist_opt_config, + validate_dist_opt_session, +) +from megatron.lite.runtime.backends.mlite.config import MegatronLiteConfig +from megatron.lite.runtime.contracts.config import ParallelConfig + + +def _engine_cfg(*, model_name: str, pp: int = 1, vpp: int = 1) -> MegatronLiteConfig: + return MegatronLiteConfig(model_name=model_name, parallel=ParallelConfig(pp=pp, vpp=vpp)) + + +def test_dist_opt_validation_accepts_model_agnostic_config(): + validate_dist_opt_config(_engine_cfg(model_name="synthetic_custom_model", pp=1, vpp=1)) + + +def test_dist_opt_validation_keeps_vpp_parallel_constraint(): + with pytest.raises(ValueError, match="dist_opt requires pp>1 when vpp>1"): + validate_dist_opt_config(_engine_cfg(model_name="synthetic_custom_model", pp=1, vpp=2)) + + +def test_validate_dist_opt_session_alias_matches_config_validator(): + assert validate_dist_opt_session is validate_dist_opt_config + validate_dist_opt_session(_engine_cfg(model_name="another_synthetic_model", pp=2, vpp=2)) diff --git a/experimental/lite/tests/unit/primitive/test_fsdp2_offload_gpu.py b/experimental/lite/tests/unit/primitive/test_fsdp2_offload_gpu.py new file mode 100644 index 00000000000..abc42bf6469 --- /dev/null +++ b/experimental/lite/tests/unit/primitive/test_fsdp2_offload_gpu.py @@ -0,0 +1,161 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +from __future__ import annotations + +import os +from types import SimpleNamespace + +import pytest +import torch +import torch.distributed as dist +import torch.nn as nn + +from megatron.lite.primitive.optimizers.fsdp2 import ( + FSDP2Config, + build_fsdp2_adamw, + build_fsdp2_device_mesh, + fsdp2_available, + wrap_fsdp2, +) +from megatron.lite.primitive.optimizers.fsdp2.adamw import iter_torch_optimizers, to_local_tensor +from megatron.lite.primitive.parallel.state import ParallelState +from megatron.lite.runtime.backends.mlite.runtime import MegatronLiteRuntime +from megatron.lite.runtime.contracts.handle import ModelHandle + + +class TinyUnit(nn.Module): + def __init__(self): + super().__init__() + self.linear = nn.Linear(8, 8) + self.act = nn.GELU() + + def forward(self, x): + return self.act(self.linear(x)) + + +class TinyModel(nn.Module): + def __init__(self): + super().__init__() + self.unit0 = TinyUnit() + self.unit1 = TinyUnit() + self.out = nn.Linear(8, 4) + + def forward(self, x): + return self.out(self.unit1(self.unit0(x))) + + +@pytest.fixture(scope="module", autouse=True) +def _single_node_cuda_dist(): + if not torch.cuda.is_available(): + pytest.skip("CUDA is required for FSDP2 offload tests.") + if not fsdp2_available(): + pytest.skip("Installed PyTorch does not expose FSDP2 fully_shard.") + + os.environ.setdefault("RANK", "0") + os.environ.setdefault("WORLD_SIZE", "1") + os.environ.setdefault("LOCAL_RANK", "0") + os.environ.setdefault("MASTER_ADDR", "127.0.0.1") + os.environ.setdefault("MASTER_PORT", "29501") + + torch.cuda.set_device(int(os.environ["LOCAL_RANK"])) + created_pg = False + if not dist.is_initialized(): + dist.init_process_group(backend="nccl", init_method="env://") + created_pg = True + yield + if created_pg and dist.is_initialized(): + dist.destroy_process_group() + + +def _parallel_state() -> ParallelState: + rank = dist.get_rank() + world_size = dist.get_world_size() + return ParallelState( + dp_group=dist.group.WORLD, + dp_cp_group=dist.group.WORLD, + dp_size=world_size, + dp_cp_size=world_size, + dp_rank=rank, + dp_cp_rank=rank, + ) + + +def _build_fsdp2_model(dtype: torch.dtype = torch.bfloat16) -> tuple[nn.Module, ParallelState]: + torch.manual_seed(1234) + model = TinyModel().cuda().to(dtype=dtype) + ps = _parallel_state() + config = FSDP2Config(unit_modules=(TinyUnit,), reshard_after_forward=True) + mesh = build_fsdp2_device_mesh(ps, config) + return wrap_fsdp2(model, ps, config, mesh=mesh), ps + + +def _build_optimizer(model: nn.Module, ps: ParallelState, *, offload_fraction: float): + return build_fsdp2_adamw( + [model], + SimpleNamespace( + optimizer="adam", + lr=1.0e-3, + weight_decay=0.0, + adam_beta1=0.9, + adam_beta2=0.95, + adam_eps=1.0e-8, + clip_grad=1.0, + offload_fraction=offload_fraction, + ), + ps, + use_fp32_master=True, + ) + + +def _local_param_devices(model: nn.Module) -> set[str]: + return {to_local_tensor(param.detach()).device.type for param in model.parameters()} + + +def _optimizer_state_devices(optimizer) -> set[str]: + devices: set[str] = set() + for child in iter_torch_optimizers(optimizer.optimizer): + for param_state in getattr(child, "state", {}).values(): + if not isinstance(param_state, dict): + continue + for value in param_state.values(): + if isinstance(value, torch.Tensor): + devices.add(to_local_tensor(value).device.type) + return devices + + +def test_fsdp2_runtime_model_and_optimizer_offload_roundtrip_single_gpu(): + model, ps = _build_fsdp2_model() + optimizer = _build_optimizer(model, ps, offload_fraction=0.0) + handle = ModelHandle( + model=model, optimizer=optimizer, parallel_state=ps, _extras={"model_chunks": [model]} + ) + runtime = MegatronLiteRuntime.__new__(MegatronLiteRuntime) + + assert _local_param_devices(model) == {"cuda"} + assert _optimizer_state_devices(optimizer) == {"cuda"} + + runtime.to(handle, "cpu", model=True, optimizer=True, grad=True) + assert _local_param_devices(model) == {"cpu"} + assert _optimizer_state_devices(optimizer) == {"cpu"} + + runtime.to(handle, "cuda", model=True, optimizer=True, grad=True) + assert _local_param_devices(model) == {"cuda"} + assert _optimizer_state_devices(optimizer) == {"cuda"} + + +def test_fsdp2_offload_fraction_keeps_optimizer_update_state_on_cpu_single_gpu(): + model, ps = _build_fsdp2_model() + optimizer = _build_optimizer(model, ps, offload_fraction=1.0) + + assert _optimizer_state_devices(optimizer) == {"cpu"} + + x = torch.randn(4, 8, device="cuda", dtype=torch.bfloat16) + target = torch.randn(4, 4, device="cuda", dtype=torch.bfloat16) + optimizer.zero_grad() + loss = torch.nn.functional.mse_loss(model(x).float(), target.float()) + loss.backward() + success, grad_norm, _ = optimizer.step() + + assert success + assert torch.isfinite(torch.tensor(grad_norm)) + assert _local_param_devices(model) == {"cuda"} + assert _optimizer_state_devices(optimizer) == {"cpu"} diff --git a/experimental/lite/tests/unit/primitive/test_fsdp2_unit.py b/experimental/lite/tests/unit/primitive/test_fsdp2_unit.py new file mode 100644 index 00000000000..00142e3f25b --- /dev/null +++ b/experimental/lite/tests/unit/primitive/test_fsdp2_unit.py @@ -0,0 +1,470 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +from __future__ import annotations + +import copy +import importlib +from types import SimpleNamespace + +import pytest +import torch +import torch.distributed as dist +import torch.nn as nn + +from megatron.lite.primitive.optimizers.fsdp2 import ( + FSDP2Config, + FSDP2Optimizer, + all_reduce_scalar_, + clip_grads_with_sharded_norm_, + fsdp2_available, +) +from megatron.lite.primitive.optimizers.fsdp2.adamw import build_adamw_optimizer +from megatron.lite.primitive.optimizers.fsdp2.wrap import build_fsdp2_shard_placement_fn +from megatron.lite.primitive.parallel.state import ParallelState + +pytestmark = pytest.mark.mlite + +fsdp2_wrap = importlib.import_module("megatron.lite.primitive.optimizers.fsdp2.wrap") +fsdp2_optimizer = importlib.import_module("megatron.lite.primitive.optimizers.fsdp2.optimizer") +fsdp2_grad_clip = importlib.import_module("megatron.lite.primitive.optimizers.fsdp2.grad_clip") + + +class ToyBlock(nn.Module): + def __init__(self): + super().__init__() + self.proj = nn.Linear(4, 4) + + def forward(self, x): + return self.proj(x) + + +class ToyModel(nn.Module): + def __init__(self): + super().__init__() + self.block = ToyBlock() + self.out = nn.Linear(4, 2) + + def forward(self, x): + return self.out(self.block(x)) + + +class TwoBlockModel(nn.Module): + def __init__(self): + super().__init__() + self.block0 = ToyBlock() + self.block1 = ToyBlock() + self.out = nn.Linear(4, 2) + + def forward(self, x): + return self.out(self.block1(self.block0(x))) + + +class NestedToyBlock(ToyBlock): + def __init__(self): + super().__init__() + self.inner = ToyBlock() + + +def test_fsdp2_config_validates_empty_wrap_surface(): + with pytest.raises(ValueError, match="wrap_root=True"): + FSDP2Config(wrap_root=False) + + +@pytest.mark.parametrize("field", ["mesh_dim_name", "device_type"]) +def test_fsdp2_config_rejects_empty_names(field: str): + with pytest.raises(ValueError, match=field): + FSDP2Config(**{field: ""}) + + +def test_fsdp2_config_normalizes_unit_and_leaf_modules(): + cfg = FSDP2Config(unit_modules=[nn.Linear], leaf_module_names=["embed"]) + + assert cfg.unit_modules == (nn.Linear,) + assert cfg.leaf_module_names == ("embed",) + assert isinstance(fsdp2_available(), bool) + + +def test_fsdp2_optimizer_offloads_dtensor_state_without_extra_knob(monkeypatch): + model = ToyModel() + torch_optimizer = torch.optim.AdamW(model.parameters(), lr=1.0e-3) + optimizer = FSDP2Optimizer(torch_optimizer, model.parameters()) + calls: list[bool] = [] + + def fake_move_optimizer_state_to_cpu(_optimizer, _offloaded_state, *, include_dtensor_state): + calls.append(include_dtensor_state) + + monkeypatch.setattr( + fsdp2_optimizer, "move_optimizer_state_to_cpu", fake_move_optimizer_state_to_cpu + ) + + optimizer.offload_state_to_cpu() + + assert calls == [True] + assert not hasattr(optimizer, "optimizer_offload_dtensor_state") + + +def test_fsdp2_shard_placement_prefers_first_divisible_dimension(): + placement_for_two = build_fsdp2_shard_placement_fn(2) + placement_for_three = build_fsdp2_shard_placement_fn(3) + + assert placement_for_two(nn.Parameter(torch.empty(3, 4))).dim == 1 + assert placement_for_three(nn.Parameter(torch.empty(3, 4))).dim == 0 + + +def test_fsdp2_shard_placement_rejects_invalid_group_size(): + with pytest.raises(ValueError, match="positive"): + build_fsdp2_shard_placement_fn(0) + + +def test_fsdp2_rejects_invalid_unit_path(): + with pytest.raises(ValueError, match="Invalid FSDP2 unit module path"): + fsdp2_wrap._resolve_unit_module_types(("Linear",)) + + +def test_fsdp2_rejects_non_module_unit_path(): + with pytest.raises(TypeError, match="does not resolve"): + fsdp2_wrap._resolve_unit_module_types(("math.sqrt",)) + + +def test_wrap_fsdp2_requires_distributed_when_mesh_is_not_provided(monkeypatch): + monkeypatch.setattr(fsdp2_wrap, "_load_fully_shard", lambda: lambda module, **kwargs: module) + + with pytest.raises(RuntimeError, match="torch.distributed"): + fsdp2_wrap.wrap_fsdp2(ToyModel(), ParallelState(), FSDP2Config()) + + +def test_wrap_fsdp2_wraps_units_then_root_and_preserves_param_attrs(monkeypatch): + model = ToyModel() + model.block.proj.weight.tensor_model_parallel = True + calls: list[nn.Module] = [] + + def fake_fully_shard(module, **kwargs): + calls.append(module) + for param in module.parameters(): + vars(param).clear() + module._fake_fsdp2_kwargs = kwargs + return module + + monkeypatch.setattr(fsdp2_wrap, "_load_fully_shard", lambda: fake_fully_shard) + + result = fsdp2_wrap.wrap_fsdp2( + model, + ParallelState(), + FSDP2Config(unit_modules=(ToyBlock,), reshard_after_forward=False), + mesh=SimpleNamespace(name="mesh"), + ) + + assert result is model + assert calls == [model.block, model] + assert model.block.proj.weight.tensor_model_parallel is True + assert model._fake_fsdp2_kwargs["reshard_after_forward"] is False + assert model._fake_fsdp2_kwargs["mesh"].name == "mesh" + + +def test_wrap_fsdp2_accepts_unit_module_import_paths(monkeypatch): + model = ToyModel() + calls: list[nn.Module] = [] + + def fake_fully_shard(module, **kwargs): + calls.append(module) + return module + + monkeypatch.setattr(fsdp2_wrap, "_load_fully_shard", lambda: fake_fully_shard) + + fsdp2_wrap.wrap_fsdp2( + model, + ParallelState(), + FSDP2Config(unit_modules=("torch.nn.modules.linear.Linear",), wrap_root=False), + mesh=SimpleNamespace(name="mesh"), + ) + + assert calls == [model.block.proj, model.out] + + +def test_wrap_fsdp2_uses_container_order_without_nested_unit_duplicates(monkeypatch): + model = nn.Module() + model.layers = nn.ModuleDict({"10": NestedToyBlock(), "2": ToyBlock(), "11": ToyBlock()}) + calls: list[nn.Module] = [] + + def fake_fully_shard(module, **kwargs): + calls.append(module) + module._fake_fsdp2_kwargs = kwargs + return module + + monkeypatch.setattr(fsdp2_wrap, "_load_fully_shard", lambda: fake_fully_shard) + + fsdp2_wrap.wrap_fsdp2( + model, + ParallelState(), + FSDP2Config(unit_modules=(ToyBlock,), reshard_after_forward=True), + mesh=SimpleNamespace(name="mesh"), + ) + + assert calls == [model.layers["10"], model.layers["2"], model.layers["11"], model] + assert model.layers["10"]._fake_fsdp2_kwargs["reshard_after_forward"] is True + assert not hasattr(model.layers["10"].inner, "_fake_fsdp2_kwargs") + assert model.layers["11"]._fake_fsdp2_kwargs["reshard_after_forward"] is False + + +def test_wrap_fsdp2_configures_default_forward_prefetch(monkeypatch): + model = TwoBlockModel() + calls: list[nn.Module] = [] + + def fake_fully_shard(module, **kwargs): + calls.append(module) + module._forward_prefetch = None + module._backward_prefetch = None + module._fake_fsdp2_kwargs = kwargs + + def set_forward_prefetch(modules, *, _module=module): + _module._forward_prefetch = list(modules) + + def set_backward_prefetch(modules, *, _module=module): + _module._backward_prefetch = list(modules) + + module.set_modules_to_forward_prefetch = set_forward_prefetch + module.set_modules_to_backward_prefetch = set_backward_prefetch + return module + + monkeypatch.setattr(fsdp2_wrap, "_load_fully_shard", lambda: fake_fully_shard) + + fsdp2_wrap.wrap_fsdp2( + model, + ParallelState(), + FSDP2Config(unit_modules=(ToyBlock,), reshard_after_forward=True), + mesh=SimpleNamespace(name="mesh"), + ) + + assert calls == [model.block0, model.block1, model] + assert model.block0._fake_fsdp2_kwargs["reshard_after_forward"] is True + assert model.block1._fake_fsdp2_kwargs["reshard_after_forward"] is False + assert model._fake_fsdp2_kwargs["reshard_after_forward"] is False + assert model._forward_prefetch == [model.block0] + assert model.block0._forward_prefetch == [model.block1] + assert model.block1._backward_prefetch is None + + +def test_wrap_fsdp2_prefetch_depths(monkeypatch): + model = nn.Sequential(ToyBlock(), ToyBlock(), ToyBlock()) + + def fake_fully_shard(module, **kwargs): + module._forward_prefetch = None + module._backward_prefetch = None + + def set_forward_prefetch(modules, *, _module=module): + _module._forward_prefetch = list(modules) + + def set_backward_prefetch(modules, *, _module=module): + _module._backward_prefetch = list(modules) + + module.set_modules_to_forward_prefetch = set_forward_prefetch + module.set_modules_to_backward_prefetch = set_backward_prefetch + return module + + monkeypatch.setattr(fsdp2_wrap, "_load_fully_shard", lambda: fake_fully_shard) + + fsdp2_wrap.wrap_fsdp2( + model, + ParallelState(), + FSDP2Config( + unit_modules=(ToyBlock,), + wrap_root=False, + forward_prefetch_depth=2, + backward_prefetch_depth=2, + ), + mesh=SimpleNamespace(name="mesh"), + ) + + assert model[0]._forward_prefetch == [model[1], model[2]] + assert model[1]._forward_prefetch == [model[2]] + assert model[2]._backward_prefetch == [model[1], model[0]] + + +def test_clip_grads_with_sharded_norm_scales_cpu_grads_once(): + p0 = nn.Parameter(torch.ones(2)) + p1 = nn.Parameter(torch.ones(2)) + p0.grad = torch.tensor([3.0, 4.0]) + p1.grad = torch.tensor([0.0, 12.0]) + + clip_grads_with_sharded_norm_([p0, p1], max_norm=6.5, total_norm=torch.tensor(13.0)) + + scale = 6.5 / (13.0 + 1.0e-6) + torch.testing.assert_close(p0.grad, torch.tensor([3.0, 4.0]) * scale) + torch.testing.assert_close(p1.grad, torch.tensor([0.0, 12.0]) * scale) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is required for NCCL scalar test.") +def test_all_reduce_scalar_moves_cpu_value_to_nccl_device(monkeypatch): + group = object() + reduced_devices: list[str] = [] + + def fake_all_reduce(value, *, op, group): + assert group is fake_all_reduce.group + assert op == dist.ReduceOp.SUM + reduced_devices.append(value.device.type) + value.add_(5.0) + + fake_all_reduce.group = group + monkeypatch.setattr(fsdp2_grad_clip.dist, "get_backend", lambda _group: "nccl") + monkeypatch.setattr(fsdp2_grad_clip.dist, "all_reduce", fake_all_reduce) + + value = torch.tensor(7.0) + all_reduce_scalar_(value, op=dist.ReduceOp.SUM, group=group) + + assert reduced_devices == ["cuda"] + assert value.device.type == "cpu" + torch.testing.assert_close(value, torch.tensor(12.0)) + + +def test_fsdp2_optimizer_uses_scalar_all_reduce_for_all_norm_groups(monkeypatch): + groups = SimpleNamespace( + dp_cp=object(), + tp=object(), + replicated=object(), + expert=object(), + pp=object(), + ) + reduced_groups: list[object] = [] + + def fake_all_reduce_scalar(value, *, op, group): + assert value.ndim == 0 + assert op == dist.ReduceOp.SUM + reduced_groups.append(group) + + monkeypatch.setattr(fsdp2_optimizer, "all_reduce_scalar_", fake_all_reduce_scalar) + monkeypatch.setattr(fsdp2_optimizer.dist, "is_initialized", lambda: True) + monkeypatch.setattr(fsdp2_optimizer.dist, "get_world_size", lambda _group: 2) + + sharded = nn.Parameter(torch.tensor([1.0])) + replicated = nn.Parameter(torch.tensor([1.0])) + expert = nn.Parameter(torch.tensor([1.0])) + tp_replicated = nn.Parameter(torch.tensor([1.0])) + sharded.grad = torch.tensor([2.0]) + replicated.grad = torch.tensor([3.0]) + expert.grad = torch.tensor([4.0]) + tp_replicated.grad = torch.tensor([5.0]) + + optimizer = FSDP2Optimizer( + torch.optim.SGD([sharded, replicated, expert, tp_replicated], lr=0.0), + [sharded, replicated, expert, tp_replicated], + ParallelState( + dp_cp_group=groups.dp_cp, + tp_group=groups.tp, + pp_group=groups.pp, + ), + clip_grad=100.0, + replicated_grad_params=[replicated], + replicated_grad_norm_group=groups.replicated, + expert_sharded_grad_params=[expert], + expert_sharded_grad_norm_group=groups.expert, + tp_replicated_grad_params=[tp_replicated], + ) + + assert optimizer.clip_grad_norm() == pytest.approx( + (2.0**2 + 3.0**2 + 4.0**2 + 5.0**2) ** 0.5 + ) + assert reduced_groups == [ + groups.dp_cp, + groups.tp, + groups.replicated, + groups.expert, + groups.pp, + ] + + +def test_fp32_adamw_state_dict_roundtrip_cpu(): + param = nn.Parameter(torch.tensor([1.0, -2.0], dtype=torch.bfloat16)) + optimizer = build_adamw_optimizer( + [{"params": [param], "weight_decay": 0.0}], + all_params=[param], + lr=0.1, + weight_decay=0.0, + betas=(0.9, 0.99), + eps=1.0e-8, + foreach=False, + use_fp32_master=True, + cpu_update=False, + model_param_dtypes={id(param): torch.bfloat16}, + opt=SimpleNamespace(), + ) + param.grad = torch.tensor([0.5, -0.25], dtype=torch.bfloat16) + optimizer.step() + state = optimizer.state_dict() + expected = {key: state[key][0].clone() for key in ("master_params", "exp_avgs", "exp_avg_sqs")} + for key, value in expected.items(): + state[key][0] = SimpleNamespace(_local_tensor=value) + + loaded_param = nn.Parameter(torch.tensor([9.0, 9.0], dtype=torch.bfloat16)) + loaded_optimizer = build_adamw_optimizer( + [{"params": [loaded_param], "weight_decay": 0.0}], + all_params=[loaded_param], + lr=0.1, + weight_decay=0.0, + betas=(0.9, 0.99), + eps=1.0e-8, + foreach=False, + use_fp32_master=True, + cpu_update=False, + model_param_dtypes={id(loaded_param): torch.bfloat16}, + opt=SimpleNamespace(), + ) + loaded_optimizer.load_state_dict(state) + loaded_state = loaded_optimizer.state_dict() + + assert loaded_state["step_count"] == state["step_count"] + torch.testing.assert_close(loaded_param, expected["master_params"].to(torch.bfloat16)) + for key, value in expected.items(): + torch.testing.assert_close(loaded_state[key][0], value) + assert loaded_state["steps"] == state["steps"] + + +@pytest.mark.parametrize("cpu_update", [False, True]) +def test_fp32_adamw_load_matches_uninterrupted_next_step_cpu(cpu_update: bool): + def build(initial_value: torch.Tensor): + param = nn.Parameter(initial_value.clone().to(dtype=torch.bfloat16)) + optimizer = build_adamw_optimizer( + [{"params": [param], "weight_decay": 0.0}], + all_params=[param], + lr=0.1, + weight_decay=0.0, + betas=(0.9, 0.99), + eps=1.0e-8, + foreach=False, + use_fp32_master=True, + cpu_update=cpu_update, + model_param_dtypes={id(param): torch.bfloat16}, + opt=SimpleNamespace(), + ) + return param, optimizer + + initial = torch.tensor([1.0, -2.0], dtype=torch.float32) + first_grad = torch.tensor([0.5, -0.25], dtype=torch.bfloat16) + second_grad = torch.tensor([-0.125, 0.375], dtype=torch.bfloat16) + + ckpt_param, ckpt_optimizer = build(initial) + direct_param, direct_optimizer = build(initial) + loaded_param, loaded_optimizer = build(initial) + + ckpt_param.grad = first_grad.clone() + ckpt_optimizer.step() + direct_param.grad = first_grad.clone() + direct_optimizer.step() + + saved_param = ckpt_param.detach().clone() + saved_state = copy.deepcopy(ckpt_optimizer.state_dict()) + + with torch.no_grad(): + loaded_param.copy_(saved_param) + loaded_optimizer.load_state_dict(saved_state) + + direct_param.grad = second_grad.clone() + direct_optimizer.step() + loaded_param.grad = second_grad.clone() + loaded_optimizer.step() + + torch.testing.assert_close(loaded_param, direct_param, atol=0.0, rtol=0.0) + direct_state = direct_optimizer.state_dict() + loaded_state = loaded_optimizer.state_dict() + assert loaded_state["step_count"] == direct_state["step_count"] + for key in ("master_params", "exp_avgs", "exp_avg_sqs"): + torch.testing.assert_close(loaded_state[key][0], direct_state[key][0], atol=0.0, rtol=0.0) + assert loaded_state["steps"] == direct_state["steps"] diff --git a/experimental/lite/tests/unit/primitive/test_module_primitives_independent_unit.py b/experimental/lite/tests/unit/primitive/test_module_primitives_independent_unit.py new file mode 100644 index 00000000000..9641cc26715 --- /dev/null +++ b/experimental/lite/tests/unit/primitive/test_module_primitives_independent_unit.py @@ -0,0 +1,131 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +from __future__ import annotations + +import pytest +import torch +import torch.nn as nn + +from megatron.lite.primitive.modules.lora import ( + GroupedLinearLoRA, + LinearLoRA, + SharedGroupedLinearLoRA, + freeze_non_lora_params, + normalize_lora_config, + trainable_param_stats, +) + +pytestmark = pytest.mark.mlite + + +def test_lora_config_aliases_and_trainable_param_accounting(): + cfg = normalize_lora_config({"enabled": True, "rank": 2, "alpha": 6, "targets": ["qkv", "fc2"]}) + + assert cfg.enabled + assert cfg.scale == 3.0 + assert cfg.targets() == {"linear_qkv", "linear_fc2"} + assert cfg.targets_module("qkv") + assert cfg.targets_module("linear_fc2") + assert not normalize_lora_config({"enabled": False, "rank": 8}).enabled + with pytest.raises(TypeError, match="LoRA config"): + normalize_lora_config(object()) + + class TinyAdapterModel(nn.Module): + def __init__(self): + super().__init__() + self.base = nn.Linear(3, 2) + self.lora_adapter = nn.Linear(3, 2) + + model = TinyAdapterModel() + stats = freeze_non_lora_params(model) + + assert stats["lora_tensors"] == 2 + assert stats["frozen_tensors"] == 2 + assert not model.base.weight.requires_grad + assert model.lora_adapter.weight.requires_grad + assert trainable_param_stats(model) == { + "trainable_tensors": 2, + "trainable_numel": model.lora_adapter.weight.numel() + model.lora_adapter.bias.numel(), + } + + +def test_linear_lora_forward_backward_matches_low_rank_delta(): + layer = LinearLoRA(3, 2, rank=2, alpha=4, dropout=0.0) + with torch.no_grad(): + layer.lora_a.copy_(torch.tensor([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]])) + layer.lora_b.copy_(torch.tensor([[1.0, 2.0], [3.0, 4.0]])) + + x = torch.tensor([[1.0, 2.0, 3.0]], requires_grad=True) + output = layer(x) + + torch.testing.assert_close(output, torch.tensor([[10.0, 22.0]])) + output.sum().backward() + torch.testing.assert_close(x.grad, torch.tensor([[8.0, 12.0, 0.0]])) + + +def test_grouped_lora_respects_per_expert_splits(): + layer = GroupedLinearLoRA(2, 2, 2, rank=1, alpha=1, dropout=0.0) + with torch.no_grad(): + layer.lora_a.copy_(torch.tensor([[[1.0, 0.0]], [[0.0, 1.0]]])) + layer.lora_b.copy_(torch.tensor([[[2.0], [3.0]], [[5.0], [7.0]]])) + + x = torch.tensor([[2.0, 9.0], [4.0, 1.0], [6.0, 3.0]]) + output = layer(x, [1, 2]) + + torch.testing.assert_close(output, torch.tensor([[4.0, 6.0], [5.0, 7.0], [15.0, 21.0]])) + with pytest.raises(ValueError, match="expected 2 splits"): + layer(x, [3]) + + +def test_shared_grouped_lora_uses_one_adapter_for_all_experts(): + layer = SharedGroupedLinearLoRA(2, 2, 2, rank=1, alpha=2, dropout=0.0) + with torch.no_grad(): + layer.lora_a.copy_(torch.tensor([[1.0, -1.0]])) + layer.lora_b.copy_(torch.tensor([[2.0], [3.0]])) + + output = layer(torch.tensor([[3.0, 1.0], [4.0, 7.0]]), [1, 1]) + + torch.testing.assert_close(output, torch.tensor([[8.0, 12.0], [-12.0, -18.0]])) + + +def test_mrope_interleaves_text_height_and_width_sections(): + from megatron.lite.primitive.modules.mrope import MultimodalRotaryEmbedding + + base = torch.arange(3 * 2 * 6, dtype=torch.float32).reshape(3, 2, 6) + + interleaved = MultimodalRotaryEmbedding._apply_interleaved_mrope(base, mrope_section=[1, 1, 1]) + + expected = base[0].clone() + expected[..., 1] = base[1, ..., 1] + expected[..., 2] = base[2, ..., 2] + torch.testing.assert_close(interleaved, expected) + + +def test_mtp_aux_loss_scaler_threads_independent_gradient(transformer_engine_import_stub): + transformer_engine_import_stub() + from megatron.lite.primitive.modules.mtp import MTPLossAutoScaler + + MTPLossAutoScaler.set_loss_scale(torch.tensor(0.125)) + output = torch.tensor([1.0, 2.0, 3.0], requires_grad=True) + mtp_loss = torch.tensor(4.0, requires_grad=True) + + MTPLossAutoScaler.apply(output * 3.0, mtp_loss).sum().backward() + + torch.testing.assert_close(output.grad, torch.full_like(output, 3.0)) + torch.testing.assert_close(mtp_loss.grad, torch.tensor(0.125)) + MTPLossAutoScaler.main_loss_backward_scale = 1.0 + + +def test_gated_delta_static_helpers_are_finite_and_shape_stable(transformer_engine_import_stub): + transformer_engine_import_stub() + from megatron.lite.primitive.modules.gated_delta_net import GatedDeltaNet + + alpha = torch.tensor([[[0.0, 1.0], [-1.0, 2.0]]]) + beta = torch.tensor([[[0.0, 2.0], [-2.0, 4.0]]]) + + g, beta_sigmoid = GatedDeltaNet._compute_g_and_beta(torch.zeros(2), torch.ones(2), alpha, beta) + + assert g.shape == alpha.shape + assert beta_sigmoid.shape == beta.shape + assert torch.isfinite(g).all() + assert torch.isfinite(beta_sigmoid).all() + assert torch.all(g < 0) diff --git a/experimental/lite/tests/unit/primitive/test_ops_data_trainstep_unit.py b/experimental/lite/tests/unit/primitive/test_ops_data_trainstep_unit.py new file mode 100644 index 00000000000..599595f35b8 --- /dev/null +++ b/experimental/lite/tests/unit/primitive/test_ops_data_trainstep_unit.py @@ -0,0 +1,193 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +from __future__ import annotations + +import pytest +import torch +import torch.nn as nn +import torch.nn.functional as F + +from megatron.lite.primitive.data import _resolve_thd_padding, fixed_batches +from megatron.lite.primitive.deterministic import deterministic_requested +from megatron.lite.primitive.ops.cross_entropy import vocab_parallel_cross_entropy +from megatron.lite.primitive.ops.gated_delta_rule import l2norm, torch_chunk_gated_delta_rule +from megatron.lite.primitive.ops.linear_cross_entropy import linear_cross_entropy +from megatron.lite.primitive.ops.logprob import vocab_parallel_log_probs_from_logits +from megatron.lite.primitive.recompute import apply_recompute, parse_recompute_spec +from megatron.lite.primitive.train_step import compute_and_clip_grad_norm, run_microbatch_loop +from megatron.lite.primitive.utils import ensure_divisible + +pytestmark = pytest.mark.mlite + + +def test_vocab_parallel_cross_entropy_matches_torch_cross_entropy(): + logits = torch.randn(2, 3, 5, dtype=torch.float64, requires_grad=True) + labels = torch.tensor([[0, 4, 2], [3, 1, 0]]) + + loss = vocab_parallel_cross_entropy(logits, labels) + expected = F.cross_entropy( + logits.float().reshape(-1, 5), labels.reshape(-1), reduction="none" + ).view_as(labels) + + torch.testing.assert_close(loss, expected) + loss.sum().backward() + assert logits.grad is not None + assert torch.isfinite(logits.grad).all() + + +def test_logprob_selects_labels_in_batch_sequence_or_sequence_batch_layout(): + logits = torch.randn(2, 3, 4) + labels_bsh = torch.tensor([[0, 1], [2, 3], [1, 0]]) + + selected = vocab_parallel_log_probs_from_logits(logits, labels_bsh) + expected = ( + torch.log_softmax(logits.float(), dim=-1) + .gather(-1, labels_bsh.transpose(0, 1).unsqueeze(-1)) + .squeeze(-1) + .transpose(0, 1) + .contiguous() + ) + + torch.testing.assert_close(selected, expected) + with pytest.raises(ValueError, match="Could not align"): + vocab_parallel_log_probs_from_logits(logits, torch.zeros(4, 2, dtype=torch.long)) + + +def test_linear_cross_entropy_fallback_matches_explicit_matmul(): + hidden = torch.tensor([[1.0, -2.0, 0.5], [0.25, 1.5, -0.5]]) + weight = torch.tensor( + [[0.5, -1.0, 0.25], [1.0, 0.0, -0.5], [-0.5, 0.75, 1.0], [0.25, 0.5, -1.5]] + ) + labels = torch.tensor([0, 3]) + + log_probs, entropy = linear_cross_entropy(hidden, weight, labels, temperature=2.0) + logits = hidden.matmul(weight.t()) / 2.0 + expected_loss = F.cross_entropy(logits, labels, reduction="none") + expected_entropy = torch.distributions.Categorical(logits=logits).entropy() + + torch.testing.assert_close(log_probs, -expected_loss) + torch.testing.assert_close(entropy, expected_entropy) + + +def test_gated_delta_rule_math_helpers_return_finite_stateful_outputs(): + x = torch.tensor([[3.0, 4.0], [0.0, 5.0]]) + normalized = l2norm(x, dim=-1, eps=0.0) + torch.testing.assert_close(normalized.norm(dim=-1), torch.ones(2)) + + query = torch.randn(1, 3, 1, 2) + key = torch.randn(1, 3, 1, 2) + value = torch.randn(1, 3, 1, 2) + g = -torch.rand(1, 3, 1) + beta = torch.rand(1, 3, 1) + + output, final_state = torch_chunk_gated_delta_rule( + query, + key, + value, + g, + beta, + chunk_size=2, + output_final_state=True, + use_qk_l2norm_in_kernel=False, + ) + + assert output.shape == value.shape + assert final_state is not None + assert final_state.shape == (1, 1, 2, 2) + assert torch.isfinite(output).all() + assert torch.isfinite(final_state).all() + + +def test_data_padding_env_and_fixed_batches_are_deterministic(monkeypatch): + monkeypatch.delenv("MEGATRON_LITE_THD_PAD_TO_ALIGNMENT", raising=False) + monkeypatch.delenv("MEGATRON_LITE_THD_PAD_MULTIPLE", raising=False) + assert _resolve_thd_padding(seq_len=7, cp_size=2) == (8, 4, True) + assert _resolve_thd_padding(seq_len=7, cp_size=1) == (7, 1, False) + + monkeypatch.setenv("MEGATRON_LITE_THD_PAD_TO_ALIGNMENT", "0") + monkeypatch.setenv("MEGATRON_LITE_THD_PAD_MULTIPLE", "8") + assert _resolve_thd_padding(seq_len=7, cp_size=2) == (7, 8, False) + monkeypatch.setenv("MEGATRON_LITE_THD_PAD_MULTIPLE", "bad") + with pytest.raises(ValueError, match="PAD_MULTIPLE"): + _resolve_thd_padding(seq_len=7, cp_size=2) + + batches_a = fixed_batches(11, seq_len=4, num_steps=2, batch_size=2, device="cpu", seed=123) + batches_b = fixed_batches(11, seq_len=4, num_steps=2, batch_size=2, device="cpu", seed=123) + for (ids_a, labels_a), (ids_b, labels_b) in zip(batches_a, batches_b, strict=True): + torch.testing.assert_close(ids_a, ids_b) + torch.testing.assert_close(labels_a, labels_b) + + +def test_deterministic_env_request_parser(monkeypatch): + monkeypatch.setenv("MEGATRON_LITE_DETERMINISTIC", "yes") + assert deterministic_requested() + monkeypatch.setenv("MEGATRON_LITE_DETERMINISTIC", "0") + assert not deterministic_requested() + + +def test_recompute_parser_and_wrapper_replays_forward_on_backward(): + assert parse_recompute_spec(None) == [] + assert parse_recompute_spec("none") == [] + assert parse_recompute_spec("full") == ["full"] + assert parse_recompute_spec("attn,mlp") == ["attn", "mlp"] + assert parse_recompute_spec(["attn"]) == ["attn"] + + class CountingModule(nn.Module): + def __init__(self): + super().__init__() + self.calls = 0 + + def forward(self, x): + self.calls += 1 + return x * x + + class Layer(nn.Module): + def __init__(self): + super().__init__() + self.inner = CountingModule() + + layer = Layer() + apply_recompute( + nn.ModuleList([layer]), + ["inner"], + {"inner": lambda module: module.inner}, + no_rng_modules={"inner"}, + ) + + x = torch.tensor([2.0, -3.0], requires_grad=True) + layer.inner(x).sum().backward() + + assert layer.inner.calls == 2 + torch.testing.assert_close(x.grad, torch.tensor([4.0, -6.0])) + + +def test_train_step_microbatch_loop_and_grad_clip_cpu_contract(): + model = nn.Linear(2, 1, bias=False) + with torch.no_grad(): + model.weight.fill_(0.5) + data = iter( + [ + {"x": torch.tensor([[1.0, 2.0]]), "y": torch.tensor([[1.0]])}, + {"x": torch.tensor([[3.0, 4.0]]), "y": torch.tensor([[2.0]])}, + ] + ) + + def forward_fn(module, batch): + return {"loss": F.mse_loss(module(batch["x"]), batch["y"])} + + output = run_microbatch_loop(model, data, 2, forward_fn) + + assert output is not None + assert output["loss"].ndim == 0 + assert model.weight.grad is not None + assert torch.isfinite(model.weight.grad).all() + + grad_norm = compute_and_clip_grad_norm(model, optimizer=None, max_norm=0.25, use_dist_opt=False) + + assert torch.isfinite(grad_norm) + assert model.weight.grad.norm() <= 0.25 + 1.0e-6 + + +def test_utils_ensure_divisible_returns_quotient_and_reports_context(): + assert ensure_divisible(12, 3, "tp") == 4 + with pytest.raises(ValueError, match=r"10 is not divisible by 4 \(tp\)"): + ensure_divisible(10, 4, "tp") diff --git a/experimental/lite/tests/unit/primitive/test_parallel_dimensions_independent_unit.py b/experimental/lite/tests/unit/primitive/test_parallel_dimensions_independent_unit.py new file mode 100644 index 00000000000..fe3ecfb7b94 --- /dev/null +++ b/experimental/lite/tests/unit/primitive/test_parallel_dimensions_independent_unit.py @@ -0,0 +1,131 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +from __future__ import annotations + +from types import SimpleNamespace + +import pytest +import torch + +from megatron.lite.primitive.parallel.cp import split_packed_for_cp +from megatron.lite.primitive.parallel.pipeline import _num_microbatches_from_config +from megatron.lite.primitive.parallel.pp import build_pipeline_chunk_layout +from megatron.lite.primitive.parallel.sp import ( + gather_for_non_sp_head, + gather_from_sequence_parallel, + scatter_to_sequence_parallel, +) +from megatron.lite.primitive.parallel.state import ParallelState + +pytestmark = pytest.mark.mlite + + +def test_tp_vocab_embedding_and_output_single_rank_contract(transformer_engine_import_stub): + transformer_engine_import_stub() + from megatron.lite.primitive.parallel.linear import ( + VocabParallelEmbedding, + VocabParallelOutput, + pad_vocab_for_tp, + ) + + ps = ParallelState(tp_size=1, tp_rank=0) + + assert pad_vocab_for_tp(151936, 2) == 151936 + assert pad_vocab_for_tp(129, 2) == 256 + + embedding = VocabParallelEmbedding(5, 3, ps, deterministic=True) + with torch.no_grad(): + values = torch.arange(embedding.local_vocab * 3, dtype=torch.float32).view( + embedding.local_vocab, 3 + ) + embedding.embedding.weight.copy_(values) + + input_ids = torch.tensor([[0, 4, 1]]) + output = embedding(input_ids) + + assert output.shape == (3, 1, 3) + torch.testing.assert_close(output[:, 0], values[input_ids[0]]) + + head = VocabParallelOutput(5, 3, ps) + logits = head(torch.ones(2, 1, 3, dtype=torch.bfloat16)) + gathered = head.gather(logits) + + assert logits.shape == (2, 1, 128) + assert gathered.shape == (2, 1, 5) + + +def test_sp_helpers_are_identity_for_single_rank_tp(): + ps = ParallelState(tp_size=1, tp_rank=0) + x = torch.randn(4, 2, 3, requires_grad=True) + + assert scatter_to_sequence_parallel(x, ps) is x + assert gather_from_sequence_parallel(x, ps) is x + assert gather_for_non_sp_head(x, ps) is x + + +def test_cp_packed_split_handles_each_sample_independently(): + input_ids = torch.arange(16) + position_ids = torch.arange(100, 116) + cu_seqlens = torch.tensor([0, 8, 16], dtype=torch.int32) + + rank0 = split_packed_for_cp( + input_ids, position_ids, cu_seqlens, max_seqlen=8, cp_rank=0, cp_size=2 + ) + rank1 = split_packed_for_cp( + input_ids, position_ids, cu_seqlens, max_seqlen=8, cp_rank=1, cp_size=2 + ) + + torch.testing.assert_close(rank0[0], torch.tensor([0, 1, 6, 7, 8, 9, 14, 15])) + torch.testing.assert_close(rank0[1], torch.tensor([100, 101, 106, 107, 108, 109, 114, 115])) + torch.testing.assert_close(rank0[2], torch.tensor([0, 4, 8], dtype=torch.int32)) + assert rank0[3] == 4 + + torch.testing.assert_close(rank1[0], torch.tensor([2, 3, 4, 5, 10, 11, 12, 13])) + torch.testing.assert_close(rank1[1], torch.tensor([102, 103, 104, 105, 110, 111, 112, 113])) + torch.testing.assert_close(rank1[2], torch.tensor([0, 4, 8], dtype=torch.int32)) + assert rank1[3] == 4 + + +def test_pp_layout_auto_balances_non_divisible_layer_counts(): + # Non-divisible counts no longer raise "not divisible": mcore's layout balances + # them. 7/pp2 with embedding/loss accounting -> [4, 3]. (Needs megatron.core.) + pytest.importorskip("megatron.core.transformer.pipeline_parallel_layer_layout") + rank0 = ParallelState(pp_size=2, pp_rank=0, pp_is_first=True, pp_is_last=False) + rank1 = ParallelState(pp_size=2, pp_rank=1, pp_is_first=False, pp_is_last=True) + + assert build_pipeline_chunk_layout(7, rank0).layer_indices == [0, 1, 2, 3] + assert build_pipeline_chunk_layout(7, rank1).layer_indices == [4, 5, 6] + + +def test_dp_dimension_controls_dense_microbatch_contract(): + ps = ParallelState(dp_size=4, dp_rank=2, dp_cp_size=8, dp_cp_rank=5) + config = SimpleNamespace(gbs=32, mbs=2, num_microbatches=None) + + assert _num_microbatches_from_config(config, ps) == 4 + + config.num_microbatches = 7 + assert _num_microbatches_from_config(config, ps) == 7 + assert ps.dp_rank == 2 + assert ps.dp_cp_size == 8 + assert ps.dp_cp_rank == 5 + + +def test_ep_token_dispatcher_local_roundtrip_is_independent_of_deepep(): + from megatron.lite.primitive.modules.dispatcher import TokenDispatcher + + ps = ParallelState(ep_size=1, ep_rank=0) + dispatcher = TokenDispatcher(num_experts=3, hidden_size=2, ps=ps, use_deepep=False) + hidden = torch.tensor([[1.0, 10.0], [2.0, 20.0], [3.0, 30.0], [4.0, 40.0]], requires_grad=True) + topk_indices = torch.tensor([[0], [2], [1], [2]]) + topk_scores = torch.ones(4, 1) + + dispatched, tokens_per_expert, dispatched_probs = dispatcher.dispatch( + hidden, topk_scores, topk_indices + ) + combined = dispatcher.combine(dispatched) + + torch.testing.assert_close(tokens_per_expert, torch.tensor([1, 1, 2])) + torch.testing.assert_close(dispatched_probs, torch.ones(4)) + torch.testing.assert_close(combined, hidden) + + combined.sum().backward() + torch.testing.assert_close(hidden.grad, torch.ones_like(hidden)) diff --git a/experimental/lite/tests/unit/primitive/test_parallel_unit.py b/experimental/lite/tests/unit/primitive/test_parallel_unit.py new file mode 100644 index 00000000000..add7ff1e7be --- /dev/null +++ b/experimental/lite/tests/unit/primitive/test_parallel_unit.py @@ -0,0 +1,368 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +from __future__ import annotations + +import pytest +import torch +from megatron.lite.primitive.parallel import ( + ParallelState, + build_pipeline_chunk_layout, + pack_nested_thd, + parallel_state_from_model, + prepare_packed_thd_for_context_parallel, + reconstruct_packed_from_cp_parts, + roll_packed_thd_left, + split_packed_to_cp_local, + zigzag_position_ids_for_cp, + zigzag_reconstruct_from_cp_parts, + zigzag_slice_for_cp, + zigzag_split_for_cp, +) + +pytestmark = pytest.mark.mlite + + +def test_cp_zigzag_split_slice_and_reconstruct_match(): + tensor = torch.arange(16).reshape(1, 8, 2) + parts = [zigzag_split_for_cp(tensor, rank, cp_size=2, seq_dim=1) for rank in range(2)] + + assert torch.equal(parts[0], tensor[:, [0, 1, 6, 7], :]) + assert torch.equal(parts[1], tensor[:, [2, 3, 4, 5], :]) + assert torch.equal(zigzag_slice_for_cp(tensor, 0, cp_size=2, seq_dim=1), parts[0]) + assert torch.equal(zigzag_slice_for_cp(tensor, 1, cp_size=2, seq_dim=1), parts[1]) + assert torch.equal(zigzag_reconstruct_from_cp_parts(parts, seq_dim=1), tensor) + + +def test_cp_position_ids_follow_zigzag_order(): + assert torch.equal( + zigzag_position_ids_for_cp(8, cp_rank=0, cp_size=2, device=torch.device("cpu")), + torch.tensor([[0, 1, 6, 7]]), + ) + assert torch.equal( + zigzag_position_ids_for_cp(8, cp_rank=1, cp_size=2, device=torch.device("cpu")), + torch.tensor([[2, 3, 4, 5]]), + ) + + +# The pipeline layout wires to Megatron-core's layer-layout machinery, so these +# tests need megatron.core importable (present in the mlite GPU/smoke containers). +_mcore_layout = pytest.importorskip( + "megatron.core.transformer.pipeline_parallel_layer_layout" +) + + +def _ranks(pp_size: int, pp_layout=None) -> list[ParallelState]: + return [ + ParallelState( + pp_size=pp_size, + pp_rank=r, + pp_is_first=(r == 0), + pp_is_last=(r == pp_size - 1), + pp_layout=pp_layout, + ) + for r in range(pp_size) + ] + + +def _layout_indices(num_layers: int, pp_size: int, *, pp_layout=None, **kw) -> list[list[int]]: + return [ + build_pipeline_chunk_layout(num_layers, ps, **kw).layer_indices + for ps in _ranks(pp_size, pp_layout) + ] + + +def _mtp_flags(num_layers: int, pp_size: int, *, pp_layout=None, **kw) -> list[bool]: + return [ + build_pipeline_chunk_layout(num_layers, ps, **kw).has_mtp + for ps in _ranks(pp_size, pp_layout) + ] + + +def _assert_full_contiguous_cover(indices: list[list[int]], num_layers: int) -> None: + """Every decoder 0..N-1 placed exactly once, in order, each stage a contiguous run.""" + assert [i for stage in indices for i in stage] == list(range(num_layers)) + for stage in indices: + assert not stage or stage == list(range(stage[0], stage[0] + len(stage))) + + +def test_pp_layout_marks_stage_boundaries(): + rank0, rank1 = _ranks(2) + assert build_pipeline_chunk_layout(8, rank0).layer_indices == [0, 1, 2, 3] + assert build_pipeline_chunk_layout(8, rank0).has_embed is True + assert build_pipeline_chunk_layout(8, rank0).has_head is False + assert build_pipeline_chunk_layout(8, rank1).layer_indices == [4, 5, 6, 7] + assert build_pipeline_chunk_layout(8, rank1).has_embed is False + assert build_pipeline_chunk_layout(8, rank1).has_head is True + + +def test_pp_layout_vpp_is_not_supported_yet(): + # pp-only: requesting VPP raises rather than silently mis-splitting. + rank0 = ParallelState(pp_size=2, pp_rank=0, pp_is_first=True, pp_is_last=False) + with pytest.raises(NotImplementedError): + build_pipeline_chunk_layout(8, rank0, vpp=2) + with pytest.raises(NotImplementedError): + build_pipeline_chunk_layout(8, rank0, vpp=2, vpp_chunk_id=1) + + +def test_pp_layout_auto_balances_non_divisible_counts(): + # account_for_embedding/loss: the embedding/loss stages each give up a decoder, + # so 6/pp4 balances to [1,2,2,1] (not [2,2,1,1]) and never raises "not divisible". + assert _layout_indices(6, 4) == [[0], [1, 2], [3, 4], [5]] + assert _layout_indices(6, 4, vpp=1) == _layout_indices(6, 4) # vpp=1 == pp-only + # An MTP head occupies the last stage's slot -> it gives up its decoder: [2,2,2,0]. + assert _layout_indices(6, 4, num_mtp_layers=1) == [[0, 1], [2, 3], [4, 5], []] + + +def test_pp_layout_accounts_for_head_tail_even_when_divisible(): + # Embedding/loss occupy head/tail slots, so 8/pp4 is [2,3,2,1] not [2,2,2,2]. + assert _layout_indices(8, 4) == [[0, 1], [2, 3, 4], [5, 6], [7]] + assert _layout_indices(8, 4, num_mtp_layers=1) == [[0, 1], [2, 3, 4], [5, 6, 7], []] + assert _layout_indices(8, 2) == [[0, 1, 2, 3], [4, 5, 6, 7]] + + +# --- Correctness matrix: real delivery counts x MTP{off,on} x PP{2,4,8} --- +_REAL_LAYERS = {"deepseek_v4": 43, "kimi_k2": 61, "glm5": 78} +_CORRECTNESS_CASES = [(m, pp, mtp) for m in _REAL_LAYERS for pp in (2, 4, 8) for mtp in (0, 1)] + + +def _mcore_reference_decoder_ids(num_layers: int, pp: int, mtp: int) -> list[list[int]]: + """Per-stage decoder ids from mcore's PipelineParallelLayerLayout, built directly + from the canonical unit sequence — the independent reference the glue must match.""" + from megatron.core.transformer.enums import LayerType + from megatron.core.transformer.pipeline_parallel_layer_layout import ( + PipelineParallelLayerLayout, + ) + + units = ["embedding"] + ["decoder"] * num_layers + ["mtp"] * mtp + ["loss"] + base, rem = divmod(len(units), pp) + rows, pos = [], 0 + for size in (base + (1 if s < rem else 0) for s in range(pp)): + rows.append(units[pos : pos + size]) + pos += size + ref = PipelineParallelLayerLayout(rows, pipeline_model_parallel_size=pp) + return [ref.get_layer_id_list(LayerType.decoder, vp_stage=0, pp_rank=r) for r in range(pp)] + + +@pytest.mark.parametrize("model, pp, mtp", _CORRECTNESS_CASES) +def test_auto_layout_is_bit_equal_to_mcore(model, pp, mtp): + # Faithful reuse: the glue's per-stage decoder ids are exactly mcore's for the same + # canonical layout, so correctness is inherited from mcore (not re-derived). + n = _REAL_LAYERS[model] + assert _layout_indices(n, pp, num_mtp_layers=mtp) == _mcore_reference_decoder_ids(n, pp, mtp) + + +@pytest.mark.parametrize("model, pp, mtp", _CORRECTNESS_CASES) +def test_auto_layout_is_legal_and_balanced(model, pp, mtp): + n = _REAL_LAYERS[model] + chunks = [build_pipeline_chunk_layout(n, ps, num_mtp_layers=mtp) for ps in _ranks(pp)] + # complete + ordered, no missing/dup decoder -> sum == num_layers + assert [i for c in chunks for i in c.layer_indices] == list(range(n)) + # embedding only on stage 0; loss/head only on the last stage + assert [c.has_embed for c in chunks] == [r == 0 for r in range(pp)] + assert [c.has_head for c in chunks] == [r == pp - 1 for r in range(pp)] + # MTP placed exactly `mtp` times, on the final (head) stage + assert sum(c.has_mtp for c in chunks) == mtp and chunks[-1].has_mtp == bool(mtp) + # balanced: per-stage unit cells (decoders + embedding/loss/mtp slots) differ by <=1 + cells = [len(c.layer_indices) + c.has_embed + c.has_head + (mtp if c.has_mtp else 0) for c in chunks] + assert max(cells) - min(cells) <= 1 + + +def test_pp_layout_custom_string_mode_is_used_verbatim(): + # Advanced users pin an explicit mcore layout string (custom mode) instead of the + # auto split. "Ettt|t|t|tL" front-loads 3 decoders onto stage 0; mcore parses + # E=embedding t=decoder L=loss. This is NOT the auto [1,2,2,1] for 6/pp4. + custom = _layout_indices(6, 4, pp_layout="Ettt|t|t|tL") + assert [len(stage) for stage in custom] == [3, 1, 1, 1] + _assert_full_contiguous_cover(custom, 6) + assert custom != _layout_indices(6, 4) # custom overrides auto + + # A custom string carrying MTP after the decoders validates against num_mtp_layers. + mtp_custom = _layout_indices(6, 4, pp_layout="Ett|tt|t|tmL", num_mtp_layers=1) + assert [len(stage) for stage in mtp_custom] == [2, 2, 1, 1] + _assert_full_contiguous_cover(mtp_custom, 6) + + +def test_pp_layout_custom_string_rejects_vpp_multi_chunk(): + # A custom layout with more stages than pp implies VPP, which is not supported. + with pytest.raises(NotImplementedError): + _layout_indices(6, 2, pp_layout="Et|t|t|ttL") # 4 stages over pp2 -> vpp=2 + + +def test_pp_layout_marks_mtp_stage_from_the_layout_not_a_fixed_rank(): + # auto: the MTP slot sits just before loss, so it lands on the final (head) stage + # -- has_mtp marks exactly that stage, and shifts the decoder split [2,2,2,0]. + assert _mtp_flags(6, 4, num_mtp_layers=1) == [False, False, False, True] + assert _layout_indices(6, 4, num_mtp_layers=1) == [[0, 1], [2, 3], [4, 5], []] + # no MTP requested -> no stage owns MTP. + assert _mtp_flags(6, 4) == [False, False, False, False] + # single stage owns the MTP too. + assert build_pipeline_chunk_layout( + 5, ParallelState(pp_size=1, pp_rank=0, pp_is_first=True, pp_is_last=True), num_mtp_layers=1 + ).has_mtp is True + + +def test_pp_layout_custom_string_mtp_lands_on_the_designated_stage(): + # The `m` token co-located with loss on the final stage: has_mtp marks that stage + # (== the head stage), driven by the layout, not a hard-coded rank. + flags = _mtp_flags(6, 4, pp_layout="Ett|tt|t|tmL", num_mtp_layers=1) + assert flags == [False, False, False, True] + assert _layout_indices(6, 4, pp_layout="Ett|tt|t|tmL", num_mtp_layers=1) == [ + [0, 1], [2, 3], [4], [5] + ] + + +@pytest.mark.xfail( + reason="Standalone MTP (a custom pp_layout placing `m` off the final/head stage) is a " + "planned follow-up: mlite's MTP shares the output head on the loss stage, so it " + "currently raises NotImplementedError instead of building MTP on the `m` stage. " + "When cross-stage standalone MTP lands, this should pass and the marker be removed.", + strict=True, + raises=NotImplementedError, +) +def test_pp_layout_custom_string_standalone_mtp_builds_on_designated_stage(): + # DESIRED (follow-up): `m` on a non-final stage builds MTP on that stage. + # "E|ttttttm|L" over pp3 -> [E], [t*6, m], [L]; MTP belongs on the middle stage. + assert _mtp_flags(6, 3, pp_layout="E|ttttttm|L", num_mtp_layers=1) == [False, True, False] + + +def test_pp_layout_single_stage_owns_all_layers(): + ps = ParallelState(pp_size=1, pp_rank=0, pp_is_first=True, pp_is_last=True) + layout = build_pipeline_chunk_layout(5, ps) + assert layout.layer_indices == [0, 1, 2, 3, 4] + assert layout.has_embed and layout.has_head + + +def test_virtual_pipeline_rank_is_tracked_on_lite_parallel_state(): + from megatron.lite.primitive.parallel.pipeline import _set_virtual_pipeline_rank + + ps = ParallelState(pp_size=2, pp_rank=1, pp_is_first=False, pp_is_last=True) + + _set_virtual_pipeline_rank(ps, chunk_id=1, num_chunks=2) + + assert ps.virtual_pipeline_size == 2 + assert ps.virtual_pipeline_rank == 1 + + _set_virtual_pipeline_rank(ps, chunk_id=None, num_chunks=2) + + assert ps.virtual_pipeline_size is None + assert ps.virtual_pipeline_rank is None + + +def test_thd_roll_keeps_sequence_boundaries(): + cu_seqlens = torch.tensor([0, 4, 8], dtype=torch.int32) + rolled, token_sum = roll_packed_thd_left(torch.arange(8), cu_seqlens_padded=cu_seqlens, dims=0) + + assert torch.equal(rolled, torch.tensor([1, 2, 3, 0, 5, 6, 7, 0])) + assert token_sum.item() == 24 + + +def test_thd_cp_split_and_reconstruct_roundtrip(): + cu_seqlens = torch.tensor([0, 8], dtype=torch.int32) + tensor = torch.arange(8) + parts = [ + split_packed_to_cp_local( + tensor, cu_seqlens_padded=cu_seqlens, cp_size=2, cp_rank=rank, dim=0 + ) + for rank in range(2) + ] + + assert torch.equal(parts[0], torch.tensor([0, 1, 6, 7])) + assert torch.equal(parts[1], torch.tensor([2, 3, 4, 5])) + assert torch.equal( + reconstruct_packed_from_cp_parts(parts, cu_seqlens_padded=cu_seqlens, cp_size=2, dim=0), + tensor, + ) + + +def test_plain_thd_batch_is_split_by_protocol_context_parallel_helper(): + ids = torch.nested.as_nested_tensor( + [torch.arange(1, 6), torch.arange(11, 18)], + layout=torch.jagged, + ) + labels = torch.nested.as_nested_tensor( + [torch.arange(101, 106), torch.arange(111, 118)], + layout=torch.jagged, + ) + loss_mask = torch.nested.as_nested_tensor( + [torch.ones(5), torch.ones(7)], + layout=torch.jagged, + ) + packed = pack_nested_thd( + ids, + cp_size=2, + split_cp=False, + labels=labels, + loss_mask=loss_mask, + ) + + assert packed.input_ids.shape == (1, 16) + assert packed.cp_size == 2 + assert packed.packed_seq_params.local_cp_size is None + + local_params, local_tensors = prepare_packed_thd_for_context_parallel( + packed.packed_seq_params, + (packed.input_ids, packed.labels, packed.loss_mask, packed.position_ids), + cp_size=2, + cp_rank=0, + ) + + expected_ids = split_packed_to_cp_local( + packed.input_ids, + cu_seqlens_padded=packed.cu_seqlens_padded, + cp_size=2, + cp_rank=0, + dim=1, + ) + expected_pos = split_packed_to_cp_local( + packed.position_ids, + cu_seqlens_padded=packed.cu_seqlens_padded, + cp_size=2, + cp_rank=0, + dim=1, + ) + local_ids, local_labels, local_loss_mask, local_pos = local_tensors + assert torch.equal(local_ids, expected_ids) + assert torch.equal(local_pos, expected_pos) + assert local_labels is not None + assert local_loss_mask is not None + assert local_params.local_cp_size == 2 + assert local_params.cp_rank == 0 + + +def test_parallel_state_from_model_unwraps_ddp_style_module(): + class Model: + ps = ParallelState(cp_size=2, cp_rank=1) + + class Wrapper: + module = Model() + + assert parallel_state_from_model(Wrapper()).cp_rank == 1 + + +def test_protocol_context_parallel_helper_keeps_pre_split_thd_batch_idempotent(): + ids = torch.nested.as_nested_tensor([torch.arange(8)], layout=torch.jagged) + packed = pack_nested_thd(ids, cp_size=2, cp_rank=1) + + local_params, local_tensors = prepare_packed_thd_for_context_parallel( + packed.packed_seq_params, + (packed.input_ids, packed.position_ids), + cp_size=2, + cp_rank=1, + ) + + assert local_params is packed.packed_seq_params + assert torch.equal(local_tensors[0], packed.input_ids) + assert local_params.local_cp_size == 2 + + +def test_protocol_context_parallel_helper_is_noop_without_packed_thd_params(): + tensor = torch.arange(8) + + local_params, local_tensors = prepare_packed_thd_for_context_parallel( + None, + (tensor,), + cp_size=2, + cp_rank=0, + ) + + assert local_params is None + assert local_tensors[0] is tensor diff --git a/experimental/lite/tests/unit/primitive/test_runtime_config_unit.py b/experimental/lite/tests/unit/primitive/test_runtime_config_unit.py new file mode 100644 index 00000000000..6e91f51c1b3 --- /dev/null +++ b/experimental/lite/tests/unit/primitive/test_runtime_config_unit.py @@ -0,0 +1,77 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +from __future__ import annotations + +import pytest + +from megatron.lite.runtime import RuntimeConfig, create_runtime +from megatron.lite.runtime.backends.mlite.config import DebugConfig, MegatronLiteConfig +from megatron.lite.runtime.contracts.config import OptimizerConfig, ParallelConfig + +pytestmark = pytest.mark.mlite + + +def test_mlite_config_defaults_are_stable(): + cfg = MegatronLiteConfig(model_name="qwen3_moe") + + assert cfg.model_name == "qwen3_moe" + assert cfg.impl == "lite" + assert cfg.parallel.tp == 1 + assert cfg.parallel.ep == 1 + assert cfg.parallel.pp == 1 + assert cfg.parallel.cp == 1 + assert isinstance(cfg.optimizer, OptimizerConfig) + assert isinstance(cfg.debug, DebugConfig) + + +def test_mlite_config_from_dict_preserves_parallel_optimizer_and_impl_cfg(): + cfg = MegatronLiteConfig.from_dict( + "/models/qwen", + { + "model_name": "qwen3_moe", + "impl": "lite", + "tp": 2, + "ep": 4, + "pp": 2, + "cp": 2, + "optimizer": { + "lr": 1.0e-4, + "weight_decay": 0.1, + "adam_beta1": 0.9, + "offload_fraction": 1.0, + }, + "impl_cfg": {"attn_impl": "mcore", "moe_impl": "ml"}, + "use_thd": True, + "precision_aware_opt": True, + }, + ) + + assert cfg.hf_path == "/models/qwen" + assert cfg.parallel == ParallelConfig(tp=2, etp=None, ep=4, pp=2, vpp=1, cp=2) + assert cfg.optimizer.lr == 1.0e-4 + assert cfg.optimizer.weight_decay == 0.1 + assert cfg.optimizer.adam_beta1 == 0.9 + assert cfg.optimizer.offload_fraction == 1.0 + assert cfg.impl_cfg["attn_impl"] == "mcore" + assert cfg.impl_cfg["moe_impl"] == "ml" + assert cfg.impl_cfg["use_thd"] is True + assert cfg.impl_cfg["precision_aware_opt"] is True + + +def test_mlite_config_rejects_num_microbatches_in_backend_config(): + with pytest.raises(ValueError, match="num_microbatches"): + MegatronLiteConfig.from_dict( + "/models/qwen", {"model_name": "qwen3_moe", "num_microbatches": 2} + ) + + +def test_create_runtime_uses_mlite_backend_registry(): + runtime = create_runtime( + RuntimeConfig( + backend="mlite", + hf_path="/models/qwen", + backend_cfg={"model_name": "qwen3_moe", "load_hf_weights": False}, + ) + ) + + assert type(runtime).__name__ == "MegatronLiteRuntime" + assert runtime.tier == "rl_best" diff --git a/experimental/lite/tests/unit/primitive/test_training_checkpoint.py b/experimental/lite/tests/unit/primitive/test_training_checkpoint.py new file mode 100644 index 00000000000..f788a85dcfe --- /dev/null +++ b/experimental/lite/tests/unit/primitive/test_training_checkpoint.py @@ -0,0 +1,390 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +from __future__ import annotations + +import copy +from types import SimpleNamespace + +import pytest +import torch +from torch.distributed.tensor import Replicate, Shard + +pytest.importorskip("megatron.core.dist_checkpointing") + +from megatron.core.dist_checkpointing.strategies.torch import ( + _replace_state_dict_keys_with_sharded_keys, +) +from megatron.lite.primitive.ckpt import dcp +from megatron.lite.primitive.ckpt.distckpt import ( + _dist_opt_checkpoint_metadata, + _model_sharded_state_dict, + _rank_offsets_and_replica_id, + _single_or_all_model_state, + _synchronize_native_optimizer_steps, + attach_model_sharded_state_dict, +) +from megatron.lite.primitive.parallel import ParallelState +from megatron.lite.primitive.protocols import default_expert_classifier, default_placement_fn +from megatron.lite.runtime.backends.mlite.runtime import MegatronLiteRuntime +from megatron.lite.runtime.contracts.handle import ModelHandle + + +@pytest.mark.parametrize("groups, expected", [([], False), ([object(), object()], True), ([object(), None], False)]) +def test_dist_opt_checkpoint_memory_efficient_metadata(groups, expected) -> None: + opts = [ + SimpleNamespace( + data_parallel_group_gloo=group, + sharded_state_dict=lambda: None, + gbuf_ranges={}, + buffers=[], + optimizer=object(), + ) + for group in groups + ] + metadata = _dist_opt_checkpoint_metadata(SimpleNamespace(chained_optimizers=opts)) + assert metadata["distrib_optim_fully_reshardable_mem_efficient"] is expected + + +def _assert_state_equal(actual, expected) -> None: + if torch.is_tensor(expected): + assert torch.equal(actual, expected) + elif isinstance(expected, dict): + assert actual.keys() == expected.keys() + for key, value in expected.items(): + _assert_state_equal(actual[key], value) + elif isinstance(expected, list): + assert len(actual) == len(expected) + for actual_item, expected_item in zip(actual, expected, strict=True): + _assert_state_equal(actual_item, expected_item) + else: + assert actual == expected + + +def test_optimizer_checkpoint_roundtrips_rank_local_state(tmp_path) -> None: + model = torch.nn.Linear(4, 2) + optimizer = torch.optim.AdamW(model.parameters(), lr=0.01) + + loss = model(torch.ones(3, 4)).sum() + loss.backward() + optimizer.step() + + expected = copy.deepcopy(optimizer.state_dict()) + dcp._save_optimizer_checkpoint(optimizer, str(tmp_path)) + + for state in optimizer.state.values(): + for value in state.values(): + if torch.is_tensor(value): + value.zero_() + + dcp._load_optimizer_checkpoint(optimizer, str(tmp_path)) + + assert (tmp_path / "optimizer_rank_0.pt").exists() + _assert_state_equal(optimizer.state_dict(), expected) + + +class FakeDistOpt: + def __init__(self): + self.save_model_sd = None + self.load_model_sd = None + self.loaded_state = None + + def sharded_state_dict(self, model_sd, is_loading: bool = False, metadata=None): + assert metadata == DISTOPT_METADATA + if is_loading: + self.load_model_sd = model_sd + else: + self.save_model_sd = model_sd + return {"is_loading": is_loading} + + def load_state_dict(self, state): + self.loaded_state = state + + +class FakeWrapper(torch.nn.Module): + def __init__(self, module): + super().__init__() + self.module = module + self.wrapper_load_called = False + + def forward(self, *args, **kwargs): + return self.module(*args, **kwargs) + + def load_state_dict(self, *args, **kwargs): + self.wrapper_load_called = True + return super().load_state_dict(*args, **kwargs) + + +DISTOPT_METADATA = { + "distrib_optim_sharding_type": "fully_reshardable", + "distrib_optim_fully_reshardable_mem_efficient": False, + "chained_optim_avoid_prefix": True, +} + + +def test_dist_opt_checkpoint_dispatches_to_mcore_distckpt(monkeypatch, tmp_path) -> None: + model = torch.nn.Linear(4, 2) + optimizer = FakeDistOpt() + ps = ParallelState(pp_rank=1, tp_rank=2, dp_cp_rank=3) + attach_model_sharded_state_dict([model], ps) + saved = {} + + def fake_save(state_dict, checkpoint_dir, **kwargs): + saved["state_dict"] = state_dict + saved["checkpoint_dir"] = checkpoint_dir + saved["kwargs"] = kwargs + + monkeypatch.setattr("megatron.lite.primitive.ckpt.distckpt.dist_checkpointing.save", fake_save) + + dcp.save_training_checkpoint(model, optimizer, 5, str(tmp_path), use_dcp=True) + + model_sd = saved["state_dict"]["model"] + assert set(model_sd) == {"weight", "bias"} + assert model_sd["weight"].replica_id == (0, 2, 3) + assert optimizer.save_model_sd is model_sd + assert saved["state_dict"]["optimizer"] == {"is_loading": False} + assert saved["state_dict"]["step"] == 5 + assert saved["checkpoint_dir"] == str(tmp_path / "step_5") + assert saved["kwargs"]["validate_access_integrity"] is False + assert saved["kwargs"]["content_metadata"] == DISTOPT_METADATA + assert not (tmp_path / "step_5" / "optimizer_rank_0.pt").exists() + + +def test_dist_opt_checkpoint_offsets_cover_tp_pp_ep_etp_topology() -> None: + ps = ParallelState( + pp_size=2, + pp_rank=1, + tp_size=2, + tp_rank=1, + ep_size=2, + ep_rank=1, + etp_size=2, + etp_rank=1, + dp_size=2, + dp_rank=0, + cp_size=1, + cp_rank=0, + dp_cp_rank=0, + expert_dp_size=1, + expert_dp_rank=0, + ) + + dense_offsets, dense_replica = _rank_offsets_and_replica_id( + [Replicate(), Replicate(), Replicate(), Shard(0)], ps, expert=False + ) + expert_offsets, expert_replica = _rank_offsets_and_replica_id( + [Replicate(), Replicate(), Shard(0), Shard(0)], ps, expert=True + ) + + assert dense_offsets == ((0, 1, 2),) + assert dense_replica == (0, 0, 0) + assert expert_offsets == ((0, 3, 4),) + assert expert_replica == (0, 0, 0) + + +def test_dist_opt_replica_id_groups_sharded_axes_by_placement() -> None: + placements = [Replicate(), Replicate(), Replicate(), Shard(0)] + rank_offsets0, replica_id0 = _rank_offsets_and_replica_id( + placements, ParallelState(tp_size=2, tp_rank=0), expert=False + ) + rank_offsets1, replica_id1 = _rank_offsets_and_replica_id( + placements, ParallelState(tp_size=2, tp_rank=1), expert=False + ) + + assert rank_offsets0 == ((0, 0, 2),) + assert rank_offsets1 == ((0, 1, 2),) + assert replica_id0 == replica_id1 == (0, 0, 0) + + expert_offsets, expert_replica_id = _rank_offsets_and_replica_id( + [Replicate(), Replicate(), Shard(0), Shard(1)], + ParallelState(ep_size=2, ep_rank=1, etp_size=2, etp_rank=1), + expert=True, + ) + + assert expert_offsets == ((0, 1, 2), (1, 1, 2)) + assert expert_replica_id == (0, 0, 0) + + +def test_dist_opt_replica_id_does_not_treat_pp_as_a_replica_axis() -> None: + rank_offsets, replica_id = _rank_offsets_and_replica_id( + [Replicate(), Replicate(), Replicate(), Shard(0)], + ParallelState(pp_size=2, pp_rank=1, tp_size=2, tp_rank=1), + expert=False, + ) + + assert rank_offsets == ((0, 1, 2),) + assert replica_id == (0, 0, 0) + + _rank_offsets, replica_id = _rank_offsets_and_replica_id( + [Replicate(), Replicate(), Replicate(), Replicate()], + ParallelState(pp_size=2, pp_rank=1, tp_size=2, tp_rank=0), + expert=False, + ) + + assert replica_id == (0, 0, 0) + + +def test_dist_opt_pp_rank_one_model_keys_survive_torch_dist_main_replica_filter() -> None: + ps = ParallelState(pp_size=2, pp_rank=1, pp_is_first=False, pp_is_last=True) + model = torch.nn.Linear(4, 2) + attach_model_sharded_state_dict([model], ps) + + model_sd = _model_sharded_state_dict(model) + filtered_sd, _flat_mapping, _rename_mapping = _replace_state_dict_keys_with_sharded_keys( + model_sd, keep_only_main_replica=True + ) + + assert set(filtered_sd) == {"model_pp1.weight", "model_pp1.bias"} + + +def test_dist_opt_model_state_keys_are_pp_and_vpp_aware() -> None: + ps = ParallelState(pp_size=2, pp_rank=1, pp_is_first=False, pp_is_last=True) + single_chunk = torch.nn.Linear(4, 2) + attach_model_sharded_state_dict([single_chunk], ps) + + single_sd = _model_sharded_state_dict(single_chunk) + + assert set(single_sd) == {"model_pp1"} + assert set(single_sd["model_pp1"]) == {"weight", "bias"} + assert single_sd["model_pp1"]["weight"].key == "model_pp1.weight" + assert _single_or_all_model_state(single_sd) is single_sd + + chunks = [torch.nn.Linear(4, 2), torch.nn.Linear(4, 2)] + attach_model_sharded_state_dict(chunks, ps) + + vpp_sd = _model_sharded_state_dict(chunks) + + assert set(vpp_sd) == {"model_pp1_vpp0", "model_pp1_vpp1"} + assert set(vpp_sd["model_pp1_vpp0"]) == {"weight", "bias"} + assert set(vpp_sd["model_pp1_vpp1"]) == {"weight", "bias"} + assert vpp_sd["model_pp1_vpp0"]["weight"].key == "model_pp1_vpp0.weight" + assert vpp_sd["model_pp1_vpp1"]["weight"].key == "model_pp1_vpp1.weight" + assert _single_or_all_model_state(vpp_sd) is vpp_sd + + +def test_dist_opt_checkpoint_loads_from_mcore_distckpt(monkeypatch, tmp_path) -> None: + wrapped_module = torch.nn.Linear(4, 2) + model = FakeWrapper(wrapped_module) + optimizer = FakeDistOpt() + attach_model_sharded_state_dict([model], ParallelState()) + expected_weight = torch.full_like(wrapped_module.weight, 3.0) + expected_bias = torch.full_like(wrapped_module.bias, -2.0) + + def fake_load(sharded_state_dict, checkpoint_dir, **kwargs): + assert set(sharded_state_dict["model"]) == {"weight", "bias"} + assert optimizer.load_model_sd is sharded_state_dict["model"] + assert sharded_state_dict["optimizer"] == {"is_loading": True} + assert checkpoint_dir == str(tmp_path / "step_5") + assert kwargs["validate_access_integrity"] is False + return { + "step": 5, + "model": {"weight": expected_weight, "bias": expected_bias}, + "optimizer": {"loaded": True}, + } + + monkeypatch.setattr("megatron.lite.primitive.ckpt.distckpt.dist_checkpointing.load", fake_load) + + step = dcp.load_training_checkpoint(model, optimizer, str(tmp_path / "step_5"), use_dcp=True) + + assert step == 5 + assert not model.wrapper_load_called + torch.testing.assert_close(wrapped_module.weight, expected_weight) + torch.testing.assert_close(wrapped_module.bias, expected_bias) + assert optimizer.loaded_state == {"loaded": True} + + +def test_dist_opt_step_sync_traverses_multi_optimizer_chain_without_optimizer_property() -> None: + class FakeTorchOptimizer: + def __init__(self, steps): + self.state = { + object(): {"step": torch.tensor(step, dtype=torch.int64)} for step in steps + } + + class FakeDistOpt: + def __init__(self, steps): + self.optimizer = FakeTorchOptimizer(steps) + + class FakeChainedOptimizer: + def __init__(self): + self.chained_optimizers = [FakeDistOpt([1, 3]), FakeDistOpt([2, 4])] + + @property + def optimizer(self): + raise AssertionError( + "ChainedOptimizer has more than one optimizer when accessing self.optimizer" + ) + + chained = FakeChainedOptimizer() + + _synchronize_native_optimizer_steps(chained) + + for child in chained.chained_optimizers: + steps = [int(state["step"].item()) for state in child.optimizer.state.values()] + assert steps == [max(steps)] * len(steps) + + +def test_runtime_checkpoint_api_passes_current_training_checkpoint_signature( + monkeypatch, tmp_path +) -> None: + calls = {} + + def fake_save(model, optimizer, step, path, config, ps, **kwargs): + calls["save"] = (model, optimizer, step, path, config, ps, kwargs) + + def fake_load(model, optimizer, path, config, ps, **kwargs): + calls["load"] = (model, optimizer, path, config, ps, kwargs) + return 7 + + monkeypatch.setattr("megatron.lite.primitive.ckpt.save_training_checkpoint", fake_save) + monkeypatch.setattr("megatron.lite.primitive.ckpt.load_training_checkpoint", fake_load) + + runtime = MegatronLiteRuntime.__new__(MegatronLiteRuntime) + model = torch.nn.Linear(1, 1) + optimizer = object() + parallel = SimpleNamespace(tp=1, etp=1, ep=1, pp=1, cp=1) + ps = object() + handle = ModelHandle( + model=model, + optimizer=optimizer, + parallel_state=ps, + config=SimpleNamespace(parallel=parallel), + ) + + runtime.save_checkpoint( + handle, str(tmp_path), global_step=7, save_model=True, save_optimizer=False + ) + loaded_step = runtime.load_checkpoint( + handle, str(tmp_path), load_model=False, load_optimizer=True + ) + + assert calls["save"] == ( + model, + optimizer, + 7, + str(tmp_path), + parallel, + ps, + { + "get_placements": default_placement_fn, + "is_expert": default_expert_classifier, + "use_dcp": True, + "save_rng": True, + "save_model": True, + "save_optimizer": False, + }, + ) + assert calls["load"] == ( + model, + optimizer, + str(tmp_path), + parallel, + ps, + { + "get_placements": default_placement_fn, + "is_expert": default_expert_classifier, + "use_dcp": True, + "load_rng": True, + "load_parameter_state_update_legacy_format": False, + "load_model": False, + "load_optimizer": True, + }, + ) + assert loaded_step == 7 diff --git a/experimental/lite/tests/unit/runtime/test_bridge_backend.py b/experimental/lite/tests/unit/runtime/test_bridge_backend.py new file mode 100644 index 00000000000..90368fe0c56 --- /dev/null +++ b/experimental/lite/tests/unit/runtime/test_bridge_backend.py @@ -0,0 +1,190 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Unit tests for the Megatron-Bridge runtime backend surface.""" + +from __future__ import annotations + +import os +import sys +import types + +import pytest + + +def test_bridge_runtime_registered_and_lazy_constructible(): + from megatron.lite.runtime import RuntimeConfig, create_runtime + from megatron.lite.runtime.backends import RUNTIME_REGISTRY + from megatron.lite.runtime.backends.bridge.config import BridgeConfig + from megatron.lite.runtime.backends.bridge.runtime import BridgeRuntime + + assert RUNTIME_REGISTRY["bridge"] == "megatron.lite.runtime.backends.bridge" + + runtime = create_runtime( + RuntimeConfig( + backend="bridge", + hf_path="/tmp/hf-model", + backend_cfg=BridgeConfig(model_name="qwen3_5"), + ) + ) + + assert isinstance(runtime, BridgeRuntime) + assert runtime.tier == "rl_best" + + +def test_mbridge_runtime_registered_and_lazy_constructible(): + from megatron.lite.runtime import RuntimeConfig, create_runtime + from megatron.lite.runtime.backends import RUNTIME_REGISTRY + from megatron.lite.runtime.backends.bridge.config import BridgeConfig + from megatron.lite.runtime.backends.mbridge.runtime import MBridgeRuntime + + assert RUNTIME_REGISTRY["mbridge"] == "megatron.lite.runtime.backends.mbridge" + + runtime = create_runtime( + RuntimeConfig( + backend="mbridge", + hf_path="/tmp/hf-model", + backend_cfg=BridgeConfig(model_name="qwen3_5"), + ) + ) + + assert isinstance(runtime, MBridgeRuntime) + assert runtime.tier == "rl_best" + + +def test_bridge_config_from_dict_accepts_nested_and_flat_parallel_fields(): + from megatron.lite.runtime.backends.bridge.config import BridgeConfig + + cfg = BridgeConfig.from_dict( + { + "model_name": "qwen3_5", + "parallel": {"tp": 2, "pp": 1}, + "ep": 4, + "optimizer": {"lr": 2e-4, "weight_decay": 0.2}, + "lr_scheduler": {"total_training_steps": 16}, + "override_transformer_config": {"attention_backend": "unfused"}, + } + ) + + assert cfg.model_name == "qwen3_5" + assert cfg.parallel.tp == 2 + assert cfg.parallel.ep == 4 + assert cfg.optimizer.lr == 2e-4 + assert cfg.optimizer.weight_decay == 0.2 + assert cfg.optimizer.total_training_steps == 16 + assert cfg.override_transformer_config == {"attention_backend": "unfused"} + + +def test_bridge_config_from_dict_rejects_num_microbatches(): + from megatron.lite.runtime.backends.bridge.config import BridgeConfig + + with pytest.raises(ValueError, match="num_microbatches"): + BridgeConfig.from_dict({"num_microbatches": 2}) + + +def test_bridge_builds_mcore_ddp_config_object(monkeypatch): + from megatron.lite.runtime.backends.bridge.config import BridgeConfig + from megatron.lite.runtime.backends.bridge.runtime import _build_ddp_config + + class _FakeDDPConfig: + def __init__(self, **kwargs): + for key, value in kwargs.items(): + setattr(self, key, value) + + monkeypatch.setitem( + sys.modules, + "megatron.core.distributed", + types.SimpleNamespace(DistributedDataParallelConfig=_FakeDDPConfig), + ) + + ddp_config = _build_ddp_config( + BridgeConfig(override_ddp_config={"overlap_grad_reduce": True, "bucket_size": 1024}) + ) + + assert isinstance(ddp_config, _FakeDDPConfig) + assert ddp_config.use_distributed_optimizer is True + assert ddp_config.grad_reduce_in_fp32 is True + assert ddp_config.overlap_grad_reduce is True + assert ddp_config.bucket_size == 1024 + + +def test_bridge_deterministic_provider_sets_te_env(monkeypatch): + from megatron.lite.runtime.backends.bridge.config import BridgeConfig + from megatron.lite.runtime.backends.bridge.runtime import _configure_provider + + monkeypatch.setenv("MEGATRON_LITE_DETERMINISTIC", "1") + monkeypatch.delenv("NVTE_ALLOW_NONDETERMINISTIC_ALGO", raising=False) + provider = types.SimpleNamespace() + + _configure_provider(provider, BridgeConfig()) + + assert provider.deterministic_mode is True + assert "NVTE_ALLOW_NONDETERMINISTIC_ALGO" in os.environ + assert os.environ["NVTE_ALLOW_NONDETERMINISTIC_ALGO"] == "0" + + +def test_bridge_registers_qwen35_moe_compat_aliases(monkeypatch): + from megatron.lite.runtime.backends.bridge.runtime import _register_bridge_compat_aliases + + registered = [] + + class _FakeMapping: + def __init__(self, *args, **kwargs): + self.args = args + self.kwargs = kwargs + + @classmethod + def register_module_type(cls, *args, **kwargs): + return None + + class _FakeQwen3NextBridge: + def provider_bridge(self, hf_pretrained): + return types.SimpleNamespace() + + class _FakeDispatcher: + _exact_types = {} + + model_bridge = types.SimpleNamespace( + get_model_bridge=_FakeDispatcher(), + register_bridge_implementation=lambda **kwargs: registered.append(kwargs), + ) + mapping_registry_mod = types.SimpleNamespace(MegatronMappingRegistry=lambda *items: list(items)) + param_mapping_mod = types.SimpleNamespace( + AutoMapping=_FakeMapping, + GDNConv1dMapping=_FakeMapping, + GatedMLPMapping=_FakeMapping, + QKVMapping=_FakeMapping, + ReplicatedMapping=_FakeMapping, + RMSNorm2ZeroCenteredRMSNormMapping=_FakeMapping, + merge_gdn_linear_weights=lambda *args, **kwargs: None, + split_gdn_linear_weights=lambda *args, **kwargs: (None, None), + ) + qwen_bridge_mod = types.SimpleNamespace(Qwen3NextBridge=_FakeQwen3NextBridge) + common_utils_mod = types.SimpleNamespace(extract_expert_number_from_param=lambda name: 0) + gpt_mod = types.SimpleNamespace(GPTModel=object) + + monkeypatch.setitem( + sys.modules, + "megatron.bridge.models.conversion", + types.SimpleNamespace(model_bridge=model_bridge), + ) + monkeypatch.setitem( + sys.modules, "megatron.bridge.models.conversion.mapping_registry", mapping_registry_mod + ) + monkeypatch.setitem( + sys.modules, "megatron.bridge.models.conversion.param_mapping", param_mapping_mod + ) + monkeypatch.setitem( + sys.modules, "megatron.bridge.models.qwen.qwen3_next_bridge", qwen_bridge_mod + ) + monkeypatch.setitem(sys.modules, "megatron.bridge.utils.common_utils", common_utils_mod) + monkeypatch.setitem(sys.modules, "megatron.core.models.gpt.gpt_model", gpt_mod) + + _register_bridge_compat_aliases() + + assert [item["source"] for item in registered] == [ + "Qwen3_5MoeForConditionalGeneration", + "Qwen3_5MoeForCausalLM", + ] + assert all(issubclass(item["bridge_class"], _FakeQwen3NextBridge) for item in registered) + assert all(item["bridge_class"] is not _FakeQwen3NextBridge for item in registered) + assert all("provider_bridge" in item["bridge_class"].__dict__ for item in registered) + assert all("mapping_registry" in item["bridge_class"].__dict__ for item in registered) diff --git a/experimental/lite/tests/unit/runtime/test_layering_contracts.py b/experimental/lite/tests/unit/runtime/test_layering_contracts.py new file mode 100644 index 00000000000..c336579dd3f --- /dev/null +++ b/experimental/lite/tests/unit/runtime/test_layering_contracts.py @@ -0,0 +1,235 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Static layering guards for MLite public data boundaries and imports. + +Five layers — bench, verl_mlite, runtime, model, primitive — have directional +import boundaries (``test_layer_import_boundaries``). On top of that the bench +and verl connector layers must hand the runtime only a model-agnostic batch: +``packed_seq_params`` / ``position_ids`` are *transient* THD metadata that may be +materialised only at the immediate forward boundary, inside the explicitly marked +allow range in the bridge runtime. + +The guard enforces the contract; it does not police prose. Substring scans ignore +string/comment content (so a docstring may still document a dependency), and the +import denylist honours :data:`IMPORT_ALLOWLIST` for vetted, optional cross-layer +imports whose reason is recorded inline. +""" + +from __future__ import annotations + +import ast +import io +import tokenize +from collections.abc import Iterable +from pathlib import Path + +LITE_ROOT = Path(__file__).resolve().parents[3] +BENCH_ROOT = LITE_ROOT / "examples" / "bench" +VERL_MLITE_ROOT = LITE_ROOT / "examples" / "verl" / "verl_mlite" +RUNTIME_ROOT = LITE_ROOT / "megatron" / "lite" / "runtime" +MODEL_ROOT = LITE_ROOT / "megatron" / "lite" / "model" +PRIMITIVE_ROOT = LITE_ROOT / "megatron" / "lite" / "primitive" +BRIDGE_RUNTIME = RUNTIME_ROOT / "backends" / "bridge" / "runtime.py" + +ALLOW_BEGIN = "MLITE_LAYERING_ALLOW_BRIDGE_FORWARD_METADATA_BEGIN" +ALLOW_END = "MLITE_LAYERING_ALLOW_BRIDGE_FORWARD_METADATA_END" +LAYER_ROOTS = { + "bench": BENCH_ROOT, + "verl_mlite": VERL_MLITE_ROOT, + "runtime": RUNTIME_ROOT, + "model": MODEL_ROOT, + "primitive": PRIMITIVE_ROOT, +} +MODEL_PACKAGE_PREFIXES = ( + "megatron.lite.model.deepseek_v4", + "megatron.lite.model.glm5", + "megatron.lite.model.kimi_k2", + "megatron.lite.model.qwen3_5", + "megatron.lite.model.qwen3_moe", +) +MODEL_NAME_TERMS = {"deepseek_v4", "glm5", "kimi_k2", "qwen3", "qwen3_5", "qwen3_moe"} +DENIED_IMPORT_PREFIXES = { + "bench": ("examples.verl", "verl", "verl_mlite", "megatron.lite.model"), + "verl_mlite": ("examples.bench", *MODEL_PACKAGE_PREFIXES), + "runtime": ("examples", "verl", "verl_mlite", *MODEL_PACKAGE_PREFIXES), + "model": ("examples", "verl", "verl_mlite", "megatron.lite.runtime.backends"), + "primitive": ( + "examples", + "verl", + "verl_mlite", + "megatron.lite.model", + "megatron.lite.runtime.backends", + "megatron.lite.runtime.megatron_utils", + ), +} + +# Vetted cross-layer imports that override the denylist. Each entry is a single +# file -> {allowed module prefix: reason}. Use this ONLY for a sanctioned, +# self-contained optional dependency — never to paper over a structural leak. +IMPORT_ALLOWLIST: dict[str, dict[str, str]] = { + # The primitive linear-cross-entropy op uses VERL's Triton-backed fused kernel + # as an optional CUDA fast path and falls back to the local torch implementation + # when the kernel (or verl) is not importable. It is a legitimate optional + # dependency on a single leaf op, not a connector back-edge, so it overrides the + # primitive `verl` denylist. + "megatron/lite/primitive/ops/linear_cross_entropy.py": { + "verl.utils.kernel.linear_cross_entropy": ( + "optional VERL fused linear-cross-entropy kernel fast-path with local fallback" + ), + }, +} + + +def _python_files(root: Path) -> list[Path]: + return sorted(path for path in root.rglob("*.py") if path.is_file()) + + +def _matches_prefix(module: str, prefix: str) -> bool: + return module == prefix or module.startswith(prefix + ".") + + +def _allowlisted_import(rel_path: str, module: str) -> bool: + for allowed in IMPORT_ALLOWLIST.get(rel_path, {}): + if _matches_prefix(module, allowed): + return True + return False + + +def _imported_modules(path: Path) -> list[tuple[int, str]]: + tree = ast.parse(path.read_text(encoding="utf-8")) + imports: list[tuple[int, str]] = [] + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imports.extend((node.lineno, alias.name) for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: + imports.append((node.lineno, node.module)) + return imports + + +def _code_lines(path: Path) -> list[str]: + """Return source lines with string/comment spans blanked out. + + The contract guard targets executable references, not documentation: a + docstring or comment may legitimately mention ``packed_seq_params`` or a model + name to explain provenance. Blanking string/comment tokens keeps those out of + the substring scan while preserving exact line numbers. Falls back to raw lines + if the file does not tokenize (fail toward flagging, never toward hiding). + """ + text = path.read_text(encoding="utf-8") + lines = text.splitlines() + blanked = [list(line) for line in lines] + masked_types = {tokenize.STRING, tokenize.COMMENT} + fstring_middle = getattr(tokenize, "FSTRING_MIDDLE", None) + if fstring_middle is not None: + masked_types.add(fstring_middle) + try: + tokens = list(tokenize.generate_tokens(io.StringIO(text).readline)) + except (tokenize.TokenError, IndentationError, SyntaxError): + return lines + for tok in tokens: + if tok.type not in masked_types: + continue + (srow, scol), (erow, ecol) = tok.start, tok.end + for row in range(srow, erow + 1): + chars = blanked[row - 1] + start = scol if row == srow else 0 + end = ecol if row == erow else len(chars) + for col in range(start, min(end, len(chars))): + chars[col] = " " + return ["".join(chars) for chars in blanked] + + +def _allow_ranges(path: Path) -> list[range]: + if path != BRIDGE_RUNTIME: + return [] + + ranges: list[range] = [] + start: int | None = None + for lineno, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): + if ALLOW_BEGIN in line: + if start is not None: + raise AssertionError(f"nested allow range in {path}") + start = lineno + elif ALLOW_END in line: + if start is None: + raise AssertionError(f"unmatched allow range end in {path}:{lineno}") + ranges.append(range(start, lineno + 1)) + start = None + if start is not None: + raise AssertionError(f"unclosed allow range in {path}") + return ranges + + +def _violations(paths: Iterable[Path], denied_terms: set[str]) -> list[str]: + found: list[str] = [] + for path in paths: + ranges = _allow_ranges(path) + for lineno, line in enumerate(_code_lines(path), start=1): + if any(lineno in allowed for allowed in ranges): + continue + for term in sorted(denied_terms): + if term in line: + rel = path.relative_to(LITE_ROOT) + found.append(f"{rel}:{lineno}: {term}") + return found + + +def _import_violations(layer: str) -> list[str]: + denied = DENIED_IMPORT_PREFIXES[layer] + found: list[str] = [] + for path in _python_files(LAYER_ROOTS[layer]): + rel = path.relative_to(LITE_ROOT).as_posix() + for lineno, module in _imported_modules(path): + if _allowlisted_import(rel, module): + continue + for prefix in denied: + if _matches_prefix(module, prefix): + found.append(f"{rel}:{lineno}: {module} matches denied {prefix}") + return found + + +def test_layer_import_boundaries() -> None: + violations = [] + for layer in LAYER_ROOTS: + violations.extend(_import_violations(layer)) + assert violations == [] + + +def test_import_allowlist_entries_are_live() -> None: + """Each allowlisted import must still exist — stop the allowlist from rotting.""" + stale: list[str] = [] + for rel_path, allowed in IMPORT_ALLOWLIST.items(): + path = LITE_ROOT / rel_path + if not path.is_file(): + stale.append(f"{rel_path}: file missing") + continue + modules = {module for _, module in _imported_modules(path)} + for allowed_module in allowed: + if not any(_matches_prefix(module, allowed_module) for module in modules): + stale.append(f"{rel_path}: no import matches allowlisted {allowed_module}") + assert stale == [] + + +def test_bench_layer_does_not_see_model_internal_batch_fields() -> None: + violations = _violations( + _python_files(BENCH_ROOT), + {"packed_seq_params", "position_ids", "to_bridge_dict"}, + ) + assert violations == [] + + +def test_verl_mlite_layer_does_not_see_model_internal_batch_fields() -> None: + violations = _violations( + _python_files(VERL_MLITE_ROOT), + {"packed_seq_params", "position_ids", "to_bridge_dict"}, + ) + assert violations == [] + + +def test_runtime_packed_seq_params_is_bridge_forward_transient_only() -> None: + violations = _violations(_python_files(RUNTIME_ROOT), {"packed_seq_params"}) + assert violations == [] + + +def test_primitive_layer_is_model_name_agnostic() -> None: + violations = _violations(_python_files(PRIMITIVE_ROOT), MODEL_NAME_TERMS) + assert violations == [] diff --git a/experimental/lite/tests/unit/runtime/test_packed_batch_bridge.py b/experimental/lite/tests/unit/runtime/test_packed_batch_bridge.py new file mode 100644 index 00000000000..0ef4629679b --- /dev/null +++ b/experimental/lite/tests/unit/runtime/test_packed_batch_bridge.py @@ -0,0 +1,142 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""Unit coverage for the PackedBatch -> bridge forward metadata boundary. + +CPU-only. The public ``PackedBatch`` contract stays free of transient THD metadata +(``packed_seq_params``); the bridge runtime renders Megatron-Core THD kwargs at the +forward call using the same canonical packing as the native lite protocols, so the +mlite-vs-bridge comparison is fair and context-parallel-correct. +""" + +from __future__ import annotations + +import torch +from examples.bench.session import _infinite_packed_batches +from megatron.lite.primitive.parallel.thd import ( + pack_nested_thd, + reconstruct_packed_from_cp_parts, +) +from megatron.lite.runtime.backends.bridge.runtime import ( + _bridge_forward_kwargs_from_packed_batch, + _nested_from_packed, +) +from megatron.lite.runtime.contracts.data import Batch, PackedBatch + + +def _packed_batch() -> PackedBatch: + return PackedBatch( + input_ids=torch.arange(8, dtype=torch.long), + labels=torch.arange(100, 108, dtype=torch.long), + seq_lens=torch.tensor([3, 5], dtype=torch.int64), + ) + + +def test_packed_batch_contract_carries_no_transient_thd_metadata() -> None: + # The contract keeps its optional extensibility slots (position_ids for custom + # layouts, routed_experts for router replay, extras for multimodal) but must + # never carry the transient Megatron-Core THD metadata the bridge derives. + batch = _packed_batch() + assert hasattr(batch, "position_ids") + assert hasattr(batch, "routed_experts") + assert hasattr(batch, "extras") + assert batch.position_ids is None + assert not hasattr(batch, "packed_seq_params") + assert not hasattr(batch, "to_bridge_dict") + + +def test_packed_batch_is_batch_subclass() -> None: + assert issubclass(PackedBatch, Batch) + + +def test_bridge_forward_kwargs_are_transient_bridge_metadata() -> None: + batch = _packed_batch() + out = _bridge_forward_kwargs_from_packed_batch(batch) + + assert set(out) == {"input_ids", "labels", "position_ids", "packed_seq_params"} + assert out["input_ids"].shape == (1, 8) + assert out["labels"].shape == (1, 8) + assert out["position_ids"].shape == (1, 8) + # tp=cp=1 -> no padding, input ids unchanged. + assert torch.equal(out["input_ids"].reshape(-1), batch.input_ids) + # Labels are rolled one position left per sequence (last token zeroed), exactly + # like the native pack_thd_forward_kwargs path, so both backends train on the + # same shifted targets. + assert torch.equal( + out["labels"].reshape(-1), + torch.tensor([101, 102, 0, 104, 105, 106, 107, 0]), + ) + assert torch.equal( + out["position_ids"].reshape(-1), + torch.tensor([0, 1, 2, 0, 1, 2, 3, 4]), + ) + + psp = out["packed_seq_params"] + assert psp.qkv_format == "thd" + assert torch.equal(psp.cu_seqlens_q, torch.tensor([0, 3, 8], dtype=torch.int32)) + assert psp.max_seqlen_q == 5 + + +def test_bridge_forward_kwargs_carry_loss_mask_only_inside_bridge() -> None: + batch = _packed_batch() + batch.loss_mask = torch.tensor([1, 1, 0, 1, 1, 1, 0, 1], dtype=torch.long) + out = _bridge_forward_kwargs_from_packed_batch(batch) + assert "loss_mask" in out + # loss_mask is rolled with the labels so it masks the shifted targets. + assert torch.equal( + out["loss_mask"].reshape(-1), + torch.tensor([1, 0, 0, 1, 1, 0, 1, 0]), + ) + + +def test_bridge_forward_kwargs_are_context_parallel_correct() -> None: + # cp_size=2 must pad to the zigzag (2*cp) alignment, keep full cu_seqlens, and + # hand each rank only its half of the tokens — the exact behaviour the previous + # hand-rolled implementation lacked. + batch = _packed_batch() + seq_lens = batch.seq_lens + + # Full CP-aligned reference (padded but not yet CP-split). + ref = pack_nested_thd( + _nested_from_packed(batch.input_ids, seq_lens), + tp_size=1, + cp_size=2, + cp_rank=0, + split_cp=False, + ) + full_padded = int(ref.cu_seqlens_padded[-1].item()) + assert full_padded == 12 # [3->4, 5->8] padded to 2*cp alignment + + rank0 = _bridge_forward_kwargs_from_packed_batch(batch, cp_size=2, cp_rank=0) + rank1 = _bridge_forward_kwargs_from_packed_batch(batch, cp_size=2, cp_rank=1) + + for local in (rank0, rank1): + assert local["input_ids"].shape == (1, full_padded // 2) + assert local["position_ids"].shape == (1, full_padded // 2) + psp = local["packed_seq_params"] + # cu_seqlens stays full (CP-aligned), not the unpadded [0, 3, 8]. + assert torch.equal(psp.cu_seqlens_q, ref.cu_seqlens_padded) + assert int(getattr(psp, "local_cp_size", 1)) == 2 + + # The two rank-local zigzag shards reconstruct the full padded sequence. + recon = reconstruct_packed_from_cp_parts( + [rank0["input_ids"][0], rank1["input_ids"][0]], + cu_seqlens_padded=ref.cu_seqlens_padded, + cp_size=2, + dim=0, + ) + assert torch.equal(recon, ref.input_ids[0]) + + +def test_infinite_packed_batches_shape_and_determinism() -> None: + gen_a = _infinite_packed_batches(vocab_size=32, seq_len=6, device="cpu", seed=7) + gen_b = _infinite_packed_batches(vocab_size=32, seq_len=6, device="cpu", seed=7) + + a = next(gen_a) + b = next(gen_b) + assert isinstance(a, PackedBatch) + assert a.input_ids.shape == (6,) + assert a.labels.shape == (6,) + assert torch.equal(a.seq_lens, torch.tensor([6], dtype=torch.int64)) + # No transient THD metadata baked into bench data. + assert a.position_ids is None + assert torch.equal(a.input_ids, b.input_ids) + assert torch.equal(a.labels, b.labels) diff --git a/experimental/lite/tests/unit/runtime/test_runtime_backend_unit.py b/experimental/lite/tests/unit/runtime/test_runtime_backend_unit.py new file mode 100644 index 00000000000..a93c4176abd --- /dev/null +++ b/experimental/lite/tests/unit/runtime/test_runtime_backend_unit.py @@ -0,0 +1,482 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +from __future__ import annotations + +import os +import subprocess +import sys +import types +from dataclasses import dataclass +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +import torch +import torch.nn as nn + +from megatron.lite.runtime import create_runtime +from megatron.lite.runtime.backends.mlite.config import MegatronLiteConfig +from megatron.lite.runtime.backends.mlite.runtime import ( + MegatronLiteRuntime, + _apply_attention_backend_env, + _build_impl_cfg, + _pipeline_callbacks, +) +from megatron.lite.runtime.contracts.config import OptimizerConfig, ParallelConfig, RuntimeConfig +from megatron.lite.runtime.contracts.handle import ModelHandle +from megatron.lite.runtime.contracts.loss import LossContext, get_loss_context, use_loss_context + +pytestmark = pytest.mark.mlite + + +def test_runtime_returns_loss_separately_from_microbatch_metrics(): + model = nn.Linear(1, 1, bias=False) + handle = ModelHandle( + model=model, + parallel_state=types.SimpleNamespace(pp_size=1), + _extras={"forward_step": lambda module, batch: {"value": module(batch["x"])}}, + ) + batches = iter([{"x": torch.ones(1, 1), "micro": i} for i in range(2)]) + result = MegatronLiteRuntime.__new__(MegatronLiteRuntime).forward_backward( + handle, + batches, + lambda out, batch: (out["value"].sum(), {"micro": batch["micro"]}), + num_microbatches=2, + ) + + assert result.metrics == {"micro": [0, 1]} + assert result.model_output.loss is not None + + +def test_pipeline_callbacks_accept_wrapped_and_presplit_context(): + context = LossContext(source_batch="source") + seen = [] + forward, loss = _pipeline_callbacks( + lambda _model, batch: seen.append((batch, get_loss_context())) + or {"loss": torch.tensor(1.0)}, + lambda out, batch, ctx: (out["loss"], {"batch": batch, "source": ctx.source_batch}), + ) + + output = forward(None, ("wrapped", context)) + with use_loss_context(context): + forward(None, "presplit") + _, metrics = loss(output, "presplit", context) + + assert seen == [("wrapped", context), ("presplit", context)] + assert metrics == {"batch": "presplit", "source": "source"} + + +def test_runtime_config_defaults_to_mlite_backend(): + cfg = RuntimeConfig() + + assert cfg.backend == "mlite" + assert cfg.hf_path == "" + assert isinstance(cfg.backend_cfg, dict) + + +def test_runtime_config_accepts_mlite_backend_cfg(): + cfg = RuntimeConfig( + backend="mlite", + hf_path="/models/Qwen3", + backend_cfg={"model_name": "qwen3", "impl": "lite", "tp": 2, "ep": 4}, + ) + + assert cfg.backend == "mlite" + assert cfg.backend_cfg["model_name"] == "qwen3" + assert cfg.backend_cfg["tp"] == 2 + + +def test_mlite_config_defaults_and_parallel_fields(): + cfg = MegatronLiteConfig( + model_name="qwen3_moe", parallel=ParallelConfig(tp=4, etp=1, ep=8, pp=2, vpp=2, cp=2) + ) + + assert cfg.model_name == "qwen3_moe" + assert cfg.impl == "lite" + assert cfg.parallel.tp == 4 + assert cfg.parallel.ep == 8 + assert cfg.parallel.pp == 2 + assert cfg.parallel.cp == 2 + + +def test_mlite_config_impl_cfg_optimizer_and_load_gate(): + hook = lambda cfg: cfg # noqa: E731 + cfg = MegatronLiteConfig( + model_name="qwen3_moe", + impl_cfg={"recompute": "full", "use_deepep": True}, + optimizer=OptimizerConfig(lr=1e-4, weight_decay=0.1, adam_beta1=0.9), + load_hf_weights=False, + model_config_hook=hook, + ) + + assert cfg.impl_cfg["recompute"] == "full" + assert cfg.impl_cfg["use_deepep"] is True + assert cfg.optimizer.lr == 1e-4 + assert cfg.optimizer.adam_beta1 == 0.9 + assert cfg.load_hf_weights is False + assert cfg.model_config_hook is hook + + +def test_mlite_config_from_dict_accepts_optimizer_override_config(): + cfg = MegatronLiteConfig.from_dict( + "/models/Qwen3", + { + "optimizer": { + "override_optimizer_config": { + "fsdp2_use_fp32_master": False, + "offload_fraction": 1.0, + } + } + }, + ) + + assert cfg.optimizer.override_optimizer_config == { + "fsdp2_use_fp32_master": False, + "offload_fraction": 1.0, + } + + +def test_mlite_config_from_dict_rejects_num_microbatches(): + with pytest.raises(ValueError, match="num_microbatches"): + MegatronLiteConfig.from_dict( + "/models/Qwen3", {"model_name": "qwen3", "tp": 4, "num_microbatches": 2} + ) + + +@dataclass +class _FakeImplConfig: + parallel: object + hf_path: str = "" + optimizer_config: object = None + attention_backend_override: str | None = None + + +def test_build_impl_cfg_backfills_top_level_hf_path_and_runtime_fields(): + proto = type("Proto", (), {"ImplConfig": _FakeImplConfig}) + cfg = MegatronLiteConfig( + model_name="qwen3", hf_path="/models/top", attention_backend_override="local" + ) + + impl_cfg = _build_impl_cfg(proto, cfg) + + assert impl_cfg.parallel is cfg.parallel + assert impl_cfg.hf_path == "/models/top" + assert impl_cfg.optimizer_config is cfg.optimizer + assert impl_cfg.attention_backend_override == "local" + + +def test_build_impl_cfg_preserves_explicit_impl_hf_path(): + proto = type("Proto", (), {"ImplConfig": _FakeImplConfig}) + cfg = MegatronLiteConfig( + model_name="qwen3", hf_path="/models/top", impl_cfg={"hf_path": "/models/impl"} + ) + + impl_cfg = _build_impl_cfg(proto, cfg) + + assert impl_cfg.hf_path == "/models/impl" + + +@pytest.mark.parametrize( + ("backend", "expected"), + [ + ("auto", ("1", "1", "1")), + ("flash", ("1", "0", "0")), + ("fused", ("0", "1", "0")), + ("unfused", ("0", "0", "1")), + ("local", ("0", "0", "0")), + ], +) +def test_attention_backend_override_sets_expected_env(monkeypatch, backend, expected): + for name in ("NVTE_FLASH_ATTN", "NVTE_FUSED_ATTN", "NVTE_UNFUSED_ATTN"): + monkeypatch.delenv(name, raising=False) + + _apply_attention_backend_env(backend, tag="unit") + + assert ( + os.environ["NVTE_FLASH_ATTN"], + os.environ["NVTE_FUSED_ATTN"], + os.environ["NVTE_UNFUSED_ATTN"], + ) == expected + + +def test_attention_backend_override_rejects_unknown_backend(): + with pytest.raises(ValueError, match="attention_backend_override"): + _apply_attention_backend_env("invalid", tag="unit") + + +class HookedOptimizer: + def __init__(self): + self.calls: list[str] = [] + + def offload_state_to_cpu(self): + self.calls.append("offload") + + def load_state_to_device(self): + self.calls.append("load") + + +def test_runtime_to_prefers_optimizer_specific_offload_hooks(): + optimizer = HookedOptimizer() + handle = ModelHandle(model=nn.Linear(2, 2), optimizer=optimizer, _extras={"model_chunks": []}) + runtime = MegatronLiteRuntime.__new__(MegatronLiteRuntime) + + runtime.to(handle, "cpu", model=False, optimizer=True, grad=False) + runtime.to(handle, "cuda", model=False, optimizer=True, grad=False) + + assert optimizer.calls == ["offload", "load"] + + +class _FakeStorage: + def __init__(self, size: int): + self._size = size + self.resize_calls: list[int] = [] + + def size(self): + return self._size + + def resize_(self, size: int): + self.resize_calls.append(size) + self._size = size + return self + + +class _FakeBufferData: + def __init__(self, size: int): + self._storage = _FakeStorage(size) + self.cpu_called = False + self.pinned = False + self.copied_from = None + self.copy_non_blocking = None + self.zero_calls = 0 + + @property + def data(self): + return self + + def cpu(self): + self.cpu_called = True + return self + + def pin_memory(self): + self.pinned = True + return self + + def storage(self): + return self._storage + + def copy_(self, other, *, non_blocking: bool): + self.copied_from = other + self.copy_non_blocking = non_blocking + return self + + def zero_(self): + self.zero_calls += 1 + return self + + +class _FakeBuffer: + def __init__(self): + self.param_data = _FakeBufferData(3) + self.grad_data = _FakeBufferData(5) + + +class _FakeModule: + def parameters(self): + return [] + + +class _FakeMegatronDDP: + def __init__(self): + self.buffer = _FakeBuffer() + self.buffers = [self.buffer] + self.expert_parallel_buffers = [] + self.module = _FakeModule() + self.to_calls: list[str] = [] + + def to(self, device): + self.to_calls.append(device) + raise AssertionError("DDP model chunks must use the buffer offload path") + + +class _FakeMegatronDDPSubclass(_FakeMegatronDDP): + pass + + +class _FakeNativeModel: + def __init__(self): + self.calls: list[str] = [] + + def to(self, device): + self.calls.append(device) + return self + + +def _install_fake_megatron_ddp(monkeypatch) -> None: + core = types.ModuleType("megatron.core") + distributed = types.ModuleType("megatron.core.distributed") + distributed.DistributedDataParallel = _FakeMegatronDDP + core.distributed = distributed + monkeypatch.setitem(sys.modules, "megatron.core", core) + monkeypatch.setitem(sys.modules, "megatron.core.distributed", distributed) + + +def test_megatron_ddp_detection_accepts_ddp_and_subclasses(monkeypatch): + from megatron.lite.runtime.megatron_utils import _is_megatron_ddp + + _install_fake_megatron_ddp(monkeypatch) + + assert _is_megatron_ddp(_FakeMegatronDDP()) is True + assert _is_megatron_ddp(_FakeMegatronDDPSubclass()) is True + assert _is_megatron_ddp(_FakeNativeModel()) is False + + +@pytest.mark.parametrize("model_cls", [_FakeMegatronDDP, _FakeMegatronDDPSubclass]) +def test_megatron_ddp_model_move_helpers_use_buffer_path(monkeypatch, model_cls): + from megatron.lite.runtime.megatron_utils import load_model_to_gpu, offload_model_to_cpu + + _install_fake_megatron_ddp(monkeypatch) + model = model_cls() + buffer = model.buffer + + offload_model_to_cpu([model]) + + assert model.to_calls == [] + assert buffer.param_data.cpu_called is True + assert buffer.param_data.pinned is True + assert buffer.param_data_size == 3 + assert buffer.grad_data_size == 5 + assert buffer.param_data.storage().size() == 0 + assert buffer.grad_data.storage().size() == 0 + + load_model_to_gpu([model]) + + assert model.to_calls == [] + assert buffer.param_data.storage().size() == 3 + assert buffer.grad_data.storage().size() == 5 + assert buffer.param_data.copied_from is buffer.param_data.cpu_data + assert buffer.param_data.copy_non_blocking is True + assert buffer.grad_data.zero_calls == 1 + + +def test_native_model_move_helpers_do_not_require_megatron_core(monkeypatch): + from megatron.lite.runtime.megatron_utils import load_model_to_gpu, offload_model_to_cpu + + monkeypatch.setitem(sys.modules, "megatron.core", None) + monkeypatch.setitem(sys.modules, "megatron.core.distributed", None) + model = _FakeNativeModel() + + offload_model_to_cpu([model]) + load_model_to_gpu([model]) + + assert model.calls == ["cpu", "cuda"] + + +def test_model_handle_dp_defaults(): + handle = ModelHandle(model=MagicMock()) + + assert handle.dp_rank == 0 + assert handle.dp_size == 1 + assert handle.dp_group is None + + +def test_model_handle_dp_from_parallel_state(): + ps = MagicMock() + ps.dp_rank = 3 + ps.dp_size = 8 + ps.dp_group = "fake_group" + + handle = ModelHandle(model=MagicMock(), parallel_state=ps) + + assert handle.dp_rank == 3 + assert handle.dp_size == 8 + assert handle.dp_group == "fake_group" + + +def test_model_handle_cp_range_and_config_properties(): + cfg = {"tp": 8, "ep": 4} + default_handle = ModelHandle(model=MagicMock()) + configured_handle = ModelHandle(model=MagicMock(), config=cfg, _extras={"cp_range": (1, 8)}) + + assert default_handle.cp_range == (1, 1) + assert configured_handle.cp_range == (1, 8) + assert configured_handle.config is cfg + + +def test_runtime_dispatch_creates_mlite_backend(): + with patch("megatron.lite.runtime.backends.mlite.create") as mock_create: + backend = MagicMock() + mock_create.return_value = backend + + runtime = create_runtime( + RuntimeConfig( + backend="mlite", hf_path="/models/test", backend_cfg={"model_name": "qwen3"} + ) + ) + + assert runtime is backend + mock_create.assert_called_once_with("/models/test", {"model_name": "qwen3"}) + + +def test_runtime_dispatch_unknown_backend_raises(): + with pytest.raises(KeyError): + create_runtime(RuntimeConfig(backend="nonexistent")) + + +def _run_verl_sft_dry_run(script: Path, tmp_path: Path, **env_overrides: str) -> str: + env = { + **os.environ, + "MODEL_PATH": "/tmp/mlite-model", + "TRAIN_FILES": "/tmp/mlite-train.parquet", + "OUTPUT_ROOT": str(tmp_path), + "DRY_RUN": "1", + "NUM_GPUS": "1", + "NPROC_PER_NODE": "1", + "TP_SIZE": "1", + "PP_SIZE": "1", + "CP_SIZE": "1", + "EP_SIZE": "1", + "ETP_SIZE": "1", + **env_overrides, + } + completed = subprocess.run([str(script)], env=env, text=True, capture_output=True, check=True) + return completed.stdout + + +def test_verl_sft_script_maps_offload_env_to_backend_args(tmp_path): + script = ( + Path(__file__).resolve().parents[3] + / "examples" + / "verl" + / "scripts" + / "run_qwen3moe_sft.sh" + ) + + command = _run_verl_sft_dry_run( + script, + tmp_path, + PARAM_OFFLOAD="True", + OPTIMIZER_OFFLOAD="True", + OPTIMIZER_STATE_OFFLOAD_FRACTION="0.75", + ) + + assert "engine.param_offload=True" in command + assert "engine.optimizer_offload=True" in command + assert "+optim.override_optimizer_config.offload_fraction=0.75" in command + assert "+optim.override_optimizer_config.use_precision_aware_optimizer=True" in command + + +def test_verl_sft_script_does_not_emit_optimizer_state_offload_when_disabled(tmp_path): + script = ( + Path(__file__).resolve().parents[3] + / "examples" + / "verl" + / "scripts" + / "run_qwen3moe_sft.sh" + ) + + command = _run_verl_sft_dry_run( + script, tmp_path, PARAM_OFFLOAD="False", OPTIMIZER_OFFLOAD="False" + ) + + assert "engine.param_offload=False" in command + assert "engine.optimizer_offload=False" in command + assert "override_optimizer_config.offload_fraction" not in command diff --git a/experimental/lite/tests/unit/verl/test_mlite_engine_checkpoint.py b/experimental/lite/tests/unit/verl/test_mlite_engine_checkpoint.py new file mode 100644 index 00000000000..c68ff754011 --- /dev/null +++ b/experimental/lite/tests/unit/verl/test_mlite_engine_checkpoint.py @@ -0,0 +1,187 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +from types import SimpleNamespace + +import pytest +import torch +from verl_mlite.engine.config import MegatronLiteEngineConfig +from verl_mlite.engine.mlite_engine import MegatronLiteEngine + + +class _Scheduler: + def __init__(self): + self.loaded_state = None + + def state_dict(self): + return {"step": 7, "lr": 0.25} + + def load_state_dict(self, state): + self.loaded_state = state + + +@pytest.fixture(autouse=True) +def _single_process_dist(monkeypatch): + monkeypatch.setattr("verl_mlite.engine.mlite_engine.dist.is_initialized", lambda: False) + + +def _optimizer_config() -> SimpleNamespace: + return SimpleNamespace( + optimizer="adam", + lr=1e-6, + min_lr=None, + min_lr_ratio=None, + clip_grad=1.0, + weight_decay=0.1, + lr_warmup_steps_ratio=0.0, + total_training_steps=10, + lr_warmup_steps=0, + override_optimizer_config={}, + ) + + +def _engine_config(**kwargs) -> MegatronLiteEngineConfig: + values = {"custom_backend_module": None, "impl_cfg": {"use_thd": True}} + values.update(kwargs) + return MegatronLiteEngineConfig(**values) + + +def _initialized_engine(*, checkpoint_config=None, param_offload=False): + engine = MegatronLiteEngine( + model_config=SimpleNamespace( + local_path="/tmp/qwen35", hf_config={"model_type": "qwen3_5_moe"}, mtp=None + ), + engine_config=_engine_config(param_offload=param_offload), + optimizer_config=_optimizer_config(), + checkpoint_config=checkpoint_config or {}, + ) + + def placement_fn(name): + return ["placement", name] + + def expert_classifier(name): + return name.endswith("expert") + + parallel = SimpleNamespace(tp=1, cp=1, pp=1) + parallel_state = SimpleNamespace(dp_rank=0) + module = torch.nn.Linear(2, 2) + optimizer = object() + scheduler = _Scheduler() + engine.module = module + engine.handle = SimpleNamespace( + _optimizer=optimizer, + _lr_scheduler=scheduler, + _config=SimpleNamespace(parallel=parallel), + _parallel_state=parallel_state, + _extras={ + "protocol": SimpleNamespace( + PLACEMENT_FN=placement_fn, EXPERT_CLASSIFIER=expert_classifier + ) + }, + ) + engine.runtime = object() + return ( + engine, + module, + optimizer, + scheduler, + parallel, + parallel_state, + placement_fn, + expert_classifier, + ) + + +def test_save_checkpoint_forwards_contents_scheduler_and_param_offload_reload( + tmp_path, monkeypatch +): + ( + engine, + module, + optimizer, + scheduler, + parallel, + parallel_state, + placement_fn, + expert_classifier, + ) = _initialized_engine(checkpoint_config={"save_contents": ["model"]}, param_offload=True) + to_calls = [] + save_calls = [] + sync_calls = [] + monkeypatch.setattr(engine, "to", lambda **kwargs: to_calls.append(kwargs)) + monkeypatch.setattr(torch.cuda, "synchronize", lambda: sync_calls.append(True)) + monkeypatch.setattr( + "verl_mlite.engine.mlite_engine.save_training_checkpoint", + lambda *args, **kwargs: save_calls.append((args, kwargs)), + ) + + engine.save_checkpoint(str(tmp_path), global_step=13) + + assert to_calls == [ + {"device": "cuda", "model": True, "optimizer": False, "grad": False}, + {"device": "cpu", "model": True, "optimizer": False, "grad": False}, + ] + assert sync_calls == [True] + assert len(save_calls) == 1 + save_args, save_kwargs = save_calls[0] + assert save_args == (module, optimizer, 13, str(tmp_path), parallel, parallel_state) + assert save_kwargs["get_placements"] is placement_fn + assert save_kwargs["is_expert"] is expert_classifier + assert save_kwargs["save_model"] is True + assert save_kwargs["save_optimizer"] is False + assert ( + torch.load(tmp_path / "lr_scheduler.pt", map_location="cpu", weights_only=False) + == scheduler.state_dict() + ) + + +def test_save_checkpoint_skips_when_contents_exclude_model_and_optimizer(tmp_path, monkeypatch): + engine, *_ = _initialized_engine(checkpoint_config={"save_contents": ["extra"]}) + checkpoint_path = tmp_path / "ckpt" + save_calls = [] + monkeypatch.setattr( + "verl_mlite.engine.mlite_engine.save_training_checkpoint", + lambda *args, **kwargs: save_calls.append((args, kwargs)), + ) + + engine.save_checkpoint(str(checkpoint_path), global_step=13) + + assert save_calls == [] + assert not checkpoint_path.exists() + + +def test_load_checkpoint_restores_scheduler_and_param_offload_reload(tmp_path, monkeypatch): + ( + engine, + module, + optimizer, + scheduler, + parallel, + parallel_state, + placement_fn, + expert_classifier, + ) = _initialized_engine(param_offload=True) + torch.save({"step": 23, "lr": 0.125}, tmp_path / "lr_scheduler.pt") + to_calls = [] + load_calls = [] + sync_calls = [] + monkeypatch.setattr(engine, "to", lambda **kwargs: to_calls.append(kwargs)) + monkeypatch.setattr(torch.cuda, "synchronize", lambda: sync_calls.append(True)) + monkeypatch.setattr( + "verl_mlite.engine.mlite_engine.load_training_checkpoint", + lambda *args, **kwargs: load_calls.append((args, kwargs)), + ) + + engine.load_checkpoint(str(tmp_path)) + + assert to_calls == [ + {"device": "cuda", "model": True, "optimizer": False, "grad": False}, + {"device": "cpu", "model": True, "optimizer": False, "grad": False}, + ] + assert sync_calls == [True] + assert scheduler.loaded_state == {"step": 23, "lr": 0.125} + assert len(load_calls) == 1 + load_args, load_kwargs = load_calls[0] + assert load_args == (module, optimizer, str(tmp_path), parallel, parallel_state) + assert load_kwargs["get_placements"] is placement_fn + assert load_kwargs["is_expert"] is expert_classifier + assert load_kwargs["load_model"] is True + assert load_kwargs["load_optimizer"] is True diff --git a/experimental/lite/tests/unit/verl/test_mlite_engine_config.py b/experimental/lite/tests/unit/verl/test_mlite_engine_config.py new file mode 100644 index 00000000000..0c366a62f23 --- /dev/null +++ b/experimental/lite/tests/unit/verl/test_mlite_engine_config.py @@ -0,0 +1,171 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +from types import SimpleNamespace + +import pytest +import torch + +from verl_mlite.engine.config import MegatronLiteEngineConfig +from verl_mlite.engine.mlite_engine import MegatronLiteEngine, _build_lr_scheduler +from megatron.lite.runtime.contracts import LossContext + + +def _optimizer_config(**override_optimizer_config) -> SimpleNamespace: + return SimpleNamespace( + optimizer="adam", + lr=1e-6, + min_lr=None, + min_lr_ratio=None, + clip_grad=1.0, + weight_decay=0.1, + lr_warmup_steps_ratio=0.0, + total_training_steps=10, + lr_warmup_steps=0, + lr_warmup_init=0.0, + lr_decay_steps=None, + lr_decay_style="constant", + weight_decay_incr_style="constant", + lr_wsd_decay_style="exponential", + lr_wsd_decay_steps=None, + use_checkpoint_opt_param_scheduler=False, + betas=(0.9, 0.95), + override_optimizer_config=override_optimizer_config, + ) + + +def _engine( + *, engine_config: MegatronLiteEngineConfig, optimizer_config: SimpleNamespace | None = None +) -> MegatronLiteEngine: + return MegatronLiteEngine( + model_config=SimpleNamespace( + local_path="/tmp/qwen35", hf_config={"model_type": "qwen3_5_moe"}, mtp=None + ), + engine_config=engine_config, + optimizer_config=optimizer_config or _optimizer_config(), + checkpoint_config={}, + ) + + +def _engine_config(**kwargs) -> MegatronLiteEngineConfig: + values = {"custom_backend_module": None, "impl_cfg": {"use_thd": True}} + values.update(kwargs) + return MegatronLiteEngineConfig(**values) + + +@pytest.mark.parametrize("num_microbatches", [1, 4]) +def test_verl_loss_hook_preserves_gradient_and_micro_outputs(num_microbatches): + engine = _engine(engine_config=_engine_config()) + weight = torch.nn.Parameter(torch.tensor(1.0)) + outputs = [] + engine._build_verl_model_output = lambda **_kwargs: {"log_probs": weight * 3} + engine.get_data_parallel_group = lambda: None + + hook = engine._make_runtime_loss_fn( + lambda model_output, **_kwargs: (model_output["log_probs"] / num_microbatches, {}), + num_microbatches=num_microbatches, + output_lst=outputs, + ) + for _ in range(num_microbatches): + loss, _ = hook({}, object(), LossContext(source_batch=object())) + (loss / num_microbatches).backward() + + torch.testing.assert_close(weight.grad, torch.tensor(3.0)) + assert [output["loss"] for output in outputs] == [3.0 / num_microbatches] * num_microbatches + + +def test_optimizer_offload_enables_full_optimizer_state_offload_by_default() -> None: + engine = _engine( + engine_config=_engine_config(optimizer_offload=True), + optimizer_config=_optimizer_config( + use_precision_aware_optimizer=True, decoupled_weight_decay=True + ), + ) + + optimizer = engine._build_mlite_optimizer_config() + + assert optimizer.offload_fraction == 1.0 + assert optimizer.use_precision_aware_optimizer is True + assert optimizer.decoupled_weight_decay is True + assert optimizer.adam_beta1 == 0.9 + assert optimizer.adam_beta2 == 0.95 + + +def test_explicit_optimizer_offload_fraction_overrides_engine_default() -> None: + engine = _engine( + engine_config=_engine_config(optimizer_offload=True), + optimizer_config=_optimizer_config(offload_fraction=0.25), + ) + + optimizer = engine._build_mlite_optimizer_config() + + assert optimizer.offload_fraction == 0.25 + + +def test_optimizer_cpu_offload_alias_maps_to_full_offload_fraction() -> None: + engine = _engine( + engine_config=_engine_config(optimizer_offload=False), + optimizer_config=_optimizer_config(optimizer_cpu_offload=True), + ) + + optimizer = engine._build_mlite_optimizer_config() + + assert optimizer.offload_fraction == 1.0 + + +def test_mlite_config_threads_rl_parallel_and_impl_settings() -> None: + engine = _engine( + engine_config=_engine_config( + tp=2, + ep=8, + etp=1, + pp=1, + cp=1, + optimizer_offload=True, + attention_backend_override="flash", + impl_cfg={"use_thd": True, "deterministic": False}, + ) + ) + + config = engine._build_mlite_config() + + assert config.model_name == "qwen3_5" + assert config.impl == "lite" + assert config.parallel.tp == 2 + assert config.parallel.ep == 8 + assert config.parallel.etp == 1 + assert config.optimizer.offload_fraction == 1.0 + assert config.attention_backend_override == "flash" + assert config.impl_cfg["use_thd"] is True + assert config.impl_cfg["deterministic"] is False + + +def test_local_lr_scheduler_warmup_decay_and_state_roundtrip() -> None: + optimizer = SimpleNamespace(param_groups=[{"lr": 0.0, "weight_decay": 0.1}]) + opt = SimpleNamespace( + total_training_steps=4, + lr_warmup_steps=1, + lr_warmup_steps_ratio=0.0, + lr_warmup_init=0.0, + lr=1.0, + min_lr=0.1, + lr_decay_steps=4, + lr_decay_style="linear", + weight_decay=0.1, + weight_decay_incr_style="constant", + lr_wsd_decay_steps=None, + lr_wsd_decay_style="exponential", + ) + + scheduler = _build_lr_scheduler(optimizer, opt) + + assert optimizer.param_groups[0]["lr"] == 0.0 + scheduler.step(1) + assert optimizer.param_groups[0]["lr"] == 1.0 + scheduler.step(1) + assert optimizer.param_groups[0]["lr"] == pytest.approx(0.7) + + state = scheduler.state_dict() + scheduler.step(10) + scheduler.load_state_dict(state) + + assert scheduler.state_dict() == state + assert optimizer.param_groups[0]["lr"] == pytest.approx(0.7) diff --git a/experimental/lite/tests/unit/verl/test_mlite_engine_cp_smoke.py b/experimental/lite/tests/unit/verl/test_mlite_engine_cp_smoke.py new file mode 100644 index 00000000000..343eb3120ad --- /dev/null +++ b/experimental/lite/tests/unit/verl/test_mlite_engine_cp_smoke.py @@ -0,0 +1,288 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +from __future__ import annotations + +import json +from types import MethodType, SimpleNamespace + +import pytest + +pytestmark = [ + pytest.mark.mlite, + pytest.mark.smoke, + pytest.mark.gpu, + pytest.mark.distributed, +] + + +def _init_dist_or_skip(): + import os + + import torch + import torch.distributed as dist + + if not torch.cuda.is_available(): + pytest.skip("CUDA is required for MLite VERL CP smoke.") + if "RANK" not in os.environ or "WORLD_SIZE" not in os.environ: + pytest.skip("Run with torchrun so CP ranks are available.") + + local_rank = int(os.environ.get("LOCAL_RANK", "0")) + torch.cuda.set_device(local_rank) + if not dist.is_initialized(): + dist.init_process_group("nccl") + if dist.get_world_size() < 2: + pytest.skip("MLite VERL CP smoke requires at least 2 ranks.") + return torch.device("cuda", local_rank) + + +def _write_kimi_config(path) -> None: + config = { + "model_type": "deepseek_v3", + "num_hidden_layers": 2, + "hidden_size": 64, + "num_attention_heads": 4, + "num_key_value_heads": 4, + "vocab_size": 128, + "intermediate_size": 96, + "moe_intermediate_size": 16, + "n_routed_experts": 4, + "n_shared_experts": 1, + "num_experts_per_tok": 2, + "n_group": 2, + "topk_group": 1, + "first_k_dense_replace": 1, + "q_lora_rank": 16, + "kv_lora_rank": 12, + "qk_nope_head_dim": 8, + "qk_rope_head_dim": 8, + "v_head_dim": 8, + "max_position_embeddings": 128, + "rope_theta": 10000.0, + "rope_scaling": { + "type": "yarn", + "factor": 1.0, + "original_max_position_embeddings": 128, + "beta_fast": 1.0, + "beta_slow": 1.0, + "mscale": 1.0, + "mscale_all_dim": 1.0, + }, + } + path.mkdir(parents=True, exist_ok=True) + (path / "config.json").write_text(json.dumps(config), encoding="utf-8") + + +def _write_glm5_config(path) -> None: + config = { + "model_type": "glm_moe_dsa", + "num_hidden_layers": 2, + "hidden_size": 128, + "num_attention_heads": 64, + "num_key_value_heads": 64, + "head_dim": 256, + "vocab_size": 32, + "max_position_embeddings": 64, + "initializer_range": 0.002, + "q_lora_rank": 16, + "kv_lora_rank": 512, + "qk_head_dim": 256, + "qk_nope_head_dim": 192, + "qk_rope_head_dim": 64, + "v_head_dim": 256, + "index_head_dim": 128, + "index_n_heads": 32, + "index_topk": 512, + "intermediate_size": 20, + "moe_intermediate_size": 6, + "first_k_dense_replace": 1, + "n_routed_experts": 3, + "n_shared_experts": 1, + "num_experts_per_tok": 3, + "num_nextn_predict_layers": 1, + "mlp_layer_types": ["dense", "dense"], + } + path.mkdir(parents=True, exist_ok=True) + (path / "config.json").write_text(json.dumps(config), encoding="utf-8") + + +def _write_deepseek_v4_config(path) -> None: + config = { + "model_type": "deepseek_v4", + "num_hidden_layers": 2, + "hidden_size": 128, + "num_attention_heads": 4, + "num_key_value_heads": 1, + "head_dim": 32, + "vocab_size": 128, + "max_position_embeddings": 128, + "initializer_range": 0.02, + "q_lora_rank": 64, + "qk_rope_head_dim": 16, + "o_lora_rank": 64, + "o_groups": 4, + "index_head_dim": 16, + "index_n_heads": 4, + "index_topk": 4, + "moe_intermediate_size": 32, + "n_routed_experts": 4, + "n_shared_experts": 1, + "num_experts_per_tok": 2, + "num_hash_layers": 1, + "num_nextn_predict_layers": 0, + "compress_ratios": [4, 4], + "compress_rope_theta": 160000.0, + "hc_mult": 2, + "hc_eps": 1e-6, + "hc_sinkhorn_iters": 4, + "rms_norm_eps": 1e-6, + "rope_theta": 10000.0, + "sliding_window": 128, + "swiglu_limit": 10.0, + "scoring_func": "sqrtsoftplus", + "topk_method": "noaux_tc", + "norm_topk_prob": True, + "routed_scaling_factor": 1.0, + } + path.mkdir(parents=True, exist_ok=True) + (path / "config.json").write_text(json.dumps(config), encoding="utf-8") + + +def _optimizer_config() -> SimpleNamespace: + return SimpleNamespace( + optimizer="adam", + lr=1e-6, + min_lr=None, + min_lr_ratio=None, + clip_grad=1.0, + weight_decay=0.0, + lr_warmup_steps_ratio=0.0, + total_training_steps=1, + lr_warmup_steps=0, + lr_warmup_init=0.0, + lr_decay_steps=None, + lr_decay_style="constant", + weight_decay_incr_style="constant", + lr_wsd_decay_style="exponential", + lr_wsd_decay_steps=None, + use_checkpoint_opt_param_scheduler=False, + betas=(0.9, 0.95), + override_optimizer_config={}, + ) + + +@pytest.mark.parametrize( + ("model_name", "model_type", "write_config", "vocab_size", "lengths"), + [ + ("kimi_k2", "deepseek_v3", _write_kimi_config, 128, [5, 7, 9]), + ("glm5", "glm_moe_dsa", _write_glm5_config, 32, [16, 20, 24]), + ("deepseek_v4", "deepseek_v4", _write_deepseek_v4_config, 128, [16, 20, 24]), + ], +) +def test_mlite_engine_runtime_thd_cp_uses_typed_packed_batch( + tmp_path, model_name, model_type, write_config, vocab_size, lengths +): + torch = pytest.importorskip("torch") + dist = pytest.importorskip("torch.distributed") + TensorDict = pytest.importorskip("tensordict").TensorDict + from verl_mlite.compat import apply_runtime_patches + + apply_runtime_patches() + from verl_mlite.engine.config import MegatronLiteEngineConfig + from verl_mlite.engine.mlite_engine import MegatronLiteEngine + from megatron.lite.runtime.contracts import PackedBatch + + device = _init_dist_or_skip() + world = dist.get_world_size() + rank = dist.get_rank() + hf_path = tmp_path / f"tiny-{model_name}" + write_config(hf_path) + + engine = MegatronLiteEngine( + model_config=SimpleNamespace( + local_path=str(hf_path), + hf_config={"model_type": model_type}, + mtp=None, + ), + engine_config=MegatronLiteEngineConfig( + model_name=model_name, + cp=world, + impl_cfg={ + "use_thd": True, + "optimizer": None, + "deterministic": False, + "mtp_enable": False, + }, + use_fused_kernels=False, + ), + optimizer_config=_optimizer_config(), + checkpoint_config={}, + ) + original_build_config = engine._build_mlite_config + + def _build_config_without_loading_weights(self): + config = original_build_config() + config.load_hf_weights = False + return config + + engine._build_mlite_config = MethodType(_build_config_without_loading_weights, engine) + engine.initialize() + engine.optimizer_zero_grad() + + input_ids = torch.nested.as_nested_tensor( + [ + torch.randint(0, vocab_size, (length,), device=device, dtype=torch.long) + for length in lengths + ], + layout=torch.jagged, + ) + loss_mask = torch.nested.as_nested_tensor( + [torch.ones(length, device=device, dtype=torch.float32) for length in lengths], + layout=torch.jagged, + ) + micro_batch = TensorDict( + {"input_ids": input_ids, "loss_mask": loss_mask}, + batch_size=[len(lengths)], + device=device, + ) + + runtime_batch = engine._make_runtime_batch(micro_batch) + loss_context = engine._make_runtime_loss_context(micro_batch, loss_scale=1.0) + assert isinstance(runtime_batch, PackedBatch) + # Connector batch is model-agnostic: true seq lengths, no padding, no extras. + assert runtime_batch.seq_lens.tolist() == lengths + assert int(runtime_batch.input_ids.numel()) == sum(lengths) + assert not runtime_batch.extras + + with engine.train_mode(): + result = engine.runtime.forward_backward( + engine.handle, + iter([(runtime_batch, loss_context)]), + loss_fn=None, + num_microbatches=1, + forward_only=False, + ) + + assert torch.isfinite(result.model_output.loss) + assert result.model_output.log_probs is not None + grad_norm = torch.zeros((), dtype=torch.float32, device=device) + for param in engine.module.parameters(): + if param.grad is not None: + grad_norm = grad_norm + param.grad.detach().float().norm() + dist.all_reduce(grad_norm, op=dist.ReduceOp.SUM) + assert torch.isfinite(grad_norm) + assert grad_norm.item() > 0.0 + + verl_output = engine._build_verl_model_output( + raw_output={"log_probs": result.model_output.log_probs}, + runtime_batch=runtime_batch, + ) + nested_log_probs = verl_output["log_probs"] + assert [int(x) for x in nested_log_probs.offsets().diff().cpu()] == lengths + + if rank == 0: + print( + "NON_SKIP_VERL_MLITE_RUNTIME_THD_CP_SMOKE_PASSED " + f"model={model_name} " + f"world_size={world} lengths={lengths} " + f"loss={float(result.model_output.loss.detach().item()):.6e} " + f"grad_norm_sum={float(grad_norm.item()):.6e}" + ) diff --git a/greptile.json b/greptile.json index 38013ea8869..ff38ad6fb1e 100644 --- a/greptile.json +++ b/greptile.json @@ -2,11 +2,10 @@ "labels": [], "comment": "Disclaimer: This is AI-generated.", "commentTypes": ["logic", "syntax", "style"], - "instructions": "Only comment if the PR description is unchanged from the default template, if a docstring is missing, or if there is a typo.", + "instructions": "Only comment if the PR description is unchanged from the default template, if a docstring is missing, if there is a typo, or if there is an important or critical bug.", "ignoreKeywords": "rename\nlinter\nprettier\ngreptile-ignor", "ignorePatterns": "greptile.json\ntesting/**/*.py\n*.md\n*.txt\n*.json", "patternRepositories": ["NVIDIA/Megatron-LM"], - "triggerOnUpdates": true, "shouldUpdateDescription": false, "disabledLabels": ["docs"], "includeAuthors": [], @@ -37,4 +36,4 @@ "defaultOpen": false }, "statusCommentsEnabled": false - } \ No newline at end of file + } diff --git a/hybrid_builders.py b/hybrid_builders.py index 05b219277ef..d95002e1a21 100644 --- a/hybrid_builders.py +++ b/hybrid_builders.py @@ -2,8 +2,8 @@ from megatron.core.models.hybrid.hybrid_layer_specs import hybrid_inference_stack_spec from megatron.core.models.hybrid.hybrid_model import HybridModel -from megatron.core.transformer import TransformerConfig -from megatron.core.transformer.spec_utils import import_module +from megatron.core.transformer import MLATransformerConfig, TransformerConfig +from megatron.core.transformer.spec_utils import ModuleSpec, import_module from megatron.training import print_rank_0 from megatron.training.arguments import core_transformer_config_from_args from model_provider import count_parameters_in_layer @@ -13,6 +13,52 @@ def hybrid_builder(args, pre_process, post_process, vp_stage=None, config=None, print_rank_0('building Hybrid model ...') if config is None: config = core_transformer_config_from_args(args, TransformerConfig) + # MLA (and DSv4 hybrid) require MLATransformerConfig so that its __post_init__ runs + # the dsv4_hybrid derivation. The hybrid pretrain path can hand us a plain + # TransformerConfig, which silently skips that derivation; rebuild as MLA to match GPT. + if args.multi_latent_attention and not isinstance(config, MLATransformerConfig): + config = core_transformer_config_from_args(args) + # DSv4-hybrid head-dim contract: qk_head_dim and kv_lora_rank are derived from + # v_head_dim and qk_pos_emb_head_dim (MLATransformerConfig.__post_init__ does this for the + # GPT path). The hybrid config can reach here without that derivation applied, which breaks + # the MLA up-proj / fused-rope contract (q head dim must equal qk_head_dim + qk_pos_emb_head + # _dim == v_head_dim). Apply it for any DSv4 MLA attention: experimental_attention_variant + # == dsv4_hybrid, OR the layer pattern uses a DSv4 attention symbol (D/C/H/W). Idempotent. + _pattern = getattr(args, "hybrid_layer_pattern", None) or "" + _uses_dsv4_attn = ( + getattr(args, "experimental_attention_variant", None) == "dsv4_hybrid" + or any(sym in _pattern for sym in ("C", "H", "W")) + ) + if _uses_dsv4_attn: + derived = config.v_head_dim - config.qk_pos_emb_head_dim + if config.qk_head_dim != derived or config.kv_lora_rank != derived: + print_rank_0( + f"[hybrid dsv4] deriving qk_head_dim/kv_lora_rank = {config.v_head_dim} - " + f"{config.qk_pos_emb_head_dim} = {derived} (was qk_head_dim={config.qk_head_dim}, " + f"kv_lora_rank={config.kv_lora_rank})" + ) + config.qk_head_dim = derived + config.kv_lora_rank = derived + # 'C'/'H'/'W' layers carry their compress ratio via the spec, but array-driven 'D' layers + # AND the indexer-loss logger (which counts ratio==4 layers) read + # config.csa_compress_ratios. When not given explicitly, derive it from the pattern + # symbols (C->4, H->128, W/D/other->0) so the array is consistent with the symbols and + # the indexer loss is normalized correctly; pad MTP depths with 0. An explicit + # --csa-compress-ratios is always respected. + if config.csa_compress_ratios is None: + ratio_map = {"C": 4, "H": 128} + # One entry per ACTUAL layer: main layers, then every MTP layer of every MTP depth + # (a depth can hold multiple hybrid layers, e.g. "/MD-E"), mirroring the arguments.py + # derivation. Padding by mtp_num_layers (depth count) would be too short and an MTP + # attention that isn't first would IndexError at num_layers + layer_number - 1. + sections = _pattern.split("/") + ratios = [ratio_map.get(c, 0) for c in sections[0].replace("|", "")] + for mtp_sec in sections[1:]: + ratios += [ratio_map.get(c, 0) for c in mtp_sec.replace("|", "")] + config.csa_compress_ratios = ratios + print_rank_0( + f"[hybrid dsv4] derived csa_compress_ratios from pattern symbols: {ratios}" + ) if config.transformer_impl == "inference_optimized": hybrid_stack_spec = hybrid_inference_stack_spec @@ -21,6 +67,11 @@ def hybrid_builder(args, pre_process, post_process, vp_stage=None, config=None, ), "inference_fuse_tp_communication is not supported for HybridModel" elif args.spec is not None: hybrid_stack_spec = import_module(args.spec) + # Allow config-aware specs: if --spec resolves to a callable (not a ModuleSpec), + # call it with config to build the stack spec (e.g. hybrid_dsv4_stack_spec, which + # wires the DSv4 CompressedSparseAttention into the 'D' layer per config). + if not isinstance(hybrid_stack_spec, ModuleSpec) and callable(hybrid_stack_spec): + hybrid_stack_spec = hybrid_stack_spec(config) else: raise ValueError("You must provide a valid hybrid layer spec via --spec") diff --git a/megatron/core/MSC_Integration.md b/megatron/core/MSC_Integration.md index da8b5c982b8..cd44d6afbb2 100644 --- a/megatron/core/MSC_Integration.md +++ b/megatron/core/MSC_Integration.md @@ -125,14 +125,16 @@ python pretrain_gpt.py \ **Notes:** Only the `torch_dist` checkpoint format is currently supported when saving to or loading from MSC URLs. -## Disable MSC +## Enable MSC -By default, MSC integration is automatically enabled when the `multi-storage-client` library is installed. MSC is also used for regular filesystem paths (like `/filesystem_mountpoint/path` in `--data-path`, `--save`, or `--load`) even when not using explicit MSC URLs. MSC functions as a very thin abstraction layer with negligible performance impact when used with regular paths, so there's typically no need to disable it. If you need to disable MSC, you can do so using the `--disable-msc` flag: +MSC integration is opt-in: even when the `multi-storage-client` library is installed, MSC is **disabled by default**. To opt in, pass the `--enable-msc` flag. Once enabled, MSC is also used for regular filesystem paths (like `/filesystem_mountpoint/path` in `--data-path`, `--save`, or `--load`), not just explicit `msc://` URLs. ```bash -python pretrain_gpt.py --disable-msc +python pretrain_gpt.py --enable-msc ``` +> **Note:** When MSC is enabled, the dist-checkpointing loader uses `msc.torch.MultiStorageFileSystemReader` instead of `CachedMetadataFileSystemReader`. This means `ckpt_assume_constant_structure=True` (and any other path that requests `cache_metadata=True`) will be silently overridden — metadata is re-read on every load. A warning is emitted in this case. + ## Performance Considerations When using object storage with MSC, there are a few important performance implications to keep in mind: diff --git a/megatron/core/_rank_utils.py b/megatron/core/_rank_utils.py index 6b1a35ca798..68aaa7bcbbd 100644 --- a/megatron/core/_rank_utils.py +++ b/megatron/core/_rank_utils.py @@ -4,16 +4,22 @@ import logging import os +import warnings from typing import Any import torch +from megatron.core._slurm_utils import resolve_slurm_rank, resolve_slurm_world_size + def safe_get_rank() -> int: - """Safely get the rank of the current process. + """Get the distributed rank safely, even if torch.distributed is not initialized. - Returns the rank from torch.distributed if initialized, otherwise falls back - to the RANK environment variable, defaulting to 0. + Fallback order: + 1. torch.distributed.get_rank() (if initialized) + 2. RANK environment variable (torchrun/torchelastic) + 3. SLURM_PROCID environment variable (SLURM) + 4. Default: 0 (with warning) Returns: int: The rank of the current process. @@ -23,11 +29,51 @@ def safe_get_rank() -> int: # If torch.distributed is not initialized, try to read environment variables. try: - return int(os.environ.get("RANK", 0)) + if "RANK" in os.environ: + return int(os.environ["RANK"]) + + slurm_rank = resolve_slurm_rank() + if slurm_rank is not None: + return slurm_rank + + warnings.warn( + "Could not determine rank from torch.distributed, RANK, or SLURM_PROCID. " + "Defaulting to rank 0." + ) + return 0 except (ValueError, TypeError): return 0 +def safe_get_world_size() -> int: + """Get the distributed world size safely, even if torch.distributed is not initialized. + + Fallback order: + 1. torch.distributed.get_world_size() (if initialized) + 2. WORLD_SIZE environment variable (torchrun/torchelastic) + 3. SLURM_NTASKS environment variable (SLURM) + 4. Default: 1 (with warning) + + Returns: + The total number of processes in the distributed job. + """ + if torch.distributed.is_initialized(): + return torch.distributed.get_world_size() + + if "WORLD_SIZE" in os.environ: + return int(os.environ["WORLD_SIZE"]) + + slurm_world_size = resolve_slurm_world_size() + if slurm_world_size is not None: + return slurm_world_size + + warnings.warn( + "Could not determine world size from torch.distributed, WORLD_SIZE, or SLURM_NTASKS. " + "Defaulting to world size 1." + ) + return 1 + + def log_single_rank(logger: logging.Logger, *args: Any, rank: int = 0, **kwargs: Any) -> None: """Log a message only on a single rank. diff --git a/megatron/core/_slurm_utils.py b/megatron/core/_slurm_utils.py new file mode 100644 index 00000000000..e6d21acc39b --- /dev/null +++ b/megatron/core/_slurm_utils.py @@ -0,0 +1,51 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Utilities for detecting and configuring SLURM cluster environments. + +This module provides functionality to detect SLURM environments and extract +distributed training configuration from SLURM environment variables. +""" + +import os + + +def is_slurm_job() -> bool: + """Detect if running in a SLURM environment. + + Returns: + True if SLURM job detected, False otherwise. + """ + return "SLURM_NTASKS" in os.environ + + +def resolve_slurm_rank() -> int | None: + """Get the global rank from SLURM environment. + + Returns: + The global rank, or None if not in SLURM environment. + """ + if not is_slurm_job(): + return None + return int(os.environ["SLURM_PROCID"]) if "SLURM_PROCID" in os.environ else None + + +def resolve_slurm_world_size() -> int | None: + """Get the world size from SLURM environment. + + Returns: + The world size, or None if not in SLURM environment. + """ + if not is_slurm_job(): + return None + return int(os.environ["SLURM_NTASKS"]) if "SLURM_NTASKS" in os.environ else None + + +def resolve_slurm_local_rank() -> int | None: + """Get the local rank from SLURM environment. + + Returns: + The local rank, or None if not in SLURM environment. + """ + if not is_slurm_job(): + return None + return int(os.environ["SLURM_LOCALID"]) if "SLURM_LOCALID" in os.environ else None diff --git a/megatron/core/datasets/data_schedule.py b/megatron/core/datasets/data_schedule.py index 8afdd6b0f39..d5bf07053cc 100644 --- a/megatron/core/datasets/data_schedule.py +++ b/megatron/core/datasets/data_schedule.py @@ -9,20 +9,63 @@ align_sample_id_groups, broadcast_scalars, broadcast_tensor, - broadcast_to_pp_group, build_packed_microbatches, create_data_iterator, - dcp_get_total_workload, - dcp_gpus_needed, - dcp_make_buckets_equal, get_batch_and_global_seqlens, get_cp_slice_for_thd, - next_hdp_group, + next_hdp_group_packing_aware, reroute_samples_to_dcp_ranks, ) -from megatron.core.packed_seq_params import PackedSeqParams +from megatron.core.packed_seq_params import ( + PackedSeqParams, + get_thd_padding_kwargs, + pad_sequence_for_thd, +) from megatron.core.pipeline_parallel.hybrid_cp_schedule import BalancedCPScheduler from megatron.core.process_groups_config import ProcessGroupCollection +from megatron.core.transformer.multi_token_prediction import mtp_on_this_rank + + +def _build_thd_padding_mask( + cu_seqlens: torch.Tensor, cu_seqlens_padded: torch.Tensor +) -> torch.Tensor: + """Build a 1D THD padding mask from scheduler sequence metadata.""" + assert cu_seqlens.dim() == 1 + assert cu_seqlens_padded.dim() == 1 + assert cu_seqlens.numel() == cu_seqlens_padded.numel() + + total_tokens = int(cu_seqlens_padded[-1].item()) + if total_tokens == 0: + return torch.empty((0,), dtype=torch.bool, device=cu_seqlens.device) + + num_sequences = cu_seqlens.numel() - 1 + if num_sequences <= 0: + return torch.ones((total_tokens,), dtype=torch.bool, device=cu_seqlens.device) + + positions = torch.arange( + total_tokens, dtype=cu_seqlens_padded.dtype, device=cu_seqlens_padded.device + ) + seq_indices = torch.searchsorted(cu_seqlens_padded[1:].contiguous(), positions, right=True) + + valid_lengths = (cu_seqlens[1:] - cu_seqlens[:-1]).clamp(min=0) + valid_ends = cu_seqlens_padded[:-1] + valid_lengths + return positions >= valid_ends[seq_indices] + + +def _sanitize_thd_padding_values(batch: Dict[str, Any], padding_mask: torch.Tensor) -> None: + """Replace padded token-like slots with safe neutral values in-place.""" + assert padding_mask.dim() == 1 + pad_values = {'tokens': 0, 'labels': 0, 'loss_mask': 0.0, 'position_ids': 0} + for key, pad_value in pad_values.items(): + tensor = batch.get(key) + if tensor is None: + continue + assert tensor.dim() == 1, f"{key} must be 1D before CP slicing, got {tensor.dim()}D" + assert tensor.numel() == padding_mask.numel(), ( + f"{key} length ({tensor.numel()}) must match padding_mask length " + f"({padding_mask.numel()}) before CP slicing." + ) + batch[key] = tensor.masked_fill(padding_mask, pad_value) class BasePackingScheduler: @@ -34,6 +77,7 @@ def __init__( cp_size: int, dp_size: int, microbatch_group_size_per_vp_stage: Optional[int], + max_num_seqs: Optional[int] = None, ): """ Args: @@ -42,11 +86,15 @@ def __init__( dp_size: The data parallel size. microbatch_group_size_per_vp_stage: The microbatch group size per virtual pipeline stage, only used when enabling VPP, otherwise None. + max_num_seqs: Optional cap on the number of real packed sequences + per microbatch. This excludes any dummy sequence later appended for + THD padding. """ self.max_seqlen_per_dp_cp_rank = max_seqlen_per_dp_cp_rank self.cp_size = cp_size self.dp_size = dp_size self.microbatch_group_size_per_vp_stage = microbatch_group_size_per_vp_stage + self.max_num_seqs = max_num_seqs def get_required_sample_keys(self): """Return the required key of each batch.""" @@ -118,7 +166,9 @@ def get_groups_and_subsamples(self, sample_id_seqlens): single_microbatch = [] for i in range(len(sample_id_seqlens)): - if sum_seqlen + sample_id_seqlens[i][1] <= self.max_seq_len_all_ranks: + if sum_seqlen + sample_id_seqlens[i][1] <= self.max_seq_len_all_ranks and ( + self.max_num_seqs is None or len(single_microbatch) < self.max_num_seqs + ): single_microbatch.append(i) sum_seqlen += sample_id_seqlens[i][1] else: @@ -178,14 +228,17 @@ def run( Steps: 1. Fetch batches and gather global sequence lengths 2. Check required sample keys - 3. Schedule samples into groups - 4. Reroute samples to DCP ranks - 5. Build packed microbatches - 6. Calculate FLOPs info - 7. Broadcast to PP group (for middle PP stages) - 8. Broadcast to TP group (for non-TP-0 ranks) + 3. Strip data fields not needed by this PP stage + 4. Schedule samples into groups + 5. Reroute samples to DCP ranks + 6. Build packed microbatches + 7. Calculate FLOPs info + 8. Broadcast scalars to TP group (for non-TP-0 ranks) 9. Handle VPP if enabled + Note: There is no PP-group broadcast. In packed-sequence mode + is_dataset_built_on_rank returns True for every PP stage on TP rank 0 + Args: data_iterator: The data iterator. num_microbatches: The number of microbatches to fetch. @@ -204,20 +257,20 @@ def run( """ total_dcp_gpus = dp_cp_group.size() + is_first_pp = pp_group.rank() == 0 + is_last_pp = pp_group.rank() == pp_group.size() - 1 + + mtp_on_this_pp = mtp_on_this_rank(config, ignore_virtual=True) + vpp_size = config.virtual_pipeline_model_parallel_size or 1 # Handle VPP: extract the correct data_iterator for this PP stage. # When VPP is enabled, data_iterator is a list with one entry per VPP stage. # We only need one data_iterator to run the schedule (all VPP stages on the # same PP rank share the same underlying dataset), so pick the first non-None. - # Record which VPP stages had data so create_data_iterator knows which ones - # need full samples vs metadata only. - vpp_has_data = None - if ( - config.virtual_pipeline_model_parallel_size is not None - and config.virtual_pipeline_model_parallel_size > 1 - ): - assert len(data_iterator) == config.virtual_pipeline_model_parallel_size - vpp_has_data = [di is not None for di in data_iterator] + # Determine which VPP stages need full data based on pipeline position and MTP. + vpp_needs_data = None + if vpp_size > 1: + assert len(data_iterator) == vpp_size extracted = None for di in data_iterator: if di is not None: @@ -225,8 +278,25 @@ def run( break data_iterator = extracted - # data_iterator is not None on TP rank 0 for PP stages that need data - # (first stage, last stage, or any stage with MTP). + # Only first VPP on first PP and last VPP on last PP need full data. + # MTP VPP stages also need full data (both tokens and labels). + # Middle VPP stages only need metadata (cu_seqlens, max_seqlen, etc.). + vpp_needs_data = [False] * vpp_size + if is_first_pp: + vpp_needs_data[0] = True + if is_last_pp: + vpp_needs_data[-1] = True + if mtp_on_this_pp: + for vp_i in range(vpp_size): + if mtp_on_this_rank(config, ignore_virtual=False, vp_stage=vp_i): + vpp_needs_data[vp_i] = True + + # In packed-sequence mode is_dataset_built_on_rank returns True for every + # PP stage on TP rank 0, so data_iterator is not None on TP rank 0 of + # every PP stage (and every stage independently fetches data and computes + # the global seqlen stats). vpp_needs_data / keys_to_keep below only decide + # which data fields are kept per stage; they do not affect whether + # data_iterator is None. if data_iterator is not None: assert tp_group.rank() == 0, "Only TP rank 0 should have data_iterator" @@ -241,7 +311,24 @@ def run( key in batch[0] ), f"Batch missing required key {key}, provided keys: {batch[0].keys()}" - # Step 3: Schedule samples into groups + # Step 3: Strip data fields not needed by this PP stage to avoid + # unnecessary all-to-all communication. First PP needs tokens/position_ids, + # last PP needs labels/loss_mask. MTP stages need all four. + # NOTE: this assumes _unpack_batch produces only the six keys below + # (tokens, position_ids, labels, loss_mask, original_seq_len, + # padded_seq_len). Any custom dataset metadata key outside this set + # would be silently dropped here; extend keys_to_keep if needed. + keys_to_keep = {'original_seq_len', 'padded_seq_len'} + if is_first_pp or mtp_on_this_pp: + keys_to_keep.update(['tokens', 'position_ids']) + if is_last_pp or mtp_on_this_pp: + keys_to_keep.update(['labels', 'loss_mask']) + for sample in batch: + for key in list(sample.keys()): + if key not in keys_to_keep: + del sample[key] + + # Step 4: Schedule samples into groups sample_id_groups = self.get_groups_and_subsamples(global_id_seqlens) # Validate scheduling result @@ -254,7 +341,7 @@ def run( f"global_id_seqlens length: {len(global_id_seqlens)}" ) - # Step 4: Reroute samples to DCP ranks + # Step 5: Reroute samples to DCP ranks samples_this_rank_with_id = reroute_samples_to_dcp_ranks( batch, global_ids_this_rank, @@ -270,12 +357,12 @@ def run( dcp_rank = dp_cp_group.rank() num_micro_batches = len(sample_id_groups) - # Step 5: Build packed microbatches + # Step 6: Build packed microbatches new_samples = build_packed_microbatches( samples_this_rank_with_id, sample_id_groups, dcp_rank, dev, self.is_dynamic_cp ) - # Step 6: Calculate FLOPs info + # Step 7: Calculate FLOPs info seqlen_sum_this_global_batch = float(sum(seqlens_gathered)) seqlen_squared_sum_this_global_batch = float( sum(seqlen**2 for seqlen in seqlens_gathered) @@ -288,24 +375,7 @@ def run( seqlen_squared_sum_this_global_batch, ) = (None, None, None, None) - # Step 7: Broadcast to PP group (for middle PP stages) - if tp_group.rank() == 0: - ( - new_samples, - num_micro_batches, - seqlen_sum_this_global_batch, - seqlen_squared_sum_this_global_batch, - ) = broadcast_to_pp_group( - new_samples, - num_micro_batches, - seqlen_sum_this_global_batch, - seqlen_squared_sum_this_global_batch, - pp_group, - dev, - is_dynamic_cp=self.is_dynamic_cp, - ) - - # Step 8: Broadcast to TP group (for non-TP-0 ranks) + # Broadcast to TP group (for non-TP-0 ranks) (num_micro_batches, seqlen_sum_this_global_batch, seqlen_squared_sum_this_global_batch) = ( broadcast_scalars( [ @@ -319,9 +389,9 @@ def run( ) num_micro_batches = int(num_micro_batches) - # Step 9: create data_iterator and handle VPP if enabled + # Step 8: Broadcast to TP group and create data_iterator new_data_iterator = create_data_iterator( - new_samples, tp_group, config, vpp_has_data, self.is_dynamic_cp + new_samples, tp_group, config, vpp_needs_data, self.is_dynamic_cp ) return ( @@ -351,28 +421,17 @@ def get_groups_and_subsamples(self, sample_id_seqlens): """ mslpr = self.max_seq_len_per_rank min_cp = self.min_cp_size - workload_fn = lambda seq_len, cp_size=None: dcp_get_total_workload( - seq_len, mslpr, cp_size, min_cp - ) - gpus_fn = lambda seq_len: dcp_gpus_needed(seq_len, mslpr, min_cp) - buckets_fn = lambda sample_seqlens, compute_est: dcp_make_buckets_equal( - sample_seqlens, compute_est, mslpr, min_cp - ) - groups = [] sample_id_groups = [] sample_id_seqlens = sorted(sample_id_seqlens, key=lambda x: x[1], reverse=True) + while sample_id_seqlens: - mb, sample_id_seqlens, exec_times, sample_ids = next_hdp_group( + _, sample_id_seqlens, _, sample_ids = next_hdp_group_packing_aware( sample_id_seqlens, - workload_fn, self.total_hdp_gpus, - gpus_needed_fn=gpus_fn, - make_buckets_equal_fn=buckets_fn, max_seq_len_per_rank=mslpr, - get_total_workload_fn=workload_fn, + min_cp_size=min_cp, ) - groups.append(mb) sample_id_groups.append(sample_ids) if ( @@ -392,6 +451,35 @@ def get_groups_and_subsamples(self, sample_id_seqlens): } +def _get_scheduler_max_real_num_seqs(config) -> Optional[int]: + """Return the scheduler cap for real THD sequences. + + ``thd_max_packed_sequences`` is the final static THD capacity, including the + optional dummy sequence appended for a padding tail. The dp_balanced + scheduler only packs real sequences, so reserve one slot when dummy-tail + padding is enabled. + """ + max_num_seqs = getattr(config, 'thd_max_packed_sequences', None) + if max_num_seqs is None: + return None + + max_num_seqs = int(max_num_seqs) + if max_num_seqs < 1: + raise ValueError(f"thd_max_packed_sequences must be >= 1, got {max_num_seqs}.") + + if getattr(config, 'pad_packed_seq_alignment', None) is not None and getattr( + config, 'pad_packed_seq_by_appending_dummy_seq', True + ): + if max_num_seqs < 2: + raise ValueError( + "thd_max_packed_sequences must be >= 2 when THD padding appends a dummy " + "sequence, because thd_max_packed_sequences includes that dummy sequence." + ) + return max_num_seqs - 1 + + return max_num_seqs + + def wrap_data_iterator( data_iterator, config, num_microbatches, pg_collection: Optional[ProcessGroupCollection] = None ): @@ -434,6 +522,12 @@ def wrap_data_iterator( if scheduler_type == 'default_dynamic_cp': scheduler_kwargs['min_cp_size'] = config.min_dynamic_context_parallel_size + scheduler_max_num_seqs = ( + _get_scheduler_max_real_num_seqs(config) + if scheduler_type == 'dp_balanced' + else getattr(config, 'thd_max_packed_sequences', None) + ) + scheduler = scheduler_map[scheduler_type]( config.max_seqlen_per_dp_cp_rank, cp_size, @@ -443,6 +537,7 @@ def wrap_data_iterator( if config.virtual_pipeline_model_parallel_size is None else config.microbatch_group_size_per_vp_stage ), + max_num_seqs=scheduler_max_num_seqs, **scheduler_kwargs, ) @@ -470,6 +565,7 @@ def get_batch_on_this_rank_for_sequence_packing( vp_stage: Optional[int] = None, dynamic_cp: bool = False, pg_collection: Optional[ProcessGroupCollection] = None, + config=None, ): """ Get a batch of data for sequence packing. @@ -477,8 +573,11 @@ def get_batch_on_this_rank_for_sequence_packing( data_iterator (Iterator): The data iterator to get the batch from. mtp_on_this_rank (bool): Whether to use multi-token prediction. vp_stage (Optional[int]): The stage of the pipeline. + config: Model parallel config used for optional THD packed-sequence padding. + When None or config.pad_packed_seq_alignment is None, no padding is applied. Returns: - tuple of (tokens, labels, loss_mask, attention_mask, position_ids, packed_seq_params) + tuple of (tokens, labels, loss_mask, attention_mask, position_ids, + packed_seq_params, padding_mask) """ if pg_collection is None: @@ -531,10 +630,21 @@ def get_batch_on_this_rank_for_sequence_packing( group_size=local_cp_size_val ) - # Partition tokens, position_ids, labels, loss_mask for context parallel. - # Only TP rank 0 on stages that have data (first/last PP stage or MTP stage) needs this. - if is_tp_rank_0 and (is_first_or_last_stage or mtp_on_this_rank): - get_cp_slice_for_thd(batch, cp_group) + # Build padding_mask before CP slicing while tensors still have the full + # packed length represented by cu_seqlens_padded[-1]. + if is_tp_rank_0: + batch['padding_mask'] = _build_thd_padding_mask( + batch['cu_seqlens'], batch['cu_seqlens_padded'] + ) + _sanitize_thd_padding_values(batch, batch['padding_mask']) + + # Partition sequence tensors for context parallelism. Padding mask is needed + # on every PP stage, while data tensors are only needed on first/last/MTP stages. + if is_tp_rank_0: + cp_slice_keys = ['padding_mask'] + if is_first_or_last_stage or mtp_on_this_rank: + cp_slice_keys.extend(['tokens', 'position_ids', 'labels', 'loss_mask']) + get_cp_slice_for_thd(batch, cp_group, keys=cp_slice_keys) # Broadcast cu_seqlens_size because we need it to create placeholder for cu_seqlens and # cu_seqlens_padded for non TP 0 ranks. @@ -545,17 +655,19 @@ def get_batch_on_this_rank_for_sequence_packing( broadcast_tensor(cu_seqlen_size, tp_src_rank, tp_group) cu_seqlen_size = cu_seqlen_size.item() - # Broadcast total_tokens because we need it to create placeholder for tokens, position_ids, - # labels, loss_mask for non TP 0 ranks. Only first stage, last stage, - # and stage with mtp need this. - - if is_first_or_last_stage or mtp_on_this_rank: - if is_tp_rank_0: - total_tokens = torch.tensor(batch['tokens'].size(0), dtype=torch.int32, device=dev) - else: - total_tokens = torch.empty(1, dtype=torch.int32, device=dev) - broadcast_tensor(total_tokens, tp_src_rank, tp_group) - total_tokens = total_tokens.item() + # Broadcast total_tokens because padding_mask is prepared on every PP stage. + # Tokens/labels/loss_mask/position_ids use the same length on stages that own them. + if is_tp_rank_0: + # Under VPP, the last PP stage has labels but no tokens, so derive + # total_tokens from cu_seqlens_padded, which is present on every + # stage. cu_seqlens_padded keeps the pre-CP packed length; divide + # by cp_size to match the already CP-sliced sequence tensors. + cp_world = cp_group.size() + total_tokens = (batch['cu_seqlens_padded'][-1].to(torch.int32) // cp_world).reshape(1) + else: + total_tokens = torch.empty(1, dtype=torch.int32, device=dev) + broadcast_tensor(total_tokens, tp_src_rank, tp_group) + total_tokens = total_tokens.item() # Step1: Prepare "tokens", "position_ids" for first stage and stage with mtp on all TP ranks. if is_first_stage or mtp_on_this_rank: @@ -587,7 +699,14 @@ def get_batch_on_this_rank_for_sequence_packing( batch['labels'] = None batch['loss_mask'] = None - # Step3: Prepare "cu_seqlens", "cu_seqlens_padded", "max_seqlen" on all ranks. + # Step3: Prepare "padding_mask" on all TP ranks. + if is_tp_rank_0: + assert batch['padding_mask'].dtype == torch.bool + batch['padding_mask'] = batch['padding_mask'].view(1, total_tokens) + else: + batch['padding_mask'] = torch.empty([1, total_tokens], dtype=torch.bool, device=dev) + + # Step4: Prepare "cu_seqlens", "cu_seqlens_padded", "max_seqlen" on all ranks. if is_tp_rank_0: assert batch['cu_seqlens'].dtype == torch.int32 assert batch['cu_seqlens_padded'].dtype == torch.int32 @@ -603,7 +722,7 @@ def get_batch_on_this_rank_for_sequence_packing( batch['cu_seqlens_padded'] = torch.empty([cu_seqlen_size], dtype=torch.int32, device=dev) batch['max_seqlen'] = torch.empty(1, dtype=torch.int32, device=dev) - # Step4: Prepare "local_cp_size" if dynamic context parallel is enabled. + # Step5: Prepare "local_cp_size" if dynamic context parallel is enabled. if dynamic_cp: if is_tp_rank_0: if type(batch['local_cp_size']) == int: @@ -623,6 +742,7 @@ def get_batch_on_this_rank_for_sequence_packing( broadcast_tensor(batch['position_ids'], tp_src_rank, tp_group) broadcast_tensor(batch['labels'], tp_src_rank, tp_group) broadcast_tensor(batch['loss_mask'], tp_src_rank, tp_group) + broadcast_tensor(batch['padding_mask'], tp_src_rank, tp_group) broadcast_tensor(batch['cu_seqlens'], tp_src_rank, tp_group) broadcast_tensor(batch['cu_seqlens_padded'], tp_src_rank, tp_group) broadcast_tensor(batch['max_seqlen'], tp_src_rank, tp_group) @@ -633,6 +753,7 @@ def get_batch_on_this_rank_for_sequence_packing( position_ids = batch['position_ids'] labels = batch['labels'] loss_mask = batch['loss_mask'] + padding_mask = batch['padding_mask'] cu_seqlens = batch['cu_seqlens'] cu_seqlens_padded = batch['cu_seqlens_padded'] max_seqlen = batch['max_seqlen'].item() @@ -643,23 +764,55 @@ def get_batch_on_this_rank_for_sequence_packing( else None ) - # Transformer Engine has a bug of cu_seqlens, we must treat cu_seqlens_padded as cu_seqlens to - # get the correct result. - # TODO: Revert this workaround once TE fixes the issue. + # cu_seqlens_q/kv hold the original (unpadded) boundaries so downstream + # loss paths (e.g. CSA indexer KL) can identify padding rows. + # cu_seqlens_q/kv_padded hold the padded boundaries consumed by attention + # kernels and THD partitioning. packed_seq_params = PackedSeqParams( qkv_format="thd", - cu_seqlens_q=cu_seqlens_padded, - cu_seqlens_kv=cu_seqlens_padded, + cu_seqlens_q=cu_seqlens, + cu_seqlens_kv=cu_seqlens, cu_seqlens_q_padded=cu_seqlens_padded, cu_seqlens_kv_padded=cu_seqlens_padded, max_seqlen_q=max_seqlen, max_seqlen_kv=max_seqlen, local_cp_size=local_cp_size, cp_group=cp_group, + pad_between_seqs=False, + ) + + # Pad the already-packed THD tensors at the end when requested. CUDA Graph + # additionally pads cu_seqlens tensors to thd_max_packed_sequences + 1 entries. + pad_alignment = ( + getattr(config, 'pad_packed_seq_alignment', None) if config is not None else None ) + if pad_alignment is not None and packed_seq_params is not None: + alignment, target_len, max_num_seqs = get_thd_padding_kwargs( + pad_alignment, + getattr(config, 'max_seqlen_per_dp_cp_rank', None), + getattr(config, 'thd_max_packed_sequences', None), + getattr(config, 'cuda_graph_impl', 'none') != 'none', + ) + tokens, labels, loss_mask, position_ids, packed_seq_params, padding_mask = ( + pad_sequence_for_thd( + tokens, + labels, + loss_mask, + position_ids, + packed_seq_params, + alignment=alignment, + target_len=target_len, + max_num_seqs=max_num_seqs, + pad_by_appending_dummy_seq=getattr( + config, 'pad_packed_seq_by_appending_dummy_seq', True + ), + padding_mask=padding_mask, + cp_group=cp_group, + ) + ) # "attention_mask" is not valid for sequence packing, so set it to None. - return tokens, labels, loss_mask, None, position_ids, packed_seq_params + return tokens, labels, loss_mask, None, position_ids, packed_seq_params, padding_mask class HybridCPDataLoaderWrapper: diff --git a/megatron/core/datasets/data_schedule_utils.py b/megatron/core/datasets/data_schedule_utils.py index c59b1742c0a..190f898cfc3 100644 --- a/megatron/core/datasets/data_schedule_utils.py +++ b/megatron/core/datasets/data_schedule_utils.py @@ -1,18 +1,18 @@ # Copyright (c) 2025 NVIDIA CORPORATION. All rights reserved. -from collections import deque from functools import lru_cache from math import ceil, log2 -from typing import Callable, Dict, List, Optional, Tuple +from typing import Dict, List, Optional, Sequence, Tuple -import numpy as np import torch from megatron.core.extensions.transformer_engine import get_thd_partitioned_indices from megatron.core.rerun_state_machine import RerunDataIterator +_DYNAMIC_CP_WORKLOAD_CAP_DELTA = 0.05 -def get_cp_slice_for_thd(batch, cp_group): + +def get_cp_slice_for_thd(batch, cp_group, keys: Optional[Sequence[str]] = None): """Partition sequence data for context parallelism in THD format. Uses TE's THD partitioned indices to split the packed sequence across CP ranks. @@ -21,19 +21,24 @@ def get_cp_slice_for_thd(batch, cp_group): Args: batch: Dict with packed sequence data. cp_group: Context parallel process group. + keys: Sequence data keys to slice. Defaults to the original THD data tensors. """ cp_size = cp_group.size() if cp_size <= 1: return cp_rank = cp_group.rank() - total_tokens = batch['tokens'].size(0) - # Transformer Engine has a bug of cu_seqlens, we must treat cu_seqlens_padded as - # cu_seqlens to get the correct result. - # TODO: Revert this workaround once TE fixes the issue. + # Partition with padded cumulative lengths so CP slices match the THD + # sequence boundaries consumed by attention kernels. cu_seqlens = batch["cu_seqlens_padded"] + # Use cu_seqlens_padded[-1] for total_tokens instead of batch['tokens'].size(0): + # under VPP, the last PP stage has labels/loss_mask but no tokens, so + # batch['tokens'] is None on that stage. cu_seqlens_padded is always populated. + total_tokens = int(cu_seqlens[-1].item()) index = get_thd_partitioned_indices(cu_seqlens, total_tokens, cp_size, cp_rank) - for key in ['tokens', 'position_ids', 'labels', 'loss_mask']: - if key in batch: + if keys is None: + keys = ('tokens', 'position_ids', 'labels', 'loss_mask') + for key in keys: + if key in batch and batch[key] is not None: batch[key] = batch[key].index_select(0, index) @@ -43,11 +48,36 @@ def _unpack_batch(batch: List[Dict[str, torch.Tensor]]) -> List[Dict[str, torch. Since each sub-sample may be routed to different DPxCP ranks, we unpack the sample here to avoid unnecessarily transferring the entire packed sample. + + Two input shapes are accepted: + + * **Pre-packed** (e.g. :class:`SFTDataset`): each sample carries a + ``cu_seqlens`` tensor and the tokens of multiple sub-samples + concatenated together. We slice them apart and synthesize + ``original_seq_len`` / ``padded_seq_len`` from the cu_seqlens deltas. + + * **Already unpacked** (e.g. :class:`VarlenDataset`): each sample is a + single sub-sample that already carries ``padded_seq_len`` (and + usually ``original_seq_len``). We just normalize the leading batch + dimension introduced by the default collate_fn and return as-is. """ + # Short-circuit for datasets that already emit one sub-sample per index. + if batch and "padded_seq_len" in batch[0]: + for sample in batch: + for key in sample.keys(): + if sample[key].ndim == 2 and sample[key].shape[0] == 1: + # Drop the redundant batch dim added by collate_fn. + sample[key] = sample[key].squeeze(0) + if "original_seq_len" not in sample: + sample["original_seq_len"] = sample["padded_seq_len"].clone() + return batch + batch_unpacked = [] - dev = batch[0]["tokens"].device + dev = batch[0]["cu_seqlens"].device original_seq_lens = [] padded_seq_lens = [] + # Determine which data fields exist in the batch + data_keys = [k for k in ["tokens", "labels", "loss_mask", "position_ids"] if k in batch[0]] for sample in batch: for key in sample.keys(): if len(sample[key].shape) == 2: @@ -63,7 +93,7 @@ def _unpack_batch(batch: List[Dict[str, torch.Tensor]]) -> List[Dict[str, torch. end_idx = sample["cu_seqlens"][sub_sample + 1] if end_idx - start_idx == 0: continue - for key in ["tokens", "labels", "loss_mask", "position_ids"]: + for key in data_keys: sub_sample_dict[key] = sample[key][start_idx:end_idx] # Since sft_dataset.py does not provide cu_seqlens_original, # we assume original_seq_len equals padded_seq_len here. @@ -151,16 +181,10 @@ def _pack_sequences( def _pack_tensors(tensors): return torch.cat([t.reshape(-1) for t in tensors], dim=0) - tokens = _pack_tensors([sample["tokens"] for sample in samples]) - labels = _pack_tensors([sample["labels"] for sample in samples]) - loss_mask = _pack_tensors([sample["loss_mask"] for sample in samples]) - position_ids = _pack_tensors([sample["position_ids"] for sample in samples]) - new_sample = {} - new_sample["tokens"] = tokens - new_sample["labels"] = labels - new_sample["loss_mask"] = loss_mask - new_sample["position_ids"] = position_ids + for key in ['tokens', 'labels', 'loss_mask', 'position_ids']: + if key in samples[0]: + new_sample[key] = _pack_tensors([sample[key] for sample in samples]) padded_lengths = padded_lengths.to(device=dev, dtype=torch.int32, non_blocking=True).reshape(-1) cu_seqlens_padded = torch.empty(padded_lengths.numel() + 1, device=dev, dtype=torch.int32) @@ -191,103 +215,6 @@ def broadcast_tensor(item, src_rank, group) -> None: torch.distributed.broadcast(item, src_rank, group=group) -def broadcast_to_pp_group( - new_samples, - num_micro_batches, - seqlen_sum_this_global_batch, - seqlen_squared_sum_this_global_batch, - pp_group, - dev, - is_dynamic_cp: bool = False, -): - """ - Broadcast num_micro_batches, seqlen_sum_this_global_batch, - seqlen_squared_sum_this_global_batch and metadata to middle PP stages. - Before this broadcast, the new_samples on middle PP stages are None, - after this broadcast, the new_samples on middle PP stages contain the metadata but - without tokens, labels, loss_mask, position_ids. - """ - - pp_src_rank = torch.distributed.get_process_group_ranks(pp_group)[0] - - if pp_group.size() > 2: - if pp_group.rank() == 0: - tensor_list = [ - torch.tensor( - [ - num_micro_batches, - seqlen_sum_this_global_batch, - seqlen_squared_sum_this_global_batch, - ], - dtype=torch.float32, - ).cuda() - ] - for sample in new_samples: - tensor_list.append(sample["max_seqlen"].unsqueeze(0)) - - if is_dynamic_cp: - for sample in new_samples: - tensor_list.append(sample["local_cp_size"].unsqueeze(0)) - - for sample in new_samples: - tensor_list.append(sample["cu_seqlens"]) - tensor_list.append(sample["cu_seqlens_padded"]) - info_to_broadcast = torch.cat(tensor_list, dim=0).to(device=dev, dtype=torch.float32) - info_length_tensor = torch.tensor(info_to_broadcast.shape[0], dtype=torch.int32).cuda() - broadcast_tensor(info_length_tensor, pp_src_rank, pp_group) - broadcast_tensor(info_to_broadcast, pp_src_rank, pp_group) - else: - info_length_tensor = torch.tensor(0, dtype=torch.int32).cuda() - broadcast_tensor(info_length_tensor, pp_src_rank, pp_group) - info_to_broadcast = torch.empty(info_length_tensor.item(), dtype=torch.float32).cuda() - broadcast_tensor(info_to_broadcast, pp_src_rank, pp_group) - if pp_group.rank() != pp_group.size() - 1: - # middle PP stages receive the broadcasted info and unpack it - info_numpy = info_to_broadcast.cpu().numpy() - num_micro_batches = int(info_numpy[0]) - seqlen_sum_this_global_batch = info_numpy[1] - seqlen_squared_sum_this_global_batch = info_numpy[2] - max_seqlens = info_to_broadcast[3 : 3 + num_micro_batches] - local_cp_sizes = ( - info_to_broadcast[3 + num_micro_batches : 3 + 2 * num_micro_batches] - if is_dynamic_cp - else None - ) - cu_seqlens_list = [] - cu_seqlens_padded_list = [] - # cu_seqlens always starts with 0, and the other metadata values - # (num_micro_batches, seqlen_sum, seqlen_squared_sum, max_seqlens) - # are always positive, so we can use 0 as the delimiter to locate - # the start of each cu_seqlens / cu_seqlens_padded tensor. - # This avoids an extra broadcast for the lengths of cu_seqlens. - indices = np.where(info_numpy == 0)[0] - for i in range(num_micro_batches): - cu_seqlens_list.append(info_to_broadcast[indices[i * 2] : indices[i * 2 + 1]]) - if i == num_micro_batches - 1: - cu_seqlens_padded_list.append(info_to_broadcast[indices[i * 2 + 1] :]) - else: - cu_seqlens_padded_list.append( - info_to_broadcast[indices[i * 2 + 1] : indices[i * 2 + 2]] - ) - - new_samples = [] - for i in range(num_micro_batches): - new_sample = {} - new_sample["max_seqlen"] = max_seqlens[i].to(torch.int32) - new_sample["cu_seqlens"] = cu_seqlens_list[i].to(torch.int32) - new_sample["cu_seqlens_padded"] = cu_seqlens_padded_list[i].to(torch.int32) - if is_dynamic_cp: - new_sample["local_cp_size"] = local_cp_sizes[i].to(torch.int32) - new_samples.append(new_sample) - - return ( - new_samples, - num_micro_batches, - seqlen_sum_this_global_batch, - seqlen_squared_sum_this_global_batch, - ) - - def broadcast_scalars(values: List, group, dev, dtype=torch.float32) -> List: """ Broadcast scalar values from rank 0 to all ranks in the group. @@ -321,21 +248,21 @@ def broadcast_scalars(values: List, group, dev, dtype=torch.float32) -> List: def create_data_iterator( - new_samples, tp_group, config, vpp_has_data=None, is_dynamic_cp: bool = False + new_samples, tp_group, config, vpp_needs_data=None, is_dynamic_cp: bool = False ): """Handle virtual pipeline parallelism. For VPP, each PP rank needs a list of data iterators (one per VPP stage). - VPP stages that originally had a data_iterator (indicated by vpp_has_data) - get full samples; others get metadata only (cu_seqlens, cu_seqlens_padded, + VPP stages that need full data (first/last pipeline stage, or MTP) get + full samples; others get metadata only (cu_seqlens, cu_seqlens_padded, max_seqlen). Args: new_samples: The packed samples after scheduling. tp_group: Tensor parallel process group. config: Model parallel config. - vpp_has_data: A list of booleans (one per VPP stage) indicating which - VPP stages originally had a data_iterator. None if VPP is disabled. + vpp_needs_data: A list of booleans (one per VPP stage) indicating which + VPP stages need full samples (data fields). None if VPP is disabled. """ if ( config.virtual_pipeline_model_parallel_size is not None @@ -346,14 +273,19 @@ def create_data_iterator( metadata_keys = ["max_seqlen", "cu_seqlens", "cu_seqlens_padded"] if is_dynamic_cp: metadata_keys.append("local_cp_size") - metadata = [ - {k: sample[k] for k in metadata_keys if k in sample} for sample in new_samples - ] new_data_iterator = [] for i in range(vpp_size): - if vpp_has_data is not None and vpp_has_data[i]: - new_data_iterator.append(RerunDataIterator(iter(new_samples))) + if vpp_needs_data is not None and vpp_needs_data[i]: + # Give each data-carrying VPP stage its own shallow-copied + # sample dicts. + samples_copy = [dict(sample) for sample in new_samples] + new_data_iterator.append(RerunDataIterator(iter(samples_copy))) else: + # Create independent metadata dicts to avoid shared-reference mutation + metadata = [ + {k: sample[k] for k in metadata_keys if k in sample} + for sample in new_samples + ] new_data_iterator.append(RerunDataIterator(iter(metadata))) else: new_data_iterator = [None for _ in range(vpp_size)] @@ -599,28 +531,25 @@ def get_batch_and_global_seqlens(data_iterator, num_microbatches, dp_group): # ============================================================================= -def next_hdp_group( +def next_hdp_group_packing_aware( sample_seqlens: List[Tuple[int, int]], - compute_estimator: Callable[[int], float], total_gpus: int, - gpus_needed_fn: Callable[[int], int], - make_buckets_equal_fn: Callable, - max_seq_len_per_rank: float, - get_total_workload_fn: Callable, - delta: float = 0.05, - strategy: str = "dp", - eps_bucket: float = 0.10, + max_seq_len_per_rank: int, + min_cp_size: int = 1, ) -> Tuple[List[List[int]], List[Tuple[int, int]], List[float], List[List[int]]]: - """Form one balanced micro-batch group across DPxCP ranks. - - This is a standalone version of the scheduling algorithm extracted from - DefaultDynamicCPScheduler so it can live in a utils module. - - Extra args compared to the method version: - gpus_needed_fn: callable(seq_len) -> int - make_buckets_equal_fn: callable(sample_seqlens, compute_estimator) -> list[deque] - max_seq_len_per_rank: max tokens per rank for packing - get_total_workload_fn: callable(seq_len, cp_size) -> float + """Form one DCP microbatch with packing-aware CP group selection. + + This differs from the legacy DCP scheduler in two ways: + 1. Short sequences may use a larger CP group than their minimum required + CP size when that lowers the critical-path rank workload. + 2. Candidate placements are bounded by ``tall * max_seq_len_per_rank``, + the per-rank workload upper bound for packing sequences no longer than + the local tallest sequence in the microbatch. + + The scheduler keeps the legacy invariant that each returned microbatch has + no empty DPxCP rank after the fill step. For non-power-of-two DPxCP layouts, + it falls back to the full DPxCP group if power-of-two expansion cannot fill + every rank. """ if not sample_seqlens: return ( @@ -630,241 +559,216 @@ def next_hdp_group( [[] for _ in range(total_gpus)], ) - buckets = make_buckets_equal_fn(sample_seqlens, compute_estimator) + def cp_min_fn(seq_len: int) -> int: + return dcp_gpus_needed(seq_len, max_seq_len_per_rank, min_cp_size) - micro_batches = [[] for _ in range(total_gpus)] - exec_times = [0.0 for _ in range(total_gpus)] - sample_ids_per_gpu = [[] for _ in range(total_gpus)] - packing_sequence_len = {} + def workload(seq_len: int, cp_size: int) -> float: + return (seq_len * seq_len) / cp_size - gpu_group_id = [None] * total_gpus - group_members = {} - group_size = {} - next_gid = 0 + sample_seqlens = sorted(sample_seqlens, key=lambda x: x[1], reverse=True) + local_tall = sample_seqlens[0][1] + cap = float(local_tall) * float(max_seq_len_per_rank) * (1.0 + _DYNAMIC_CP_WORKLOAD_CAP_DELTA) - pp_cursor = 0 - prev_needed = None - check_balance = False + micro_batches: List[List[int]] = [[] for _ in range(total_gpus)] + exec_times: List[float] = [0.0 for _ in range(total_gpus)] + sample_ids_per_gpu: List[List[int]] = [[] for _ in range(total_gpus)] + packing_sequence_len: Dict[int, float] = {} - while buckets: - sample_seq_tuple = bucket_idx = None - needed = None - - scan_order = ( - range(len(buckets)) - if strategy == "dp" - else [(pp_cursor + i) % len(buckets) for i in range(len(buckets))] - ) - - for idx in scan_order: - if not buckets[idx]: - continue - cand_tuple = buckets[idx][0] - cand_seq_len = cand_tuple[1] - needed = gpus_needed_fn(cand_seq_len) - - candidate_gids = [gid for gid, sz in group_size.items() if sz == needed] - free_ranks = [r for r, gid in enumerate(gpu_group_id) if gid is None] - if candidate_gids or len(free_ranks) >= needed: - sample_seq_tuple, bucket_idx = cand_tuple, idx - break - - if sample_seq_tuple is None: - break - - if strategy == "pp": - pp_cursor = (bucket_idx + 1) % len(buckets) - - sample_id, seq_len = sample_seq_tuple - needed = gpus_needed_fn(seq_len) - if prev_needed is None: - prev_needed = needed - - candidate_gids = [ - gid - for gid, sz in group_size.items() - if sz == needed and packing_sequence_len[gid] + seq_len / needed <= max_seq_len_per_rank - ] - if candidate_gids: - best_gid, best_load = min( - ((gid, max(exec_times[r] for r in group_members[gid])) for gid in candidate_gids), - key=lambda t: t[1], - ) - else: - best_gid, best_load = None, float("inf") + gpu_group_id: List[Optional[int]] = [None] * total_gpus + group_members: Dict[int, List[int]] = {} + group_size: Dict[int, int] = {} + next_gid = 0 - free_ranks = [r for r, gid in enumerate(gpu_group_id) if gid is None] - if len(free_ranks) >= needed: - free_sorted = sorted(free_ranks, key=lambda r: exec_times[r]) - new_members = free_sorted[:needed] - new_load = exec_times[new_members[-1]] + sample_id, seq_len = sample_seqlens[0] + cp_size = cp_min_fn(seq_len) + assert cp_size <= total_gpus, ( + f"Sequence length {seq_len} requires CP size {cp_size}, " + f"but only {total_gpus} DPxCP ranks are available." + ) + group_id = next_gid + next_gid += 1 + members = list(range(cp_size)) + group_members[group_id] = members + group_size[group_id] = cp_size + packing_sequence_len[group_id] = seq_len / cp_size + per_gpu_cost = workload(seq_len, cp_size) + for rank in members: + gpu_group_id[rank] = group_id + micro_batches[rank].append(seq_len) + exec_times[rank] += per_gpu_cost + sample_ids_per_gpu[rank].append(sample_id) + + leftovers: List[Tuple[int, int]] = [] + for sample_id, seq_len in sample_seqlens[1:]: + min_needed = cp_min_fn(seq_len) + best = None + + cp_size = min_needed + while cp_size <= total_gpus: + per_gpu_cost = workload(seq_len, cp_size) + + for group_id, size in list(group_size.items()): + if size != cp_size: + continue + if packing_sequence_len.get(group_id, 0) + seq_len / cp_size > max_seq_len_per_rank: + continue + members = group_members[group_id] + member_set = set(members) + projected_max = max( + time + per_gpu_cost if rank in member_set else time + for rank, time in enumerate(exec_times) + ) + if projected_max <= cap and (best is None or projected_max < best[0]): + best = (projected_max, cp_size, "add", group_id, None) - if new_load < best_load: - best_gid = None - chosen_members = new_members - else: - chosen_members = group_members[best_gid] + free_ranks = [ + rank + for rank, assigned_group_id in enumerate(gpu_group_id) + if assigned_group_id is None + ] + if len(free_ranks) >= cp_size: + chosen_members = sorted(free_ranks, key=lambda rank: exec_times[rank])[:cp_size] + chosen_set = set(chosen_members) + projected_max = max( + time + per_gpu_cost if rank in chosen_set else time + for rank, time in enumerate(exec_times) + ) + if projected_max <= cap and (best is None or projected_max < best[0]): + best = (projected_max, cp_size, "new", None, chosen_members) + + cp_size *= 2 + + if best is None: + leftovers.append((sample_id, seq_len)) + continue + + _, selected_cp_size, action, group_id, chosen_members = best + per_gpu_cost = workload(seq_len, selected_cp_size) + if action == "add": + members = group_members[group_id] + packing_sequence_len[group_id] += seq_len / selected_cp_size + for rank in members: + micro_batches[rank].append(seq_len) + exec_times[rank] += per_gpu_cost + sample_ids_per_gpu[rank].append(sample_id) else: - if best_gid is None: - break - chosen_members = group_members[best_gid] - - if best_gid is None: - best_gid = next_gid + group_id = next_gid next_gid += 1 - group_members[best_gid] = chosen_members - group_size[best_gid] = needed - for r in chosen_members: - gpu_group_id[r] = best_gid - - per_gpu_cost = compute_estimator(seq_len) + group_members[group_id] = chosen_members + group_size[group_id] = selected_cp_size + packing_sequence_len[group_id] = seq_len / selected_cp_size + for rank in chosen_members: + gpu_group_id[rank] = group_id + micro_batches[rank].append(seq_len) + exec_times[rank] += per_gpu_cost + sample_ids_per_gpu[rank].append(sample_id) + + def fill_empty_gpus_once() -> bool: + nonlocal micro_batches, exec_times, sample_ids_per_gpu + + empty_ranks = [rank for rank, micro_batch in enumerate(micro_batches) if not micro_batch] + if not empty_ranks: + return False + assert all( + not micro_batches[rank] for rank in range(empty_ranks[0], total_gpus) + ), "fill_empty_gpus_once assumes empty ranks are contiguous at the tail" - packing_sequence_len[best_gid] = packing_sequence_len.get(best_gid, 0) + seq_len / needed - for r in chosen_members: - micro_batches[r].append(seq_len) - exec_times[r] += per_gpu_cost - sample_ids_per_gpu[r].append(sample_id) + existing_group_sizes = set(group_size.values()) + if not existing_group_sizes: + return False + min_group_size = min(existing_group_sizes) + next_power = min(min_group_size * 2, total_gpus) - buckets[bucket_idx].popleft() + for group_id, size in list(group_size.items()): + if size != min_group_size: + continue - while buckets and not buckets[0]: - buckets.pop(0) - pp_cursor %= max(1, len(buckets)) + members = group_members[group_id] + needed_count = next_power - min_group_size + group_start_rank = members[0] + group_end_rank = members[-1] + empty_rank = empty_ranks[0] + if group_end_rank + 1 > empty_rank or group_end_rank + needed_count >= total_gpus: + continue - if needed < prev_needed: - check_balance = True + work_to_push = micro_batches[group_end_rank + 1 : empty_rank] + exec_times_to_push = exec_times[group_end_rank + 1 : empty_rank] + sample_ids_to_push = sample_ids_per_gpu[group_end_rank + 1 : empty_rank] - if ( - check_balance - and buckets - and max(exec_times) - min(exec_times) <= delta * max(exec_times) - ): - break + new_micro_batches: List[List[int]] = [[] for _ in range(total_gpus)] + new_exec_times: List[float] = [0.0 for _ in range(total_gpus)] + new_sample_ids_per_gpu: List[List[int]] = [[] for _ in range(total_gpus)] - leftovers = [] - for b in buckets: - for sample_seq_tuple in b: - leftovers.append(sample_seq_tuple) - - def trim_overload(): - while True: - cur_max = max(exec_times) - cur_min = min(exec_times) - cur_slack = cur_max - cur_min - if cur_slack <= delta * cur_max: - break - if cur_min == 0: - break - - max_r = exec_times.index(cur_max) - gid = gpu_group_id[max_r] - members = group_members[gid] - - if not micro_batches[max_r] or len(micro_batches[max_r]) <= 1: - break - - seq = micro_batches[max_r][-1] - per_gpu_cost = compute_estimator(seq) - - proj_times = exec_times[:] - for r in members: - proj_times[r] -= per_gpu_cost - - proj_slack = max(proj_times) - min(proj_times) - - if proj_slack < cur_slack: - sample_id_to_remove = sample_ids_per_gpu[max_r][-1] - for r in members: - micro_batches[r].pop() - exec_times[r] -= per_gpu_cost - sample_ids_per_gpu[r].pop() - leftovers.append((sample_id_to_remove, seq)) - else: - break + for rank in range(group_start_rank): + new_micro_batches[rank] = micro_batches[rank] + new_exec_times[rank] = exec_times[rank] + new_sample_ids_per_gpu[rank] = sample_ids_per_gpu[rank] - # TODO(tailaim): uncomment this to support different ranks have different num_microbatches - # trim_overload() + for rank in range(group_start_rank, group_end_rank + needed_count + 1): + new_micro_batches[rank] = list(micro_batches[group_end_rank]) + new_exec_times[rank] = sum( + workload(length, next_power) for length in micro_batches[group_end_rank] + ) + new_sample_ids_per_gpu[rank] = list(sample_ids_per_gpu[group_end_rank]) - total_work_before = sum(len(mb) for mb in micro_batches) + for idx, work in enumerate(work_to_push): + target_rank = group_end_rank + needed_count + 1 + idx + new_micro_batches[target_rank] = work + new_exec_times[target_rank] = exec_times_to_push[idx] + new_sample_ids_per_gpu[target_rank] = sample_ids_to_push[idx] - def fill_empty_gpus(micro_batches, exec_times, sample_ids_per_gpu, group_members, group_size): - empty_gpus = [i for i in range(total_gpus) if not micro_batches[i]] - if not empty_gpus: - return (micro_batches, exec_times, sample_ids_per_gpu, group_members, group_size) + group_size[group_id] = next_power + group_members[group_id] = list( + range(group_start_rank, group_end_rank + needed_count + 1) + ) + for other_group_id in list(group_size.keys()): + if other_group_id == group_id: + continue + if min(group_members[other_group_id]) > group_end_rank: + group_members[other_group_id] = [ + rank + needed_count for rank in group_members[other_group_id] + ] - existing_group_sizes = set(group_size.values()) - assert ( - existing_group_sizes - ), "There should be at least one group existing, cannot redistribute, " - "try to increase 'max-seqlen-per-dp-cp-rank'." + micro_batches = new_micro_batches + exec_times = new_exec_times + sample_ids_per_gpu = new_sample_ids_per_gpu + return True - min_group_size = min(existing_group_sizes) - next_power = min(min_group_size * 2, total_gpus) + return False - for gid, size in group_size.items(): - if size == min_group_size: - members = group_members[gid] - needed_count = next_power - min_group_size - group_start_gpu = members[0] - group_end_gpu = members[-1] - empty_gpu = [idx for idx, work in enumerate(micro_batches) if not work][0] - assert not all( - work for work in micro_batches[empty_gpu : empty_gpu + needed_count] - ), "Empty GPUs were detected but not enough to expand." - work_to_push = micro_batches[group_end_gpu + 1 : empty_gpu] - exec_times_to_push = exec_times[group_end_gpu + 1 : empty_gpu] - sample_ids_to_push = sample_ids_per_gpu[group_end_gpu + 1 : empty_gpu] - - new_micro_batches = [[]] * len(micro_batches) - new_exec_times = [0.0] * len(exec_times) - new_sample_ids_per_gpu = [[]] * len(sample_ids_per_gpu) - - for i in range(group_start_gpu): - new_micro_batches[i] = micro_batches[i] - new_exec_times[i] = exec_times[i] - new_sample_ids_per_gpu[i] = sample_ids_per_gpu[i] - - for i in range(group_start_gpu, group_end_gpu + needed_count + 1): - new_micro_batches[i] = micro_batches[group_end_gpu] - new_exec_times[i] = get_total_workload_fn( - micro_batches[group_end_gpu][0], next_power - ) - new_sample_ids_per_gpu[i] = sample_ids_per_gpu[group_end_gpu] + def fill_with_full_dpxcp_group() -> None: + nonlocal micro_batches, exec_times, sample_ids_per_gpu, leftovers - for i, work in enumerate(work_to_push): - new_micro_batches[group_end_gpu + needed_count + 1 + i] = work - new_exec_times[group_end_gpu + needed_count + 1 + i] = exec_times_to_push[i] - new_sample_ids_per_gpu[group_end_gpu + needed_count + 1 + i] = ( - sample_ids_to_push[i] - ) + selected: List[Tuple[int, int]] = [] + next_leftovers: List[Tuple[int, int]] = [] + packed_sequence_len = 0.0 - group_size[gid] = next_power - group_members[gid] = list(range(members[0], members[-1] + needed_count + 1)) - for pushed_gid in group_size.keys(): - if pushed_gid > gid: - group_members[pushed_gid] = [ - x + needed_count for x in group_members[pushed_gid] - ] - - return ( - new_micro_batches, - new_exec_times, - new_sample_ids_per_gpu, - group_members, - group_size, - ) + for sample_id, seq_len in sample_seqlens: + per_rank_len = seq_len / total_gpus + if packed_sequence_len + per_rank_len <= max_seq_len_per_rank: + selected.append((sample_id, seq_len)) + packed_sequence_len += per_rank_len + else: + next_leftovers.append((sample_id, seq_len)) - empty_gpus = any([not micro_batches[i] for i in range(total_gpus)]) - while empty_gpus: - micro_batches, exec_times, sample_ids_per_gpu, group_members, group_size = fill_empty_gpus( - micro_batches, exec_times, sample_ids_per_gpu, group_members, group_size + assert selected, ( + "At least one sequence should fit in the full DPxCP group; " + "try to increase 'max-seqlen-per-dp-cp-rank'." ) - empty_gpus = any([not micro_batches[i] for i in range(total_gpus)]) - total_work_after = sum(len(mb) for mb in micro_batches) - assert ( - total_work_after >= total_work_before - ), f"Samples were removed: {total_work_before} -> {total_work_after}" + selected_ids = [sample_id for sample_id, _ in selected] + selected_lens = [seq_len for _, seq_len in selected] + per_rank_work = sum(workload(seq_len, total_gpus) for _, seq_len in selected) + + micro_batches = [list(selected_lens) for _ in range(total_gpus)] + exec_times = [per_rank_work for _ in range(total_gpus)] + sample_ids_per_gpu = [list(selected_ids) for _ in range(total_gpus)] + leftovers = next_leftovers + + while any(not micro_batch for micro_batch in micro_batches): + if not fill_empty_gpus_once(): + fill_with_full_dpxcp_group() + break return micro_batches, leftovers, exec_times, sample_ids_per_gpu @@ -972,49 +876,3 @@ def dcp_gpus_needed(seq_len: int, max_seq_len_per_rank: int, min_cp_size: int = """Number of GPUs needed, rounded up to the next power of 2, lower-bounded by min_cp_size.""" raw = max(1, 2 ** ceil(log2(seq_len / max_seq_len_per_rank))) return max(min_cp_size, raw) - - -@lru_cache(maxsize=128) -def dcp_get_total_workload( - seq_length: int, max_seq_len_per_rank: int, cp_size: Optional[int] = None, min_cp_size: int = 1 -) -> float: - """Estimate workload of a sub-sample for scheduling balance.""" - if cp_size is None: - cp_size = dcp_gpus_needed(seq_length, max_seq_len_per_rank, min_cp_size) - return (seq_length * seq_length) / cp_size - - -def dcp_make_buckets_equal( - sample_seqlens: List[Tuple[int, int]], - compute_estimator: Callable, - max_seq_len_per_rank: int, - min_cp_size: int = 1, -) -> List[deque]: - """Split samples into buckets of roughly equal work, one per unique CP size.""" - seqlens = [seq_len for _, seq_len in sample_seqlens] - k = len({dcp_gpus_needed(L, max_seq_len_per_rank, min_cp_size) for L in seqlens}) - - work = [] - for _, s in sample_seqlens: - cp_size = dcp_gpus_needed(s, max_seq_len_per_rank, min_cp_size) - work.append(compute_estimator(s, cp_size)) - total_work = sum(work) - target = total_work / k - buckets, cur, cur_work = [], [], 0.0 - remaining_k = k - - for i, (sample_id, seq_len) in enumerate(sample_seqlens): - w = compute_estimator(seq_len) - projected = cur_work + w - if cur and ( - projected > target * 1.1 or len(sample_seqlens) - i <= remaining_k - len(buckets) - ): - buckets.append(deque(cur)) - cur, cur_work = [], 0.0 - remaining_k -= 1 - cur.append((sample_id, seq_len)) - cur_work += w - - if cur: - buckets.append(deque(cur)) - return buckets diff --git a/megatron/core/datasets/gpt_dataset.py b/megatron/core/datasets/gpt_dataset.py index 418a02719df..e34d930adbf 100644 --- a/megatron/core/datasets/gpt_dataset.py +++ b/megatron/core/datasets/gpt_dataset.py @@ -79,6 +79,18 @@ class GPTDatasetConfig(BlendedMegatronDatasetConfig): sft_mock_dataset_config_json: Optional[str] = None """This config provides the necessary information for the mock dataset.""" + varlen_mock_dataset_config_json: Optional[str] = None + """Mock-dataset config (same JSON schema as ``sft_mock_dataset_config_json``) + used by the ``--use-varlen-dataset`` path; kept separate so the varlen path + does not implicitly inherit SFT-specific knobs.""" + + varlen_sbhd_validation: bool = False + """When True, :class:`VarlenDataset.__getitem__` emits SBHD samples padded + to ``sequence_length`` (no ``cu_seqlens`` / ``original_seq_len`` / + ``padded_seq_len``), bypassing the packed-sequence path. Used to obtain a + SBHD reference run that mirrors the THD path's tokenization but skips all + packing — useful for THD numerical-correctness validation.""" + def __post_init__(self) -> None: """Do asserts and set fields post init""" super().__post_init__() @@ -89,6 +101,12 @@ def __post_init__(self) -> None: assert self.reset_attention_mask is not None assert self.eod_mask_loss is not None + if self.varlen_sbhd_validation: + assert not self.dynamic_context_parallel, ( + "--varlen-sbhd-validation is incompatible with " + "--dynamic-context-parallel (SBHD mode is not packed)." + ) + self.token_dtype_code = ( None if self.tokenizer.vocab_size is None diff --git a/megatron/core/datasets/readme.md b/megatron/core/datasets/readme.md index 64d0e3dd5bd..af3bccdebf4 100644 --- a/megatron/core/datasets/readme.md +++ b/megatron/core/datasets/readme.md @@ -219,7 +219,6 @@ Phase 1 │ (once per global │ ────────► ├─ get_groups_and_subsamples() └─ PackedSeqParams(...) ├─ reroute_samples_to_dcp_ranks() [utils] ├─ build_packed_microbatches() [utils] - ├─ broadcast_to_pp_group() [utils] ├─ broadcast_scalars() [utils] └─ create_data_iterator() [utils] ``` @@ -249,7 +248,6 @@ Utility functions consumed by the schedulers above: | `get_batch_and_global_seqlens()` | Fetch `num_microbatches` batches from the data iterator and all-gather sequence lengths across DP ranks. | | `reroute_samples_to_dcp_ranks()` | All-to-all communication to transfer sub-samples to their scheduled DP×CP rank. | | `build_packed_microbatches()` | Concatenate sub-samples within each microbatch group and produce `cu_seqlens`. | -| `broadcast_to_pp_group()` | Broadcast packed samples and metadata from the first/last PP stage to middle stages. | | `broadcast_scalars()` | Broadcast scalar values (e.g. `num_microbatches`, FLOPs stats) across a process group. | | `broadcast_tensor()` | Broadcast a single tensor within a process group. | | `create_data_iterator()` | Wrap packed sample lists into a data iterator; handles VPP stage splitting. | diff --git a/megatron/core/dist_checkpointing/strategies/nvrx.py b/megatron/core/dist_checkpointing/strategies/nvrx.py index 1df26f00ff4..1d609e1186f 100644 --- a/megatron/core/dist_checkpointing/strategies/nvrx.py +++ b/megatron/core/dist_checkpointing/strategies/nvrx.py @@ -5,6 +5,15 @@ from importlib import import_module from typing import Any, Callable, Dict +try: + from packaging.version import Version as PkgVersion + + HAVE_PACKAGING = True +except ImportError: + HAVE_PACKAGING = False + +NVRX_MIN_VERSION = "0.6.0" + def has_nvrx_async_support() -> bool: """Checks whether the NVRx async checkpointing symbols Megatron uses are importable.""" @@ -32,6 +41,10 @@ def has_nvrx_async_support() -> bool: getattr(state_dict_saver, "save_state_dict_async_finalize", None), getattr(state_dict_saver, "save_state_dict_async_plan", None), ) + assert ( + is_nvrx_min_version() + ), f"Minimum required nvidia-resiliency-ext package version is {NVRX_MIN_VERSION}." + return all(symbol is not None for symbol in required_symbols) and hasattr( filesystem_async, "_results_queue" ) @@ -53,3 +66,27 @@ def make_nvrx_async_request( async_fn_kwargs=async_fn_kwargs or {}, preload_fn=preload_fn, ) + + +def is_nvrx_min_version(version: str = NVRX_MIN_VERSION) -> bool: + """Check if minimum version of `NVRx` is installed.""" + if not HAVE_PACKAGING: + raise ImportError( + "packaging is not installed. Please install it with `pip install packaging`." + ) + + try: + import nvidia_resiliency_ext as nvrx + + HAVE_NVRX = True + except (ImportError, ModuleNotFoundError): + HAVE_NVRX = False + + nvrx_version = str(nvrx.__version__) if HAVE_NVRX else "0.0.0" + + # Compare the release segment only so that a pinned pre-release / dev build + # of the required version (e.g. "0.6.0.dev33", which PEP 440 orders *below* + # "0.6.0") still satisfies a ">= 0.6.0" requirement. The required-symbol + # check in has_nvrx_async_support() remains the authoritative guard that the + # specific async-checkpoint APIs are actually present. + return PkgVersion(nvrx_version).release >= PkgVersion(version).release diff --git a/megatron/core/dist_checkpointing/strategies/torch.py b/megatron/core/dist_checkpointing/strategies/torch.py index 7943561700f..31782acb851 100644 --- a/megatron/core/dist_checkpointing/strategies/torch.py +++ b/megatron/core/dist_checkpointing/strategies/torch.py @@ -831,6 +831,14 @@ def _get_filesystem_reader( ) -> FileSystemReader: if MultiStorageClientFeature.is_enabled(): msc = MultiStorageClientFeature.import_package() + if cache_metadata: + warnings.warn( + "MSC is enabled: returning msc.torch.MultiStorageFileSystemReader instead of " + "CachedMetadataFileSystemReader. The cache_metadata=True request " + "(e.g. ckpt_assume_constant_structure=True) will be ignored and metadata " + "will be re-read on every load. Pass --enable-msc only when this is intended.", + stacklevel=2, + ) return msc.torch.MultiStorageFileSystemReader(checkpoint_dir, thread_count=2) if cache_metadata: diff --git a/megatron/core/dist_checkpointing/validation.py b/megatron/core/dist_checkpointing/validation.py index 5c9e6ba3caa..b0cbae618a7 100644 --- a/megatron/core/dist_checkpointing/validation.py +++ b/megatron/core/dist_checkpointing/validation.py @@ -207,7 +207,12 @@ def verify_checkpoint(checkpoint_dir: str): Args: checkpoint_dir (str): checkpoint directory """ - if not Path(checkpoint_dir).exists(): + if MultiStorageClientFeature.is_enabled(): + msc = MultiStorageClientFeature.import_package() + isdir = msc.os.path.isdir(str(checkpoint_dir), strict=False) + else: + isdir = os.path.isdir(checkpoint_dir) + if not isdir: raise CheckpointingException(f'Checkpoint directory {checkpoint_dir} does not exist') if not check_is_distributed_checkpoint(checkpoint_dir): diff --git a/megatron/core/distributed/distributed_data_parallel.py b/megatron/core/distributed/distributed_data_parallel.py index a711f1405d1..e313113a448 100644 --- a/megatron/core/distributed/distributed_data_parallel.py +++ b/megatron/core/distributed/distributed_data_parallel.py @@ -7,7 +7,6 @@ import torch from ..config_logger import has_config_logger_enabled, log_config_to_disk -from ..fp8_utils import is_float8tensor, post_all_gather_processing from ..optimizer.param_layout import FullParamLayout from ..process_groups_config import ProcessGroupCollection from ..transformer.cuda_graphs import is_graph_capturing @@ -165,6 +164,8 @@ def __init__( param_indices == layout.param_indices ), f"param_indices for {buffer_key} do not match between grouping and layout" + self.full_param_layout = full_param_layout + # Compute gradient scaling factors. if config.calculate_per_token_loss: assert ( @@ -304,6 +305,23 @@ def __init__( bucket_groups[num_bucket_groups - i - 1] ) + # Set `previous_grad_reduce_bucket_group` so each bucket group can drain its predecessor's + # reduce-scatter at dispatch time. Only needed for reduce_scatter_with_fp32_accumulation, + # which holds an intermediate all-to-all output tensor pinned until .wait() runs; without + # this draining, all such tensors stay live until end-of-step. The fp32-accum path asserts + # num_distributed_optimizer_instances == 1 elsewhere, so we only link in that case. + # Grad-reduce dispatches happen in forward order of bucket_groups during backward (buckets + # closer to the output finish their gradients first), so bucket_groups[i]'s immediate + # predecessor in dispatch order is bucket_groups[i-1]. + if ( + self.ddp_config.overlap_grad_reduce + and self.ddp_config.reduce_scatter_with_fp32_accumulation + and self.ddp_config.num_distributed_optimizer_instances == 1 + ): + for bucket_groups in [self.bucket_groups, self.expert_parallel_bucket_groups]: + for i in range(1, len(bucket_groups)): + bucket_groups[i].previous_grad_reduce_bucket_group = bucket_groups[i - 1] + # Create map from param to bucket group, used in pre_hook. for bucket_groups in [self.bucket_groups, self.expert_parallel_bucket_groups]: for bucket_group in bucket_groups: @@ -471,6 +489,24 @@ def no_sync(self): for bucket_group in self.bucket_groups + self.expert_parallel_bucket_groups: bucket_group.is_last_microbatch = True + def _start_bucket_group_param_sync( + self, bucket_group: '_ParamAndGradBucketGroup', force_sync: bool + ) -> None: + """Dispatch one bucket group's param all-gather + run the FP8 / MXFP8 / FP4 + post-all-gather work the synchronous path needs. + + Factored out of :meth:`start_param_sync` so callers that own a subset + of bucket groups (e.g. a chained ``LayerWiseDistributedOptimizer`` + + ``DistributedOptimizer`` pair) can sync only their own buckets without + losing the post-processing that follows the collective. + """ + bucket_group.start_param_sync(force_sync=force_sync) + + if self.ddp_config.overlap_param_gather: + return + + bucket_group._post_param_sync() + def start_param_sync(self, *unused, force_sync: bool = False, force_dispatch: bool = False): """ Initiates param sync (all-gather) communication operations for all model parameters. @@ -491,43 +527,7 @@ def start_param_sync(self, *unused, force_sync: bool = False, force_dispatch: bo return for bucket_group in self.bucket_groups + self.expert_parallel_bucket_groups: - bucket_group.start_param_sync(force_sync=force_sync) - - if not self.ddp_config.overlap_param_gather: - # For MXFP8 params, we need to copy the all-gathered param data from the buffer to - # the param.data, since param buffer is not mapped to model params for MXFP8 case. - # The paramaters are cast from bf16 to MXFP8 during copy. - # In the case of "overlap_param_gather=True", the param copy is done - # in "finish_param_sync" stage after zeroing the shared gardient buffers. - if self.ddp_config.reuse_grad_buf_for_mxfp8_param_ag: - for bucket in bucket_group.buckets: - is_bf16_weight_bucket = False - for param in bucket.params: - # Skip copying since bf16 weights in the mxfp8 model - # are already mapped to param.data. - if not is_float8tensor(param): - is_bf16_weight_bucket = True - break - param_start, param_end = bucket.param_to_index[param] - param_slice = bucket.param_data.view(-1)[param_start:param_end] - param.data.copy_(param_slice.view(param.data.shape)) - if is_bf16_weight_bucket: - continue - # All-gathered params are not needed after being copied to param.data. - # Zero out the param buffer (shared with grad buffer) for gradient - # accumulation. We cannot zero out the entire grad buffer because one grad - # buffer may correspond to multiple param buffers. If we zero out the entire - # grad buffer, it would clear the data of those param buffers that have not - # yet completed AG. - bucket.param_data.zero_() - else: - fp8_params = [] - for bucket in bucket_group.buckets: - for param in bucket.params: - if is_float8tensor(param): - fp8_params.append(param) - if len(fp8_params) > 0: - post_all_gather_processing(fp8_params) + self._start_bucket_group_param_sync(bucket_group, force_sync=force_sync) def start_grad_sync(self, *unused): """ diff --git a/megatron/core/distributed/distributed_data_parallel_config.py b/megatron/core/distributed/distributed_data_parallel_config.py index 4c56ddb84b0..7666f05f816 100644 --- a/megatron/core/distributed/distributed_data_parallel_config.py +++ b/megatron/core/distributed/distributed_data_parallel_config.py @@ -1,10 +1,12 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. from dataclasses import dataclass, field from typing import Optional, Tuple import torch +from ..utils import is_torch_min_version + @dataclass class DistributedDataParallelConfig: @@ -34,6 +36,12 @@ class DistributedDataParallelConfig: enabled. Defaults to 1, which means DistOpt is across entire DP domain. """ + use_layer_wise_param_layout: bool = False + """Layer-wise (Muon) optimizer only. When True, LayerWise-managed buffers use + the shard-aligned padded LayerWise param layout. When False (default), the compact + decoupled layout is selected instead. + """ + check_for_nan_in_grad: bool = False """ If true, check for NaNs and Infs in gradients _before_ communication collective. @@ -48,6 +56,11 @@ class DistributedDataParallelConfig: value of max(40000000, 1000000 * dp_size) parameters (larger DP sizes need larger buckets to ensure collectives do not become latency-bound).""" + num_buckets: Optional[int] = None + """Number of buckets for data-parallel communication. Should only specify one of + `bucket_size` and `num_buckets`. If `num_buckets` is specified, `bucket_size` + will be determined at runtime.""" + pad_buckets_for_high_nccl_busbw: bool = False """If true, make sure the bucket size is divisible by a large power of 2 (2^16) to ensure NCCL collectives have high bus bandwidth at large DP counts, since NCCL @@ -165,7 +178,9 @@ class DistributedDataParallelConfig: If True, use all-gather during the initial Megatron-FSDP parameter synchronization step. This can increase overlap between the first parameter all-gather and computation, helping to better hide the - initial communication cost. + initial communication cost. Should be deactivated when using + full-iteration CG, or partial CG if AG/RS is launched beyond the + CG capture scope but is waited on during the capture scope. """ outer_dp_sharding_strategy: str = 'no_shard' @@ -228,6 +243,38 @@ class DistributedDataParallelConfig: """If true, use the `fully_shard` API for FSDP sharding the model. """ + megatron_fsdp_prefetch_recompute_forward_weights: bool = False + """If set to True, Megatron-FSDP prefetches rowwise weights needed by activation + recomputation during backward before prefetching backward transpose weights. + """ + + megatron_fsdp_cache_param_bucket_views: bool = False + """If set to True, Megatron-FSDP caches parameter bucket views to reduce repeated + Python-side view setup when attaching module parameters to all-gather buckets. + """ + + megatron_fsdp_cuda_graph_mode: bool = False + """If set to True, Megatron-FSDP will practice CUDA graph-safe operations, such as + not dereferencing `param.grad` after the optimizer step to preserve references for + CUDA graph replay. Can affect memory utilization in some cases, such as when the + gradient shard is not a view of the Megatron-FSDP sharded gradient buffer, so + FusedAdam(use_decoupled_grad=True) + megatron_fsdp_use_decoupled_grad=True or + setting megatron_fsdp_main_params_dtype == megatron_fsdp_main_grads_dtype is + recommended to avoid casting the gradient to the parameter precision and creating + a casted-copy of the gradient shard that cannot be dereferenced due to replay. + """ + + megatron_fsdp_enable_fine_grained_param_gather: bool = False + """If set to True, enables fine-grained parameter gathering for Megatron-FSDP. + This feature increases the overlap between parameter all-gather and forward computation, + at the cost of more frequent communication calls. + For MXFP8, this approach helps save memory during fine-grained activation + recomputation, because MXFP8 forward and backward passes use different + parameter representations (rowwise data for forward, colwise data for backward). + In this mode, only the rowwise parameters of modules involved in recomputation + will be unsharded. + """ + def __post_init__(self): import os @@ -235,7 +282,21 @@ def __post_init__(self): if self.reuse_grad_buf_for_mxfp8_param_ag: assert self.fp8_param_gather, "Reuse grad buffer only when keeping params in MXFP8." - if self.nccl_ub: + if self.megatron_fsdp_prefetch_recompute_forward_weights: + assert ( + self.use_megatron_fsdp + ), "megatron_fsdp_prefetch_recompute_forward_weights requires use_megatron_fsdp." + assert self.data_parallel_sharding_strategy == "optim_grads_params", ( + "megatron_fsdp_prefetch_recompute_forward_weights is only supported with " + "data_parallel_sharding_strategy='optim_grads_params'." + ) + + if self.megatron_fsdp_cache_param_bucket_views: + assert ( + self.use_megatron_fsdp + ), "megatron_fsdp_cache_param_bucket_views requires use_megatron_fsdp." + + if self.nccl_ub and not is_torch_min_version("2.11.0a0"): if 'expandable_segments:True' in os.getenv('PYTORCH_CUDA_ALLOC_CONF', '').split(','): raise ValueError( "PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True is currently not supported " @@ -247,3 +308,7 @@ def __post_init__(self): "Only need to explicitly specify param_name patterns for FP32 local accumulation " "if .main_grads aren't already in FP32" ) + + if self.num_buckets is not None: + assert self.bucket_size is None, "Cannot specify both num_buckets and bucket_size" + assert self.num_buckets > 0, "num_buckets must be greater than 0" diff --git a/megatron/core/distributed/finalize_model_grads.py b/megatron/core/distributed/finalize_model_grads.py index cc1d0d80e7b..e494bbde2b6 100644 --- a/megatron/core/distributed/finalize_model_grads.py +++ b/megatron/core/distributed/finalize_model_grads.py @@ -332,7 +332,11 @@ def reset_model_temporary_tensors(config: TransformerConfig, model: List[torch.n module.reset_global_aux_loss_tracker() -def _update_router_expert_bias(model: List[torch.nn.Module], config: TransformerConfig): +def _update_router_expert_bias( + model: List[torch.nn.Module], + config: TransformerConfig, + tp_dp_cp_group: Optional[torch.distributed.ProcessGroup] = None, +): """ Update the expert bias of the router for a global batch. This requires all-reduce of local_tokens_per_expert across TPxCPxDP ranks @@ -349,6 +353,7 @@ def _update_router_expert_bias(model: List[torch.nn.Module], config: Transformer hasattr(module, 'expert_bias') and module.training and module.expert_bias is not None + and not getattr(module, 'frozen_expert_bias', False) ): tokens_per_expert_list.append(module.local_tokens_per_expert) expert_bias_list.append(module.expert_bias) @@ -358,7 +363,10 @@ def _update_router_expert_bias(model: List[torch.nn.Module], config: Transformer stacked_tokens_per_expert = torch.stack(tokens_per_expert_list, dim=0) stacked_expert_bias = torch.stack(expert_bias_list, dim=0) stacked_updated_expert_bias = get_updated_expert_bias( - stacked_tokens_per_expert, stacked_expert_bias, config.moe_router_bias_update_rate + stacked_tokens_per_expert, + stacked_expert_bias, + config.moe_router_bias_update_rate, + tp_dp_cp_group=tp_dp_cp_group, ) for expert_bias, updated_expert_bias in zip(expert_bias_list, stacked_updated_expert_bias): @@ -456,6 +464,7 @@ def finalize_model_grads( """ config = get_model_config(model[0]) + tp_dp_cp_group = None if pg_collection is not None: assert hasattr(pg_collection, 'tp') assert hasattr(pg_collection, 'pp') @@ -474,6 +483,11 @@ def finalize_model_grads( "If you don't need pos_embd_group, you need to explicitly set it to None." ) assert hasattr(pg_collection, 'dp_cp') + if config.moe_router_enable_expert_bias: + assert hasattr(pg_collection, 'tp_dp_cp') and pg_collection.tp_dp_cp is not None, ( + "pg_collection must have tp_dp_cp when " "moe_router_enable_expert_bias is enabled." + ) + tp_dp_cp_group = pg_collection.tp_dp_cp tp_group = pg_collection.tp pp_group = pg_collection.pp embd_group = pg_collection.embd @@ -527,7 +541,11 @@ def finalize_model_grads( config.timers('embedding-grads-all-reduce').stop() if config.moe_router_enable_expert_bias: - _update_router_expert_bias(model, config) + if pg_collection is None: + tp_dp_cp_group = parallel_state.get_tensor_and_data_parallel_group( + with_context_parallel=True + ) + _update_router_expert_bias(model, config, tp_dp_cp_group=tp_dp_cp_group) reset_model_temporary_tensors(config, model) @@ -544,7 +562,10 @@ def finalize_model_grads( # all-reduce across DP ranks. torch.distributed.all_reduce(num_tokens, group=dp_cp_group) + + # Clamp to avoid div-by-zero without a host-side branch on a device tensor, + # which would otherwise cause a sync that is illegal during CUDA graph capture. + safe_num_tokens = torch.clamp(num_tokens, min=1) + scaling = 1.0 / safe_num_tokens for model_chunk in model: - if num_tokens > 0: - scaling = 1.0 / num_tokens - model_chunk.scale_gradients(scaling) + model_chunk.scale_gradients(scaling) diff --git a/megatron/core/distributed/fsdp/mcore_fsdp_adapter.py b/megatron/core/distributed/fsdp/mcore_fsdp_adapter.py index 6f389b11058..1dc673891ff 100644 --- a/megatron/core/distributed/fsdp/mcore_fsdp_adapter.py +++ b/megatron/core/distributed/fsdp/mcore_fsdp_adapter.py @@ -215,6 +215,7 @@ def __init__( enable_fine_grained_param_gather_hook=( (config.fp8_recipe == "mxfp8" and ddp_config.fp8_param_gather) or config.overlap_moe_expert_parallel_comm + or self.ddp_config.megatron_fsdp_enable_fine_grained_param_gather ), enable_fine_grained_param_gather_backward_hook=( config.overlap_moe_expert_parallel_comm diff --git a/megatron/core/distributed/fsdp/src/megatron_fsdp/distributed_data_parallel_config.py b/megatron/core/distributed/fsdp/src/megatron_fsdp/distributed_data_parallel_config.py index 6c360b92fbf..fb97d95a33c 100644 --- a/megatron/core/distributed/fsdp/src/megatron_fsdp/distributed_data_parallel_config.py +++ b/megatron/core/distributed/fsdp/src/megatron_fsdp/distributed_data_parallel_config.py @@ -1,10 +1,12 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. from dataclasses import dataclass from typing import Optional import torch +from .utils import is_torch_min_version + @dataclass class DistributedDataParallelConfig: @@ -90,7 +92,9 @@ class DistributedDataParallelConfig: If True, use all-gather during the initial Megatron-FSDP parameter synchronization step. This can increase overlap between the first parameter all-gather and computation, helping to better hide the - initial communication cost. + initial communication cost. Should be deactivated when using + full-iteration CG, or partial CG if AG/RS is launched beyond the + CG capture scope but is waited on during the capture scope. """ fsdp_db_use_persist_buf_on_alloc_fail: bool = False @@ -159,11 +163,49 @@ class DistributedDataParallelConfig: """If true, use the `fully_shard` API for FSDP sharding the model. """ + megatron_fsdp_prefetch_recompute_forward_weights: bool = False + """If set to True, Megatron-FSDP prefetches rowwise weights needed by activation + recomputation during backward before prefetching backward transpose weights. + """ + + megatron_fsdp_cache_param_bucket_views: bool = False + """If set to True, Megatron-FSDP caches parameter bucket views to reduce repeated + Python-side view setup when attaching module parameters to all-gather buckets. + """ + + megatron_fsdp_cuda_graph_mode: bool = False + """If set to True, Megatron-FSDP will practice CUDA graph-safe operations, such as + not dereferencing `param.grad` after the optimizer step to preserve references for + CUDA graph replay. Can affect memory utilization in some cases, such as when the + gradient shard is not a view of the Megatron-FSDP sharded gradient buffer, so + FusedAdam(use_decoupled_grad=True) + megatron_fsdp_use_decoupled_grad=True or + setting megatron_fsdp_main_params_dtype == megatron_fsdp_main_grads_dtype is + recommended to avoid casting the gradient to the parameter precision and creating + a casted-copy of the gradient shard that cannot be dereferenced due to replay. + """ + + megatron_fsdp_enable_fine_grained_param_gather: bool = False + """If set to True, enables fine-grained parameter gathering for Megatron-FSDP. + This feature increases the overlap between parameter all-gather and forward computation, + at the cost of more frequent communication calls. + For MXFP8, this approach helps save memory during fine-grained activation + recomputation, because MXFP8 forward and backward passes use different + parameter representations (rowwise data for forward, colwise data for backward). + In this mode, only the rowwise parameters of modules involved in recomputation + will be unsharded. + """ + def __post_init__(self): import os """Check the validity of the config.""" - if self.nccl_ub: + if self.megatron_fsdp_prefetch_recompute_forward_weights: + assert self.data_parallel_sharding_strategy == "optim_grads_params", ( + "megatron_fsdp_prefetch_recompute_forward_weights is only supported with " + "data_parallel_sharding_strategy='optim_grads_params'." + ) + + if self.nccl_ub and not is_torch_min_version("2.11.0a0"): if 'expandable_segments:True' in os.getenv('PYTORCH_CUDA_ALLOC_CONF', '').split(','): raise ValueError( "PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True is currently not supported " diff --git a/.github/workflows/sync-skills.yml b/megatron/core/distributed/fsdp/src/megatron_fsdp/experimental/__init__.py similarity index 68% rename from .github/workflows/sync-skills.yml rename to megatron/core/distributed/fsdp/src/megatron_fsdp/experimental/__init__.py index 75b8c20dca0..1bd55b7d995 100644 --- a/.github/workflows/sync-skills.yml +++ b/megatron/core/distributed/fsdp/src/megatron_fsdp/experimental/__init__.py @@ -11,19 +11,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -name: Sync skills → agent dirs -on: - workflow_dispatch: - push: - branches: - - main - paths: - - "skills/**" - - "AGENTS.md" +"""Experimental Megatron-FSDP implementation.""" -jobs: - sync: - uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_sync_skills.yml@v0.91.0 - secrets: - PAT: ${{ secrets.PAT }} +from .dbuffer import DBuffer +from .placement import Flat, Partial, Placement, Replicate + +__all__ = ["DBuffer", "Flat", "Partial", "Placement", "Replicate"] diff --git a/megatron/core/distributed/fsdp/src/megatron_fsdp/experimental/dbuffer.py b/megatron/core/distributed/fsdp/src/megatron_fsdp/experimental/dbuffer.py new file mode 100644 index 00000000000..51f52451089 --- /dev/null +++ b/megatron/core/distributed/fsdp/src/megatron_fsdp/experimental/dbuffer.py @@ -0,0 +1,454 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Distributed tensor buffers for Megatron-FSDP.""" + +import dataclasses +from collections.abc import Iterable + +import torch +import torch.distributed as dist +import torch.distributed.tensor as dist_tensor +from torch.distributed import DeviceMesh +from torch.distributed.tensor import DTensor + +from .layout import GlobalLayout, Shape, non_leading_numel +from .placement import Flat, Partial, Placement, Replicate + + +@dataclasses.dataclass(frozen=True) +class _OwnedRange: + numel: int + tensor_relative_offset: int + buffer_relative_offset: int + + +def _validate_mesh_axis(mesh: DeviceMesh, axis: int) -> None: + if not isinstance(axis, int) or isinstance(axis, bool): + raise TypeError(f"Mesh axis must be an int, got {type(axis).__name__}.") + if axis < 0 or axis >= mesh.ndim: + raise ValueError(f"Mesh axis {axis} is out of bounds for mesh ndim {mesh.ndim}.") + + +def _validate_placements(placements: Iterable[Placement]) -> None: + seen_flat = False + for placement in placements: + if not isinstance(placement, (Replicate, Partial, Flat)): + raise TypeError(f"Unsupported DBuffer placement: {placement!r}.") + if isinstance(placement, Flat): + seen_flat = True + elif seen_flat: + raise ValueError( + "Flat placements must be a suffix of the placement list so each " + "local buffer is a contiguous global-buffer range." + ) + + +class DBuffer: + """A distributed buffer holding a group of logical tensors. + + DBuffer is analogous to DTensor, but manages a group of logical tensors in + one local storage tensor. It stores enough metadata to return per-tensor + views, redistribute the buffer across mesh axes, and materialize per-tensor + DTensors for optimizer state or distributed checkpointing. + """ + + # DBuffer owns only the data-parallel sub-mesh. Higher-level callers, such as + # ParameterGroup, should extend returned DTensors with tensor-parallel mesh axes + # because TP sharding metadata lives on nn.Parameter in MCore/TransformerEngine. + mesh: DeviceMesh + placements: tuple[Placement, ...] + layout: GlobalLayout + offset: int + local_buffer: torch.Tensor + + def __init__( + self, + mesh: DeviceMesh, + placements: Iterable[Placement], + tensor_shapes: Iterable[Shape], + dtype: torch.dtype, + device: torch.device | str, + ) -> None: + """Create a DBuffer and allocate its local buffer. + + Args: + mesh: Device mesh whose dimensions correspond to ``placements``. + placements: Per-mesh-axis DBuffer placements. + tensor_shapes: Global shapes for each logical tensor in this buffer. + dtype: Dtype for the local buffer. + device: Device for the local buffer. + """ + placements = tuple(placements) + if len(placements) != mesh.ndim: + raise ValueError( + f"Expected {mesh.ndim} placements for device mesh, got {len(placements)}." + ) + _validate_placements(placements) + + self.mesh = mesh + self.placements = placements + + tensor_shapes = tuple(torch.Size(shape) for shape in tensor_shapes) + self.layout = GlobalLayout.build(tensor_shapes, dp_size=self.mesh.size()) + + self.offset, local_numel = self.layout.get_local_range(self.mesh, self.placements) + self.local_buffer = torch.empty(local_numel, dtype=dtype, device=device) + + @property + def dtype(self) -> torch.dtype: + """Dtype of the local buffer.""" + return self.local_buffer.dtype + + @property + def device(self) -> torch.device: + """Device of the local buffer.""" + return self.local_buffer.device + + def _get_owned_range(self, tensor_index: int) -> _OwnedRange | None: + """Return this buffer's owned range for logical tensor ``tensor_index``.""" + tensor_start = self.layout.tensor_to_offset[tensor_index] + tensor_end = tensor_start + self.layout.tensor_shapes[tensor_index].numel() + buffer_start = self.offset + buffer_end = self.offset + self.local_buffer.numel() + + overlap_start = max(tensor_start, buffer_start) + overlap_end = min(tensor_end, buffer_end) + if overlap_start >= overlap_end: + return None + + return _OwnedRange( + numel=overlap_end - overlap_start, + tensor_relative_offset=overlap_start - tensor_start, + buffer_relative_offset=overlap_start - buffer_start, + ) + + @classmethod + def from_local( + cls, + local_buffer: torch.Tensor, + mesh: DeviceMesh, + placements: Iterable[Placement], + tensor_shapes: Iterable[Shape], + ) -> "DBuffer": + """Create a DBuffer from an existing local buffer. + + Args: + local_buffer: Contiguous local tensor storage for this rank. DBuffer + uses it directly in collectives such as all-gather and + reduce-scatter, which are efficient with contiguous tensors. + mesh: Device mesh whose dimensions correspond to ``placements``. + placements: Per-mesh-axis DBuffer placements. + tensor_shapes: Global shapes for each logical tensor in this buffer. + + Returns: + A DBuffer that reuses ``local_buffer`` without allocating storage. + """ + placements = tuple(placements) + if len(placements) != mesh.ndim: + raise ValueError( + f"Expected {mesh.ndim} placements for device mesh, got {len(placements)}." + ) + _validate_placements(placements) + if local_buffer.dim() != 1: + raise ValueError("local_buffer must be a flat 1D tensor.") + if not local_buffer.is_contiguous(): + raise ValueError("local_buffer must be contiguous for collective operations.") + + tensor_shapes = tuple(torch.Size(shape) for shape in tensor_shapes) + layout = GlobalLayout.build(tensor_shapes, dp_size=mesh.size()) + offset, local_numel = layout.get_local_range(mesh, placements) + if local_buffer.numel() != local_numel: + raise ValueError( + f"Expected local_buffer with {local_numel} elements, got " + f"{local_buffer.numel()}." + ) + + buffer = cls.__new__(cls) + buffer.mesh = mesh + buffer.placements = placements + buffer.layout = layout + buffer.offset = offset + buffer.local_buffer = local_buffer + return buffer + + @classmethod + def distribute_tensors( + cls, tensors: Iterable[torch.Tensor], mesh: DeviceMesh, placements: Iterable[Placement] + ) -> "DBuffer": + """Distribute full local tensor values into a DBuffer. + + Args: + tensors: Full tensor values available on this rank. + mesh: Device mesh whose dimensions correspond to ``placements``. + placements: Per-mesh-axis DBuffer placements. + + Returns: + A DBuffer whose local storage matches ``placements``. + """ + tensors = tuple(tensor.detach().contiguous() for tensor in tensors) + if not tensors: + raise ValueError("DBuffer.distribute_tensors() requires at least one tensor.") + + dtype = tensors[0].dtype + for tensor in tensors: + if tensor.dtype != dtype: + raise ValueError("All tensors in a DBuffer must have the same dtype.") + + tensor_shapes = tuple(tensor.shape for tensor in tensors) + buffer = cls( + mesh=mesh, + placements=placements, + tensor_shapes=tensor_shapes, + dtype=dtype, + device=mesh.device_type, + ) + # Only logical tensor ranges are initialized. Padding and layout gaps are not + # observable through get_local_tensor() and can remain unspecified. + for index, tensor in enumerate(tensors): + owned_range = buffer._get_owned_range(index) + if owned_range is None: + continue + + source_slice = tensor.view(-1).narrow( + 0, owned_range.tensor_relative_offset, owned_range.numel + ) + buffer.local_buffer.narrow( + 0, owned_range.buffer_relative_offset, owned_range.numel + ).copy_(source_slice) + return buffer + + def _create_or_validate_out( + self, placements: Iterable[Placement], out: "DBuffer | None" + ) -> "DBuffer": + placements = tuple(placements) + if out is None: + return DBuffer( + mesh=self.mesh, + placements=placements, + tensor_shapes=self.layout.tensor_shapes, + dtype=self.dtype, + device=self.device, + ) + + if out.mesh != self.mesh: + raise ValueError(f"Expected out mesh {self.mesh!r}, got {out.mesh!r}.") + if out.placements != placements: + raise ValueError(f"Expected out placements {placements!r}, got {out.placements!r}.") + if out.layout != self.layout: + raise ValueError(f"Expected out layout {self.layout!r}, got {out.layout!r}.") + if out.dtype != self.dtype: + raise ValueError(f"Expected out dtype {self.dtype}, got {out.dtype}.") + if out.device != self.device: + raise ValueError(f"Expected out device {self.device}, got {out.device}.") + return out + + def redistribute( + self, new_placements: Iterable[Placement], *, out: "DBuffer | None" = None + ) -> "DBuffer": + """Redistribute this buffer to ``new_placements``. + + This dispatcher supports the one-axis transitions: + Flat -> Replicate, Partial -> Replicate, Partial -> Flat, and + Replicate -> Flat. Other placement changes are intentionally unsupported. + """ + new_placements = tuple(new_placements) + if len(new_placements) != self.mesh.ndim: + raise ValueError( + f"Expected {self.mesh.ndim} placements for device mesh, got " + f"{len(new_placements)}." + ) + _validate_placements(new_placements) + + changed_axis: int | None = None + for axis, (old_placement, new_placement) in enumerate( + zip(self.placements, new_placements, strict=True) + ): + if old_placement == new_placement: + continue + if changed_axis is not None: + raise NotImplementedError( + "redistribute() currently supports one placement change, " + f"got changed axes {changed_axis} and {axis}." + ) + changed_axis = axis + + if changed_axis is None: + if out is None: + return self + out = self._create_or_validate_out(new_placements, out) + out.local_buffer.copy_(self.local_buffer) + return out + + axis = changed_axis + old_placement = self.placements[axis] + new_placement = new_placements[axis] + if isinstance(old_placement, Flat) and isinstance(new_placement, Replicate): + return self.allgather(axis, out=out) + if isinstance(old_placement, Partial) and isinstance(new_placement, Replicate): + return self.allreduce(axis, out=out) + if isinstance(old_placement, Partial) and isinstance(new_placement, Flat): + return self.reduce_scatter(axis, new_placement, out=out) + if isinstance(old_placement, Replicate) and isinstance(new_placement, Flat): + return self.scatter(axis, new_placement, out=out) + raise NotImplementedError( + "Unsupported DBuffer placement transition on axis " + f"{axis}: {old_placement!r} -> {new_placement!r}." + ) + + def allgather(self, mesh_axis: int, *, out: "DBuffer | None" = None) -> "DBuffer": + """All-gather a sharded axis into Replicate placement.""" + _validate_mesh_axis(self.mesh, mesh_axis) + if not isinstance(self.placements[mesh_axis], Flat): + raise ValueError( + f"allgather() currently requires Flat placement on axis {mesh_axis!r}." + ) + + placements = list(self.placements) + placements[mesh_axis] = Replicate() + _validate_placements(placements) + out = self._create_or_validate_out(placements, out) + dist.all_gather_into_tensor( + output_tensor=out.local_buffer, + input_tensor=self.local_buffer, + group=self.mesh.get_group(mesh_axis), + ) + return out + + def allreduce(self, mesh_axis: int, *, out: "DBuffer | None" = None) -> "DBuffer": + """All-reduce a Partial axis into Replicate placement.""" + _validate_mesh_axis(self.mesh, mesh_axis) + axis = mesh_axis + partial_placement = self.placements[axis] + if not isinstance(partial_placement, Partial): + raise ValueError(f"allreduce() requires Partial placement on axis {mesh_axis!r}.") + + placements = list(self.placements) + placements[axis] = Replicate() + out = self._create_or_validate_out(placements, out) + out.local_buffer.copy_(self.local_buffer) + dist.all_reduce( + out.local_buffer, op=partial_placement.reduce_op, group=self.mesh.get_group(axis) + ) + return out + + def reduce_scatter( + self, mesh_axis: int, new_placement: Placement, *, out: "DBuffer | None" = None + ) -> "DBuffer": + """Reduce-scatter a Partial axis into ``new_placement``.""" + _validate_mesh_axis(self.mesh, mesh_axis) + axis = mesh_axis + if not isinstance(new_placement, Flat): + raise NotImplementedError("DBuffer currently supports reduce_scatter() to Flat only.") + partial_placement = self.placements[axis] + if not isinstance(partial_placement, Partial): + raise ValueError(f"reduce_scatter() requires Partial placement on axis {mesh_axis!r}.") + + placements = list(self.placements) + placements[axis] = new_placement + _validate_placements(placements) + out = self._create_or_validate_out(placements, out) + dist.reduce_scatter_tensor( + output=out.local_buffer, + input=self.local_buffer, + op=partial_placement.reduce_op, + group=self.mesh.get_group(axis), + ) + return out + + def scatter( + self, mesh_axis: int, new_placement: Placement, *, out: "DBuffer | None" = None + ) -> "DBuffer": + """Locally chunk a Replicate axis into ``new_placement``.""" + _validate_mesh_axis(self.mesh, mesh_axis) + axis = mesh_axis + if not isinstance(new_placement, Flat): + raise NotImplementedError("DBuffer currently supports scatter() to Flat only.") + if not isinstance(self.placements[axis], Replicate): + raise ValueError(f"scatter() requires Replicate placement on axis {mesh_axis!r}.") + + placements = list(self.placements) + placements[axis] = new_placement + _validate_placements(placements) + + if out is None: + destination_offset, destination_numel = self.layout.get_local_range( + self.mesh, placements + ) + else: + out = self._create_or_validate_out(placements, out) + destination_offset = out.offset + destination_numel = out.local_buffer.numel() + + local_buffer_offset = destination_offset - self.offset + if ( + local_buffer_offset < 0 + or local_buffer_offset + destination_numel > self.local_buffer.numel() + ): + raise RuntimeError("scatter() destination is not contained in the source local buffer.") + local_slice = self.local_buffer.narrow(0, local_buffer_offset, destination_numel) + if out is None: + return DBuffer.from_local(local_slice, self.mesh, placements, self.layout.tensor_shapes) + + out.local_buffer.copy_(local_slice) + return out + + def get_local_tensor(self, index: int) -> torch.Tensor: + """Return this rank's local view for logical tensor ``index``. + + Flat placements shard dim 0, so the returned view preserves all + non-leading dimensions and only changes the leading dimension. + """ + shape = self.layout.tensor_shapes[index] + owned_range = self._get_owned_range(index) + + row_size = non_leading_numel(shape) + if owned_range is None: + empty_shape = torch.Size((0, *shape[1:])) + return torch.empty(empty_shape, dtype=self.dtype, device=self.device) + + if owned_range.tensor_relative_offset % row_size != 0 or owned_range.numel % row_size != 0: + raise RuntimeError( + f"Local tensor shard for tensor {index} does not preserve dim-0 boundaries." + ) + local_shape = torch.Size((owned_range.numel // row_size, *shape[1:])) + return self.local_buffer.narrow( + 0, owned_range.buffer_relative_offset, owned_range.numel + ).view(local_shape) + + def get_dtensor(self, index: int) -> DTensor: + """Return logical tensor ``index`` as a DTensor.""" + torch_placements = [] + for placement in self.placements: + if isinstance(placement, Replicate): + torch_placements.append(dist_tensor.Replicate()) + elif isinstance(placement, Flat): + torch_placements.append(dist_tensor.Shard(0)) + elif isinstance(placement, Partial): + raise ValueError("Partial DBuffer placements cannot be represented as DTensor.") + else: + raise TypeError(f"Unsupported placement for DTensor conversion: {placement!r}.") + + local_tensor = self.get_local_tensor(index) + tensor_shape = self.layout.tensor_shapes[index] + # DBuffer uses contiguous flat storage, and Flat only shards dim 0, so + # the local view's stride matches the logical global tensor stride. + return DTensor.from_local( + local_tensor=local_tensor, + device_mesh=self.mesh, + placements=tuple(torch_placements), + run_check=False, + shape=tensor_shape, + stride=local_tensor.stride(), + ) diff --git a/megatron/core/distributed/fsdp/src/megatron_fsdp/experimental/layout.py b/megatron/core/distributed/fsdp/src/megatron_fsdp/experimental/layout.py new file mode 100644 index 00000000000..11070495e4d --- /dev/null +++ b/megatron/core/distributed/fsdp/src/megatron_fsdp/experimental/layout.py @@ -0,0 +1,251 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Global tensor layout metadata for DBuffer.""" + +import dataclasses +import math +from collections.abc import Iterable +from typing import TypeAlias + +import torch +from torch.distributed import DeviceMesh + +from .placement import Flat, Placement + +Shape: TypeAlias = torch.Size | Iterable[int] + + +@dataclasses.dataclass(frozen=True) +class GlobalLayout: + """Global tensor layout in element coordinates.""" + + tensor_shapes: tuple[torch.Size, ...] + tensor_to_offset: tuple[int, ...] + size: int + + @classmethod + def build(cls, shapes: Iterable[Shape], dp_size: int) -> "GlobalLayout": + """Compute global tensor element offsets and padded size. + + This is a DBuffer-specific reimplementation of + ``param_and_grad_buffer.build_data_parallel_buffer_index``. It keeps only + the global offset construction and final padding so each rank-local shard + size is a multiple of ``chunk_size``; DBuffer derives rank-local slices + later through DTensor placements. + + The computed layout is compatible with Flat, TensorAtomic, and BlockAtomic, + even though the latter two are not implemented. + + ``chunk_size`` is the least common multiple of each tensor's row size + (``shape[1:].numel()``). For example, with shapes P0=(2, 6), P1=(4, 4), + P2=(4, 4), P3=(1, 2), P4=(1, 6), ``chunk_size = LCM(6, 4, 4, 2, 6) = 12`` + and a 5-rank DP layout has equal-size rank shards: + + ``` + rank 0 [ 0, 12): | P0 row 0 | P0 row 1 | + rank 1 [12, 24): | P1 row 0 | P1 row 1 | P1 row 2 | + rank 2 [24, 36): | P1 row 3 | P3 | gap | P2 row 0 | + rank 3 [36, 48): | P2 row 1 | P2 row 2 | P2 row 3 | + rank 4 [48, 60): | P4 | pad | + ``` + + The diagram uses four character columns per element; segment widths are + proportional. Every chunk boundary is aligned to each tensor's row size, + so each DP shard owns full rows even when fragments fill regular-tensor + padding gaps. + + Args: + shapes: Logical tensor shapes in tensor-id order. + dp_size: Data-parallel shard count for this global layout. + + Returns: + Global layout with row-aligned tensor offsets and a total size padded + to a multiple of ``chunk_size * dp_size``, so every rank-local shard + length is a multiple of ``chunk_size``. + """ + if dp_size <= 0: + raise ValueError(f"DP size must be positive, got {dp_size}.") + + tensor_shapes = tuple(torch.Size(shape) for shape in shapes) + chunk_size = 1 + for shape in tensor_shapes: + row_size = non_leading_numel(shape) + if row_size <= 0: + raise ValueError( + f"Cannot compute a layout for zero-sized non-leading dims: {shape}." + ) + chunk_size = math.lcm(chunk_size, row_size) + + # chunk_size is the packing unit. Since every tensor row size divides it, + # DP shard boundaries that are multiples of chunk_size avoid splitting dim-0 rows. + UNASSIGNED_OFFSET = -1 + tensor_to_offset: list[int] = [UNASSIGNED_OFFSET] * len(tensor_shapes) + fragment_items = [] + regular_items = [] + for tensor_id, shape in enumerate(tensor_shapes): + if shape.numel() < chunk_size: + fragment_items.append((tensor_id, shape)) + else: + regular_items.append((tensor_id, shape)) + + # Regular tensors anchor the layout. Fragments are held back to fill padding + # gaps left by regular tensors whose sizes are not exact multiples of chunk_size. + fragment_items.sort(key=lambda id_shape: id_shape[1].numel(), reverse=True) + + next_offset = 0 + while regular_items: + tensor_id, shape = regular_items.pop(0) + tensor_numel = shape.numel() + tensor_to_offset[tensor_id] = next_offset + + if tensor_numel % chunk_size == 0: + next_offset += tensor_numel + continue + + gap_offset = next_offset + tensor_numel + next_offset += _pad_to_multiple(tensor_numel, chunk_size) + fragment_gap_end = next_offset + remainder = tensor_numel % chunk_size + + # Try to pair this non-divisible regular tensor with a conjugate regular + # tensor whose remainder fits in the same chunk_size interval. The + # conjugate starts in the gap and then continues with full chunk_size + # intervals after this one. + conjugate_item = None + for candidate_item in regular_items[:]: + _, candidate_shape = candidate_item + candidate_numel = candidate_shape.numel() + candidate_remainder = candidate_numel % chunk_size + if candidate_remainder == 0: + continue + if remainder + candidate_remainder <= chunk_size: + conjugate_item = candidate_item + regular_items.remove(candidate_item) + break + + if conjugate_item is not None: + conjugate_id, conjugate_shape = conjugate_item + conjugate_numel = conjugate_shape.numel() + conjugate_remainder = conjugate_numel % chunk_size + conjugate_offset = next_offset - conjugate_remainder + tensor_to_offset[conjugate_id] = conjugate_offset + fragment_gap_end = conjugate_offset + next_offset += (conjugate_numel // chunk_size) * chunk_size + + # Fill any remaining gap with fragments, keeping each fragment aligned to + # its own row size so dim-0 rows remain contiguous within DP shards. + for fragment in fragment_items[:]: + frag_id, frag_shape = fragment + frag_numel = frag_shape.numel() + aligned_gap_offset = _pad_to_multiple(gap_offset, non_leading_numel(frag_shape)) + if aligned_gap_offset + frag_numel > fragment_gap_end: + continue + tensor_to_offset[frag_id] = aligned_gap_offset + gap_offset = aligned_gap_offset + frag_numel + fragment_items.remove(fragment) + + # Fragments that did not fit into regular-tensor gaps are appended at the tail. + for frag_id, frag_shape in fragment_items: + next_offset = _pad_to_multiple(next_offset, non_leading_numel(frag_shape)) + tensor_to_offset[frag_id] = next_offset + next_offset += frag_shape.numel() + + return cls( + tensor_shapes=tensor_shapes, + tensor_to_offset=tuple(tensor_to_offset), + size=_pad_to_multiple(next_offset, chunk_size * dp_size), + ) + + def __post_init__(self) -> None: + """Validate tensor offsets are row-aligned, in bounds, and non-overlapping.""" + + @dataclasses.dataclass(frozen=True) + class TensorRange: + """Contiguous global element range occupied by one logical tensor.""" + + start: int + end: int + tensor_id: int + + if self.size < 0: + raise AssertionError(f"Global layout size {self.size} is negative.") + if len(self.tensor_shapes) != len(self.tensor_to_offset): + raise AssertionError( + "Global layout has mismatched tensor shapes and offsets: " + f"{len(self.tensor_shapes)} shapes and {len(self.tensor_to_offset)} offsets." + ) + + tensor_ranges: list[TensorRange] = [] + for tensor_id, (shape, start) in enumerate( + zip(self.tensor_shapes, self.tensor_to_offset, strict=True) + ): + if start < 0: + raise AssertionError(f"Tensor {tensor_id} offset {start} is negative.") + + row_size = non_leading_numel(shape) + if row_size <= 0: + raise AssertionError(f"Tensor {tensor_id} has invalid row size {row_size}.") + if start % row_size != 0: + raise AssertionError( + f"Tensor {tensor_id} offset {start} is not aligned to row size {row_size}." + ) + + end = start + shape.numel() + if end > self.size: + raise AssertionError( + f"Tensor {tensor_id} range [{start}, {end}) exceeds " + f"layout size {self.size}." + ) + tensor_ranges.append(TensorRange(start, end, tensor_id)) + + previous_range: TensorRange | None = None + for current_range in sorted(tensor_ranges, key=lambda tensor_range: tensor_range.start): + if previous_range is not None and current_range.start < previous_range.end: + raise AssertionError( + "Global layout tensors overlap: " + f"tensor {previous_range.tensor_id} " + f"[{previous_range.start}, {previous_range.end}) and " + f"tensor {current_range.tensor_id} " + f"[{current_range.start}, {current_range.end})." + ) + previous_range = current_range + + def get_local_range(self, mesh: DeviceMesh, placements: Iterable[Placement]) -> tuple[int, int]: + """Return this rank's local element offset and length for ``placements``.""" + offset = 0 + numel = self.size + for axis, placement in reversed(tuple(enumerate(placements))): + if not isinstance(placement, Flat): + continue + axis_size = mesh.size(axis) + if numel % axis_size != 0: + raise ValueError( + f"Local range size {numel} is not divisible by Flat axis size {axis_size}." + ) + shard_size = numel // axis_size + offset += mesh.get_local_rank(axis) * shard_size + numel = shard_size + return offset, numel + + +def non_leading_numel(shape: torch.Size) -> int: + """Return the number of elements after dim 0 for a non-scalar shape.""" + if len(shape) == 0: + raise ValueError(f"DBuffer layout does not support 0D tensor shapes: {shape}.") + return shape[1:].numel() + + +def _pad_to_multiple(value: int, multiple: int) -> int: + return ((value + multiple - 1) // multiple) * multiple diff --git a/megatron/core/distributed/fsdp/src/megatron_fsdp/experimental/placement.py b/megatron/core/distributed/fsdp/src/megatron_fsdp/experimental/placement.py new file mode 100644 index 00000000000..1b561c9634d --- /dev/null +++ b/megatron/core/distributed/fsdp/src/megatron_fsdp/experimental/placement.py @@ -0,0 +1,55 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""DBuffer placement definitions. + +These placement concepts are borrowed from PyTorch DTensor placements: +``Replicate`` and ``Partial`` mirror DTensor's placements. ``Flat`` is the +only sharded DBuffer placement implemented so far; it stores dim-0 shards in a +flattened local buffer. + +============= ============= ==================== +Source Destination DBuffer operation +============= ============= ==================== +sharded ``Replicate`` ``allgather()`` +``Partial`` sharded ``reduce_scatter()`` +``Partial`` ``Replicate`` ``allreduce()`` +``Replicate`` sharded ``scatter()`` (local) +============= ============= ==================== +""" + +import dataclasses + +import torch.distributed as dist + + +class Placement: + """Base class for DBuffer placements.""" + + +@dataclasses.dataclass(frozen=True) +class Replicate(Placement): + """Replicated local buffer placement.""" + + +@dataclasses.dataclass(frozen=True) +class Partial(Placement): + """Unreduced replicated local buffer placement.""" + + reduce_op: dist.ReduceOp.RedOpType = dist.ReduceOp.SUM + + +@dataclasses.dataclass(frozen=True) +class Flat(Placement): + """Flat per-unit dim-0 sharded local buffer placement.""" diff --git a/megatron/core/distributed/fsdp/src/megatron_fsdp/fully_shard.py b/megatron/core/distributed/fsdp/src/megatron_fsdp/fully_shard.py index a1f7fabd50a..fc6367bc02b 100644 --- a/megatron/core/distributed/fsdp/src/megatron_fsdp/fully_shard.py +++ b/megatron/core/distributed/fsdp/src/megatron_fsdp/fully_shard.py @@ -1,16 +1,4 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. import logging import types @@ -103,7 +91,10 @@ def fully_shard_model( fsdp_db_use_persist_buf_on_alloc_fail: bool = False, disable_symmetric_registration: bool = False, enable_fine_grained_param_gather: bool = False, + prefetch_recompute_forward_weights: bool = False, + cache_param_bucket_views: bool = False, use_decoupled_grad: bool = False, + cuda_graph_mode: bool = False, ) -> torch.nn.Module: """ Fully-shard the model for Megatron-FSDP. This wraps the model in a MegatronFSDP @@ -261,10 +252,29 @@ class that schedules the sharding lifecycle of the model parameters and gradient unshards parameters per-Module instead of unsharding all sub-modules of an FSDP unit module simultaneously. Defaults to False. + prefetch_recompute_forward_weights (bool): + Whether to prefetch rowwise weights needed by activation recomputation during + backward before prefetching backward transpose weights. Defaults to False. + + cache_param_bucket_views (bool): + Whether to cache parameter bucket views to reduce repeated Python-side view setup + when attaching module parameters to all-gather buckets. Defaults to False. + use_decoupled_grad (bool): If true, reduced gradients are installed into `Parameter.decoupled_grad` instead of `Parameter.grad`. Defaults to False. + cuda_graph_mode (bool): + If true, Megatron-FSDP will practice CUDA graph-safe operations, such as + not dereferencing `param.grad` after the optimizer step to preserve references + for CUDA graph replay. Can affect memory utilization in some cases, such as + when the gradient shard is not a view of the Megatron-FSDP sharded gradient + buffer, so `FusedAdam(use_decoupled_grad=True) + use_decoupled_grad=True` or + setting `megatron_fsdp_main_params_dtype == megatron_fsdp_main_grads_dtype` + is recommended to avoid casting the gradient to the parameter precision and + creating a casted-copy of the gradient shard that cannot be dereferenced due + to replay. Defaults to False. + Returns: model (MegatronFSDP): The wrapped Megatron-FSDP model configured for FSDP. """ @@ -346,6 +356,17 @@ class that schedules the sharding lifecycle of the model parameters and gradient "Meta device initialization (init_model_with_meta_device=True) is not " "supported or necessary for the 'no_shard' / 0 sharding strategy." ) + if prefetch_recompute_forward_weights: + if zero_dp_strategy != "optim_grads_params": + raise ValueError( + "prefetch_recompute_forward_weights is only supported with " + "zero_dp_strategy='optim_grads_params'." + ) + if not fsdp_unit_modules: + raise ValueError( + "prefetch_recompute_forward_weights requires fsdp_unit_modules to define " + "the Megatron-FSDP unit-level backward prefetch order." + ) # DDP Config for Megatron FSDP. ddp_config = DistributedDataParallelConfig( @@ -359,7 +380,10 @@ class that schedules the sharding lifecycle of the model parameters and gradient fsdp_double_buffer=fsdp_double_buffer or nccl_ub, fsdp_db_use_persist_buf_on_alloc_fail=fsdp_db_use_persist_buf_on_alloc_fail, disable_symmetric_registration=disable_symmetric_registration, + megatron_fsdp_prefetch_recompute_forward_weights=prefetch_recompute_forward_weights, + megatron_fsdp_cache_param_bucket_views=cache_param_bucket_views, megatron_fsdp_use_decoupled_grad=use_decoupled_grad, + megatron_fsdp_cuda_graph_mode=cuda_graph_mode, ) # Create FSDPDistributedIndex. @@ -665,7 +689,10 @@ def fully_shard( fsdp_db_use_persist_buf_on_alloc_fail: bool = False, disable_symmetric_registration: bool = False, enable_fine_grained_param_gather: bool = False, + prefetch_recompute_forward_weights: bool = False, + cache_param_bucket_views: bool = False, use_decoupled_grad: bool = False, + cuda_graph_mode: bool = False, ) -> tuple[MegatronFSDP, torch.optim.Optimizer]: """ Fully shard the model and the optimizer for Megatron-FSDP. @@ -716,7 +743,10 @@ def fully_shard( fsdp_db_use_persist_buf_on_alloc_fail=fsdp_db_use_persist_buf_on_alloc_fail, disable_symmetric_registration=disable_symmetric_registration, enable_fine_grained_param_gather=enable_fine_grained_param_gather, + prefetch_recompute_forward_weights=prefetch_recompute_forward_weights, + cache_param_bucket_views=cache_param_bucket_views, use_decoupled_grad=use_decoupled_grad, + cuda_graph_mode=cuda_graph_mode, ) # Extend optimizer methods to support Megatron-FSDP operations. diff --git a/megatron/core/distributed/fsdp/src/megatron_fsdp/megatron_fsdp.py b/megatron/core/distributed/fsdp/src/megatron_fsdp/megatron_fsdp.py index 3b46521e45b..f914b429968 100644 --- a/megatron/core/distributed/fsdp/src/megatron_fsdp/megatron_fsdp.py +++ b/megatron/core/distributed/fsdp/src/megatron_fsdp/megatron_fsdp.py @@ -1,16 +1,4 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License.import functools +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. import functools import importlib @@ -272,6 +260,9 @@ def __init__( self.enable_fine_grained_param_gather_backward_hook = ( enable_fine_grained_param_gather_backward_hook ) + self.prefetch_recompute_forward_weights = ( + self.ddp_config.megatron_fsdp_prefetch_recompute_forward_weights + ) self.report_nan_in_param_grad = report_nan_in_param_grad # FSDPDistributedIndex stores the process groups and meshes used by Megatron-FSDP. @@ -340,15 +331,8 @@ def __init__( self._init_fsdp_param_and_grad_buffer() self._register_fsdp_hooks(self.module) self.microbatch_count = 0 - - # Add a reference from the distributed parameters to self for API - # accessibility, e.g. when attaching MegatronFSDP scheduled ops - # to the distributed optimizer.step() and optimizer.zero_grad(). self.is_param_fsdp_distributed = False self._replace_param_with_distributed_if_needed() - for param in self.module.parameters(): - # Attach MegatronFSDP reference to the parameter. - setattr(param, "_megatron_fsdp_model", self) def _check_module_parameter_types(self): """ @@ -601,13 +585,10 @@ def _grad_acc(param): # Sharded Gradient Buffer gbuf = group.hfsdp_helper_gbuf if group.hfsdp_helper_gbuf else group.main_grad_buffer if gbuf.is_data_distributed: + # If TransformerEngine gradient accumulation is fused, then param.get_main_grad() + # already holds the wgrad and param.grad_added_to_main_grad=True. if not param.grad_added_to_main_grad: - # Get `main_grad` will allocate bucket, check that the currently - # used main_grad buffer does not exceed the scope of two FSDP Unit - # Modules, i.e., the buffer limit imposed by double-buffer allocator. - if self.ddp_config.fsdp_double_buffer: - self.grad_reduce_pipeline._enforce_double_buffer_limit([group_id]) - + # Allocate a unsharded gradient buffer. param.main_grad = param.get_main_grad() if param.grad is not None: if self.report_nan_in_param_grad: @@ -617,7 +598,6 @@ def _grad_acc(param): param.main_grad.copy_(to_local_if_dtensor(param.grad)) del param.grad else: - # Prepare for fused wgrad accumulation. param.main_grad.zero_() # Unsharded Gradient Buffer else: @@ -878,9 +858,29 @@ def _pre_backward_param_unshard(module: nn.Module, *unused): param_list = list(module.parameters(recurse=False)) # All-gather / unshard the module parameters before the backward pass. - self.all_gather_and_wait_parameters_ready( - param_list, prefetch_order=PrefetchOrder.BACKWARD_PASS_ORDER, bwd=True - ) + if self.prefetch_recompute_forward_weights: + self.all_gather_and_wait_parameters_ready( + param_list, + prefetch=False, + prefetch_order=PrefetchOrder.BACKWARD_PASS_ORDER, + bwd=True, + ) + self.all_gather_and_wait_parameters_ready( + param_list, + prefetch=True, + prefetch_order=PrefetchOrder.BACKWARD_PASS_ORDER, + bwd=False, + ) + self.all_gather_and_wait_parameters_ready( + param_list, + prefetch=True, + prefetch_order=PrefetchOrder.BACKWARD_PASS_ORDER, + bwd=True, + ) + else: + self.all_gather_and_wait_parameters_ready( + param_list, prefetch_order=PrefetchOrder.BACKWARD_PASS_ORDER, bwd=True + ) self._root_pre_backward_hook_issued = False @@ -1289,7 +1289,7 @@ def synchronize_param_gather(self): """ Synchronize parameter all-gather operations for all model parameters. """ - self.all_gather_pipeline.reset() + self.all_gather_pipeline.reset(preserve_non_fsdp_units=True) self._replace_param_with_distributed_if_needed() def synchronize_gradient_reduce(self): @@ -1358,6 +1358,8 @@ def _replace_param_with_distributed_if_needed(self): # DTensor parameter is managed by Megatron FSDP. if not hasattr(dist_param, "__fsdp_param__"): dist_param.__fsdp_param__ = True + if not hasattr(dist_param, "_megatron_fsdp_model"): + dist_param._megatron_fsdp_model = self _replace_module_parameter(self.module, name, dist_param) # Handle shared weights @@ -1370,6 +1372,8 @@ def _replace_param_with_raw_if_needed(self): for name, _ in self.module.named_parameters(): assert name in self.raw_param, f"Raw parameter {name} not found in module." + if not hasattr(self.raw_param[name], "_megatron_fsdp_model"): + self.raw_param[name]._megatron_fsdp_model = self _replace_module_parameter(self.module, name, self.raw_param[name]) # Handle shared weights diff --git a/megatron/core/distributed/fsdp/src/megatron_fsdp/package_info.py b/megatron/core/distributed/fsdp/src/megatron_fsdp/package_info.py index c3c7fc18ca5..c6f2355eddb 100644 --- a/megatron/core/distributed/fsdp/src/megatron_fsdp/package_info.py +++ b/megatron/core/distributed/fsdp/src/megatron_fsdp/package_info.py @@ -2,7 +2,7 @@ MAJOR = 0 -MINOR = 5 +MINOR = 6 PATCH = 0 PRE_RELEASE = 'rc0' diff --git a/megatron/core/distributed/fsdp/src/megatron_fsdp/param_and_grad_buffer.py b/megatron/core/distributed/fsdp/src/megatron_fsdp/param_and_grad_buffer.py index 9190ac04167..4a0b854abde 100644 --- a/megatron/core/distributed/fsdp/src/megatron_fsdp/param_and_grad_buffer.py +++ b/megatron/core/distributed/fsdp/src/megatron_fsdp/param_and_grad_buffer.py @@ -1,16 +1,4 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # TODO: Split this file into smaller files. @@ -59,6 +47,18 @@ logger = logging.getLogger(__name__) +def _same_tensor_view(a: Optional[torch.Tensor], b: torch.Tensor) -> bool: + if a is None: + return False + return ( + a.data_ptr() == b.data_ptr() + and a.dtype == b.dtype + and a.shape == b.shape + and a.stride() == b.stride() + and a.storage_offset() == b.storage_offset() + ) + + try: # Default to Megatron-LM FW. from megatron.core.distributed.distributed_data_parallel_config import ( @@ -724,7 +724,7 @@ def __init__( # Fallback allocator used if the fixed pool allocator cannot fulfill a request. self.fallback_to_persistent_buffer = fallback_to_persistent_buffer - self.backup_allocator = TemporaryBucketAllocator() + self.backup_allocator = StorageResizeBasedBucketAllocator() def _is_two_bucket_group_equal(self, group_a, group_b): # Check if two bucket groups are equivalent in dtype and size. @@ -906,7 +906,7 @@ def __init__( # Build the data parallel buffer index, which contains information # on where each parameter / gradient tensor will be stored in this # distributed buffer. - (self.item_index_map, self.bucket_index, self.shard_bucket_index) = ( + self.item_index_map, self.bucket_index, self.shard_bucket_index = ( build_data_parallel_buffer_index( [to_local_if_dtensor(p).shape for p in self.params], self.dp_rank, @@ -924,6 +924,8 @@ def __init__( # Count all parameters in this buffer and store their enumerated index. self.param_idx = {p: i for i, p in enumerate(self.params)} + self.cache_param_bucket_views = ddp_config.megatron_fsdp_cache_param_bucket_views + self._param_bucket_view_cache = {} def init_data(self, data: torch.Tensor): """Allocate a buffer Tensor to persistently store the data for this @@ -970,6 +972,30 @@ def fetch_bucket( # Need to set parameter data after resize model weight buffer data-storage. if set_param_data: + self.set_param_data_from_bucket(bucket) + return bucket + + def _bucket_view_cache_key(self, bucket: Bucket): + return ( + bucket.data.data_ptr(), + bucket.data.numel(), + bucket.data.dtype, + str(bucket.data.device), + self.is_transpose_buffer, + ) + + def _build_param_bucket_view_entries(self, bucket: Bucket): + entries = [] + for p in self.params: + item_id = self.param_idx[p] + p = to_local_if_dtensor(p) + data = self.get_item_from_bucket(bucket, item_id).view(p.shape) + entries.append((p, data, is_float8tensor(p))) + return entries + + def set_param_data_from_bucket(self, bucket: Bucket) -> None: + """Attach module parameter tensors to their views in an all-gather bucket.""" + if not self.cache_param_bucket_views: for p in self.params: item_id = self.param_idx[p] p = to_local_if_dtensor(p) @@ -978,7 +1004,24 @@ def fetch_bucket( fp8_set_raw_data(p, data, self.is_transpose_buffer) else: p.data = data - return bucket + return + + cache_key = self._bucket_view_cache_key(bucket) + entries = self._param_bucket_view_cache.get(cache_key) + if entries is None: + entries = self._build_param_bucket_view_entries(bucket) + self._param_bucket_view_cache[cache_key] = entries + + for p, data, is_fp8 in entries: + if is_fp8: + old_data = fp8_get_raw_data(p, self.is_transpose_buffer) + if _same_tensor_view(old_data, data): + continue + fp8_set_raw_data(p, data, self.is_transpose_buffer) + else: + if _same_tensor_view(p.data, data): + continue + p.data = data def allocate_bucket_storage( self, @@ -1404,6 +1447,29 @@ def _does_param_require_new_bucket(param): is_expert_parameter = lambda n, p: ".experts." in n + def _should_split_from_grouped_expert_bucket( + is_expert_param: bool, + param: torch.nn.Parameter, + param_chunk_size_factor: int, + chunk_size_factor: int, + same_factor_params: List[torch.nn.Parameter], + ) -> bool: + """ + Split grouped expert (>=3D) tensors with heterogeneous chunk size + factors into separate buckets to avoid LCM-inflated bucket alignment + padding. + """ + # Non-expert groups keep the original LCM/fragment merge. + if not is_expert_param: + return False + # Param already aligns with bucket chunk size factor (always true for + # the first param after sort); no split needed. + if param_chunk_size_factor == chunk_size_factor: + return False + return to_local_if_dtensor(param).dim() >= 3 or any( + to_local_if_dtensor(p).dim() >= 3 for p in same_factor_params + ) + # Step 1: Group the parameters according to their execution order and attributes. # FSDP unit module parameters are split into multiple parameter sub-groups. # All parameters in the module are assigned a parameter group, even non-FSDP modules. @@ -1503,17 +1569,27 @@ def _does_param_require_new_bucket(param): remaining_params = [] for param in params: param_shape = to_local_if_dtensor(param).shape + param_chunk_size_factor = param_shape[1:].numel() + if _should_split_from_grouped_expert_bucket( + group.is_expert_param, + param, + param_chunk_size_factor, + chunk_size_factor, + same_factor_params, + ): + remaining_params.append(param) + continue if ( - param_shape[1:].numel() == chunk_size_factor + param_chunk_size_factor == chunk_size_factor or ( - chunk_size_factor % param_shape[1:].numel() == 0 + chunk_size_factor % param_chunk_size_factor == 0 and param_shape.numel() % chunk_size_factor == 0 ) or (param_shape.numel() < chunk_size_factor) ): same_factor_params.append(param) else: - lcm_chunk_size_factor = math.lcm(chunk_size_factor, param_shape[1:].numel()) + lcm_chunk_size_factor = math.lcm(chunk_size_factor, param_chunk_size_factor) chunk_size_factor = lcm_chunk_size_factor same_factor_params.append(param) # Create a new parameter group with the same chunk size factor. @@ -1767,7 +1843,7 @@ def __init__( ) # Get the parameter groups. - (self.parameter_groups, self.param_to_param_group, self.bucket_to_bucket_group) = ( + self.parameter_groups, self.param_to_param_group, self.bucket_to_bucket_group = ( _get_parameter_groups(module, bucketing_policy, meta_device_init_fp8_params) ) self._init_each_parameter_group_buffers(meta_device_init_fp8_params) @@ -2134,6 +2210,7 @@ def _init_each_parameter_group_buffers(self, meta_device_init_fp8_params): name="fsdp_fp8_transpose_params", fsdp_param_groups=self.parameter_groups, size=UB_BUFFER_NUM, + fallback_to_persistent_buffer=self.ddp_config.fsdp_db_use_persist_buf_on_alloc_fail, ) self.main_grad_alloc = FixedPoolAllocator( name="fsdp_grads", @@ -2691,18 +2768,23 @@ def _alloc(dtype, size): p._item_id = item_id def main_grad_getter(p): + # Get gradient buffer and item ID. + gbuf = p._gbuf + item_id = p._item_id + # Need to free a bucket for the incoming bucket if double-buffering. + p._megatron_fsdp_model.grad_reduce_pipeline._enforce_double_buffer_limit( + [gbuf.bucket_id] + ) # Make sure main_grad memory is allocated when initially accessed. # When gradients are sharded, we can pre-allocate a communication # bucket to avoid casting to a communication data-type. Otherwise, # return the item backed by the main gradient buffer required to # support un-sharded gradient accumulation at high precision. - bucket = p._gbuf.fetch_bucket( + bucket = gbuf.fetch_bucket( dtype=( self.mp_policy.grad_comm_dtype if p._gbuf.is_data_distributed else None ) ) - gbuf = p._gbuf - item_id = p._item_id # View it as p.shape so you can insert the param.grad into # the bucket seamlessly. return gbuf.get_item_from_bucket(bucket, item_id).view( @@ -2784,14 +2866,31 @@ def zero_grad(self): """ Zero out the underlying grad_buffer and reset all buckets in preparation for the next iteration of training. - """ - for name, param in self.optimizer_named_parameters: - param.grad = None - if hasattr(param, "decoupled_grad"): - param.decoupled_grad = None - if name in self.dist_main_grad: - self.dist_main_grad[name]._local_tensor = None + Gradient shards are dereferenced to free memory. However, dereferencing is + not compatible with (FWD-BWD / full-iteration) CUDA graph-ability, because + we need to preserve this reference to the sharded gradient generated during + CUDA graph replay (`setattr` in `update_main_grads` not executed during + CUDA graph replay, as it is not a CUDA kernel). + + If the gradient is decoupled (precision-aware) or is equivalent to the + distributed optimizer parameter precision, the gradient shard is a view of + the Megatron-FSDP sharded gradient buffer. If not, then not dereferencing + this gradient shard will increase memory utilization as this gradient is a + persistent casted-copy of the accumulated gradient. + """ + if not self.ddp_config.megatron_fsdp_cuda_graph_mode: + # Dereference the sharded gradient to reclaim memory + # unless a full-iteration CUDA graph is utilized. + for name, param in self.optimizer_named_parameters: + param.grad = None + if hasattr(param, "decoupled_grad"): + param.decoupled_grad = None + if name in self.dist_main_grad: + self.dist_main_grad[name]._local_tensor = None + + # Zero the Megatron-FSDP sharded gradient buffer. If param.grad or param.decoupled_grad + # is a view of this buffer, they will be zero'd as well. for group in self.parameter_groups: if group.main_grad_buffer: group.main_grad_buffer.data.zero_() @@ -2898,6 +2997,7 @@ def set_param_attribute(): "is_embedding_or_output_parameter", "is_embedding_parameter", "_tensor_parallel_mode", + "_megatron_fsdp_model", ]: if hasattr(orig_param, attr_name): setattr(param, attr_name, getattr(orig_param, attr_name)) @@ -2930,11 +3030,6 @@ def update_main_grads(self): from the main gradient buffer. If the model parameters are sharded, we only need to update the gradient shard associated with the model parameter shard, as both are sharded symmetrically. - - Checks if high-precision main weights are utilized for optimization. - Otherwise, falls back to low-precision model weights, and further - falls back to the original module parameters not managed by cFSDP - in the case of no sharding / cFSDP OFF. """ for name, param in self.optimizer_named_parameters: orig_param = param.orig_param @@ -2955,11 +3050,11 @@ def update_main_grads(self): optimizer_grad = group.main_grad_buffer.get_item( item_id, only_shard=sharded_optimizer_state ) - if group.main_weight_buffer is not None: - if not self.use_decoupled_grad: - # Convert the gradient to the main weight buffer dtype. - # TODO(@cspades): Why this is necessary? Casted below. - optimizer_grad = optimizer_grad.to(param.dtype) + if group.main_weight_buffer is not None and not self.use_decoupled_grad: + # Convert the gradient to the main weight data-type for optimization. + # Not needed for decoupled gradients, because the precision-aware + # optimizer can apply gradients to parameters of different precision! + optimizer_grad = optimizer_grad.to(param.dtype) if name not in self.dist_main_grad: # Register the gradient as a distributed tensor. @@ -2988,7 +3083,7 @@ def update_main_grads(self): setattr(param, "decoupled_grad", grad) else: # Attach the gradient to the optimizer parameter. - setattr(param, "grad", grad.to(param.dtype) if grad is not None else None) + setattr(param, "grad", grad) @property def num_buckets(self): @@ -3386,13 +3481,15 @@ class BucketStatus(Enum): Attributes: EMPTY (int): The bucket is empty and not in use. + PRESERVED (int): The bucket storage is retained but not ready for use. COMMUNICATING (int): The bucket is currently being used for communication. READY_TO_USE (int): The bucket is filled with data and ready for use. """ EMPTY = 1 - COMMUNICATING = 2 - READY_TO_USE = 3 + PRESERVED = 2 + COMMUNICATING = 3 + READY_TO_USE = 4 class GradReducePipeline: @@ -3549,7 +3646,7 @@ def _enforce_double_buffer_limit(self, add_buckets): for _, _, bucket_id in reversed(self.grad_reduce_queue): fsdp_unit_id = param_groups[bucket_id].fsdp_unit_id double_buf_units.add(fsdp_unit_id) - if len(double_buf_units) > 1: + if len(double_buf_units) > 2: keep_n -= 1 with torch.cuda.stream(self.rs_stream): @@ -3914,8 +4011,14 @@ def num_buckets(self): """Return the number of buckets.""" return self.buffer.num_buckets - def reset(self): - """Reset the pipeline state.""" + def reset(self, preserve_non_fsdp_units: bool = True): + """Reset the pipeline state. + + Non-FSDP-unit buckets are preserved by default because their params may + be read across module boundaries. Setting preserve_non_fsdp_units=False + releases all bucket storage and is intended only for debugging when the + model will not be reused. + """ if len(self.param_gather_event_map) > 0: warnings.warn( ( @@ -3925,14 +4028,28 @@ def reset(self): UserWarning, ) while len(self.param_gather_event_map) > 0: - (bucket_id, bwd) = next(iter(self.param_gather_event_map)) + bucket_id, bwd = next(iter(self.param_gather_event_map)) self.wait_bucket_ready(bucket_id, bwd) + for bucket_id in range(self.num_buckets): + is_unit_bucket = self.buffer.parameter_groups[bucket_id].fsdp_unit_id is not None for bwd in [False, True]: - self.bucket_can_be_released[self.get_bucket_key(bucket_id, bwd)] = True + bucket_key = self.get_bucket_key(bucket_id, bwd) + # If preserve_non_fsdp_units is set, then do not release buckets + # associated with FSDP non-units. Instead, mark the bucket as PRESERVED + # (not NEW) so a later all-gather refreshes preserved non-unit bucket + # storage in place. + if preserve_non_fsdp_units and not is_unit_bucket: + self.bucket_status[bucket_key] = BucketStatus.PRESERVED + else: + self.bucket_can_be_released[bucket_key] = True self.recycle_unused_buckets() - assert all([status is BucketStatus.EMPTY for status in self.bucket_status.values()]), ( + expected_statuses = (BucketStatus.EMPTY,) + if preserve_non_fsdp_units: + expected_statuses += (BucketStatus.PRESERVED,) + + assert all(status in expected_statuses for status in self.bucket_status.values()), ( f"There are still working buckets, it is not safe to reset. " f"bucket_status: {self.bucket_status}." ) @@ -3987,10 +4104,6 @@ def all_gather_params( "but double buffers can support no more than 2 FSDP units." ) - # Do not release the buckets that are being all-gathered. - for bucket_id in ag_buckets: - self.bucket_can_be_released[self.get_bucket_key(bucket_id, bwd)] = False - # If prefetch is enabled, we will add prefetch buckets to ag_buckets. if prefetch: @@ -4060,11 +4173,18 @@ def need_skip_prefetch(bucket_id): ag_buckets = list(sorted(set(ag_buckets))) bucket_id = next_bucket_id(ag_buckets) - # Only all-gather on buckets that have not been allocated yet. + # Do not release the buckets that are requested by this call, even if + # they are already ready and do not need a new all-gather. + for bucket_id in ag_buckets: + self.bucket_can_be_released[self.get_bucket_key(bucket_id, bwd)] = False + + # Only all-gather on buckets that have not been allocated yet or whose + # persistent storage was preserved but is not ready for use. ag_buckets = [ bucket_id for bucket_id in ag_buckets - if self.bucket_status[self.get_bucket_key(bucket_id, bwd)] == BucketStatus.EMPTY + if self.bucket_status[self.get_bucket_key(bucket_id, bwd)] + in (BucketStatus.EMPTY, BucketStatus.PRESERVED) ] if len(ag_buckets) == 0: return @@ -4138,12 +4258,13 @@ def wait_bucket_ready(self, bucket_id, bwd, empty_ok=False): if self.bucket_status[bucket_key] == BucketStatus.READY_TO_USE: # Already ready to use. return - if self.bucket_status[bucket_key] == BucketStatus.EMPTY: + if self.bucket_status[bucket_key] in (BucketStatus.EMPTY, BucketStatus.PRESERVED): if empty_ok: return - # Bucket shouldn't be empty, this implies that the bucket - # was not allocated or NCCL operations are not complete. - raise ValueError(f"Bucket {bucket_id} is empty.") + # Bucket should not be empty or merely preserved here; this implies that + # the bucket was not allocated, was not made ready for use, or NCCL + # operations are not complete. + raise ValueError(f"Bucket {bucket_id} is {self.bucket_status[bucket_key].name}.") # Wait for asynchronous / overlapped NCCL operations to complete. param_gather_event, mark_bucket_ready_to_use = self.param_gather_event_map.pop(bucket_key) @@ -4234,7 +4355,10 @@ def async_bucket_gather(self, bucket_id, bwd) -> None: bucket_key = self.get_bucket_key(bucket_id, bwd) self.bucket_can_be_released[bucket_key] = False - if self.bucket_status[bucket_key] != BucketStatus.EMPTY: + if self.bucket_status[bucket_key] in ( + BucketStatus.COMMUNICATING, + BucketStatus.READY_TO_USE, + ): return self.bucket_status[bucket_key] = BucketStatus.COMMUNICATING diff --git a/megatron/core/distributed/fsdp/src/megatron_fsdp/uneven_dtensor.py b/megatron/core/distributed/fsdp/src/megatron_fsdp/uneven_dtensor.py index d1ae1c97101..fc1135deabf 100644 --- a/megatron/core/distributed/fsdp/src/megatron_fsdp/uneven_dtensor.py +++ b/megatron/core/distributed/fsdp/src/megatron_fsdp/uneven_dtensor.py @@ -489,11 +489,13 @@ def split_dtensor( for size in split_size_or_sections: split_points.append(split_points[-1] + size) - # One collective call — result reused for all splits below. - assert hasattr(dtensor._local_tensor, "__create_chunk_list__"), ( - "DTensor local tensor is missing chunk metadata." - ) - chunk_meta = dtensor._local_tensor.__create_chunk_list__()[0] + # Chunk metadata is reused for all splits below. Use the cached closure + # when present (fast path, no collective); otherwise compute it once via a + # single collective. + if hasattr(dtensor._local_tensor, "__create_chunk_list__"): + chunk_meta = dtensor._local_tensor.__create_chunk_list__()[0] + else: + chunk_meta = gather_and_compute_chunk_metadata(dtensor) chunk_slice = slice(chunk_meta.offsets[dim], chunk_meta.offsets[dim] + chunk_meta.sizes[dim]) local_offset = chunk_meta.offsets[dim] local_tensor = dtensor.to_local() diff --git a/megatron/core/distributed/fsdp/src/megatron_fsdp/utils.py b/megatron/core/distributed/fsdp/src/megatron_fsdp/utils.py index 53b4aedec08..4e856ab0801 100644 --- a/megatron/core/distributed/fsdp/src/megatron_fsdp/utils.py +++ b/megatron/core/distributed/fsdp/src/megatron_fsdp/utils.py @@ -46,6 +46,13 @@ HAVE_TE = False +try: + _torch_version = PkgVersion(torch.__version__) +except Exception: + # This is a WAR for building docs, where torch is not actually imported + _torch_version = PkgVersion("0.0.0") + + _MODEL_PARALLEL_RNG_TRACKER_NAME = "model-parallel-rng" @@ -78,6 +85,13 @@ def is_te_min_version(vers, check_equality=True): return te_version > PkgVersion(vers) +def is_torch_min_version(version, check_equality=True): + """Check if minimum version of `torch` is installed.""" + if check_equality: + return _torch_version >= PkgVersion(version) + return _torch_version > PkgVersion(version) + + def is_submodule(module, parent_module, strict=True): """ Check if a module is a submodule of another module. @@ -839,6 +853,10 @@ def get_mcore_tensor_parallel_partition_dim(param: torch.Tensor) -> Optional[int return 0 elif param._tensor_parallel_mode == "row": return 1 + if getattr(param, "tensor_model_parallel", False): + partition_dim = getattr(param, "partition_dim", None) + if partition_dim is not None and partition_dim >= 0: + return int(partition_dim) return None diff --git a/megatron/core/distributed/fsdp/src/pyproject.toml b/megatron/core/distributed/fsdp/src/pyproject.toml index 783030cc809..2845a14ab62 100644 --- a/megatron/core/distributed/fsdp/src/pyproject.toml +++ b/megatron/core/distributed/fsdp/src/pyproject.toml @@ -1,7 +1,7 @@ # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. [build-system] -requires = ["setuptools<80.0.0", "pybind11"] +requires = ["setuptools>=80", "pybind11"] build-backend = "setuptools.build_meta" [tool.setuptools] diff --git a/megatron/core/distributed/param_and_grad_buffer.py b/megatron/core/distributed/param_and_grad_buffer.py index 26d5ba81b3d..329a673259a 100644 --- a/megatron/core/distributed/param_and_grad_buffer.py +++ b/megatron/core/distributed/param_and_grad_buffer.py @@ -1,5 +1,6 @@ # Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +import dataclasses import fnmatch import functools import logging @@ -45,8 +46,6 @@ dist_all_gather_func = torch.distributed._all_gather_base dist_reduce_scatter_func = torch.distributed._reduce_scatter_base -import megatron.core.nccl_allocator as nccl_allocator - class BufferType(Enum): """ @@ -200,6 +199,12 @@ def __init__( self.params.add(param) self.next_param_gather_bucket_group = None + # Set in DistributedDataParallel.__init__ when reduce_scatter_with_fp32_accumulation is on: + # points to the bucket group whose grad-reduce was dispatched immediately before mine in + # the backward pass. start_grad_sync drains this predecessor before dispatching its own + # collective, so the predecessor's intermediate all-to-all buffer is freed before the new + # one is allocated. + self.previous_grad_reduce_bucket_group = None if self.ddp_config.num_distributed_optimizer_instances > 1: self.inter_distributed_optimizer_instance_group = None @@ -208,6 +213,16 @@ def __init__( not self.ddp_config.reduce_scatter_with_fp32_accumulation ), "RS w/ FP32 accumulation not supported with num_distributed_optimizer_instances > 1" + reduction_collective = ( + "reduce-scatter" if self.ddp_config.use_distributed_optimizer else "all-reduce" + ) + log_single_rank( + logger, + logging.INFO, + f"Using {reduction_collective} for gradient reductions because " + f"{self.ddp_config.use_distributed_optimizer=}", + ) + global dist_reduce_scatter_func if self.ddp_config.reduce_scatter_with_fp32_accumulation: dist_reduce_scatter_func = reduce_scatter_with_fp32_accumulation @@ -236,6 +251,10 @@ def __init__( self.param_gather_handle = None self.param_gather_dispatched = False self.grad_reduce_handle = None + # Per-iteration flag: True once finish_grad_sync has run this step. Lets a successor + # bucket group early-drain its predecessor without the end-of-step finalize loop + # double-waiting. Reset by `reset()`. + self.grad_reduce_finished = False # Each time a local shard is created from bucket.param_data or bucket.grad_data, it # introduces some CPU overheads. We use these two lists to cache the created local @@ -259,6 +278,7 @@ def reset(self): self.is_first_batch = False self.per_param_grad_ready_counts = {} self.is_last_microbatch = True + self.grad_reduce_finished = False def _post_param_sync(self): """Run post-processing after param all-gather completes.""" @@ -358,8 +378,9 @@ def start_param_sync(self, force_sync: bool = False): async_op = self.ddp_config.overlap_param_gather and not force_sync if not self.ddp_config.use_distributed_optimizer: - # Layer-wise optimizer path: use all_gather for variable-size - # param gather. + # Legacy layer-wise optimizer path: use all_gather for variable-size + # param gather. Once all layerwise call sites set + # ddp_config.use_distributed_optimizer=True, this branch can be removed. # # Each rank may own a different number of params per bucket, so # layerwise_param_flat_sizes can vary across ranks. PyTorch's NCCL @@ -549,6 +570,23 @@ def start_grad_sync(self, force_all_reduce: Optional[bool] = False): # already been dispatched. return + # Drain the predecessor bucket group's reduce-scatter before allocating ours. Only + # linked under reduce_scatter_with_fp32_accumulation, which holds an intermediate + # all-to-all output tensor pinned until .wait() runs. We only drain when the + # predecessor has actually been dispatched this iteration (grad_reduce_handle set): + # backward param ordering does not always match bucket linkage order (e.g. NVFP4 + # bucket layouts), so the predecessor may not have fired yet when we arrive here. + # In that case the predecessor will dispatch and drain on its own once its params + # become ready. The end-of-step finalize loop still catches any bucket that + # neither a successor nor itself drained. + if ( + self.previous_grad_reduce_bucket_group is not None + and self.previous_grad_reduce_bucket_group.grad_reduce_handle is not None + ): + self.previous_grad_reduce_bucket_group.finish_grad_sync( + force_all_reduce=force_all_reduce + ) + assert ( self.grad_reduce_handle is None ), "Should not have multiple communication calls outstanding at once" @@ -694,6 +732,14 @@ def finish_grad_sync(self, force_all_reduce: Optional[bool] = False): When ddp_config.overlap_grad_reduce is set to True, waits for asynchronous communication call to complete. When ddp_config.overlap_grad_reduce is set to False, makes synchronous call. + + When ddp_config.overlap_grad_reduce is set to True, this method is idempotent + within an iteration: a second call is a no-op. This lets a successor bucket + group early-drain its predecessor at dispatch time (see + `previous_grad_reduce_bucket_group`) while still allowing the end-of-step + finalize loop to call this on every bucket without double-waiting. The + non-overlap path preserves its original per-call dispatch+wait behaviour + because it has no predecessor draining. """ self.param_gather_dispatched = False # If overlap_grad_reduce is False, start (and finish) synchronous communication call here. @@ -701,6 +747,8 @@ def finish_grad_sync(self, force_all_reduce: Optional[bool] = False): self.start_grad_sync(force_all_reduce=force_all_reduce) self._copy_back_extra_main_grads() return + if self.grad_reduce_finished: + return # If first batch, start asynchronous communication here. register_grad_ready() launches # asynchronous communication only once self.golden_per_param_grad_ready_counts is # populated at the end of this first batch. @@ -711,6 +759,7 @@ def finish_grad_sync(self, force_all_reduce: Optional[bool] = False): if self.ddp_config.num_distributed_optimizer_instances > 1: torch.cuda.current_stream().wait_stream(self.communication_stream) self._copy_back_extra_main_grads() + self.grad_reduce_finished = True return assert self.grad_reduce_handle is not None, ( f"Communication call has not been issued for this bucket " @@ -720,6 +769,7 @@ def finish_grad_sync(self, force_all_reduce: Optional[bool] = False): self.grad_reduce_handle.wait() self.grad_reduce_handle = None self._copy_back_extra_main_grads() + self.grad_reduce_finished = True def free_overlap_buffers(self): """Free GPU buffers used by overlap param gather. @@ -811,15 +861,22 @@ def group_params_for_buffers( param_dtype = torch.uint8 grad_dtype = torch.float if grad_reduce_in_fp32 else param.dtype is_expert_parallel = not getattr(param, 'allreduce', True) + is_managed_by_layer_wise_optimizer = getattr( + param, 'is_managed_by_layer_wise_optimizer', False + ) - key = BufferKey(param_dtype, grad_dtype, is_expert_parallel) + key = BufferKey( + param_dtype, grad_dtype, is_expert_parallel, is_managed_by_layer_wise_optimizer + ) param_list = key_to_params.get(key, []) param_list.append(param) key_to_params[key] = param_list # Use param.dtype (not param_dtype) so FP8/NVFP4 params share offsets with their # logical high-precision dtype, needed for checkpoint compatibility. - offset_key = BufferKey(param.dtype, grad_dtype, is_expert_parallel) + offset_key = BufferKey( + param.dtype, grad_dtype, is_expert_parallel, is_managed_by_layer_wise_optimizer + ) offset = dtype_to_offsets.get(offset_key, 0) dtype_to_offsets[offset_key] = offset + 1 indices = key_to_indices.get(key, []) @@ -950,6 +1007,17 @@ def __init__( self.gradient_scaling_factor = gradient_scaling_factor self.nccl_ub = nccl_ub + self._is_layer_wise_buffer = bool( + self.params and getattr(self.params[0], "is_managed_by_layer_wise_optimizer", False) + ) + # Bake the per-buffer DistOpt decision into this buffer's ddp_config (single source of + # truth; bucket groups inherit it): a LayerWise (Muon) buffer on the compact decoupled + # layout disables DistributedOptimizer, while sibling buffers keep the model-level setting. + if self._is_layer_wise_buffer and not getattr( + self.ddp_config, "use_layer_wise_param_layout", True + ): + self.ddp_config = dataclasses.replace(self.ddp_config, use_distributed_optimizer=False) + # Data structures to store underlying buckets and relevant indexing data. self.buckets = [] self.param_to_bucket = {} # Param -> bucket mapping. @@ -997,6 +1065,27 @@ def __init__( # nvfp4_packed_numel_unpadded is already set by _compute_nvfp4_packed_layout. assert self.numel_unpadded <= self.numel + + # Diagnostic: log persistent buffer size vs. unpadded payload so the cost of any + # optimizer-driven padding (e.g. the LayerWise shard-aligned ``dp_size * max(shard_load)`` + # layout) is visible per buffer. Emit at INFO only when it is interesting — a + # LayerWise-managed buffer or one that actually carries padding — and DEBUG otherwise, so + # ordinary (zero-padding) buffers do not spam non-experimental runs. + _padding = self.numel - self.numel_unpadded + _pad_frac = _padding / max(self.numel_unpadded, 1) + log_on_each_pipeline_stage( + logger, + logging.INFO if (self._is_layer_wise_buffer or _padding > 0) else logging.DEBUG, + f"ParamAndGradBuffer layout: param_dtype={self.param_dtype} " + f"grad_dtype={self.grad_dtype} dp_world_size={self.data_parallel_world_size} " + f"layerwise={self._is_layer_wise_buffer} " + f"distopt={self.ddp_config.use_distributed_optimizer} " + f"numel={self.numel} numel_unpadded={self.numel_unpadded} " + f"padding={_padding} ({_pad_frac:.1%})", + tp_group=self.tp_group, + dp_cp_group=self.dp_cp_group, + ) + if self.has_nvfp4_params: assert self.nvfp4_packed_numel_unpadded <= self.nvfp4_packed_numel if self.ddp_config.use_distributed_optimizer: @@ -1009,6 +1098,7 @@ def __init__( self.param_data = None self.grad_data = None self.extra_main_grads = [] + self.nccl_mem_pool = None if self.nccl_ub: # If nccl_ub is True, use nccl_allocator to allocate memory for param_data/grad_data. @@ -1016,6 +1106,7 @@ def __init__( pool = nccl_allocator.create_nccl_mem_pool( symmetric=not self.ddp_config.disable_symmetric_registration ) + self.nccl_mem_pool = pool mem_alloc_context = functools.partial( nccl_allocator.nccl_mem, pool, @@ -1496,31 +1587,85 @@ def partition_buckets( if len(buffers) == 0: return [] - dtype_to_buffer_map = {} + # At most one fp8 (uint8) buffer is allowed; Cases 2 and 3 below branch on + # whether one is present. Non-uint8 dtypes can legitimately appear in + # multiple buffers (e.g. LayerWise-managed bf16 weights + Adam-managed bf16 + # biases share the bf16 ``param_dtype`` but live in separate buffers), so + # the uniqueness check is restricted to uint8. + fp8_buffer = None for buffer in buffers: - dtype = buffer.param_dtype - # Make sure that the param_dtype of any two buffers is different. - assert dtype not in dtype_to_buffer_map - dtype_to_buffer_map[dtype] = buffer - - # Case 1: Put all buckets into a single bucket group if force_single_bucket_group is True. + if buffer.param_dtype == torch.uint8: + assert fp8_buffer is None + fp8_buffer = buffer + + # A bucket group performs a single collective type (reduce-scatter for DistOpt buffers, + # all-reduce otherwise), so buckets merged into one group must agree on the effective + # per-buffer ``use_distributed_optimizer``. The decoupled LayerWise layout + # (``use_layer_wise_param_layout=False``) gives LayerWise (Muon) buffers + # ``use_distributed_optimizer=False`` while sibling buffers keep True; the no-fp8 Case 2 below + # keeps every bucket in its own group so they never mix, but the merging Cases 1/3 must assert + # consistency. + _ddp_config = buffers[0].ddp_config + _decouple = not getattr(_ddp_config, "use_layer_wise_param_layout", True) + + def _bucket_distopt(bucket): + """This bucket's effective ``use_distributed_optimizer``.""" + is_lw = bool( + bucket.params_list + and getattr(bucket.params_list[0], "is_managed_by_layer_wise_optimizer", False) + ) + if _decouple and is_lw: + return False + return _ddp_config.use_distributed_optimizer + + def _merged_use_distributed_optimizer(merge_buckets): + values = {_bucket_distopt(bucket) for bucket in merge_buckets} + assert len(values) == 1, ( + "Cannot merge buckets with differing effective use_distributed_optimizer into one " + "bucket group. This happens when the decoupled LayerWise layout " + "(use_layer_wise_param_layout=False) mixes LayerWise (all-reduce) and non-LayerWise " + "(reduce-scatter) buffers under a merging bucketing strategy (e.g. the fp8 merge " + "path). Disable bucket merging for the decoupled LayerWise path." + ) + return values.pop() + + # Case 1: Put all buckets into a single bucket group if force_single_bucket_group is True + # (e.g. disable_bucketing / non-first VPP chunks). A bucket group performs a single + # collective type, so when the decoupled LayerWise layout (use_layer_wise_param_layout=False) + # mixes LayerWise (all-reduce, non-DistOpt) and non-LayerWise (reduce-scatter, DistOpt) + # buffers in one chunk, we cannot + # merge them into a single group. Split by the effective per-bucket use_distributed_optimizer + # instead, preserving order. When all buckets agree (the non-decoupled case) this collapses + # to exactly one group, identical to the previous behavior. if force_single_bucket_group: - buckets = [] - ddp_config = buffers[0].ddp_config data_parallel_group = buffers[0].data_parallel_group data_parallel_world_size = buffers[0].data_parallel_world_size + ordered_distopt_values = [] + buckets_by_distopt = {} + # buffer.ddp_config already carries the per-buffer use_distributed_optimizer. + ddp_config_by_distopt = {} for buffer in buffers: - assert ddp_config == buffer.ddp_config assert data_parallel_group == buffer.data_parallel_group assert data_parallel_world_size == buffer.data_parallel_world_size - buckets.extend(buffer.buckets) - - bucket_group = _ParamAndGradBucketGroup( - buckets, ddp_config, data_parallel_group, data_parallel_world_size - ) - return [bucket_group] + distopt = buffer.ddp_config.use_distributed_optimizer + ddp_config_by_distopt.setdefault(distopt, buffer.ddp_config) + for bucket in buffer.buckets: + if distopt not in buckets_by_distopt: + buckets_by_distopt[distopt] = [] + ordered_distopt_values.append(distopt) + buckets_by_distopt[distopt].append(bucket) + + return [ + _ParamAndGradBucketGroup( + buckets_by_distopt[distopt], + ddp_config_by_distopt[distopt], + data_parallel_group, + data_parallel_world_size, + ) + for distopt in ordered_distopt_values + ] - if torch.uint8 not in dtype_to_buffer_map: + if fp8_buffer is None: # Case 2: When there is no fp8 buffer in the input buffers, let each bucket group have # only one bucket. bucket_groups = [] @@ -1537,14 +1682,14 @@ def partition_buckets( return bucket_groups else: # Case 3: When using fp8 params, merge all non-fp8 buckets into the last fp8 bucket group. - non_fp8_buckets = [] + # Track each non-fp8 bucket with its buffer's (authoritative) ddp_config. + non_fp8_buckets = [] # list of (bucket, ddp_config) for buffer in buffers: if buffer.param_dtype != torch.uint8: for bucket in buffer.buckets: - non_fp8_buckets.append(bucket) + non_fp8_buckets.append((bucket, buffer.ddp_config)) bucket_groups = [] - fp8_buffer = dtype_to_buffer_map[torch.uint8] for bucket in fp8_buffer.buckets: if len(bucket_groups) == len(fp8_buffer.buckets) - 1: # reduce_scatter_with_fp32_accumulation requires exactly one bucket @@ -1556,17 +1701,17 @@ def partition_buckets( bucket_groups.append( _ParamAndGradBucketGroup( [bucket], - buffer.ddp_config, + fp8_buffer.ddp_config, buffer.data_parallel_group, buffer.data_parallel_world_size, ) ) if non_fp8_buckets: - for non_fp8_bucket in non_fp8_buckets: + for non_fp8_bucket, non_fp8_ddp_config in non_fp8_buckets: bucket_groups.append( _ParamAndGradBucketGroup( [non_fp8_bucket], - buffer.ddp_config, + non_fp8_ddp_config, buffer.data_parallel_group, buffer.data_parallel_world_size, ) @@ -1574,14 +1719,19 @@ def partition_buckets( continue # Skip the default bucket group creation below else: - group_buckets = [bucket] + non_fp8_buckets + group_buckets = [bucket] + [b for b, _ in non_fp8_buckets] else: # The first N-1 bucket groups. group_buckets = [bucket] + # Merged buckets must share the fp8 group's effective use_distributed_optimizer. + assert ( + _merged_use_distributed_optimizer(group_buckets) + == fp8_buffer.ddp_config.use_distributed_optimizer + ) bucket_groups.append( _ParamAndGradBucketGroup( group_buckets, - buffer.ddp_config, + fp8_buffer.ddp_config, buffer.data_parallel_group, buffer.data_parallel_world_size, ) diff --git a/megatron/core/extensions/transformer_engine.py b/megatron/core/extensions/transformer_engine.py index c6a8653c4ea..64bce710993 100644 --- a/megatron/core/extensions/transformer_engine.py +++ b/megatron/core/extensions/transformer_engine.py @@ -1,5 +1,7 @@ # Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +from __future__ import annotations +import copy import dataclasses import enum import inspect @@ -32,10 +34,12 @@ ) from megatron.core.process_groups_config import ProcessGroupCollection from megatron.core.quantization.quant_config import QuantizationConfig +from megatron.core.quantization.utils import get_quant_config_or_none from megatron.core.tensor_parallel.layers import ( _initialize_affine_weight_cpu, set_tensor_model_parallel_attributes, ) +from megatron.core.tensor_parallel.mappings import gather_from_tensor_model_parallel_region from megatron.core.tensor_parallel.random import ( get_cuda_rng_tracker, get_data_parallel_rng_tracker_name, @@ -43,7 +47,7 @@ ) from megatron.core.tensor_parallel.utils import divide from megatron.core.transformer.enums import AttnMaskType -from megatron.core.transformer.mlp import MLP +from megatron.core.transformer.mlp import MLP, MLPSubmodules from megatron.core.transformer.torch_norm import LayerNormInterface from megatron.core.transformer.transformer_config import TransformerConfig from megatron.core.transformer.utils import ( @@ -63,7 +67,7 @@ try: import transformer_engine as te - from transformer_engine.pytorch.fp8 import FP8GlobalStateManager, fp8_autocast + from transformer_engine.pytorch.fp8 import FP8GlobalStateManager, fp8_autocast, fp8_model_init HAVE_TE = True except ImportError: @@ -123,6 +127,14 @@ class TEQuantizationRecipe: If an amax reduction is applicable, such as in per-tensor quantization recipe, whether to reduce only along TP groups. """ + fp8_param: bool = False + """ + If cast the initialized parameters to fp8 precision and all-gather weights in FP8. + """ + fp4_param: bool = False + """ + If cast the initialized parameters to fp4 precision and all-gather weights in FP4. + """ @classmethod def parse_from_config(cls, quant_config: Dict[Any, Any]) -> "TEQuantizationRecipe": @@ -205,6 +217,61 @@ def parse_from_config(quant_config: QuantizationConfig) -> "TEQuantizationParams raise NotImplementedError(f"Unhandled configuration type {config_type}") +def _get_fp8_model_init_for_quant_recipe(qrecipe: TEQuantizationRecipe): + if qrecipe.fp8_quantization_recipe is None and qrecipe.fp4_quantization_recipe is None: + enabled = False + quant_recipe = None + elif qrecipe.fp8_quantization_recipe is not None: + enabled = qrecipe.fp8_param + if qrecipe.fp8_format == "e4m3": + fp8_format = te.common.recipe.Format.E4M3 + elif qrecipe.fp8_format == "hybrid": + fp8_format = te.common.recipe.Format.HYBRID + else: + raise ValueError(f"Unhandled fp8_format {qrecipe.fp8_format}") + + if qrecipe.fp8_quantization_recipe == Fp8Recipe.custom: + from megatron.core.fp8_utils import _get_custom_recipe + + assert qrecipe.custom_recipe_factory is not None + quant_recipe = _get_custom_recipe(qrecipe.custom_recipe_factory) + elif qrecipe.fp8_quantization_recipe == Fp8Recipe.tensorwise: + quant_recipe = te.common.recipe.Float8CurrentScaling(fp8_format=fp8_format) + elif qrecipe.fp8_quantization_recipe == Fp8Recipe.blockwise: + quant_recipe = te.common.recipe.Float8BlockScaling(fp8_format=fp8_format) + elif qrecipe.fp8_quantization_recipe == Fp8Recipe.mxfp8: + quant_recipe = te.common.recipe.MXFP8BlockScaling(fp8_format=fp8_format) + else: + raise ValueError(f"Unhandled fp8 recipe: {qrecipe.fp8_quantization_recipe}") + else: + # Fp4 configured. + enabled = qrecipe.fp4_param + if qrecipe.fp4_quantization_recipe == Fp4Recipe.custom: + from megatron.core.fp8_utils import _get_custom_recipe + + assert qrecipe.custom_recipe_factory is not None + quant_recipe = _get_custom_recipe(qrecipe.custom_recipe_factory) + elif qrecipe.fp4_quantization_recipe == Fp4Recipe.nvfp4: + quant_recipe = te.common.recipe.NVFP4BlockScaling() + else: + raise ValueError(f"Unhandled fp4 recipe: {qrecipe.fp4_quantization_recipe}") + + return fp8_model_init( + enabled=enabled, + recipe=quant_recipe, + preserve_high_precision_init_val=torch.is_grad_enabled(), + ) + + +def _get_fp8_model_init_for_quant_params(qparams: TEQuantizationParams | None, training: bool): + if qparams is None: + return nullcontext() + elif not training and qparams.evaluation_recipe is not None: + return _get_fp8_model_init_for_quant_recipe(qparams.evaluation_recipe) + else: + return _get_fp8_model_init_for_quant_recipe(qparams.training_recipe) + + def _get_fp8_autocast_for_quant_recipe(qrecipe: TEQuantizationRecipe): if FP8GlobalStateManager.is_fp8_enabled(): if not qrecipe.override_quantized_autocast: @@ -685,7 +752,7 @@ def __init__( output_size: int, *, parallel_mode: Optional[str], - config: ModelParallelConfig, + config: TransformerConfig, init_method: Callable, bias: bool, skip_bias_add: bool, @@ -694,7 +761,12 @@ def __init__( is_expert: bool = False, symmetric_ar_type: Optional[str] = None, tp_group: Optional[torch.distributed.ProcessGroup] = None, + name: str | None = None, ): + """ + Args: + name (str | None): module instance name passed top-down from its paranet module + """ if not HAVE_TE: raise ImportError( "Transformer Engine is not installed. " @@ -816,24 +888,31 @@ def __init__( tp_size = 1 tp_group_for_te = None - super().__init__( - in_features=input_size, - out_features=output_size, - sequence_parallel=self.config.sequence_parallel, - fuse_wgrad_accumulation=self.config.gradient_accumulation_fusion, - # Pass None if not initialized for backward compatibility with the ckpt converter. - tp_group=tp_group_for_te if torch.distributed.is_initialized() else None, - tp_size=tp_size, - get_rng_state_tracker=( - get_cuda_rng_tracker if get_cuda_rng_tracker().is_initialized() else None - ), - init_method=condition_init_method(config, init_method), - bias=bias, - return_bias=self.te_return_bias, - parallel_mode=te_parallel_mode, - **extra_kwargs, - ) self.te_quant_params: Optional[TEQuantizationParams] = None + quant_config = get_quant_config_or_none(name, config.quant_recipe) + self.finish_init(quant_config) + init_quant_context = _get_fp8_model_init_for_quant_params( + self.te_quant_params, torch.is_grad_enabled() + ) + + with init_quant_context: + super().__init__( + in_features=input_size, + out_features=output_size, + sequence_parallel=self.config.sequence_parallel, + fuse_wgrad_accumulation=self.config.gradient_accumulation_fusion, + # Pass None if not initialized for backward compatibility with the ckpt converter. + tp_group=tp_group_for_te if torch.distributed.is_initialized() else None, + tp_size=tp_size, + get_rng_state_tracker=( + get_cuda_rng_tracker if get_cuda_rng_tracker().is_initialized() else None + ), + init_method=condition_init_method(config, init_method), + bias=bias, + return_bias=self.te_return_bias, + parallel_mode=te_parallel_mode, + **extra_kwargs, + ) for param in self.parameters(): if is_expert: @@ -865,7 +944,7 @@ def will_execute_quantized(self, is_context_quantized: bool) -> bool: self.te_quant_params, self.training, is_context_quantized ) - def forward(self, x): + def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor | None]: """Forward.""" _is_first_microbatch = ( None if self.disable_parameter_transpose_cache else self.is_first_microbatch @@ -926,7 +1005,12 @@ def __init__( tp_comm_buffer_name: Optional[str] = None, tp_group: Optional[torch.distributed.ProcessGroup] = None, stride: int = 1, + name: str | None = None, ): + """ + Args: + name (str | None): module instance name passed top-down from its paranet module + """ if not HAVE_TE: raise ImportError( "Transformer Engine is not installed. " @@ -1019,30 +1103,37 @@ def __init__( self.stride = stride - super().__init__( - in_features=input_size, - out_features=output_size, - eps=self.config.layernorm_epsilon, - sequence_parallel=self.config.sequence_parallel, - fuse_wgrad_accumulation=self.config.gradient_accumulation_fusion, - tp_group=tp_group if torch.distributed.is_initialized() else None, - tp_size=self.config.tensor_model_parallel_size, - get_rng_state_tracker=( - get_cuda_rng_tracker if get_cuda_rng_tracker().is_initialized() else None - ), - init_method=( - condition_init_method(config, init_method) - if not config.use_cpu_initialization - else lambda w: None - ), - bias=bias, - return_bias=self.te_return_bias, - parallel_mode="column", - return_layernorm_output=False, - zero_centered_gamma=self.config.layernorm_zero_centered_gamma, - **extra_kwargs, - ) self.te_quant_params: Optional[TEQuantizationParams] = None + quant_config = get_quant_config_or_none(name, config.quant_recipe) + self.finish_init(quant_config) + init_quant_context = _get_fp8_model_init_for_quant_params( + self.te_quant_params, torch.is_grad_enabled() + ) + + with init_quant_context: + super().__init__( + in_features=input_size, + out_features=output_size, + eps=self.config.layernorm_epsilon, + sequence_parallel=self.config.sequence_parallel, + fuse_wgrad_accumulation=self.config.gradient_accumulation_fusion, + tp_group=tp_group if torch.distributed.is_initialized() else None, + tp_size=self.config.tensor_model_parallel_size, + get_rng_state_tracker=( + get_cuda_rng_tracker if get_cuda_rng_tracker().is_initialized() else None + ), + init_method=( + condition_init_method(config, init_method) + if not config.use_cpu_initialization + else lambda w: None + ), + bias=bias, + return_bias=self.te_return_bias, + parallel_mode="column", + return_layernorm_output=False, + zero_centered_gamma=self.config.layernorm_zero_centered_gamma, + **extra_kwargs, + ) # Set proper partition_stride setattr(self.weight, 'partition_stride', stride) @@ -1143,7 +1234,7 @@ def __init__( input_size: int, output_size: int, *, - config: ModelParallelConfig, + config: TransformerConfig, init_method: Callable, gather_output: bool, bias: bool, @@ -1153,7 +1244,12 @@ def __init__( tp_comm_buffer_name: Optional[str] = None, tp_group: Optional[torch.distributed.ProcessGroup] = None, stride: int = 1, + name: str | None = None, ): + """ + Args: + name (str | None): module instance name passed top-down from its paranet module + """ if not HAVE_TE: raise ImportError( "Transformer Engine is not installed. " @@ -1185,6 +1281,7 @@ def __init__( tp_comm_buffer_name=tp_comm_buffer_name, symmetric_ar_type=config.symmetric_ar_type, tp_group=tp_group, + name=name, ) # Set proper partition_stride @@ -1244,6 +1341,135 @@ def backward_dw(self): super().backward_dw() +class TELMHeadColumnParallelLinear(TEColumnParallelLinear): + """Wrapper for ``TEColumnParallelLinear`` used as the LM-head output projection under MXFP8. + + Drop-in replacement for the ``tensor_parallel.ColumnParallelLinear`` LM head: + ``delay_wgrad_compute`` is forced off to mirror its no-op ``backward_dw``, + and ``get/set_extra_state`` match the bf16 LM head's state-dict shim. The + LM-head kwargs ``keep_master_weight_for_test``, ``skip_weight_param_allocation``, + ``defer_embedding_wgrad_compute`` buffers, and ``disable_grad_reduce`` are + accepted to preserve the ``ColumnParallelLinear`` signature but currently + raise when set non-default — TE will not support them natively, so they + would have to be implemented in this subclass, which has not been done yet. + + Active only when ``fp8_output_proj=True`` with ``fp8_recipe='mxfp8'``. + """ + + def __init__( + self, + input_size, + output_size, + *, + config, + init_method, + bias=True, + gather_output=False, + stride=1, + keep_master_weight_for_test=False, + skip_bias_add=False, + skip_weight_param_allocation: bool = False, + embedding_activation_buffer=None, + grad_output_buffer=None, + is_expert: bool = False, + tp_comm_buffer_name: Optional[str] = None, + disable_grad_reduce: bool = False, + tp_group: Optional[torch.distributed.ProcessGroup] = None, + ): + from megatron.core.fp8_utils import is_mxfp8_output_proj_active + + if not is_mxfp8_output_proj_active(config): + raise RuntimeError( + "TELMHeadColumnParallelLinear is only valid when fp8_output_proj=True, " + "fp8=True, and fp8_recipe='mxfp8'." + ) + if keep_master_weight_for_test: + raise ValueError("TE output projection does not support keep_master_weight_for_test.") + if skip_weight_param_allocation: + raise ValueError("TE output projection does not support skip_weight_param_allocation.") + if embedding_activation_buffer is not None or grad_output_buffer is not None: + raise ValueError( + "TE MXFP8 output projection does not support defer_embedding_wgrad_compute." + ) + if disable_grad_reduce: + raise ValueError("TE output projection does not support disable_grad_reduce.") + + te_config = copy.copy(config) + # Match ColumnParallelLinear.backward_dw's no-op so the LM head keeps + # the same wgrad-timing behavior it had before this subclass existed. + te_config.delay_wgrad_compute = False + + super().__init__( + input_size=input_size, + output_size=output_size, + config=te_config, + init_method=init_method, + gather_output=False, + bias=bias, + skip_bias_add=skip_bias_add, + is_expert=is_expert, + skip_weight_param_allocation=skip_weight_param_allocation, + tp_comm_buffer_name=tp_comm_buffer_name, + tp_group=tp_group, + stride=stride, + ) + + self.input_size = input_size + self.output_size = output_size + self.output_size_per_partition = divide(output_size, self.tp_size) + self.gather_output = gather_output + self.skip_bias_add = skip_bias_add + self.embedding_activation_buffer = None + self.grad_output_buffer = None + self.disable_grad_reduce = False + self.tp_group = self._tp_group + + self._register_load_state_dict_pre_hook( + lambda state_dict, prefix, *args, **kwargs: state_dict.setdefault( + f"{prefix}_extra_state" + ) + ) + + def get_extra_state(self): + """Return None to match ``ColumnParallelLinear``'s no-extra-state shim. + + Keeps the LM-head state dict compatible across the bf16 / MXFP8 swap. + """ + return None + + def set_extra_state(self, state): + """No-op to match ``ColumnParallelLinear.set_extra_state`` (ignored).""" + return + + def forward( + self, + input_: torch.Tensor, + weight: Optional[torch.Tensor] = None, + runtime_gather_output: Optional[bool] = None, + ): + """Run TE MXFP8 output projection. Returns ``(output, bias)``.""" + from megatron.core.fp8_utils import get_fp8_context + + if weight is not None and weight is not self.weight: + raise RuntimeError("TE MXFP8 output projection does not support runtime weight.") + + with get_fp8_context(self.config): + torch.cuda.nvtx.range_push("mxfp8_output_proj_telinear") + try: + output_parallel, output_bias = super().forward(input_) + finally: + torch.cuda.nvtx.range_pop() + + gather_output = self.gather_output + if runtime_gather_output is not None: + gather_output = runtime_gather_output + if gather_output: + output = gather_from_tensor_model_parallel_region(output_parallel, group=self.tp_group) + else: + output = output_parallel + return output, output_bias + + class TERowParallelLinear(TELinear): """Wrapper for the Transformer-Engine's `Linear` layer but specialized similar to megatron's `RowParallelLinear` layer.""" @@ -1253,7 +1479,7 @@ def __init__( input_size: int, output_size: int, *, - config: ModelParallelConfig, + config: TransformerConfig, init_method: Callable, bias: bool, input_is_parallel: bool, @@ -1261,7 +1487,12 @@ def __init__( is_expert: bool, tp_comm_buffer_name: Optional[str] = None, tp_group: Optional[torch.distributed.ProcessGroup] = None, + name: str | None = None, ): + """ + Args: + name (str | None): module instance name passed top-down from its paranet module + """ if not HAVE_TE: raise ImportError( "Transformer Engine is not installed. " @@ -1293,6 +1524,7 @@ def __init__( tp_comm_buffer_name=tp_comm_buffer_name, symmetric_ar_type=config.symmetric_ar_type, tp_group=tp_group, + name=name, ) if config.use_cpu_initialization: world_size = get_pg_size(tp_group) @@ -1557,6 +1789,10 @@ def forward( num_splits: Optional[int] = None, ) -> torch.Tensor: """Forward.""" + # Save TE's current CP group before potential DCP switch (for restore at end). + _te_orig_cp_group = self.cp_group + _te_orig_cp_global_ranks = self.cp_global_ranks + if packed_seq_params is not None: # If Dynamic CP group is provided, update TE DPA CP group if packed_seq_params.local_cp_size is not None: @@ -1633,6 +1869,14 @@ def forward( # Update Q K outside of TE Attention API core_attn_out, batch_max_attention_logits = core_attn_out + # The max attention logit is only used as a statistic for qk-clip + # and logging, so it never needs gradients. Detach it from the + # autograd graph, otherwise accumulating it into + # current_max_attn_logits keeps every batch's attention forward + # graph alive and leaks memory (most visibly when only + # log_max_attention_logit is set and clip_qk() never resets it). + batch_max_attention_logits = batch_max_attention_logits.detach() + # Update QK_Clip balancing eta if self.current_max_attn_logits is None: self.current_max_attn_logits = batch_max_attention_logits @@ -1647,6 +1891,19 @@ def forward( _fa_kwargs["num_splits"] = num_splits core_attn_out = super().forward(query, key, value, attention_mask, **_fa_kwargs) + # Restore TE's CP group after dynamic CP forward. + if ( + packed_seq_params is not None + and packed_seq_params.local_cp_size is not None + and self.config.context_parallel_size > 1 + ): + super().set_context_parallel_group( + _te_orig_cp_group, + _te_orig_cp_global_ranks, + TEDotProductAttention.cp_stream, + self.cp_comm_type, + ) + return core_attn_out def sharded_state_dict( @@ -1695,7 +1952,12 @@ def __init__( is_expert: bool = False, tp_comm_buffer_name: Optional[str] = None, pg_collection: Optional[ProcessGroupCollection] = None, + name: str | None = None, ): + """ + Args: + name (str | None): module instance name passed top-down from its paranet module + """ self.config = config # TE returns a zero length Tensor when bias=False and @@ -1708,11 +1970,11 @@ def __init__( self.disable_parameter_transpose_cache = self.config.disable_parameter_transpose_cache extra_kwargs = _get_extra_te_kwargs(config) + self.delay_wgrad_compute = ( self.config.delay_wgrad_compute or self.config.overlap_dispatch_backward_with_experts_wgrad ) - if self.delay_wgrad_compute: if is_te_min_version("2.3.0"): extra_kwargs["delay_wgrad_compute"] = True @@ -1757,77 +2019,57 @@ def __init__( tp_group_for_te = None if is_te_min_version("2.14.0"): - extra_kwargs["single_grouped_weight"] = getattr( - config, "moe_single_grouped_weight", False - ) - extra_kwargs["single_grouped_bias"] = getattr( - config, "moe_single_grouped_bias", False - ) + # nemo_26.04 ships TE 2.14.0+71bbefbf whose GroupedLinear.__init__ does NOT + # yet accept single_grouped_{weight,bias}, even though the version string + # passes is_te_min_version("2.14.0"). Introspect the signature instead of + # version-gating, mirroring the patch in dsv4_fused_attn / main_megatron. + # The GroupedLinear.__init__ signature is constant for a given TE install, so + # introspect once and cache at module scope rather than on every TEGroupedLinear + # instantiation (matters for large MoE models with many expert groups). + global _TE_GROUPED_LINEAR_INIT_PARAMS + try: + _gl_params = _TE_GROUPED_LINEAR_INIT_PARAMS + except NameError: + _gl_params = _TE_GROUPED_LINEAR_INIT_PARAMS = set( + inspect.signature(te.pytorch.GroupedLinear.__init__).parameters + ) + if "single_grouped_weight" in _gl_params: + extra_kwargs["single_grouped_weight"] = getattr( + config, "moe_single_grouped_weight", False + ) + if "single_grouped_bias" in _gl_params: + extra_kwargs["single_grouped_bias"] = getattr( + config, "moe_single_grouped_bias", False + ) - super().__init__( - num_gemms=num_gemms, - in_features=input_size, - out_features=output_size, - sequence_parallel=self.config.sequence_parallel, - fuse_wgrad_accumulation=self.config.gradient_accumulation_fusion, - tp_group=tp_group_for_te if torch.distributed.is_initialized() else None, - tp_size=tp_size, - get_rng_state_tracker=( - get_cuda_rng_tracker if get_cuda_rng_tracker().is_initialized() else None - ), - init_method=condition_init_method(config, init_method), - bias=bias, - return_bias=self.te_return_bias, - parallel_mode=parallel_mode, - **extra_kwargs, - ) self.te_quant_params: Optional[TEQuantizationParams] = None - for param in self.parameters(): - setattr(param, "allreduce", not (is_expert and self.expert_parallel)) - - def normalize_grouped_parameter_keys( - self, - state_dict, - prefix, - local_metadata, - strict, - missing_keys, - unexpected_keys, - error_msgs, - ): - """Make grouped checkpoint keys compatible across parameter layouts.""" - - def maybe_remap_param(param_name: str, single_grouped: bool) -> None: - grouped_key = f"{prefix}{param_name}" - indexed_keys = [ - f"{prefix}{param_name}{gemm_idx}" for gemm_idx in range(self.num_gemms) - ] - has_grouped_key = grouped_key in state_dict - has_any_indexed_key = any(key in state_dict for key in indexed_keys) - has_all_indexed_keys = all(key in state_dict for key in indexed_keys) - - if single_grouped: - if has_grouped_key or not has_all_indexed_keys: - return - state_dict[grouped_key] = torch.stack( - [state_dict.pop(key) for key in indexed_keys], dim=0 - ) - else: - if has_any_indexed_key or not has_grouped_key: - return - split_tensors = self._split_grouped_checkpoint_tensor( - state_dict.pop(grouped_key), grouped_key - ) - for gemm_idx, tensor in enumerate(split_tensors): - state_dict[f"{prefix}{param_name}{gemm_idx}"] = tensor + quant_config = get_quant_config_or_none(name, config.quant_recipe) + self.finish_init(quant_config) + init_quant_context = _get_fp8_model_init_for_quant_params( + self.te_quant_params, torch.is_grad_enabled() + ) - maybe_remap_param("weight", getattr(self, "single_grouped_weight", False)) - if self.use_bias: - maybe_remap_param("bias", getattr(self, "single_grouped_bias", False)) + with init_quant_context: + super().__init__( + num_gemms=num_gemms, + in_features=input_size, + out_features=output_size, + sequence_parallel=self.config.sequence_parallel, + fuse_wgrad_accumulation=self.config.gradient_accumulation_fusion, + tp_group=tp_group_for_te if torch.distributed.is_initialized() else None, + tp_size=tp_size, + get_rng_state_tracker=( + get_cuda_rng_tracker if get_cuda_rng_tracker().is_initialized() else None + ), + init_method=condition_init_method(config, init_method), + bias=bias, + return_bias=self.te_return_bias, + parallel_mode=parallel_mode, + **extra_kwargs, + ) - self._register_load_state_dict_pre_hook( - normalize_grouped_parameter_keys, with_module=True - ) + for param in self.parameters(): + setattr(param, "allreduce", not (is_expert and self.expert_parallel)) # Explicitly stamp partition_dim and partition_stride on expert weight # tensors when explicit_expert_comm cleared parallel_mode. TE ≤2.12 @@ -1844,6 +2086,10 @@ def maybe_remap_param(param_name: str, single_grouped: bool) -> None: setattr(weight, "partition_dim", part_dim) setattr(weight, "partition_stride", 1) + self._register_load_state_dict_pre_hook( + type(self)._normalize_grouped_parameter_keys, with_module=True + ) + def merge_extra_states( self, state_dict, @@ -1934,6 +2180,51 @@ def merge_extra_states( self._register_load_state_dict_pre_hook(merge_extra_states, with_module=True) + def _normalize_grouped_parameter_keys( + self, + state_dict, + prefix, + local_metadata, + strict, + missing_keys, + unexpected_keys, + error_msgs, + ): + """Make grouped checkpoint keys compatible across parameter layouts. + + Registered as a load_state_dict pre-hook to bridge checkpoints saved + in one layout (single grouped tensor vs per-GEMM indexed tensors) + and a model expecting the other. + """ + + def maybe_remap_param(param_name: str, single_grouped: bool) -> None: + grouped_key = f"{prefix}{param_name}" + indexed_keys = [ + f"{prefix}{param_name}{gemm_idx}" for gemm_idx in range(self.num_gemms) + ] + has_grouped_key = grouped_key in state_dict + has_any_indexed_key = any(key in state_dict for key in indexed_keys) + has_all_indexed_keys = all(key in state_dict for key in indexed_keys) + + if single_grouped: + if has_grouped_key or not has_all_indexed_keys: + return + state_dict[grouped_key] = torch.stack( + [state_dict.pop(key) for key in indexed_keys], dim=0 + ) + else: + if has_any_indexed_key or not has_grouped_key: + return + split_tensors = self._split_grouped_checkpoint_tensor( + state_dict.pop(grouped_key), grouped_key + ) + for gemm_idx, tensor in enumerate(split_tensors): + state_dict[f"{prefix}{param_name}{gemm_idx}"] = tensor + + maybe_remap_param("weight", getattr(self, "single_grouped_weight", False)) + if self.use_bias: + maybe_remap_param("bias", getattr(self, "single_grouped_bias", False)) + def _split_grouped_checkpoint_tensor( self, tensor: torch.Tensor, checkpoint_key: str ) -> list[torch.Tensor]: @@ -2161,7 +2452,12 @@ def __init__( is_expert: bool, tp_comm_buffer_name: Optional[str] = None, pg_collection: Optional[ProcessGroupCollection] = None, + name: str | None = None, ): + """ + Args: + name (str | None): module instance name passed top-down from its paranet module + """ super().__init__( num_gemms=num_gemms, input_size=input_size, @@ -2174,6 +2470,7 @@ def __init__( is_expert=is_expert, tp_comm_buffer_name=tp_comm_buffer_name, pg_collection=pg_collection, + name=name, ) def sharded_state_dict(self, prefix="", sharded_offsets=(), metadata=None): @@ -2207,7 +2504,12 @@ def __init__( is_expert: bool, tp_comm_buffer_name: Optional[str] = None, pg_collection: Optional[ProcessGroupCollection] = None, + name: str | None = None, ): + """ + Args: + name (str | None): module instance name passed top-down from its paranet module + """ super().__init__( num_gemms=num_gemms, input_size=input_size, @@ -2220,6 +2522,7 @@ def __init__( is_expert=is_expert, tp_comm_buffer_name=tp_comm_buffer_name, pg_collection=pg_collection, + name=name, ) def sharded_state_dict(self, prefix="", sharded_offsets=(), metadata=None): @@ -2527,7 +2830,34 @@ def forward(self, hidden_states: torch.Tensor, **kwargs) -> Tuple[Tensor, Option return out, bias - class TEFusedDenseMLP(TEFusedMLP): + @classmethod + def as_mlp_submodule( + cls, + submodules: MLPSubmodules, + config: TransformerConfig, + pg_collection: ProcessGroupCollection, + is_mtp_layer: bool, + is_expert: bool = False, + input_size: int | None = None, + ffn_hidden_size: int | None = None, + name: str | None = None, + ) -> MLP: + """Helper function to build an MLP as a TransformerLayer's mlp submodule.""" + del is_mtp_layer + assert hasattr( + pg_collection, 'tp' + ), 'TP process group is required for TEFusedMLP in TransformerLayer' + return cls( + config=config, + submodules=submodules, + tp_group=pg_collection.tp, + is_expert=is_expert, + input_size=input_size, + ffn_hidden_size=ffn_hidden_size, + name=name, + ) + + class TEFusedMLPWithGroupedLinear(TEFusedMLP): """Dense MLP using GroupedLinear(num_groups=1) to trigger ForwardGroupedMLP_CuTeGEMMSwiGLU_MXFP8 fusion on SM100+ with MXFP8 recipe. @@ -2561,13 +2891,11 @@ def __init__(self, *args, **kwargs): def _make_fused_impl(self) -> te.pytorch.ops.Sequential: """Construct fused module with GroupedLinear(num_groups=1) + ScaledSwiGLU.""" - fused_impl = te.pytorch.ops.Sequential() - - # Tensor parallelism configuration tp_world_size = get_tensor_model_parallel_world_size() - tp_group = None if tp_world_size > 1: - tp_group = get_tensor_model_parallel_group() + return super()._make_fused_impl() + + fused_impl = te.pytorch.ops.Sequential() # RNG state rng_state_tracker_function = None @@ -2656,18 +2984,15 @@ def _make_fused_impl(self) -> te.pytorch.ops.Sequential: # No _mxfp8_weight0 pre-computation to avoid ~28 GB persistent FP8 tensors. fused_impl.append(op) - if tp_world_size > 1: - if self.linear_fc2.sequence_parallel: - fused_impl.append(te.pytorch.ops.ReduceScatter(tp_group)) - else: - fused_impl.append(te.pytorch.ops.AllReduce(tp_group)) - self._register_hooks_on_fused_impl(fused_impl) return fused_impl def forward(self, hidden_states: torch.Tensor, **kwargs) -> Tuple[Tensor, Optional[Tensor]]: """Forward pass using GroupedLinear(num_groups=1) + ScaledSwiGLU.""" + if get_tensor_model_parallel_world_size() > 1: + return super().forward(hidden_states, **kwargs) + orig_shape = hidden_states.shape hidden_size = hidden_states.size(-1) hidden_states_2d = hidden_states.view(-1, hidden_size) @@ -2690,7 +3015,7 @@ def forward(self, hidden_states: torch.Tensor, **kwargs) -> Tuple[Tensor, Option recipe = self._recipe if self._fused_impl is None: - with te.pytorch.fp8_autocast(enabled=True, fp8_recipe=recipe): + with te.pytorch.quantized_model_init(enabled=True, recipe=recipe): self._fused_impl = (self._make_fused_impl(),) # Apply norm in BF16 OUTSIDE the MXFP8 autocast to preserve the rstd @@ -2698,7 +3023,7 @@ def forward(self, hidden_states: torch.Tensor, **kwargs) -> Tuple[Tensor, Option # gradient amplification, and causes convergence issues). normed = self._norm_seq[0](hidden_states_2d) - with te.pytorch.fp8_autocast(enabled=True, fp8_recipe=recipe): + with te.pytorch.autocast(enabled=True, recipe=recipe): out = self._fused_impl[0](normed, tokens_per_expert, scales, tokens_per_expert) out = out.view(*orig_shape[:-1], out.size(-1)) @@ -2711,9 +3036,14 @@ def forward(self, hidden_states: torch.Tensor, **kwargs) -> Tuple[Tensor, Option return out, bias + # ``TEFusedDenseMLP`` is the original dev name for the main-side + # ``TEFusedMLPWithGroupedLinear`` class. Keep it importable for dev callers. + TEFusedDenseMLP = TEFusedMLPWithGroupedLinear + else: TEFusedMLP = None # type: ignore[assignment, misc] TEFusedDenseMLP = None # type: ignore[assignment, misc] + TEFusedMLPWithGroupedLinear = None # type: ignore[assignment, misc] class TEDelayedScaling(te.common.recipe.DelayedScaling): diff --git a/megatron/core/extensions/transformer_engine_spec_provider.py b/megatron/core/extensions/transformer_engine_spec_provider.py index c365fb4835d..391ef0635e9 100644 --- a/megatron/core/extensions/transformer_engine_spec_provider.py +++ b/megatron/core/extensions/transformer_engine_spec_provider.py @@ -49,7 +49,7 @@ def column_parallel_linear(self) -> type: """Which column parallel linear module TE backend uses""" return TEColumnParallelLinear - def row_parallel_linear(self) -> type: + def row_parallel_linear(self) -> type[TERowParallelLinear]: """Which row parallel linear module TE backend uses""" return TERowParallelLinear diff --git a/megatron/core/fp8_utils.py b/megatron/core/fp8_utils.py index 971454a916b..c9335e3b9f8 100644 --- a/megatron/core/fp8_utils.py +++ b/megatron/core/fp8_utils.py @@ -529,6 +529,24 @@ def is_first_last_bf16_layer(config: TransformerConfig, layer_no: int): return False +def is_mxfp8_output_proj_active(config) -> bool: + """Return True when the LM-head output projection should run under MXFP8. + + Active when ``fp8_output_proj=True``, ``fp8=True``, ``fp8_recipe='mxfp8'``, + and Transformer Engine is installed. + """ + if not HAVE_TE: + return False + if not getattr(config, "fp8_output_proj", False): + return False + if not getattr(config, "fp8", False): + return False + + fp8_recipe = getattr(config, "fp8_recipe", None) + recipe_value = getattr(fp8_recipe, "value", fp8_recipe) + return str(recipe_value).lower() == "mxfp8" or str(fp8_recipe).lower().endswith(".mxfp8") + + if HAVE_TE: from megatron.core import parallel_state from megatron.core.extensions.transformer_engine import TEDelayedScaling diff --git a/megatron/core/full_cuda_graph.py b/megatron/core/full_cuda_graph.py index abee2bf811e..1465b20fde2 100644 --- a/megatron/core/full_cuda_graph.py +++ b/megatron/core/full_cuda_graph.py @@ -75,14 +75,21 @@ def copy_tensors_in_struct(src): def clone_tensors_in_struct(tgt, src): """Copy src to pre-existing tensors in tgt.""" if isinstance(src, tuple): - raise Exception(f"Unsupported copy for tuple yet: {type(src)}") + if not isinstance(tgt, tuple) or len(tgt) != len(src): + return copy_tensors_in_struct(src) + return tuple(clone_tensors_in_struct(t, s) for t, s in zip(tgt, src)) elif isinstance(src, list): + if not isinstance(tgt, list) or len(tgt) != len(src): + return copy_tensors_in_struct(src) for i in range(len(src)): if isinstance(src[i], (tuple, list, dict, torch.Tensor)): - clone_tensors_in_struct(tgt[i], src[i]) + tgt[i] = clone_tensors_in_struct(tgt[i], src[i]) else: tgt[i] = src[i] + return tgt elif isinstance(src, dict): + if not isinstance(tgt, dict): + return copy_tensors_in_struct(src) for k in src: if isinstance(src[k], (tuple, list, dict, torch.Tensor)): clone_tensors_in_struct(tgt[k], src[k]) diff --git a/megatron/core/fusions/fused_mhc_kernels.py b/megatron/core/fusions/fused_mhc_kernels.py index 6a19255196a..92c436690a6 100644 --- a/megatron/core/fusions/fused_mhc_kernels.py +++ b/megatron/core/fusions/fused_mhc_kernels.py @@ -1,40 +1,840 @@ # Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -"""Fused cuTile kernels for mHC (Manifold-Constrained Hyper-Connections). +"""Fused kernels for mHC (Manifold-Constrained Hyper-Connections). -Requires cuda.tile (cuTile) for optimal performance on supported GPUs -(compute capability 10.x+). Reference (non-fused) implementations live in -``megatron.core.transformer.hyper_connection`` and are used when cuTile is -unavailable or when the ``use_fused_mhc`` config flag is False. +Uses Triton and cuda.tile (cuTile) kernels when available, with PyTorch +reference implementations as fallback. Reference (non-fused) implementations +live in ``megatron.core.transformer.hyper_connection`` and are used when fused +kernels are unavailable or when the ``use_fused_mhc`` config flag is False. Four fused operations: - sinkhorn: Sinkhorn-Knopp projection to doubly stochastic matrix - h_aggregate: weighted n-stream -> 1-stream aggregation - - h_post_bda: fused H_res @ residual + H_post * (x + bias) + - h_post_bda: fused H_res.T @ residual + H_post * (x + bias) - proj_rms: fused projection + RMS normalization """ +import logging import math +import os +import shutil +import subprocess +import warnings from typing import Optional, Tuple import torch from torch import Tensor +from megatron.core._rank_utils import log_single_rank, safe_get_rank + +logger = logging.getLogger(__name__) +LOG2E = math.log2(math.e) + + +def _env_flag(name: str) -> bool: + return os.getenv(name, "0").lower() in ("1", "true", "yes", "on") + + +def _forced_backend() -> Tuple[str, Optional[Exception]]: + value = os.getenv("MHC_FORCE_BACKEND", "auto").strip().lower() + value = value.replace("-", "_").replace("+", "_") + aliases = { + "auto": "auto", + "mixed": "auto", + "default": "auto", + "native": "native", + "torch": "native", + "pytorch": "native", + "none": "native", + "triton": "triton", + "triton_native": "triton", + "cutile": "cutile", + "cu_tile": "cutile", + "cuda_tile": "cutile", + } + if value not in aliases: + valid = ", ".join(sorted(aliases)) + return "auto", ValueError( + f"Unsupported MHC_FORCE_BACKEND={value!r}; expected one of: {valid}" + ) + return aliases[value], None + + # --------------------------------------------------------------------------- # Check cuTile availability # --------------------------------------------------------------------------- _CUTILE_AVAILABLE = False +_CUTILE_EXPERIMENTAL_AVAILABLE = False +_CUTILE_DEVICE_SUPPORT_CACHE: Optional[bool] = None +_CUTILE_DEVICE_SUPPORT_ERROR: Optional[str] = None try: import cuda.tile as ct _CUTILE_AVAILABLE = True + try: + import cuda.tile_experimental as ct_experimental + + _CUTILE_EXPERIMENTAL_AVAILABLE = True + except ImportError: + pass +except ImportError: + pass + + +# --------------------------------------------------------------------------- +# Check Triton availability +# --------------------------------------------------------------------------- +_TRITON_AVAILABLE = False +try: + import triton + import triton.language as tl + + _TRITON_AVAILABLE = True except ImportError: pass +_MHC_FORCED_BACKEND, _MHC_BACKEND_VALIDATION_ERROR = _forced_backend() + + +def _record_mhc_backend_validation_error(error: Exception) -> None: + global _MHC_BACKEND_VALIDATION_ERROR + if _MHC_BACKEND_VALIDATION_ERROR is None: + _MHC_BACKEND_VALIDATION_ERROR = error + + +def _raise_mhc_backend_validation_error() -> None: + if _MHC_BACKEND_VALIDATION_ERROR is not None: + raise _MHC_BACKEND_VALIDATION_ERROR + if _MHC_FORCED_BACKEND == "cutile" and not is_cutile_available(): + raise RuntimeError( + "MHC_FORCE_BACKEND=cutile was requested, but cuTile does not support " + f"the current device: {_CUTILE_DEVICE_SUPPORT_ERROR}" + ) + + +if _MHC_FORCED_BACKEND == "native": + _TRITON_AVAILABLE = False + _CUTILE_AVAILABLE = False + _CUTILE_EXPERIMENTAL_AVAILABLE = False +elif _MHC_FORCED_BACKEND == "triton": + if not _TRITON_AVAILABLE: + _record_mhc_backend_validation_error( + RuntimeError("MHC_FORCE_BACKEND=triton was requested, but Triton is not available") + ) + _CUTILE_AVAILABLE = False + _CUTILE_EXPERIMENTAL_AVAILABLE = False +elif _MHC_FORCED_BACKEND == "cutile": + if not _CUTILE_AVAILABLE: + _record_mhc_backend_validation_error( + RuntimeError("MHC_FORCE_BACKEND=cutile was requested, but cuTile is not available") + ) + _TRITON_AVAILABLE = False + +if _env_flag("MHC_DISABLE_TRITON"): + if _MHC_FORCED_BACKEND == "triton": + _record_mhc_backend_validation_error( + ValueError("MHC_FORCE_BACKEND=triton conflicts with MHC_DISABLE_TRITON=1") + ) + _TRITON_AVAILABLE = False + +if _env_flag("MHC_DISABLE_CUTILE"): + if _MHC_FORCED_BACKEND == "cutile": + _record_mhc_backend_validation_error( + ValueError("MHC_FORCE_BACKEND=cutile conflicts with MHC_DISABLE_CUTILE=1") + ) + _CUTILE_AVAILABLE = False + _CUTILE_EXPERIMENTAL_AVAILABLE = False + + def is_cutile_available() -> bool: - """Return True if cuTile fused kernels are available.""" - return _CUTILE_AVAILABLE + """Return True if cuTile fused kernels are enabled.""" + return _CUTILE_AVAILABLE and _cutile_supports_current_device() + + +def _get_tileiras_path() -> Optional[str]: + """Return the tileiras compiler path if it can be found.""" + tileiras = shutil.which("tileiras") + if tileiras is not None: + return tileiras + + cuda_home = os.getenv("CUDA_HOME") or os.getenv("CUDA_PATH") or "/usr/local/cuda" + candidate = os.path.join(cuda_home, "bin", "tileiras") + if os.path.exists(candidate): + return candidate + return None + + +def _cutile_supports_current_device() -> bool: + """Return whether cuTile can compile for the current CUDA device.""" + global _CUTILE_DEVICE_SUPPORT_CACHE, _CUTILE_DEVICE_SUPPORT_ERROR + + if not _CUTILE_AVAILABLE: + return False + if _CUTILE_DEVICE_SUPPORT_CACHE is not None: + return _CUTILE_DEVICE_SUPPORT_CACHE + + if not torch.cuda.is_available(): + _CUTILE_DEVICE_SUPPORT_ERROR = "CUDA is not available" + _CUTILE_DEVICE_SUPPORT_CACHE = False + return False + + major, minor = torch.cuda.get_device_capability() + arch = f"sm_{major}{minor}" + tileiras = _get_tileiras_path() + if tileiras is None: + _CUTILE_DEVICE_SUPPORT_ERROR = "tileiras compiler was not found" + _CUTILE_DEVICE_SUPPORT_CACHE = False + return False + + try: + result = subprocess.run( + [tileiras, "--gpu-name", arch], capture_output=True, check=False, text=True, timeout=10 + ) + except (OSError, subprocess.SubprocessError) as exc: + _CUTILE_DEVICE_SUPPORT_ERROR = str(exc) + _CUTILE_DEVICE_SUPPORT_CACHE = False + return False + + output = f"{result.stdout}\n{result.stderr}" + if "Cannot find option named" in output and arch in output: + _CUTILE_DEVICE_SUPPORT_ERROR = output.strip().splitlines()[0] + _CUTILE_DEVICE_SUPPORT_CACHE = False + return False + + _CUTILE_DEVICE_SUPPORT_CACHE = True + return True + + +def is_triton_available() -> bool: + """Return True if Triton is enabled for supported mHC kernels.""" + return _TRITON_AVAILABLE + + +# ============================================================================ +# Triton implementations (only defined when triton is available) +# ============================================================================ + +if _TRITON_AVAILABLE: + TLOG2E = tl.constexpr(LOG2E) + + # ============================================================================ + # Sinkhorn-Knopp + # ============================================================================ + + @triton.autotune( + configs=[triton.Config({}, num_warps=nw) for nw in (1, 2, 4, 8)], key=["HC", "NUM_ITERS"] + ) + @triton.jit + def _triton_sinkhorn_fwd_kernel( + inp_ptr, out_ptr, M_init_ptr, N_batch, eps, HC: tl.constexpr, NUM_ITERS: tl.constexpr + ): + """Grid: (N_batch,). Each program handles one [HC, HC] matrix.""" + pid = tl.program_id(0) + if pid >= N_batch: + return + + base = pid * HC * HC + offs_r = tl.arange(0, HC) + offs_c = tl.arange(0, HC) + mat_ptrs = base + offs_r[:, None] * HC + offs_c[None, :] + + logits = tl.load(inp_ptr + mat_ptrs).to(tl.float32) + row_max = tl.max(logits, axis=1) + # Subtract row_max before exp to keep the exponent numerically stable. + M = tl.exp2((logits - row_max[:, None]) * TLOG2E) + tl.store(M_init_ptr + mat_ptrs, M.to(M_init_ptr.dtype.element_ty)) + + row_sum = tl.sum(M, axis=1) + M = M / row_sum[:, None] + eps + col_sum = tl.sum(M, axis=0) + M = M / (col_sum[None, :] + eps) + for _ in range(NUM_ITERS - 1): + row_sum = tl.sum(M, axis=1) + M = M / (row_sum[:, None] + eps) + col_sum = tl.sum(M, axis=0) + M = M / (col_sum[None, :] + eps) + + tl.store(out_ptr + mat_ptrs, M.to(out_ptr.dtype.element_ty)) + + @triton.autotune( + configs=[triton.Config({}, num_warps=nw) for nw in (1, 2, 4, 8)], key=["HC", "NUM_ITERS"] + ) + @triton.jit + def _triton_sinkhorn_bwd_kernel( + grad_out_ptr, + M_init_ptr, + grad_inp_ptr, + ws_M_ptr, + ws_rs_ptr, + ws_cs_ptr, + N_batch, + eps, + HC: tl.constexpr, + NUM_ITERS: tl.constexpr, + ): + """Grid: (N_batch,). Each program handles one [HC, HC] backward.""" + pid = tl.program_id(0) + if pid >= N_batch: + return + + base = pid * HC * HC + M_ws_base = pid * 2 * NUM_ITERS * HC * HC + v_ws_base = pid * NUM_ITERS + offs_r = tl.arange(0, HC) + offs_c = tl.arange(0, HC) + mat_ptrs = base + offs_r[:, None] * HC + offs_c[None, :] + + M = tl.load(M_init_ptr + mat_ptrs).to(tl.float32) + for t in range(NUM_ITERS): + ws_off = M_ws_base + (2 * t) * HC * HC + tl.store(ws_M_ptr + ws_off + offs_r[:, None] * HC + offs_c[None, :], M) + + row_sum = tl.sum(M, axis=1) + tl.store(ws_rs_ptr + (v_ws_base + t) * HC + offs_r, row_sum) + if t == 0: + M = M / row_sum[:, None] + eps + else: + M = M / (row_sum[:, None] + eps) + + ws_off = M_ws_base + (2 * t + 1) * HC * HC + tl.store(ws_M_ptr + ws_off + offs_r[:, None] * HC + offs_c[None, :], M) + + col_sum = tl.sum(M, axis=0) + tl.store(ws_cs_ptr + (v_ws_base + t) * HC + offs_c, col_sum) + M = M / (col_sum[None, :] + eps) + + # M is the final forward output. It is the right value for the first VJP + # through the last column-normalization step. + grad = tl.load(grad_out_ptr + mat_ptrs).to(tl.float32) + for t_rev in range(NUM_ITERS): + t = NUM_ITERS - 1 - t_rev + + col_s = tl.load(ws_cs_ptr + (v_ws_base + t) * HC + offs_c).to(tl.float32) + grad = grad / (col_s[None, :] + eps) + col_corr = tl.sum(grad * M, axis=0) + grad = grad - col_corr[None, :] + M = tl.load( + ws_M_ptr + + M_ws_base + + (2 * t + 1) * HC * HC + + offs_r[:, None] * HC + + offs_c[None, :] + ).to(tl.float32) + + row_s = tl.load(ws_rs_ptr + (v_ws_base + t) * HC + offs_r).to(tl.float32) + if t == 0: + grad = grad / row_s[:, None] + row_corr = tl.sum(grad * (M - eps), axis=1) + else: + grad = grad / (row_s[:, None] + eps) + row_corr = tl.sum(grad * M, axis=1) + grad = grad - row_corr[:, None] + M = tl.load( + ws_M_ptr + M_ws_base + (2 * t) * HC * HC + offs_r[:, None] * HC + offs_c[None, :] + ).to(tl.float32) + + M_init = tl.load(M_init_ptr + mat_ptrs).to(tl.float32) + grad = grad * M_init + tl.store(grad_inp_ptr + mat_ptrs, grad.to(grad_inp_ptr.dtype.element_ty)) + + def _triton_sinkhorn_fwd( + input_logits: Tensor, num_iterations: int, eps: float = 1e-6 + ) -> Tuple[Tensor, Tensor]: + original_shape = input_logits.shape + hc = original_shape[-1] + N_batch = input_logits.numel() // (hc * hc) + dev = input_logits.device + out = torch.empty(N_batch, hc, hc, dtype=input_logits.dtype, device=dev) + M_init = torch.empty(N_batch, hc, hc, dtype=input_logits.dtype, device=dev) + inp = input_logits.contiguous().view(N_batch, hc, hc) + _triton_sinkhorn_fwd_kernel[(N_batch,)](inp, out, M_init, N_batch, eps, hc, num_iterations) + return out.view(original_shape), M_init.view(original_shape) + + def _triton_sinkhorn_bwd( + grad_output: Tensor, M_init: Tensor, num_iterations: int, eps: float = 1e-6 + ) -> Tensor: + original_shape = grad_output.shape + hc = original_shape[-1] + N_batch = grad_output.numel() // (hc * hc) + dev = grad_output.device + grad_input = torch.empty(N_batch, hc, hc, dtype=grad_output.dtype, device=dev) + go = grad_output.contiguous().view(N_batch, hc, hc) + mi = M_init.contiguous().view(N_batch, hc, hc) + ws_M = torch.empty(N_batch * 2 * num_iterations * hc * hc, dtype=torch.float32, device=dev) + ws_rs = torch.empty(N_batch * num_iterations * hc, dtype=torch.float32, device=dev) + ws_cs = torch.empty(N_batch * num_iterations * hc, dtype=torch.float32, device=dev) + _triton_sinkhorn_bwd_kernel[(N_batch,)]( + go, mi, grad_input, ws_M, ws_rs, ws_cs, N_batch, eps, hc, num_iterations + ) + return grad_input.view(original_shape) + + class TritonFusedSinkhorn(torch.autograd.Function): + """Autograd wrapper for Triton fused Sinkhorn.""" + + @staticmethod + def forward(ctx, input_logits: Tensor, num_iterations: int, eps: float = 1e-6): + """Run Triton Sinkhorn forward and save initial matrix for backward.""" + out, M_init = _triton_sinkhorn_fwd(input_logits, num_iterations, eps) + ctx.save_for_backward(M_init) + ctx.num_iterations = num_iterations + ctx.eps = eps + return out + + @staticmethod + def backward(ctx, grad_output: Tensor): + """Run Triton Sinkhorn backward.""" + (M_init,) = ctx.saved_tensors + grad_input = _triton_sinkhorn_bwd(grad_output, M_init, ctx.num_iterations, ctx.eps) + return grad_input, None, None + + def triton_fused_sinkhorn( + input_logits: Tensor, num_iterations: int, eps: float = 1e-6 + ) -> Tensor: + """Apply Triton fused Sinkhorn with autograd support.""" + return TritonFusedSinkhorn.apply(input_logits, num_iterations, eps) + + # ============================================================================ + # H_aggregate forward + # ============================================================================ + + @triton.autotune( + configs=[ + triton.Config({"BLOCK_C": bc, "BLOCK_S": bs}, num_warps=nw) + for bc in (64, 128, 256, 512) + for bs in (1, 2, 4, 8) + for nw in (2, 4, 8) + ], + key=["C", "N"], + ) + @triton.jit + def _triton_h_agg_fwd_kernel( + x_ptr, + h_ptr, + out_ptr, + sb, + C: tl.constexpr, + N: tl.constexpr, + stride_x_s, + stride_x_n, + stride_x_c, + BLOCK_C: tl.constexpr, + BLOCK_S: tl.constexpr, + ): + """out[s, c] = sum_i x[s, i, c] * h[s, i].""" + pid_s = tl.program_id(0) + pid_c = tl.program_id(1) + offs_s = pid_s * BLOCK_S + tl.arange(0, BLOCK_S) + offs_c = pid_c * BLOCK_C + tl.arange(0, BLOCK_C) + mask_s = offs_s < sb + mask_c = offs_c < C + mask_2d = mask_s[:, None] & mask_c[None, :] + + acc = tl.zeros((BLOCK_S, BLOCK_C), dtype=tl.float32) + for i in tl.static_range(N): + x_i = tl.load( + x_ptr + offs_s[:, None] * stride_x_s + i * stride_x_n + offs_c[None, :], + mask=mask_2d, + other=0.0, + ).to(tl.float32) + h_i = tl.load(h_ptr + offs_s * N + i, mask=mask_s, other=0.0).to(tl.float32) + acc += h_i[:, None] * x_i + tl.store( + out_ptr + offs_s[:, None] * C + offs_c[None, :], + acc.to(out_ptr.dtype.element_ty), + mask=mask_2d, + ) + + def _triton_h_aggregate_fwd(x: Tensor, h_pre: Tensor) -> Tensor: + s, b, n, C = x.shape + sb = s * b + out = torch.empty(sb, C, dtype=x.dtype, device=x.device) + x_flat = x.contiguous().view(sb, n, C) + h_flat = h_pre.contiguous().view(sb, n) + + grid = lambda META: (triton.cdiv(sb, META["BLOCK_S"]), triton.cdiv(C, META["BLOCK_C"])) + _triton_h_agg_fwd_kernel[grid]( + x_flat, h_flat, out, sb, C, n, x_flat.stride(0), x_flat.stride(1), x_flat.stride(2) + ) + return out.view(s, b, C) + + # ============================================================================ + # H_post BDA + # ============================================================================ + + @triton.autotune( + configs=[ + triton.Config({"BLOCK_C": bc, "BLOCK_S": bs}, num_warps=nw) + for bc in (64, 128, 256, 512) + for bs in (1, 2, 4, 8) + for nw in (2, 4, 8) + ], + key=["C", "N"], + ) + @triton.jit + def _triton_hpb_fwd_kernel( + hr_ptr, + orig_ptr, + hp_ptr, + x_ptr, + bias_ptr, + out_ptr, + sb, + C: tl.constexpr, + N: tl.constexpr, + stride_hr_s, + stride_hr_i, + stride_hr_j, + stride_orig_s, + stride_orig_n, + stride_orig_c, + stride_out_s, + stride_out_n, + stride_out_c, + HAS_BIAS: tl.constexpr, + BLOCK_C: tl.constexpr, + BLOCK_S: tl.constexpr, + ): + """out = hr.T @ orig + hp * (x + bias).""" + pid_s = tl.program_id(0) + pid_c = tl.program_id(1) + offs_s = pid_s * BLOCK_S + tl.arange(0, BLOCK_S) + offs_c = pid_c * BLOCK_C + tl.arange(0, BLOCK_C) + mask_s = offs_s < sb + mask_c = offs_c < C + mask_2d = mask_s[:, None] & mask_c[None, :] + + x_tile = tl.load(x_ptr + offs_s[:, None] * C + offs_c[None, :], mask=mask_2d, other=0.0).to( + tl.float32 + ) + if HAS_BIAS: + bias_tile = tl.load(bias_ptr + offs_c, mask=mask_c, other=0.0).to(tl.float32) + x_tile += bias_tile[None, :] + + for i in tl.static_range(N): + hp_i = tl.load(hp_ptr + offs_s * N + i, mask=mask_s, other=0.0).to(tl.float32) + out_i = hp_i[:, None] * x_tile + + for j in tl.static_range(N): + hr_ji = tl.load( + hr_ptr + offs_s * stride_hr_s + j * stride_hr_i + i * stride_hr_j, + mask=mask_s, + other=0.0, + ).to(tl.float32) + orig_j = tl.load( + orig_ptr + + offs_s[:, None] * stride_orig_s + + j * stride_orig_n + + offs_c[None, :], + mask=mask_2d, + other=0.0, + ).to(tl.float32) + out_i += hr_ji[:, None] * orig_j + + tl.store( + out_ptr + offs_s[:, None] * stride_out_s + i * stride_out_n + offs_c[None, :], + out_i.to(out_ptr.dtype.element_ty), + mask=mask_2d, + ) + + def _triton_h_post_bda_fwd( + h_res: Tensor, original_residual: Tensor, h_post: Tensor, x: Tensor, bias: Optional[Tensor] + ) -> Tensor: + s, b, n, C = original_residual.shape + sb = s * b + dev = h_res.device + out = torch.empty(sb, n, C, dtype=h_res.dtype, device=dev) + hr_flat = h_res.contiguous().view(sb, n, n) + orig_flat = original_residual.contiguous().view(sb, n, C) + hp_flat = h_post.contiguous().view(sb, n) + x_flat = x.contiguous().view(sb, C) + + grid = lambda META: (triton.cdiv(sb, META["BLOCK_S"]), triton.cdiv(C, META["BLOCK_C"])) + _triton_hpb_fwd_kernel[grid]( + hr_flat, + orig_flat, + hp_flat, + x_flat, + bias if bias is not None else x_flat, + out, + sb, + C, + n, + hr_flat.stride(0), + hr_flat.stride(1), + hr_flat.stride(2), + orig_flat.stride(0), + orig_flat.stride(1), + orig_flat.stride(2), + out.stride(0), + out.stride(1), + out.stride(2), + HAS_BIAS=(bias is not None), + ) + return out.view(s, b, n, C) + + @triton.autotune( + configs=[ + triton.Config({"BLOCK_C": bc, "BLOCK_S": bs}, num_warps=nw) + for bc in (64, 128, 256, 512) + for bs in (1, 2, 4, 8) + for nw in (2, 4, 8) + ], + key=["C", "N"], + ) + @triton.jit + def _triton_hpb_bwd_g_x_orig_kernel( + go_ptr, + hr_ptr, + hp_ptr, + g_orig_ptr, + g_x_ptr, + sb, + C: tl.constexpr, + N: tl.constexpr, + stride_go_s, + stride_go_n, + stride_go_c, + stride_hr_s, + stride_hr_i, + stride_hr_j, + stride_orig_s, + stride_orig_n, + stride_orig_c, + BLOCK_C: tl.constexpr, + BLOCK_S: tl.constexpr, + ): + """g_x = hp @ go, g_orig = hr @ go.""" + pid_s = tl.program_id(0) + pid_c = tl.program_id(1) + offs_s = pid_s * BLOCK_S + tl.arange(0, BLOCK_S) + offs_c = pid_c * BLOCK_C + tl.arange(0, BLOCK_C) + mask_s = offs_s < sb + mask_c = offs_c < C + mask_2d = mask_s[:, None] & mask_c[None, :] + + g_x_acc = tl.zeros((BLOCK_S, BLOCK_C), dtype=tl.float32) + for j in tl.static_range(N): + go_j = tl.load( + go_ptr + offs_s[:, None] * stride_go_s + j * stride_go_n + offs_c[None, :], + mask=mask_2d, + other=0.0, + ).to(tl.float32) + hp_j = tl.load(hp_ptr + offs_s * N + j, mask=mask_s, other=0.0).to(tl.float32) + g_x_acc += hp_j[:, None] * go_j + tl.store( + g_x_ptr + offs_s[:, None] * C + offs_c[None, :], + g_x_acc.to(g_x_ptr.dtype.element_ty), + mask=mask_2d, + ) + + for i in tl.static_range(N): + g_orig_i = tl.zeros((BLOCK_S, BLOCK_C), dtype=tl.float32) + for j in tl.static_range(N): + go_j = tl.load( + go_ptr + offs_s[:, None] * stride_go_s + j * stride_go_n + offs_c[None, :], + mask=mask_2d, + other=0.0, + ).to(tl.float32) + hr_ij = tl.load( + hr_ptr + offs_s * stride_hr_s + i * stride_hr_i + j * stride_hr_j, + mask=mask_s, + other=0.0, + ).to(tl.float32) + g_orig_i += hr_ij[:, None] * go_j + tl.store( + g_orig_ptr + offs_s[:, None] * stride_orig_s + i * stride_orig_n + offs_c[None, :], + g_orig_i.to(g_orig_ptr.dtype.element_ty), + mask=mask_2d, + ) + + @triton.autotune( + configs=[ + triton.Config({"BLOCK_C": bc, "BLOCK_S": bs}, num_warps=nw) + for bc in (64, 128, 256, 512) + for bs in (1, 2, 4, 8) + for nw in (2, 4, 8) + ], + key=["C", "N"], + ) + @triton.jit + def _triton_hpb_bwd_g_hp_hr_kernel( + go_ptr, + orig_ptr, + x_ptr, + bias_ptr, + g_hr_ptr, + g_hp_ptr, + sb, + C: tl.constexpr, + N: tl.constexpr, + stride_go_s, + stride_go_n, + stride_go_c, + stride_orig_s, + stride_orig_n, + stride_orig_c, + stride_hr_s, + stride_hr_i, + stride_hr_j, + HAS_BIAS: tl.constexpr, + BLOCK_C: tl.constexpr, + BLOCK_S: tl.constexpr, + ): + """g_hp = sum_c go*(x+bias), g_hr = orig @ go.T.""" + pid_s = tl.program_id(0) + offs_s = pid_s * BLOCK_S + tl.arange(0, BLOCK_S) + mask_s = offs_s < sb + + g_hp_acc = tl.zeros((BLOCK_S, N), dtype=tl.float32) + g_hr_acc = tl.zeros((BLOCK_S, N * N), dtype=tl.float32) + + for c_start in range(0, C, BLOCK_C): + offs_c = c_start + tl.arange(0, BLOCK_C) + mask_c = offs_c < C + mask_2d = mask_s[:, None] & mask_c[None, :] + + x_tile = tl.load( + x_ptr + offs_s[:, None] * C + offs_c[None, :], mask=mask_2d, other=0.0 + ).to(tl.float32) + if HAS_BIAS: + bias_tile = tl.load(bias_ptr + offs_c, mask=mask_c, other=0.0).to(tl.float32) + x_tile += bias_tile[None, :] + + for i in tl.static_range(N): + go_i = tl.load( + go_ptr + offs_s[:, None] * stride_go_s + i * stride_go_n + offs_c[None, :], + mask=mask_2d, + other=0.0, + ).to(tl.float32) + dot_hp = tl.sum(go_i * x_tile, axis=1) + g_hp_acc += tl.where( + tl.arange(0, N)[None, :] == i, + dot_hp[:, None], + tl.zeros((BLOCK_S, N), dtype=tl.float32), + ) + for j in tl.static_range(N): + orig_j = tl.load( + orig_ptr + + offs_s[:, None] * stride_orig_s + + j * stride_orig_n + + offs_c[None, :], + mask=mask_2d, + other=0.0, + ).to(tl.float32) + dot_hr = tl.sum(go_i * orig_j, axis=1) + g_hr_acc += tl.where( + tl.arange(0, N * N)[None, :] == j * N + i, + dot_hr[:, None], + tl.zeros((BLOCK_S, N * N), dtype=tl.float32), + ) + + offs_n = tl.arange(0, N) + tl.store( + g_hp_ptr + offs_s[:, None] * N + offs_n[None, :], + g_hp_acc.to(g_hp_ptr.dtype.element_ty), + mask=mask_s[:, None], + ) + + # N is expected to stay small for mHC, so this simple extraction is acceptable. + nn_offs = tl.arange(0, N * N) + for i in tl.static_range(N): + for j in tl.static_range(N): + col_mask = (nn_offs == (i * N + j)).to(tl.float32) + val = tl.sum(g_hr_acc * col_mask[None, :], axis=1) + tl.store( + g_hr_ptr + offs_s * stride_hr_s + i * stride_hr_i + j * stride_hr_j, + val.to(g_hr_ptr.dtype.element_ty), + mask=mask_s, + ) + + def _triton_h_post_bda_bwd( + grad_output: Tensor, + h_res: Tensor, + original_residual: Tensor, + h_post: Tensor, + x: Tensor, + bias: Optional[Tensor], + ) -> Tuple[Tensor, Tensor, Tensor, Tensor, Optional[Tensor]]: + s, b, n, C = original_residual.shape + sb = s * b + dev = h_res.device + + g_hr = torch.empty(sb, n, n, dtype=h_res.dtype, device=dev) + g_res = torch.empty(sb, n, C, dtype=original_residual.dtype, device=dev) + g_hp = torch.empty(sb, n, dtype=h_post.dtype, device=dev) + g_x = torch.empty(sb, C, dtype=x.dtype, device=dev) + + go_flat = grad_output.contiguous().view(sb, n, C) + hr_flat = h_res.contiguous().view(sb, n, n) + orig_flat = original_residual.contiguous().view(sb, n, C) + hp_flat = h_post.contiguous().view(sb, n) + x_flat = x.contiguous().view(sb, C) + + grid_a = lambda META: (triton.cdiv(sb, META["BLOCK_S"]), triton.cdiv(C, META["BLOCK_C"])) + _triton_hpb_bwd_g_x_orig_kernel[grid_a]( + go_flat, + hr_flat, + hp_flat, + g_res, + g_x, + sb, + C, + n, + go_flat.stride(0), + go_flat.stride(1), + go_flat.stride(2), + hr_flat.stride(0), + hr_flat.stride(1), + hr_flat.stride(2), + g_res.stride(0), + g_res.stride(1), + g_res.stride(2), + ) + + grid_b = lambda META: (triton.cdiv(sb, META["BLOCK_S"]),) + _triton_hpb_bwd_g_hp_hr_kernel[grid_b]( + go_flat, + orig_flat, + x_flat, + bias if bias is not None else x_flat, + g_hr, + g_hp, + sb, + C, + n, + go_flat.stride(0), + go_flat.stride(1), + go_flat.stride(2), + orig_flat.stride(0), + orig_flat.stride(1), + orig_flat.stride(2), + g_hr.stride(0), + g_hr.stride(1), + g_hr.stride(2), + HAS_BIAS=(bias is not None), + ) + + g_bias = g_x.sum(dim=0).to(dtype=bias.dtype) if bias is not None else None + return ( + g_hr.view(s, b, n, n), + g_res.view(s, b, n, C), + g_hp.view(s, b, n), + g_x.view(s, b, C), + g_bias, + ) + + +_TRITON_IMPLS = ( + { + "sinkhorn": triton_fused_sinkhorn, + "h_aggregate_fwd": _triton_h_aggregate_fwd, + "h_post_bda_fwd": _triton_h_post_bda_fwd, + "h_post_bda_bwd": _triton_h_post_bda_bwd, + } + if _TRITON_AVAILABLE + else {"sinkhorn": None, "h_aggregate_fwd": None, "h_post_bda_fwd": None, "h_post_bda_bwd": None} +) # ============================================================================ @@ -44,7 +844,6 @@ def is_cutile_available() -> bool: if _CUTILE_AVAILABLE: ConstInt = ct.Constant[int] PAD_ZERO = ct.PaddingMode.ZERO - LOG2E = 1.4426950408889634 # -- Sinkhorn kernels ---------------------------------------------------- @@ -61,7 +860,11 @@ def _ct_sinkhorn_fwd_kernel( index=(pid, 0, 0), tile=ct.reshape(M.astype(M_init_out.dtype), (TILE_SIZE, HC, HC)), ) - for _ in range(NUM_ITERS): + row_sum = ct.sum(M, axis=2, keepdims=True) + M = M / row_sum + eps + col_sum = ct.sum(M, axis=1, keepdims=True) + M = M / (col_sum + eps) + for _ in range(NUM_ITERS - 1): row_sum = ct.sum(M, axis=2, keepdims=True) M = M / (row_sum + eps) col_sum = ct.sum(M, axis=1, keepdims=True) @@ -90,7 +893,10 @@ def _ct_sinkhorn_bwd_kernel( ct.store(ws_M, index=(M_base + 2 * t, 0, 0), tile=M) row_sum = ct.sum(M, axis=2, keepdims=True) ct.store(ws_rs, index=(v_base + t, 0, 0), tile=row_sum) - M = M / (row_sum + eps) + if t == 0: + M = M / row_sum + eps + else: + M = M / (row_sum + eps) ct.store(ws_M, index=(M_base + 2 * t + 1, 0, 0), tile=M) col_sum = ct.sum(M, axis=1, keepdims=True) ct.store(ws_cs, index=(v_base + t, 0, 0), tile=col_sum) @@ -105,60 +911,139 @@ def _ct_sinkhorn_bwd_kernel( grad = grad - col_corr M = ct.load(ws_M, index=(M_base + 2 * t + 1, 0, 0), shape=(TILE_SIZE, HC, HC)) row_s = ct.load(ws_rs, index=(v_base + t, 0, 0), shape=(TILE_SIZE, HC, 1)) - grad = grad / (row_s + eps) - row_corr = ct.sum(grad * M, axis=2, keepdims=True) + if t == 0: + grad = grad / row_s + row_corr = ct.sum(grad * (M - eps), axis=2, keepdims=True) + else: + grad = grad / (row_s + eps) + row_corr = ct.sum(grad * M, axis=2, keepdims=True) grad = grad - row_corr M = ct.load(ws_M, index=(M_base + 2 * t, 0, 0), shape=(TILE_SIZE, HC, HC)) grad = grad * M ct.store(grad_inp, index=(pid, 0, 0), tile=grad.astype(grad_inp.dtype)) + def _sinkhorn_autotune_tile_sizes(N_batch): + """Generate autotune search space for sinkhorn kernels.""" + for ts in (1, 2, 4, 8, 16, 32, 64, 128): + if ts <= N_batch: + yield ts + + _sinkhorn_fwd_best_cfg: dict = {} + _sinkhorn_bwd_best_cfg: dict = {} + def _cutile_sinkhorn_fwd( - input_logits: Tensor, num_iterations: int, eps: float = 1e-8 + input_logits: Tensor, num_iterations: int, eps: float = 1e-6 ) -> Tuple[Tensor, Tensor]: original_shape = input_logits.shape hc = original_shape[-1] N_batch = input_logits.numel() // (hc * hc) - TILE_SIZE = math.gcd(N_batch, 128) dev = input_logits.device + stream = torch.cuda.current_stream() out = torch.empty(N_batch, hc, hc, dtype=input_logits.dtype, device=dev) M_init = torch.empty(N_batch, hc, hc, dtype=input_logits.dtype, device=dev) - ct.launch( - torch.cuda.current_stream(), - (math.ceil(N_batch / TILE_SIZE), 1, 1), - _ct_sinkhorn_fwd_kernel, - (input_logits.view(N_batch, hc, hc), out, M_init, eps, hc, num_iterations, TILE_SIZE), - ) + inp = input_logits.view(N_batch, hc, hc) + + cache_key = (N_batch, hc, num_iterations) + cached = _sinkhorn_fwd_best_cfg.get(cache_key) + + if cached is not None or not _CUTILE_EXPERIMENTAL_AVAILABLE: + ts = cached if cached is not None else math.gcd(N_batch, 128) + ct.launch( + stream, + (math.ceil(N_batch / ts), 1, 1), + _ct_sinkhorn_fwd_kernel, + (inp, out, M_init, eps, hc, num_iterations, ts), + ) + else: + from types import SimpleNamespace + + configs = [ + SimpleNamespace(TILE_SIZE=ts) for ts in _sinkhorn_autotune_tile_sizes(N_batch) + ] + tuned = ct_experimental.autotune_launch( + stream, + grid_fn=lambda cfg: (math.ceil(N_batch / cfg.TILE_SIZE), 1, 1), + kernel=_ct_sinkhorn_fwd_kernel, + args_fn=lambda cfg: (inp, out, M_init, eps, hc, num_iterations, cfg.TILE_SIZE), + search_space=configs, + ) + best_ts = tuned.tuned_config.TILE_SIZE + _sinkhorn_fwd_best_cfg[cache_key] = best_ts + ct.launch( + stream, + (math.ceil(N_batch / best_ts), 1, 1), + _ct_sinkhorn_fwd_kernel, + (inp, out, M_init, eps, hc, num_iterations, best_ts), + ) + return out.view(original_shape), M_init.view(original_shape) def _cutile_sinkhorn_bwd( - grad_output: Tensor, M_init: Tensor, num_iterations: int, eps: float = 1e-8 + grad_output: Tensor, M_init: Tensor, num_iterations: int, eps: float = 1e-6 ) -> Tensor: original_shape = grad_output.shape hc = original_shape[-1] N_batch = grad_output.numel() // (hc * hc) - TILE_SIZE = math.gcd(N_batch, 128) dev = grad_output.device - ws_M = torch.empty(N_batch * 2 * num_iterations, hc, hc, dtype=torch.float32, device=dev) - ws_rs = torch.empty(N_batch * num_iterations, hc, 1, dtype=torch.float32, device=dev) - ws_cs = torch.empty(N_batch * num_iterations, 1, hc, dtype=torch.float32, device=dev) + stream = torch.cuda.current_stream() grad_input = torch.empty(N_batch, hc, hc, dtype=grad_output.dtype, device=dev) - ct.launch( - torch.cuda.current_stream(), - (math.ceil(N_batch / TILE_SIZE), 1, 1), - _ct_sinkhorn_bwd_kernel, - ( - grad_output.view(N_batch, hc, hc), - M_init.view(N_batch, hc, hc), - grad_input, - ws_M, - ws_rs, - ws_cs, - eps, - hc, - num_iterations, - TILE_SIZE, - ), - ) + go = grad_output.view(N_batch, hc, hc) + mi = M_init.view(N_batch, hc, hc) + + cache_key = (N_batch, hc, num_iterations) + cached = _sinkhorn_bwd_best_cfg.get(cache_key) + + def _alloc_and_launch(ts): + ws_M = torch.empty( + N_batch * 2 * num_iterations, hc, hc, dtype=torch.float32, device=dev + ) + ws_rs = torch.empty(N_batch * num_iterations, hc, 1, dtype=torch.float32, device=dev) + ws_cs = torch.empty(N_batch * num_iterations, 1, hc, dtype=torch.float32, device=dev) + ct.launch( + stream, + (math.ceil(N_batch / ts), 1, 1), + _ct_sinkhorn_bwd_kernel, + (go, mi, grad_input, ws_M, ws_rs, ws_cs, eps, hc, num_iterations, ts), + ) + + if cached is not None or not _CUTILE_EXPERIMENTAL_AVAILABLE: + ts = cached if cached is not None else math.gcd(N_batch, 128) + _alloc_and_launch(ts) + else: + from types import SimpleNamespace + + configs = [ + SimpleNamespace(TILE_SIZE=ts) for ts in _sinkhorn_autotune_tile_sizes(N_batch) + ] + # Allocate workspace for largest tile size (all configs share same workspace shape). + ws_M = torch.empty( + N_batch * 2 * num_iterations, hc, hc, dtype=torch.float32, device=dev + ) + ws_rs = torch.empty(N_batch * num_iterations, hc, 1, dtype=torch.float32, device=dev) + ws_cs = torch.empty(N_batch * num_iterations, 1, hc, dtype=torch.float32, device=dev) + tuned = ct_experimental.autotune_launch( + stream, + grid_fn=lambda cfg: (math.ceil(N_batch / cfg.TILE_SIZE), 1, 1), + kernel=_ct_sinkhorn_bwd_kernel, + args_fn=lambda cfg: ( + go, + mi, + grad_input, + ws_M, + ws_rs, + ws_cs, + eps, + hc, + num_iterations, + cfg.TILE_SIZE, + ), + search_space=configs, + ) + best_ts = tuned.tuned_config.TILE_SIZE + _sinkhorn_bwd_best_cfg[cache_key] = best_ts + # Re-launch with best config. + _alloc_and_launch(best_ts) + return grad_input.view(original_shape) # -- H_aggregate kernels ------------------------------------------------- @@ -197,15 +1082,17 @@ def _ct_h_agg_bwd_kernel(go, x, h_pre, gx, gh, N: ConstInt, TILE_M: ConstInt, TI def _cutile_h_aggregate_fwd(x: Tensor, h_pre: Tensor) -> Tensor: s, b, n, C = x.shape sb = s * b - TILE_SIZE = math.gcd(sb, 4) - TILE_C = math.gcd(C, 1024) + stream = torch.cuda.current_stream() out = torch.empty(sb, C, dtype=x.dtype, device=x.device) + x_flat = x.view(sb, n, C) + h_flat = h_pre.view(sb, n) + + # Autotune disabled — causes cudaErrorLaunchFailure during training. + tm, tc = math.gcd(sb, 4), math.gcd(C, 1024) ct.launch( - torch.cuda.current_stream(), - (math.ceil(sb / TILE_SIZE),), - _ct_h_agg_fwd_kernel, - (x.view(sb, n, C), h_pre.view(sb, n), out, n, TILE_SIZE, TILE_C), + stream, (math.ceil(sb / tm),), _ct_h_agg_fwd_kernel, (x_flat, h_flat, out, n, tm, tc) ) + return out.view(s, b, C) def _cutile_h_aggregate_bwd( @@ -213,25 +1100,22 @@ def _cutile_h_aggregate_bwd( ) -> Tuple[Tensor, Tensor]: s, b, n, C = x.shape sb = s * b - TILE_C = math.gcd(C, 1024) - TILE_M = math.gcd(sb, 4) + stream = torch.cuda.current_stream() gx = torch.empty(sb, n, C, dtype=x.dtype, device=x.device) - gh = torch.empty(sb, n, dtype=x.dtype, device=x.device) + gh = torch.empty(sb, n, dtype=h_pre.dtype, device=x.device) + go_flat = grad_output.view(sb, C) + x_flat = x.view(sb, n, C) + h_flat = h_pre.view(sb, n) + + # Autotune disabled — causes cudaErrorLaunchFailure during training. + tm, tc = math.gcd(sb, 4), math.gcd(C, 1024) ct.launch( - torch.cuda.current_stream(), - (math.ceil(sb / TILE_M),), + stream, + (math.ceil(sb / tm),), _ct_h_agg_bwd_kernel, - ( - grad_output.view(sb, C), - x.view(sb, n, C), - h_pre.view(sb, n), - gx, - gh, - n, - TILE_M, - TILE_C, - ), + (go_flat, x_flat, h_flat, gx, gh, n, tm, tc), ) + return gx.view(s, b, n, C), gh.view(s, b, n) # -- H_post BDA kernels -------------------------------------------------- @@ -243,28 +1127,23 @@ def _ct_hpb_fwd_kernel( pid = ct.bid(0) num_c_tiles = ct.num_tiles(x, axis=1, shape=(TILE_SIZE, TILE_C)) hp_tile = ct.load(hp, index=(pid, 0), shape=(TILE_SIZE, N), padding_mode=PAD_ZERO) - hp_2d = ct.reshape(hp_tile, (N, 1)) + hp_exp = ct.expand_dims(hp_tile, axis=2) # (TILE_SIZE, N, 1) hr_tile = ct.load(hr, index=(pid, 0, 0), shape=(TILE_SIZE, N, N), padding_mode=PAD_ZERO) - hr_2d = ct.reshape(hr_tile, (N, N)) for ct_idx in range(num_c_tiles): orig_tile = ct.load( orig, index=(pid, 0, ct_idx), shape=(TILE_SIZE, N, TILE_C), padding_mode=PAD_ZERO ) - orig_2d = ct.reshape(orig_tile, (N, TILE_C)) x_tile = ct.load( x, index=(pid, ct_idx), shape=(TILE_SIZE, TILE_C), padding_mode=PAD_ZERO ) - x_2d = ct.reshape(x_tile, (1, TILE_C)) - out_2d = hp_2d * x_2d + x_exp = ct.expand_dims(x_tile, axis=1) # (TILE_SIZE, 1, TILE_C) + out_tile = hp_exp * x_exp # (TILE_SIZE, N, TILE_C) for j in range(N): - out_2d += ct.extract(hr_2d, (0, j), shape=(N, 1)) * ct.extract( - orig_2d, (j, 0), shape=(1, TILE_C) - ) - ct.store( - out, - index=(pid, 0, ct_idx), - tile=ct.reshape(out_2d, (TILE_SIZE, N, TILE_C)).astype(out.dtype), - ) + hr_row = ct.extract(hr_tile, (0, j, 0), shape=(TILE_SIZE, 1, N)) + hr_col = ct.reshape(hr_row, (TILE_SIZE, N, 1)) + orig_row = ct.extract(orig_tile, (0, j, 0), shape=(TILE_SIZE, 1, TILE_C)) + out_tile = out_tile + hr_col * orig_row + ct.store(out, index=(pid, 0, ct_idx), tile=out_tile.astype(out.dtype)) @ct.kernel def _ct_hpb_fwd_bias_kernel( @@ -273,210 +1152,158 @@ def _ct_hpb_fwd_bias_kernel( pid = ct.bid(0) num_c_tiles = ct.num_tiles(x, axis=1, shape=(TILE_SIZE, TILE_C)) hp_tile = ct.load(hp, index=(pid, 0), shape=(TILE_SIZE, N), padding_mode=PAD_ZERO) - hp_2d = ct.reshape(hp_tile, (N, 1)) + hp_exp = ct.expand_dims(hp_tile, axis=2) # (TILE_SIZE, N, 1) hr_tile = ct.load(hr, index=(pid, 0, 0), shape=(TILE_SIZE, N, N), padding_mode=PAD_ZERO) - hr_2d = ct.reshape(hr_tile, (N, N)) for ct_idx in range(num_c_tiles): orig_tile = ct.load( orig, index=(pid, 0, ct_idx), shape=(TILE_SIZE, N, TILE_C), padding_mode=PAD_ZERO ) - orig_2d = ct.reshape(orig_tile, (N, TILE_C)) x_tile = ct.load( x, index=(pid, ct_idx), shape=(TILE_SIZE, TILE_C), padding_mode=PAD_ZERO ) bias_tile = ct.load(bias, index=(ct_idx,), shape=(TILE_C,), padding_mode=PAD_ZERO) - xb_2d = ct.reshape(x_tile, (1, TILE_C)) + ct.reshape(bias_tile, (1, TILE_C)) - out_2d = hp_2d * xb_2d + xb_exp = ct.expand_dims(x_tile + bias_tile, axis=1) # (TILE_SIZE, 1, TILE_C) + out_tile = hp_exp * xb_exp # (TILE_SIZE, N, TILE_C) for j in range(N): - out_2d += ct.extract(hr_2d, (0, j), shape=(N, 1)) * ct.extract( - orig_2d, (j, 0), shape=(1, TILE_C) - ) - ct.store( - out, - index=(pid, 0, ct_idx), - tile=ct.reshape(out_2d, (TILE_SIZE, N, TILE_C)).astype(out.dtype), - ) + hr_row = ct.extract(hr_tile, (0, j, 0), shape=(TILE_SIZE, 1, N)) + hr_col = ct.reshape(hr_row, (TILE_SIZE, N, 1)) + orig_row = ct.extract(orig_tile, (0, j, 0), shape=(TILE_SIZE, 1, TILE_C)) + out_tile = out_tile + hr_col * orig_row + ct.store(out, index=(pid, 0, ct_idx), tile=out_tile.astype(out.dtype)) @ct.kernel - def _ct_hpb_bwd_kernel( - go, - hr, - orig, - hp, - x, - g_hr, - g_orig, - g_hp, - g_x, - N: ConstInt, - TILE_C: ConstInt, - TILE_SIZE: ConstInt, + def _ct_hpb_bwd_g_x_orig_kernel( + go, hr, hp, g_orig, g_x, N: ConstInt, TILE_C: ConstInt, TILE_SIZE: ConstInt ): + """Compute g_x = hp @ go and g_orig = hr @ go. + + Grid: (ceil(sb / TILE_SIZE), ceil(C / TILE_C)). + 2D grid — no loop, no accumulators. + """ pid = ct.bid(0) - num_c_tiles = ct.cdiv(go.shape[2], TILE_C) - hp_tile = ct.load(hp, index=(pid, 0), shape=(TILE_SIZE, N)) - hp_2d = ct.reshape(hp_tile, (1, N)) + ct_idx = ct.bid(1) + hp_tile = ct.load(hp, index=(pid, 0), shape=(TILE_SIZE, N), padding_mode=PAD_ZERO) hr_tile = ct.load(hr, index=(pid, 0, 0), shape=(TILE_SIZE, N, N), padding_mode=PAD_ZERO) - hr_2d = ct.reshape(hr_tile, (N, N)) - acc_g_hp_2d = ct.full((N, 1), 0, dtype=ct.float32) - acc_g_hr_2d = ct.full((N, N), 0, dtype=ct.float32) + go_tile = ct.load( + go, index=(pid, 0, ct_idx), shape=(TILE_SIZE, N, TILE_C), padding_mode=PAD_ZERO + ) + g_x_tile = ct.full((TILE_SIZE, 1, TILE_C), 0, dtype=ct.float32) + g_orig_tile = ct.full((TILE_SIZE, N, TILE_C), 0, dtype=ct.float32) + for j in range(N): + hp_j = ct.extract(hp_tile, (0, j), shape=(TILE_SIZE, 1)) + hp_j_exp = ct.expand_dims(hp_j, axis=2) # [TS, 1, 1] + go_j = ct.extract(go_tile, (0, j, 0), shape=(TILE_SIZE, 1, TILE_C)) + g_x_tile = g_x_tile + hp_j_exp * go_j + hr_col_j = ct.extract(hr_tile, (0, 0, j), shape=(TILE_SIZE, N, 1)) + g_orig_tile = g_orig_tile + hr_col_j * go_j + ct.store( + g_x, + index=(pid, ct_idx), + tile=ct.reshape(g_x_tile, (TILE_SIZE, TILE_C)).astype(g_x.dtype), + ) + ct.store(g_orig, index=(pid, 0, ct_idx), tile=g_orig_tile.astype(g_orig.dtype)) + + @ct.kernel + def _ct_hpb_bwd_g_hp_hr_kernel( + go, orig, x, g_hr, g_hp, N: ConstInt, TILE_C: ConstInt, TILE_SIZE: ConstInt + ): + """Compute g_hp = sum(go * x) and g_hr = orig @ go.T (no bias). + + Grid: (ceil(sb / TILE_SIZE),). Loops over C-tiles. + """ + pid = ct.bid(0) + num_c_tiles = ct.cdiv(go.shape[2], TILE_C) + acc_g_hp = ct.full((TILE_SIZE, N, 1), 0, dtype=ct.float32) + acc_g_hr = ct.full((TILE_SIZE, N, N), 0, dtype=ct.float32) for ct_idx in range(num_c_tiles): x_tile = ct.load( x, index=(pid, ct_idx), shape=(TILE_SIZE, TILE_C), padding_mode=PAD_ZERO ) - x_2d = ct.reshape(x_tile, (1, TILE_C)) + x_exp = ct.expand_dims(x_tile, axis=1) # [TS, 1, TC] go_tile = ct.load( go, index=(pid, 0, ct_idx), shape=(TILE_SIZE, N, TILE_C), padding_mode=PAD_ZERO ) - go_2d = ct.reshape(go_tile, (N, TILE_C)) orig_tile = ct.load( orig, index=(pid, 0, ct_idx), shape=(TILE_SIZE, N, TILE_C), padding_mode=PAD_ZERO ) - orig_2d = ct.reshape(orig_tile, (N, TILE_C)) - g_x_2d = ct.full((1, TILE_C), 0, dtype=hp.dtype) - g_orig_2d = ct.full((N, TILE_C), 0, dtype=hp.dtype) - for j in range(N): - g_x_2d += ct.extract(hp_2d, (0, j), shape=(1, 1)).item() * ct.extract( - go_2d, (j, 0), shape=(1, TILE_C) - ) - g_orig_2d += ct.extract(hr_2d, (j, 0), shape=(1, N)).reshape((N, 1)) * ct.extract( - go_2d, (j, 0), shape=(1, TILE_C) - ) - acc_g_hp_2d += ct.sum(go_2d * x_2d, axis=1, keepdims=True) - acc_g_hr_2d += ct.sum( - ct.expand_dims(go_2d, axis=1) * ct.expand_dims(orig_2d, axis=0), axis=2 - ) - ct.store( - g_x, - index=(pid, ct_idx), - tile=ct.reshape(g_x_2d, (TILE_SIZE, TILE_C)).astype(g_x.dtype), - ) - ct.store( - g_orig, - index=(pid, 0, ct_idx), - tile=ct.reshape(g_orig_2d, (TILE_SIZE, N, TILE_C)).astype(g_orig.dtype), + acc_g_hp = acc_g_hp + ct.sum(go_tile * x_exp, axis=2, keepdims=True) + acc_g_hr = acc_g_hr + ct.sum( + ct.expand_dims(orig_tile, axis=2) * ct.expand_dims(go_tile, axis=1), axis=3 ) - ct.store( - g_hp, index=(pid, 0), tile=ct.reshape(acc_g_hp_2d, (TILE_SIZE, N)).astype(g_hp.dtype) - ) - ct.store( - g_hr, - index=(pid, 0, 0), - tile=ct.reshape(acc_g_hr_2d, (TILE_SIZE, N, N)).astype(g_hr.dtype), - ) + ct.store(g_hp, index=(pid, 0), tile=ct.reshape(acc_g_hp, (TILE_SIZE, N)).astype(g_hp.dtype)) + ct.store(g_hr, index=(pid, 0, 0), tile=acc_g_hr.astype(g_hr.dtype)) @ct.kernel - def _ct_hpb_bwd_bias_kernel( - go, - hr, - orig, - hp, - x, - bias, - g_hr, - g_orig, - g_hp, - g_x, - N: ConstInt, - TILE_C: ConstInt, - TILE_SIZE: ConstInt, + def _ct_hpb_bwd_g_hp_hr_bias_kernel( + go, orig, x, bias, g_hr, g_hp, N: ConstInt, TILE_C: ConstInt, TILE_SIZE: ConstInt ): + """Compute g_hp = sum(go * (x+bias)) and g_hr = orig @ go.T (with bias). + + Grid: (ceil(sb / TILE_SIZE),). Loops over C-tiles. + """ pid = ct.bid(0) num_c_tiles = ct.cdiv(go.shape[2], TILE_C) - hp_tile = ct.load(hp, index=(pid, 0), shape=(TILE_SIZE, N)) - hp_2d = ct.reshape(hp_tile, (1, N)) - hr_tile = ct.load(hr, index=(pid, 0, 0), shape=(TILE_SIZE, N, N), padding_mode=PAD_ZERO) - hr_2d = ct.reshape(hr_tile, (N, N)) - acc_g_hp_2d = ct.full((N, 1), 0, dtype=ct.float32) - acc_g_hr_2d = ct.full((N, N), 0, dtype=ct.float32) + acc_g_hp = ct.full((TILE_SIZE, N, 1), 0, dtype=ct.float32) + acc_g_hr = ct.full((TILE_SIZE, N, N), 0, dtype=ct.float32) for ct_idx in range(num_c_tiles): x_tile = ct.load( x, index=(pid, ct_idx), shape=(TILE_SIZE, TILE_C), padding_mode=PAD_ZERO ) bias_tile = ct.load(bias, index=(ct_idx,), shape=(TILE_C,), padding_mode=PAD_ZERO) - xb_2d = ct.reshape(x_tile, (1, TILE_C)) + ct.reshape(bias_tile, (1, TILE_C)) + xb_exp = ct.expand_dims(x_tile + bias_tile, axis=1) # [TS, 1, TC] go_tile = ct.load( go, index=(pid, 0, ct_idx), shape=(TILE_SIZE, N, TILE_C), padding_mode=PAD_ZERO ) - go_2d = ct.reshape(go_tile, (N, TILE_C)) orig_tile = ct.load( orig, index=(pid, 0, ct_idx), shape=(TILE_SIZE, N, TILE_C), padding_mode=PAD_ZERO ) - orig_2d = ct.reshape(orig_tile, (N, TILE_C)) - g_x_2d = ct.full((1, TILE_C), 0, dtype=hp.dtype) - g_orig_2d = ct.full((N, TILE_C), 0, dtype=hp.dtype) - for j in range(N): - g_x_2d += ct.extract(hp_2d, (0, j), shape=(1, 1)).item() * ct.extract( - go_2d, (j, 0), shape=(1, TILE_C) - ) - g_orig_2d += ct.extract(hr_2d, (j, 0), shape=(1, N)).reshape((N, 1)) * ct.extract( - go_2d, (j, 0), shape=(1, TILE_C) - ) - acc_g_hp_2d += ct.sum(go_2d * xb_2d, axis=1, keepdims=True) - acc_g_hr_2d += ct.sum( - ct.expand_dims(go_2d, axis=1) * ct.expand_dims(orig_2d, axis=0), axis=2 - ) - ct.store( - g_x, - index=(pid, ct_idx), - tile=ct.reshape(g_x_2d, (TILE_SIZE, TILE_C)).astype(g_x.dtype), - ) - ct.store( - g_orig, - index=(pid, 0, ct_idx), - tile=ct.reshape(g_orig_2d, (TILE_SIZE, N, TILE_C)).astype(g_orig.dtype), + acc_g_hp = acc_g_hp + ct.sum(go_tile * xb_exp, axis=2, keepdims=True) + acc_g_hr = acc_g_hr + ct.sum( + ct.expand_dims(orig_tile, axis=2) * ct.expand_dims(go_tile, axis=1), axis=3 ) - ct.store( - g_hp, index=(pid, 0), tile=ct.reshape(acc_g_hp_2d, (TILE_SIZE, N)).astype(g_hp.dtype) - ) - ct.store( - g_hr, - index=(pid, 0, 0), - tile=ct.reshape(acc_g_hr_2d, (TILE_SIZE, N, N)).astype(g_hr.dtype), - ) + ct.store(g_hp, index=(pid, 0), tile=ct.reshape(acc_g_hp, (TILE_SIZE, N)).astype(g_hp.dtype)) + ct.store(g_hr, index=(pid, 0, 0), tile=acc_g_hr.astype(g_hr.dtype)) + + # -- H_post BDA autotune configs & caches -------------------------------- + + def _hpb_autotune_configs(sb, C): + """Generate TILE_SIZE × TILE_C search space for h_post_bda kernels.""" + for tile_size in (1, 2, 4, 8): + for tile_c in (32, 64, 128, 256, 512, 1024): + if tile_c <= C and tile_size <= sb: + yield {"TILE_SIZE": tile_size, "TILE_C": tile_c} + + _hpb_fwd_best_cfg: dict = {} + _hpb_bwd_g_x_orig_best_cfg: dict = {} + _hpb_bwd_g_hp_hr_best_cfg: dict = {} def _cutile_h_post_bda_fwd( h_res: Tensor, original_residual: Tensor, h_post: Tensor, x: Tensor, bias: Optional[Tensor] ) -> Tensor: s, b, n, C = original_residual.shape sb = s * b - TILE_C = math.gcd(C, 1024) - TILE_SIZE = math.gcd(sb, 1) + stream = torch.cuda.current_stream() out = torch.empty(sb, n, C, dtype=h_res.dtype, device=h_res.device) - grid = (math.ceil(sb / TILE_SIZE),) - if bias is not None: - ct.launch( - torch.cuda.current_stream(), - grid, - _ct_hpb_fwd_bias_kernel, - ( - h_res.view(sb, n, n), - original_residual.view(sb, n, C), - h_post.view(sb, n), - x.view(sb, C), - bias, - out, - n, - TILE_C, - TILE_SIZE, - ), - ) + hr_flat = h_res.view(sb, n, n) + orig_flat = original_residual.view(sb, n, C) + hp_flat = h_post.view(sb, n) + x_flat = x.view(sb, C) + + cache_key = (sb, n, C, bias is not None) + cached = _hpb_fwd_best_cfg.get(cache_key) + kernel = _ct_hpb_fwd_bias_kernel if bias is not None else _ct_hpb_fwd_kernel + + # Autotune disabled — causes cudaErrorLaunchFailure during training. + if cached is not None: + ts, tc = cached else: - ct.launch( - torch.cuda.current_stream(), - grid, - _ct_hpb_fwd_kernel, - ( - h_res.view(sb, n, n), - original_residual.view(sb, n, C), - h_post.view(sb, n), - x.view(sb, C), - out, - n, - TILE_C, - TILE_SIZE, - ), - ) + ts, tc = 1, math.gcd(C, 1024) + args = (hr_flat, orig_flat, hp_flat, x_flat) + if bias is not None: + args = args + (bias,) + args = args + (out, n, tc, ts) + ct.launch(stream, (math.ceil(sb / ts),), kernel, args) + return out.view(s, b, n, C) def _cutile_h_post_bda_bwd( @@ -489,55 +1316,105 @@ def _cutile_h_post_bda_bwd( ) -> Tuple[Tensor, Tensor, Tensor, Tensor, Optional[Tensor]]: s, b, n, C = original_residual.shape sb = s * b - TILE_C = math.gcd(C, 1024) - TILE_SIZE = math.gcd(sb, 1) + stream = torch.cuda.current_stream() g_hr = torch.empty(sb, n, n, dtype=h_res.dtype, device=h_res.device) - g_res = torch.empty(sb, n, C, dtype=h_res.dtype, device=h_res.device) - g_hp = torch.empty(sb, n, dtype=h_res.dtype, device=h_res.device) - g_x = torch.empty(sb, C, dtype=h_res.dtype, device=h_res.device) - grid = (sb,) - if bias is not None: + g_res = torch.empty(sb, n, C, dtype=original_residual.dtype, device=h_res.device) + g_hp = torch.empty(sb, n, dtype=h_post.dtype, device=h_res.device) + g_x = torch.empty(sb, C, dtype=x.dtype, device=h_res.device) + go_flat = grad_output.view(sb, n, C) + hr_flat = h_res.view(sb, n, n) + orig_flat = original_residual.view(sb, n, C) + hp_flat = h_post.view(sb, n) + x_flat = x.view(sb, C) + + # --- Kernel A: g_x, g_orig (2D grid, no loop) --- + cache_key_a = ('hpb_bwd_g_x_orig', sb, n, C) + cached_a = _hpb_bwd_g_x_orig_best_cfg.get(cache_key_a) + + if cached_a is not None or not _CUTILE_EXPERIMENTAL_AVAILABLE: + if cached_a is not None: + ts, tc = cached_a + else: + ts, tc = 1, math.gcd(C, 1024) ct.launch( - torch.cuda.current_stream(), - grid, - _ct_hpb_bwd_bias_kernel, - ( - grad_output.view(sb, n, C), - h_res.view(sb, n, n), - original_residual.view(sb, n, C), - h_post.view(sb, n), - x.view(sb, C), - bias, - g_hr, + stream, + (math.ceil(sb / ts), math.ceil(C / tc)), + _ct_hpb_bwd_g_x_orig_kernel, + (go_flat, hr_flat, hp_flat, g_res, g_x, n, tc, ts), + ) + else: + from types import SimpleNamespace + + configs = [SimpleNamespace(**c) for c in _hpb_autotune_configs(sb, C)] + tuned = ct_experimental.autotune_launch( + stream, + grid_fn=lambda cfg: (math.ceil(sb / cfg.TILE_SIZE), math.ceil(C / cfg.TILE_C)), + kernel=_ct_hpb_bwd_g_x_orig_kernel, + args_fn=lambda cfg: ( + go_flat, + hr_flat, + hp_flat, g_res, - g_hp, g_x, n, - TILE_C, - TILE_SIZE, + cfg.TILE_C, + cfg.TILE_SIZE, ), + search_space=configs, ) - else: + best = tuned.tuned_config + _hpb_bwd_g_x_orig_best_cfg[cache_key_a] = (best.TILE_SIZE, best.TILE_C) ct.launch( - torch.cuda.current_stream(), - grid, - _ct_hpb_bwd_kernel, - ( - grad_output.view(sb, n, C), - h_res.view(sb, n, n), - original_residual.view(sb, n, C), - h_post.view(sb, n), - x.view(sb, C), - g_hr, - g_res, - g_hp, - g_x, - n, - TILE_C, - TILE_SIZE, - ), + stream, + (math.ceil(sb / best.TILE_SIZE), math.ceil(C / best.TILE_C)), + _ct_hpb_bwd_g_x_orig_kernel, + (go_flat, hr_flat, hp_flat, g_res, g_x, n, best.TILE_C, best.TILE_SIZE), ) - g_bias = g_x.sum(dim=0) if bias is not None else None + + # --- Kernel B: g_hp, g_hr (1D grid, loops C-tiles) --- + cache_key_b = ('hpb_bwd_g_hp_hr', sb, n, C, bias is not None) + cached_b = _hpb_bwd_g_hp_hr_best_cfg.get(cache_key_b) + hp_hr_kernel = ( + _ct_hpb_bwd_g_hp_hr_bias_kernel if bias is not None else _ct_hpb_bwd_g_hp_hr_kernel + ) + + if cached_b is not None or not _CUTILE_EXPERIMENTAL_AVAILABLE: + if cached_b is not None: + ts, tc = cached_b + else: + ts, tc = 1, math.gcd(C, 1024) + args = (go_flat, orig_flat, x_flat) + if bias is not None: + args = args + (bias,) + args = args + (g_hr, g_hp, n, tc, ts) + ct.launch(stream, (math.ceil(sb / ts),), hp_hr_kernel, args) + else: + from types import SimpleNamespace + + configs = [SimpleNamespace(**c) for c in _hpb_autotune_configs(sb, C)] + + def _hp_hr_args_fn(cfg): + args = (go_flat, orig_flat, x_flat) + if bias is not None: + args = args + (bias,) + return args + (g_hr, g_hp, n, cfg.TILE_C, cfg.TILE_SIZE) + + tuned = ct_experimental.autotune_launch( + stream, + grid_fn=lambda cfg: (math.ceil(sb / cfg.TILE_SIZE),), + kernel=hp_hr_kernel, + args_fn=_hp_hr_args_fn, + search_space=configs, + ) + best = tuned.tuned_config + _hpb_bwd_g_hp_hr_best_cfg[cache_key_b] = (best.TILE_SIZE, best.TILE_C) + args = (go_flat, orig_flat, x_flat) + if bias is not None: + args = args + (bias,) + args = args + (g_hr, g_hp, n, best.TILE_C, best.TILE_SIZE) + ct.launch(stream, (math.ceil(sb / best.TILE_SIZE),), hp_hr_kernel, args) + + g_bias = g_x.sum(dim=0).to(dtype=bias.dtype) if bias is not None else None return ( g_hr.view(s, b, n, n), g_res.view(s, b, n, C), @@ -549,10 +1426,9 @@ def _cutile_h_post_bda_bwd( # -- Proj RMS kernels ---------------------------------------------------- @ct.function - def _ct_rms_dnorm(a_tile, norm_tile, dr_tile, K): + def _ct_rms_dnorm(a_tile, norm_tile, dr_tile, K, eps=1e-6): inv_norm = ct.where(norm_tile > 0, 1.0 / norm_tile, 0.0) inv_sqrt_k = 1.0 / ct.sqrt(K) - eps = 1e-8 u = norm_tile * inv_sqrt_k + eps coeff = -(1.0 / (u * u)) * inv_sqrt_k return dr_tile * coeff * a_tile * inv_norm @@ -571,12 +1447,24 @@ def _ct_proj_rms_fwd_kernel( TILE_M: ConstInt, TILE_N: ConstInt, TILE_K: ConstInt, + SPLIT_K: ConstInt, ): + ''' + Grid: (num_tiles_m, num_tiles_k). + Fused matmul + norm + r: proj, norm, r in one pass over K. + R is a retained signature placeholder; r is computed after split-K + reduction from NORM. + ''' tile_m_id = ct.bid(0) + split_k_id = ct.bid(1) + num_m_tiles = ct.cdiv(M, TILE_M) num_k_tiles = ct.cdiv(K, TILE_K) + num_k_tiles_per_split = ct.cdiv(num_k_tiles, SPLIT_K) + tile_k_id_start = split_k_id * num_k_tiles_per_split + tile_k_id_end = ct.minimum(tile_k_id_start + num_k_tiles_per_split, num_k_tiles) acc = ct.full((TILE_M, TILE_N), 0.0, dtype=ct.float32) sum_sq = ct.full((TILE_M, 1), 0.0, dtype=ct.float32) - for tile_k_id in range(num_k_tiles): + for tile_k_id in range(tile_k_id_start, tile_k_id_end): a_tile = ct.load( A, index=(tile_m_id, tile_k_id), shape=(TILE_M, TILE_K), padding_mode=PAD_ZERO ) @@ -585,12 +1473,119 @@ def _ct_proj_rms_fwd_kernel( a_tile.astype(ct.tfloat32), b_tile.transpose().astype(ct.tfloat32), acc=acc ) sum_sq += ct.sum(a_tile * a_tile, axis=1, keepdims=True) - norm_tile = ct.sqrt(sum_sq) - v = norm_tile / ct.sqrt(K) + eps - r_tile = 1.0 / v - ct.store(PROJ, index=(tile_m_id, 0), tile=acc.astype(PROJ.dtype)) - ct.store(NORM, index=(tile_m_id, 0), tile=norm_tile.astype(NORM.dtype)) - ct.store(R, index=(tile_m_id, 0), tile=r_tile.astype(R.dtype)) + + bid_m_k = tile_m_id + split_k_id * num_m_tiles + ct.store(PROJ, index=(bid_m_k, 0), tile=acc.astype(PROJ.dtype)) + ct.store(NORM, index=(bid_m_k, 0), tile=sum_sq.astype(NORM.dtype)) + + # -- Sigmoid helper for cuTile kernels ------------------------------------ + + @ct.function + def _ct_sigmoid(x): + """Sigmoid via exp2: σ(x) = 1 / (1 + 2^(-x * log2(e))).""" + return 1.0 / (1.0 + ct.exp2(-x * LOG2E)) + + # -- Reduce split-K + compute_h kernel ------------------------------------ + + @ct.kernel + def _ct_reduce_compute_h_kernel( + Y_acc, + R_acc, + Bias, + Alpha_pre, + Alpha_post, + Alpha_res, + H_PRE, + H_POST, + H_RES, + R, + PROJ_OUT, + M: int, + N: int, + K: int, + n: ConstInt, + eps: float, + compute_h_eps: float, + TILE_SIZE_M: ConstInt, + TILE_SIZE_N: ConstInt, + SPLIT_K: ConstInt, + ): + """Reduce split-K partial proj/norm, compute r, and apply compute_h activations. + + Grid: (ceil(M / TILE_SIZE_M),). + TILE_SIZE_N = next_power_of_2(N) so one tile covers the full N dimension. + Alpha_{pre,post,res} are [1] tensors (scalar parameters). + """ + bid_m = ct.bid(0) + num_bid_m = ct.cdiv(M, TILE_SIZE_M) + + alpha_pre = ct.load(Alpha_pre, index=(0,), shape=(1,)).item() + alpha_post = ct.load(Alpha_post, index=(0,), shape=(1,)).item() + alpha_res = ct.load(Alpha_res, index=(0,), shape=(1,)).item() + + # 1. Reduce split-K partials for each logical output segment. + pre_accum = ct.full((TILE_SIZE_M, n), 0.0, dtype=ct.float32) + post_accum = ct.full((TILE_SIZE_M, n), 0.0, dtype=ct.float32) + r_accum = ct.full((TILE_SIZE_M, 1), 0.0, dtype=ct.float32) + + for split_idx in ct.static_iter(range(SPLIT_K)): + bid_m_k = bid_m + split_idx * num_bid_m + pre_tile = ct.load( + Y_acc, index=(bid_m_k, 0), shape=(TILE_SIZE_M, n), padding_mode=PAD_ZERO + ) + post_tile = ct.load( + Y_acc, index=(bid_m_k, 1), shape=(TILE_SIZE_M, n), padding_mode=PAD_ZERO + ) + pre_accum = pre_accum + ct.astype(pre_tile, ct.float32) + post_accum = post_accum + ct.astype(post_tile, ct.float32) + + r_tile = ct.load( + R_acc, index=(bid_m_k, 0), shape=(TILE_SIZE_M, 1), padding_mode=PAD_ZERO + ) + r_accum = r_accum + ct.astype(r_tile, ct.float32) + + # Store reduced projection segments for backward. + ct.store(PROJ_OUT, index=(bid_m, 0), tile=pre_accum.astype(PROJ_OUT.dtype)) + ct.store(PROJ_OUT, index=(bid_m, 1), tile=post_accum.astype(PROJ_OUT.dtype)) + + # 2. Compute r = norm / sqrt(K) + denom = ct.full((TILE_SIZE_M, 1), K * 1.0, dtype=ct.float32) + r_val = ct.sqrt(ct.truediv(r_accum, denom)) + + ct.store(R, index=(bid_m, 0), tile=r_val.astype(R.dtype)) + + # 3. Apply compute_h directly into split outputs. + inv_r_eps = 1.0 / (r_val + eps) + bias_pre = ct.load(Bias, index=(0, 0), shape=(1, n), padding_mode=PAD_ZERO) + bias_post = ct.load(Bias, index=(0, 1), shape=(1, n), padding_mode=PAD_ZERO) + bias_pre = ct.astype(bias_pre, ct.float32) + bias_post = ct.astype(bias_post, ct.float32) + + h_pre_linear = pre_accum * alpha_pre * inv_r_eps + bias_pre + h_post_linear = post_accum * alpha_post * inv_r_eps + bias_post + h_pre = _ct_sigmoid(h_pre_linear) + compute_h_eps + h_post = _ct_sigmoid(h_post_linear) * 2.0 + + ct.store(H_PRE, index=(bid_m, 0), tile=h_pre.astype(H_PRE.dtype)) + ct.store(H_POST, index=(bid_m, 0), tile=h_post.astype(H_POST.dtype)) + + for res_chunk in ct.static_iter(range(n)): + res_accum = ct.full((TILE_SIZE_M, n), 0.0, dtype=ct.float32) + for split_idx in ct.static_iter(range(SPLIT_K)): + bid_m_k = bid_m + split_idx * num_bid_m + res_tile = ct.load( + Y_acc, + index=(bid_m_k, 2 + res_chunk), + shape=(TILE_SIZE_M, n), + padding_mode=PAD_ZERO, + ) + res_accum = res_accum + ct.astype(res_tile, ct.float32) + + bias_res = ct.load(Bias, index=(0, 2 + res_chunk), shape=(1, n), padding_mode=PAD_ZERO) + bias_res = ct.astype(bias_res, ct.float32) + h_res = res_accum * alpha_res * inv_r_eps + bias_res + ct.store(PROJ_OUT, index=(bid_m, 2 + res_chunk), tile=res_accum.astype(PROJ_OUT.dtype)) + ct.store(H_RES, index=(bid_m, res_chunk), tile=h_res.astype(H_RES.dtype)) @ct.kernel def _ct_proj_rms_bwd_kernel( @@ -604,6 +1599,7 @@ def _ct_proj_rms_bwd_kernel( M: int, N: int, K: int, + eps: float, TILE_SIZE_M: ConstInt, TILE_SIZE_N: ConstInt, TILE_SIZE_K: ConstInt, @@ -626,7 +1622,7 @@ def _ct_proj_rms_bwd_kernel( dr_tile = ct.load( DR, index=(tile_m_id, 0), shape=(TILE_SIZE_M, 1), padding_mode=zero_pad ) - accumulator_da = accumulator_da + _ct_rms_dnorm(a_tile, norm_tile, dr_tile, K) + accumulator_da = accumulator_da + _ct_rms_dnorm(a_tile, norm_tile, dr_tile, K, eps) b_tile = ct.load( B, index=(0, tile_k_id), shape=(TILE_SIZE_N, TILE_SIZE_K), padding_mode=zero_pad ) @@ -643,7 +1639,7 @@ def _ct_proj_rms_bwd_kernel( @ct.kernel def _ct_proj_rms_bwd_small_k_kernel( - A, B, NORM, DD, DR, DA, DB, M: int, N: int, K: int, TILE_N_SIZE: ConstInt + A, B, NORM, DD, DR, DA, DB, M: int, N: int, K: int, eps: float, TILE_N_SIZE: ConstInt ): zero_pad = ct.PaddingMode.ZERO TILE_DB_SIZE_M = 128 @@ -699,7 +1695,7 @@ def _ct_proj_rms_bwd_small_k_kernel( DR, index=(dd_tile_idx, 0), shape=(TILE_DA_SIZE_M, 1), padding_mode=zero_pad ) accumulator_da = accumulator_da + _ct_rms_dnorm( - a_tile.astype(ct.float32), norm_tile, dr_tile, K + a_tile.astype(ct.float32), norm_tile, dr_tile, K, eps ) b_tile = ct.load( B, @@ -729,33 +1725,447 @@ def _next_power_of_2(n: int) -> int: n += 1 return n + def _proj_rms_fwd_autotune_configs(N): + """Generate autotune search space for proj_rms forward kernel.""" + TILE_N = _next_power_of_2(N) + tile_ms = (32, 64, 128) + tile_ks = (32, 64, 128) + split_ks = (1, 2, 4, 8, 16) + for tile_m in tile_ms: + for tile_k in tile_ks: + for split_k in split_ks: + yield {"TILE_M": tile_m, "TILE_N": TILE_N, "TILE_K": tile_k, 'SPLIT_K': split_k} + + def _default_tile_m(M: int) -> int: + """Pick a tile size that avoids unmasked stores past the M dimension.""" + for tile_m in (128, 64, 32, 16, 8, 4, 2, 1): + if tile_m <= M and M % tile_m == 0: + return tile_m + return 1 + + def _default_proj_rms_fwd_config(M: int, K: int, TILE_N: int): + """Static fallback for skinny MHC projection when autotune cache is absent.""" + split_k = 16 if K >= 16384 else 8 if K >= 8192 else 1 + return _default_tile_m(M), TILE_N, min(128, K), split_k + + # Cache the best config across calls (keyed by M, N, K). + _proj_rms_fwd_best_cfg: dict = {} + def _cutile_proj_rms_fwd( - x: Tensor, weight: Tensor, eps: float = 1e-8 + x: Tensor, weight: Tensor, eps: float = 1e-6 ) -> Tuple[Tensor, Tensor, Tensor]: M, K = x.shape N = weight.shape[0] - TILE_M = 128 TILE_N = _next_power_of_2(N) - TILE_K = 128 - num_tiles_m = math.ceil(M / TILE_M) - proj = torch.empty(M, N, dtype=x.dtype, device=x.device) - norm = torch.empty(M, 1, dtype=x.dtype, device=x.device) - r = torch.empty(M, 1, dtype=x.dtype, device=x.device) - ct.launch( - torch.cuda.current_stream(), - (num_tiles_m,), - _ct_proj_rms_fwd_kernel, - (x, weight, proj, norm, r, M, N, K, eps, TILE_M, TILE_N, TILE_K), - ) + dev = x.device + stream = torch.cuda.current_stream() + + cache_key = (M, N, K) + cached = _proj_rms_fwd_best_cfg.get(cache_key) + + if cached is not None or not _CUTILE_EXPERIMENTAL_AVAILABLE: + # Use cached best config, or fall back to default if no experimental. + if cached is not None: + tm, tn, tk, split_k = cached + else: + tm, tn, tk, split_k = _default_proj_rms_fwd_config(M, K, TILE_N) + + proj = torch.empty(split_k * M, N, dtype=x.dtype, device=dev) + norm = torch.empty(split_k * M, 1, dtype=x.dtype, device=dev) + # _ct_proj_rms_fwd_kernel keeps R in its signature; r is computed + # below from the reduced norm. + r = torch.empty(split_k * M, 1, dtype=x.dtype, device=dev) + + ct.launch( + stream, + (math.ceil(M / tm), split_k), + _ct_proj_rms_fwd_kernel, + (x, weight, proj, norm, r, M, N, K, eps, tm, tn, tk, split_k), + ) + proj = proj.view(split_k, M, N).to(torch.float32).sum(dim=0).to(dtype=x.dtype) + norm = norm.view(split_k, M, 1).to(torch.float32).sum(dim=0).to(dtype=x.dtype) + else: + # Autotune on first call for this shape. + from types import SimpleNamespace + + configs = [SimpleNamespace(**c) for c in _proj_rms_fwd_autotune_configs(N)] + # filter out configs with TILE_K > K or TILE_M > M + configs = [cfg for cfg in configs if cfg.TILE_K <= K and M % cfg.TILE_M == 0] + if len(configs) == 0: + tm, tn, tk, split_k = _default_proj_rms_fwd_config(M, K, TILE_N) + proj = torch.empty(split_k * M, N, dtype=x.dtype, device=dev) + norm = torch.empty(split_k * M, 1, dtype=x.dtype, device=dev) + # Signature placeholder for the cuTile kernel; not read. + r = torch.empty(split_k * M, 1, dtype=x.dtype, device=dev) + ct.launch( + stream, + (math.ceil(M / tm), split_k), + _ct_proj_rms_fwd_kernel, + (x, weight, proj, norm, r, M, N, K, eps, tm, tn, tk, split_k), + ) + proj = proj.view(split_k, M, N).to(torch.float32).sum(dim=0).to(dtype=x.dtype) + norm = norm.view(split_k, M, 1).to(torch.float32).sum(dim=0).to(dtype=x.dtype) + else: + mx_split_k = max(cfg.SPLIT_K for cfg in configs) + proj = torch.empty(mx_split_k * M, N, dtype=x.dtype, device=dev) + norm = torch.empty(mx_split_k * M, 1, dtype=x.dtype, device=dev) + # Signature placeholder for autotune launches; not read. + r = torch.empty(mx_split_k * M, 1, dtype=x.dtype, device=dev) + tuned = ct_experimental.autotune_launch( + stream, + grid_fn=lambda cfg: (math.ceil(M / cfg.TILE_M), cfg.SPLIT_K), + kernel=_ct_proj_rms_fwd_kernel, + args_fn=lambda cfg: ( + x, + weight, + proj, + norm, + r, + M, + N, + K, + eps, + cfg.TILE_M, + cfg.TILE_N, + cfg.TILE_K, + cfg.SPLIT_K, + ), + search_space=configs, + ) + best = tuned.tuned_config + _proj_rms_fwd_best_cfg[cache_key] = ( + best.TILE_M, + best.TILE_N, + best.TILE_K, + best.SPLIT_K, + ) + proj = torch.empty(best.SPLIT_K * M, N, dtype=x.dtype, device=dev) + norm = torch.empty(best.SPLIT_K * M, 1, dtype=x.dtype, device=dev) + # Signature placeholder for the cuTile kernel; not read. + r = torch.empty(best.SPLIT_K * M, 1, dtype=x.dtype, device=dev) + # Re-launch with best config for correct output. + ct.launch( + stream, + (math.ceil(M / best.TILE_M), best.SPLIT_K), + _ct_proj_rms_fwd_kernel, + ( + x, + weight, + proj, + norm, + r, + M, + N, + K, + eps, + best.TILE_M, + best.TILE_N, + best.TILE_K, + best.SPLIT_K, + ), + ) + + proj = proj.view(best.SPLIT_K, M, N).to(torch.float32).sum(dim=0).to(dtype=x.dtype) + norm = norm.view(best.SPLIT_K, M, 1).to(torch.float32).sum(dim=0).to(dtype=x.dtype) + norm = torch.sqrt(norm) + r = 1.0 / (norm / math.sqrt(K) + eps) return proj, norm, r + # -- Reduce + compute_h launcher ------------------------------------------ + + def _reduce_compute_h_autotune_configs(M): + """Generate autotune search space for reduce_compute_h kernel.""" + min_tile_m = 16 if M >= 16 else 1 + for tile_m in (128, 64, 32, 16, 8, 4, 2, 1): + if tile_m < min_tile_m: + continue + if tile_m <= M and M % tile_m == 0: + yield tile_m + + def _default_reduce_compute_h_tile_m(M: int) -> int: + """Pick a reduce tile size with enough blocks to cover the GPU.""" + try: + num_sms = torch.cuda.get_device_properties("cuda").multi_processor_count + except Exception: + num_sms = 128 + + valid = [tm for tm in _reduce_compute_h_autotune_configs(M)] + for tm in valid: + if math.ceil(M / tm) >= num_sms: + return tm + return valid[-1] if valid else 1 + + _reduce_compute_h_best_cfg: dict = {} + + def _cutile_reduce_compute_h( + proj_acc: Tensor, + norm_acc: Tensor, + bias: Tensor, + alpha_pre: Tensor, + alpha_post: Tensor, + alpha_res: Tensor, + n: int, + M: int, + N: int, + K: int, + eps: float, + compute_h_eps: float, + _proj_tile_m: int, + tile_n: int, + split_k: int, + ) -> Tuple[Tensor, Tensor, Tensor, Tensor, Tensor]: + """Launch reduce split-K + compute_h kernel. + + Returns: + h_pre: [M, n] sigmoid-activated pre weights + h_post: [M, n] 2*sigmoid-activated post weights + h_res: [M, n*n] residual logits + r: [M, 1] r = norm / sqrt(K) + proj_reduced: [M, N] reduced projection (for backward) + """ + dev = proj_acc.device + stream = torch.cuda.current_stream() + + bias_2d = bias.unsqueeze(0).contiguous() # [1, N] + + h_pre_out = torch.empty(M, n, dtype=proj_acc.dtype, device=dev) + h_post_out = torch.empty(M, n, dtype=proj_acc.dtype, device=dev) + h_res_out = torch.empty(M, N - 2 * n, dtype=proj_acc.dtype, device=dev) + r_out = torch.empty(M, 1, dtype=proj_acc.dtype, device=dev) + proj_out = torch.empty(M, N, dtype=proj_acc.dtype, device=dev) + + default_tm = _default_reduce_compute_h_tile_m(M) + cache_key = (M, N, K, n, split_k) + cached = _reduce_compute_h_best_cfg.get(cache_key) + + def _make_args(tm): + return ( + proj_acc, + norm_acc, + bias_2d, + alpha_pre, + alpha_post, + alpha_res, + h_pre_out, + h_post_out, + h_res_out, + r_out, + proj_out, + M, + N, + K, + n, + eps, + compute_h_eps, + tm, + tile_n, + split_k, + ) + + if cached is not None or not _CUTILE_EXPERIMENTAL_AVAILABLE: + tm = cached if cached is not None else default_tm + ct.launch(stream, (math.ceil(M / tm),), _ct_reduce_compute_h_kernel, _make_args(tm)) + else: + from types import SimpleNamespace + + configs = [SimpleNamespace(TILE_M=tm) for tm in _reduce_compute_h_autotune_configs(M)] + tuned = ct_experimental.autotune_launch( + stream, + grid_fn=lambda cfg: (math.ceil(M / cfg.TILE_M),), + kernel=_ct_reduce_compute_h_kernel, + args_fn=lambda cfg: _make_args(cfg.TILE_M), + search_space=configs, + ) + best_tm = tuned.tuned_config.TILE_M + _reduce_compute_h_best_cfg[cache_key] = best_tm + ct.launch( + stream, (math.ceil(M / best_tm),), _ct_reduce_compute_h_kernel, _make_args(best_tm) + ) + + return h_pre_out, h_post_out, h_res_out, r_out, proj_out + + # -- Combined proj_rms + compute_h forward -------------------------------- + + def _cutile_proj_rms_compute_h_fwd( + x: Tensor, + weight: Tensor, + bias: Tensor, + alpha_pre: Tensor, + alpha_post: Tensor, + alpha_res: Tensor, + n: int, + eps: float, + compute_h_eps: float, + ) -> Tuple[Tensor, Tensor, Tensor, Tensor, Tensor]: + """Fused proj_rms + compute_h forward. + + Launches the existing _ct_proj_rms_fwd_kernel (split-K matmul + partial norm), + then _ct_reduce_compute_h_kernel (reduce + r + activations). + + Returns: + h_pre: [M, n] activated pre weights + h_post: [M, n] activated post weights + h_res: [M, n*n] residual logits + r: [M, 1] r = norm / sqrt(K) + proj_reduced: [M, N] reduced projection (for backward) + """ + M, K = x.shape + N = weight.shape[0] + TILE_N = _next_power_of_2(N) + dev = x.device + stream = torch.cuda.current_stream() + + cache_key = (M, N, K) + cached = _proj_rms_fwd_best_cfg.get(cache_key) + + if cached is not None or not _CUTILE_EXPERIMENTAL_AVAILABLE: + if cached is not None: + tm, tn, tk, split_k = cached + else: + tm, tn, tk, split_k = _default_proj_rms_fwd_config(M, K, TILE_N) + + proj_acc = torch.empty(split_k * M, N, dtype=x.dtype, device=dev) + norm_acc = torch.empty(split_k * M, 1, dtype=x.dtype, device=dev) + # _ct_proj_rms_fwd_kernel keeps R in its signature; reduce_compute_h + # computes r from norm_acc. + r_placeholder = torch.empty(split_k * M, 1, dtype=x.dtype, device=dev) + + ct.launch( + stream, + (math.ceil(M / tm), split_k), + _ct_proj_rms_fwd_kernel, + (x, weight, proj_acc, norm_acc, r_placeholder, M, N, K, eps, tm, tn, tk, split_k), + ) + else: + from types import SimpleNamespace + + configs = [SimpleNamespace(**c) for c in _proj_rms_fwd_autotune_configs(N)] + configs = [cfg for cfg in configs if cfg.TILE_K <= K and M % cfg.TILE_M == 0] + if len(configs) == 0: + tm, tn, tk, split_k = _default_proj_rms_fwd_config(M, K, TILE_N) + proj_acc = torch.empty(split_k * M, N, dtype=x.dtype, device=dev) + norm_acc = torch.empty(split_k * M, 1, dtype=x.dtype, device=dev) + # Signature placeholder for the cuTile kernel; not read. + r_placeholder = torch.empty(split_k * M, 1, dtype=x.dtype, device=dev) + ct.launch( + stream, + (math.ceil(M / tm), split_k), + _ct_proj_rms_fwd_kernel, + ( + x, + weight, + proj_acc, + norm_acc, + r_placeholder, + M, + N, + K, + eps, + tm, + tn, + tk, + split_k, + ), + ) + else: + mx_split_k = max(cfg.SPLIT_K for cfg in configs) + proj_acc = torch.empty(mx_split_k * M, N, dtype=x.dtype, device=dev) + norm_acc = torch.empty(mx_split_k * M, 1, dtype=x.dtype, device=dev) + # Signature placeholder for autotune launches; not read. + r_placeholder = torch.empty(mx_split_k * M, 1, dtype=x.dtype, device=dev) + tuned = ct_experimental.autotune_launch( + stream, + grid_fn=lambda cfg: (math.ceil(M / cfg.TILE_M), cfg.SPLIT_K), + kernel=_ct_proj_rms_fwd_kernel, + args_fn=lambda cfg: ( + x, + weight, + proj_acc, + norm_acc, + r_placeholder, + M, + N, + K, + eps, + cfg.TILE_M, + cfg.TILE_N, + cfg.TILE_K, + cfg.SPLIT_K, + ), + search_space=configs, + ) + best = tuned.tuned_config + _proj_rms_fwd_best_cfg[cache_key] = ( + best.TILE_M, + best.TILE_N, + best.TILE_K, + best.SPLIT_K, + ) + tm, tn, tk, split_k = best.TILE_M, best.TILE_N, best.TILE_K, best.SPLIT_K + + proj_acc = torch.empty(split_k * M, N, dtype=x.dtype, device=dev) + norm_acc = torch.empty(split_k * M, 1, dtype=x.dtype, device=dev) + # Signature placeholder for the cuTile kernel; not read. + r_placeholder = torch.empty(split_k * M, 1, dtype=x.dtype, device=dev) + ct.launch( + stream, + (math.ceil(M / tm), split_k), + _ct_proj_rms_fwd_kernel, + ( + x, + weight, + proj_acc, + norm_acc, + r_placeholder, + M, + N, + K, + eps, + tm, + tn, + tk, + split_k, + ), + ) + + # Launch reduce + compute_h kernel + h_pre, h_post, h_res, r, proj_reduced = _cutile_reduce_compute_h( + proj_acc, + norm_acc, + bias, + alpha_pre, + alpha_post, + alpha_res, + n, + M, + N, + K, + eps, + compute_h_eps, + tm, + TILE_N, + split_k, + ) + return h_pre, h_post, h_res, r, proj_reduced + + def _proj_rms_bwd_autotune_configs(N): + """Generate autotune search space for proj_rms backward kernel (K >= 8192 path).""" + TILE_N = _next_power_of_2(N) + tile_ms = (32, 64, 128) + tile_ks = (32, 64, 128, 256) + for tile_m in tile_ms: + for tile_k in tile_ks: + yield {"TILE_SIZE_M": tile_m, "TILE_SIZE_N": TILE_N, "TILE_SIZE_K": tile_k} + + _proj_rms_bwd_best_cfg: dict = {} + def _cutile_proj_rms_bwd( grad_proj: Tensor, grad_r: Tensor, x: Tensor, weight: Tensor, norm: Tensor, - eps: float = 1e-8, + eps: float = 1e-6, ) -> Tuple[Tensor, Tensor]: M, K = x.shape N = weight.shape[0] @@ -763,67 +2173,973 @@ def _cutile_proj_rms_bwd( db = torch.empty_like(weight) TILE_SIZE_N = _next_power_of_2(N) assert TILE_SIZE_N <= 256, f"TILE_SIZE_N too large: {TILE_SIZE_N}" - num_sms = torch.cuda.get_device_properties("cuda").multi_processor_count + stream = torch.cuda.current_stream() + if K >= 8192: - TILE_SIZE_M, TILE_SIZE_K = 128, 128 - grid = (math.ceil(K / TILE_SIZE_K), 1) + cache_key = (M, N, K) + cached = _proj_rms_bwd_best_cfg.get(cache_key) + + if cached is not None or not _CUTILE_EXPERIMENTAL_AVAILABLE: + if cached is not None: + tm, tn, tk = cached + else: + tm, tn, tk = 128, TILE_SIZE_N, 128 + ct.launch( + stream, + (math.ceil(K / tk), 1), + _ct_proj_rms_bwd_kernel, + (x, weight, norm, grad_proj, grad_r, da, db, M, N, K, eps, tm, tn, tk), + ) + else: + from types import SimpleNamespace + + configs = [SimpleNamespace(**c) for c in _proj_rms_bwd_autotune_configs(N)] + tuned = ct_experimental.autotune_launch( + stream, + grid_fn=lambda cfg: (math.ceil(K / cfg.TILE_SIZE_K), 1), + kernel=_ct_proj_rms_bwd_kernel, + args_fn=lambda cfg: ( + x, + weight, + norm, + grad_proj, + grad_r, + da, + db, + M, + N, + K, + eps, + cfg.TILE_SIZE_M, + cfg.TILE_SIZE_N, + cfg.TILE_SIZE_K, + ), + search_space=configs, + ) + best = tuned.tuned_config + _proj_rms_bwd_best_cfg[cache_key] = ( + best.TILE_SIZE_M, + best.TILE_SIZE_N, + best.TILE_SIZE_K, + ) + ct.launch( + stream, + (math.ceil(K / best.TILE_SIZE_K), 1), + _ct_proj_rms_bwd_kernel, + ( + x, + weight, + norm, + grad_proj, + grad_r, + da, + db, + M, + N, + K, + eps, + best.TILE_SIZE_M, + best.TILE_SIZE_N, + best.TILE_SIZE_K, + ), + ) + else: + num_sms = torch.cuda.get_device_properties("cuda").multi_processor_count + grid = (num_sms, 2, 1) ct.launch( - torch.cuda.current_stream(), + stream, grid, - _ct_proj_rms_bwd_kernel, + _ct_proj_rms_bwd_small_k_kernel, + (x, weight, norm, grad_proj, grad_r, da, db, M, N, K, eps, TILE_SIZE_N), + ) + return da, db + + # -- Fused compute_h + proj_rms backward kernels ---------------------------- + + @ct.kernel + def _ct_fused_grad_h_proj_kernel( + GRAD_H_PRE, # [M, n] + GRAD_H_POST, # [M, n] + GRAD_H_RES, # [M, n*n] + H_PRE, # [M, n] + H_POST, # [M, n] + PROJ, # [M, N] + R, # [M, 1] + GRAD_R_EXT, # [M, 1] + Alpha_pre, # [1] + Alpha_post, # [1] + Alpha_res, # [1] + GRAD_H, # [M, TILE_SIZE_N] output + GRAD_PROJ, # [M, TILE_SIZE_N] output + GRAD_R_TOTAL, # [M, 1] output + M: int, + N: int, + n: ConstInt, + eps: float, + compute_h_eps: float, + TILE_SIZE_M: ConstInt, + TILE_SIZE_N: ConstInt, + HAS_GRAD_H_PRE: ConstInt, + HAS_GRAD_H_POST: ConstInt, + HAS_GRAD_H_RES: ConstInt, + HAS_GRAD_R_EXT: ConstInt, + ): + """Precompute grad_h, grad_proj, and grad_r_total for downstream backward kernels. + + Grid: (ceil(M / TILE_SIZE_M),). + """ + tile_m_id = ct.bid(0) + + alpha_pre = ct.load(Alpha_pre, index=(0,), shape=(1,)).item() + alpha_post = ct.load(Alpha_post, index=(0,), shape=(1,)).item() + alpha_res = ct.load(Alpha_res, index=(0,), shape=(1,)).item() + + r_tile = ct.load(R, index=(tile_m_id, 0), shape=(TILE_SIZE_M, 1), padding_mode=PAD_ZERO) + r_tile = ct.astype(r_tile, ct.float32) + + r_eps = r_tile + eps + inv_r_eps = 1.0 / r_eps + grad_r_from_h = ct.full((TILE_SIZE_M, 1), 0.0, dtype=ct.float32) + + # Clear the padded columns once inside this kernel. Valid columns are + # overwritten by the segment stores below. + zero_full = ct.full((TILE_SIZE_M, TILE_SIZE_N), 0.0, dtype=ct.float32) + ct.store(GRAD_H, index=(tile_m_id, 0), tile=zero_full.astype(GRAD_H.dtype)) + ct.store(GRAD_PROJ, index=(tile_m_id, 0), tile=zero_full.astype(GRAD_PROJ.dtype)) + + if HAS_GRAD_H_PRE: + gy_pre = ct.load( + GRAD_H_PRE, index=(tile_m_id, 0), shape=(TILE_SIZE_M, n), padding_mode=PAD_ZERO + ) + gy_pre = ct.astype(gy_pre, ct.float32) + else: + gy_pre = ct.full((TILE_SIZE_M, n), 0.0, dtype=ct.float32) + h_pre = ct.load(H_PRE, index=(tile_m_id, 0), shape=(TILE_SIZE_M, n), padding_mode=PAD_ZERO) + h_pre = ct.astype(h_pre, ct.float32) + proj_pre = ct.load( + PROJ, index=(tile_m_id, 0), shape=(TILE_SIZE_M, n), padding_mode=PAD_ZERO + ) + proj_pre = ct.astype(proj_pre, ct.float32) + sigmoid_pre = h_pre - compute_h_eps + grad_h_pre = gy_pre * sigmoid_pre * (1.0 - sigmoid_pre) + grad_proj_pre = grad_h_pre * alpha_pre * inv_r_eps + grad_r_from_h += ct.sum( + grad_h_pre * proj_pre * alpha_pre * (-inv_r_eps * inv_r_eps), axis=1, keepdims=True + ) + ct.store(GRAD_H, index=(tile_m_id, 0), tile=grad_h_pre.astype(GRAD_H.dtype)) + ct.store(GRAD_PROJ, index=(tile_m_id, 0), tile=grad_proj_pre.astype(GRAD_PROJ.dtype)) + + if HAS_GRAD_H_POST: + gy_post = ct.load( + GRAD_H_POST, index=(tile_m_id, 0), shape=(TILE_SIZE_M, n), padding_mode=PAD_ZERO + ) + gy_post = ct.astype(gy_post, ct.float32) + else: + gy_post = ct.full((TILE_SIZE_M, n), 0.0, dtype=ct.float32) + h_post = ct.load( + H_POST, index=(tile_m_id, 0), shape=(TILE_SIZE_M, n), padding_mode=PAD_ZERO + ) + h_post = ct.astype(h_post, ct.float32) + proj_post = ct.load( + PROJ, index=(tile_m_id, 1), shape=(TILE_SIZE_M, n), padding_mode=PAD_ZERO + ) + proj_post = ct.astype(proj_post, ct.float32) + sigmoid_post = h_post * 0.5 + grad_h_post = gy_post * sigmoid_post * (1.0 - sigmoid_post) * 2.0 + grad_proj_post = grad_h_post * alpha_post * inv_r_eps + grad_r_from_h += ct.sum( + grad_h_post * proj_post * alpha_post * (-inv_r_eps * inv_r_eps), axis=1, keepdims=True + ) + ct.store(GRAD_H, index=(tile_m_id, 1), tile=grad_h_post.astype(GRAD_H.dtype)) + ct.store(GRAD_PROJ, index=(tile_m_id, 1), tile=grad_proj_post.astype(GRAD_PROJ.dtype)) + + for res_chunk in ct.static_iter(range(n)): + if HAS_GRAD_H_RES: + grad_h_res = ct.load( + GRAD_H_RES, + index=(tile_m_id, res_chunk), + shape=(TILE_SIZE_M, n), + padding_mode=PAD_ZERO, + ) + grad_h_res = ct.astype(grad_h_res, ct.float32) + else: + grad_h_res = ct.full((TILE_SIZE_M, n), 0.0, dtype=ct.float32) + proj_res = ct.load( + PROJ, + index=(tile_m_id, 2 + res_chunk), + shape=(TILE_SIZE_M, n), + padding_mode=PAD_ZERO, + ) + proj_res = ct.astype(proj_res, ct.float32) + grad_proj_res = grad_h_res * alpha_res * inv_r_eps + grad_r_from_h += ct.sum( + grad_h_res * proj_res * alpha_res * (-inv_r_eps * inv_r_eps), axis=1, keepdims=True + ) + ct.store(GRAD_H, index=(tile_m_id, 2 + res_chunk), tile=grad_h_res.astype(GRAD_H.dtype)) + ct.store( + GRAD_PROJ, + index=(tile_m_id, 2 + res_chunk), + tile=grad_proj_res.astype(GRAD_PROJ.dtype), + ) + + if HAS_GRAD_R_EXT: + grad_r_ext_tile = ct.load( + GRAD_R_EXT, index=(tile_m_id, 0), shape=(TILE_SIZE_M, 1), padding_mode=PAD_ZERO + ) + grad_r_ext_tile = ct.astype(grad_r_ext_tile, ct.float32) + else: + grad_r_ext_tile = ct.full((TILE_SIZE_M, 1), 0.0, dtype=ct.float32) + grad_r_total = grad_r_from_h + grad_r_ext_tile + + ct.store(GRAD_R_TOTAL, index=(tile_m_id, 0), tile=grad_r_total.astype(GRAD_R_TOTAL.dtype)) + + @ct.kernel + def _ct_fused_grad_x_weight_kernel( + X, # [M, K] + WEIGHT, # [N, K] + GRAD_PROJ, # [M, TILE_SIZE_N] precomputed + GRAD_R_TOTAL, # [M, 1] precomputed + R, # [M, 1] + GRAD_X, # [M, K] output + GRAD_WEIGHT, # [N, K] output + M: int, + N: int, + K: int, + TILE_SIZE_M: ConstInt, + TILE_SIZE_N: ConstInt, + TILE_SIZE_K: ConstInt, + ): + """Compute grad_x and grad_weight simultaneously. + + Grid: (ceil(K / TILE_SIZE_K),). + Each block handles one K-tile and loops over all M-tiles. + Per M-tile: computes and stores grad_x, accumulates grad_weight. + """ + tile_k_id = ct.bid(0) + NUM_M_TILES = ct.cdiv(M, TILE_SIZE_M) + + # Load weight tile once — only depends on K-tile + weight_tile = ct.load( + WEIGHT, index=(0, tile_k_id), shape=(TILE_SIZE_N, TILE_SIZE_K), padding_mode=PAD_ZERO + ) + + acc_grad_weight = ct.full((TILE_SIZE_K, TILE_SIZE_N), 0.0, dtype=ct.float32) + + for tile_m_id in range(NUM_M_TILES): + grad_proj_tile = ct.load( + GRAD_PROJ, + index=(tile_m_id, 0), + shape=(TILE_SIZE_M, TILE_SIZE_N), + padding_mode=PAD_ZERO, + ) + x_tile = ct.load( + X, + index=(tile_m_id, tile_k_id), + shape=(TILE_SIZE_M, TILE_SIZE_K), + padding_mode=PAD_ZERO, + ) + grad_r_total = ct.load( + GRAD_R_TOTAL, index=(tile_m_id, 0), shape=(TILE_SIZE_M, 1), padding_mode=PAD_ZERO + ) + r_tile = ct.load(R, index=(tile_m_id, 0), shape=(TILE_SIZE_M, 1), padding_mode=PAD_ZERO) + r_tile = ct.astype(r_tile, ct.float32) + + # grad_x = grad_proj @ weight + grad_r_total * x / (r * K) + inv_rK = 1.0 / (r_tile * K) + acc_grad_x = (grad_r_total * inv_rK) * ct.astype(x_tile, ct.float32) + acc_grad_x = ct.mma( + grad_proj_tile.astype(ct.tfloat32), weight_tile.astype(ct.tfloat32), acc=acc_grad_x + ) + ct.store(GRAD_X, index=(tile_m_id, tile_k_id), tile=acc_grad_x.astype(GRAD_X.dtype)) + + # Accumulate grad_weight += x.T @ grad_proj + acc_grad_weight = ct.mma( + x_tile.transpose().astype(ct.tfloat32), + grad_proj_tile.astype(ct.tfloat32), + acc=acc_grad_weight, + ) + + ct.store( + GRAD_WEIGHT, + index=(0, tile_k_id), + tile=acc_grad_weight.transpose().astype(GRAD_WEIGHT.dtype), + ) + + @ct.kernel + def _ct_scalar_grads_partials_kernel( + GRAD_H, # [M, TILE_SIZE_N] precomputed + PROJ, # [M, N] + R, # [M, 1] + GRAD_ALPHA_PRE_PARTIALS, # [num_m_blocks, 1] output + GRAD_ALPHA_POST_PARTIALS, # [num_m_blocks, 1] output + GRAD_ALPHA_RES_PARTIALS, # [num_m_blocks, 1] output + GRAD_BIAS_PARTIALS, # [num_m_blocks, TILE_SIZE_N] output + M: int, + N: int, + n: int, + eps: float, + TILE_SIZE_M: ConstInt, + TILE_SIZE_N: ConstInt, + ): + """Compute per-M-tile scalar-gradient partials. + + Grid: (ceil(M / TILE_SIZE_M),). Each block processes one M-tile. + """ + bid_m = ct.bid(0) + + offsets = ct.arange(TILE_SIZE_N, dtype=ct.int32) + one = ct.full((TILE_SIZE_N,), 1.0, dtype=ct.float32) + zero = ct.full((TILE_SIZE_N,), 0.0, dtype=ct.float32) + mask_pre = ct.where(ct.less(offsets, n), one, zero) + mask_post = ct.where(ct.less(offsets, 2 * n), one, zero) - mask_pre + mask_res = one - mask_pre - mask_post + + mask_pre_2d = ct.reshape(mask_pre, (1, TILE_SIZE_N)) + mask_post_2d = ct.reshape(mask_post, (1, TILE_SIZE_N)) + mask_res_2d = ct.reshape(mask_res, (1, TILE_SIZE_N)) + + grad_h = ct.load( + GRAD_H, index=(bid_m, 0), shape=(TILE_SIZE_M, TILE_SIZE_N), padding_mode=PAD_ZERO + ) + proj_tile = ct.load( + PROJ, index=(bid_m, 0), shape=(TILE_SIZE_M, TILE_SIZE_N), padding_mode=PAD_ZERO + ) + proj_tile = ct.astype(proj_tile, ct.float32) + r_tile = ct.load(R, index=(bid_m, 0), shape=(TILE_SIZE_M, 1), padding_mode=PAD_ZERO) + r_tile = ct.astype(r_tile, ct.float32) + + r_eps = r_tile + eps + inv_r_eps = 1.0 / r_eps + + ga_all = grad_h * proj_tile * inv_r_eps + ga_pre = ct.reshape(ct.sum(ga_all * mask_pre_2d), (1, 1)) + ga_post = ct.reshape(ct.sum(ga_all * mask_post_2d), (1, 1)) + ga_res = ct.reshape(ct.sum(ga_all * mask_res_2d), (1, 1)) + partial_gb = ct.sum(grad_h, axis=0, keepdims=False) + ct.store( + GRAD_ALPHA_PRE_PARTIALS, + index=(bid_m, 0), + tile=ga_pre.astype(GRAD_ALPHA_PRE_PARTIALS.dtype), + ) + ct.store( + GRAD_ALPHA_POST_PARTIALS, + index=(bid_m, 0), + tile=ga_post.astype(GRAD_ALPHA_POST_PARTIALS.dtype), + ) + ct.store( + GRAD_ALPHA_RES_PARTIALS, + index=(bid_m, 0), + tile=ga_res.astype(GRAD_ALPHA_RES_PARTIALS.dtype), + ) + ct.store( + GRAD_BIAS_PARTIALS, + index=(bid_m, 0), + tile=ct.reshape(partial_gb, (1, TILE_SIZE_N)).astype(GRAD_BIAS_PARTIALS.dtype), + ) + + @ct.kernel + def _ct_scalar_grads_reduce_kernel( + GRAD_ALPHA_PRE_PARTIALS, # [num_m_blocks, 1] + GRAD_ALPHA_POST_PARTIALS, # [num_m_blocks, 1] + GRAD_ALPHA_RES_PARTIALS, # [num_m_blocks, 1] + GRAD_BIAS_PARTIALS, # [num_m_blocks, TILE_SIZE_N] + GRAD_ALPHA_PRE, # [1, 1] output + GRAD_ALPHA_POST, # [1, 1] output + GRAD_ALPHA_RES, # [1, 1] output + GRAD_BIAS, # [1, TILE_SIZE_N] output + NUM_M_BLOCKS: int, + TILE_SIZE_N: ConstInt, + ): + """Reduce scalar-gradient partials and write final dtype outputs.""" + acc_pre = ct.full((1, 1), 0.0, dtype=ct.float32) + acc_post = ct.full((1, 1), 0.0, dtype=ct.float32) + acc_res = ct.full((1, 1), 0.0, dtype=ct.float32) + acc_bias = ct.full((1, TILE_SIZE_N), 0.0, dtype=ct.float32) + + for bid_m in range(NUM_M_BLOCKS): + acc_pre += ct.load( + GRAD_ALPHA_PRE_PARTIALS, index=(bid_m, 0), shape=(1, 1), padding_mode=PAD_ZERO + ).astype(ct.float32) + acc_post += ct.load( + GRAD_ALPHA_POST_PARTIALS, index=(bid_m, 0), shape=(1, 1), padding_mode=PAD_ZERO + ).astype(ct.float32) + acc_res += ct.load( + GRAD_ALPHA_RES_PARTIALS, index=(bid_m, 0), shape=(1, 1), padding_mode=PAD_ZERO + ).astype(ct.float32) + acc_bias += ct.load( + GRAD_BIAS_PARTIALS, index=(bid_m, 0), shape=(1, TILE_SIZE_N), padding_mode=PAD_ZERO + ).astype(ct.float32) + + ct.store(GRAD_ALPHA_PRE, index=(0, 0), tile=acc_pre.astype(GRAD_ALPHA_PRE.dtype)) + ct.store(GRAD_ALPHA_POST, index=(0, 0), tile=acc_post.astype(GRAD_ALPHA_POST.dtype)) + ct.store(GRAD_ALPHA_RES, index=(0, 0), tile=acc_res.astype(GRAD_ALPHA_RES.dtype)) + ct.store(GRAD_BIAS, index=(0, 0), tile=acc_bias.astype(GRAD_BIAS.dtype)) + + @ct.kernel + def _ct_fused_compute_h_proj_rms_bwd_small_k_kernel( + X, # [M, K] + WEIGHT, # [N, K] + GRAD_PROJ, # [M, TILE_N] precomputed + GRAD_R_TOTAL, # [M, 1] precomputed + R, # [M, 1] + GRAD_X, # [M, K] output + GRAD_WEIGHT, # [N, K] output + M: int, + N: int, + K: int, + TILE_N_SIZE: ConstInt, + ): + """Fused backward (small K path) with work-stealing. + + Grid: (num_sms, 2). + bid(1)==0: grad_weight via work-stealing over K-tiles, loops M. + bid(1)==1: grad_x via work-stealing over (M×K) tiles. + Scalar gradients are computed by the separate partial/reduce kernels. + """ + zero_pad = ct.PaddingMode.ZERO + + TILE_DB_SIZE_M = 128 + TILE_DB_SIZE_K = 64 + NUM_M_TILES = ct.cdiv(M, TILE_DB_SIZE_M) + NUM_K_TILES = ct.cdiv(K, TILE_DB_SIZE_K) + + if ct.bid(1) == 0: + # --- grad_weight path --- + for tile_id in range(ct.bid(0), NUM_K_TILES, ct.num_blocks(0)): + accumulator_db = ct.full((TILE_DB_SIZE_K, TILE_N_SIZE), 0.0, dtype=ct.float32) + for m_tile in range(NUM_M_TILES): + x_tile = ct.load( + X, + index=(m_tile, tile_id), + shape=(TILE_DB_SIZE_M, TILE_DB_SIZE_K), + padding_mode=zero_pad, + ) + grad_proj_tile = ct.load( + GRAD_PROJ, + index=(m_tile, 0), + shape=(TILE_DB_SIZE_M, TILE_N_SIZE), + padding_mode=zero_pad, + ) + + accumulator_db = ct.mma( + x_tile.transpose().astype(ct.tfloat32), + grad_proj_tile.astype(ct.tfloat32), + acc=accumulator_db, + ) + + ct.store( + GRAD_WEIGHT, + index=(0, tile_id), + tile=accumulator_db.transpose().astype(GRAD_WEIGHT.dtype), + allow_tma=False, + ) + + TILE_DA_SIZE_M = 128 + TILE_DA_SIZE_K = 256 + NUM_DA_TILES = ct.cdiv(M, TILE_DA_SIZE_M) * ct.cdiv(K, TILE_DA_SIZE_K) + NUM_DA_K_TILES = ct.cdiv(K, TILE_DA_SIZE_K) + + if ct.bid(1) == 1: + # --- grad_x path --- + for tile_id in range(ct.bid(0), NUM_DA_TILES, ct.num_blocks(0)): + b_tile_idx = tile_id % NUM_DA_K_TILES + dd_tile_idx = tile_id // NUM_DA_K_TILES + + grad_proj_tile = ct.load( + GRAD_PROJ, + index=(dd_tile_idx, 0), + shape=(TILE_DA_SIZE_M, TILE_N_SIZE), + padding_mode=zero_pad, + ) + grad_r_total = ct.load( + GRAD_R_TOTAL, + index=(dd_tile_idx, 0), + shape=(TILE_DA_SIZE_M, 1), + padding_mode=zero_pad, + ) + r_tile = ct.load( + R, index=(dd_tile_idx, 0), shape=(TILE_DA_SIZE_M, 1), padding_mode=zero_pad + ) + r_tile = ct.astype(r_tile, ct.float32) + + x_tile = ct.load( + X, + index=(dd_tile_idx, b_tile_idx), + shape=(TILE_DA_SIZE_M, TILE_DA_SIZE_K), + padding_mode=zero_pad, + ) + inv_rK = 1.0 / (r_tile * K) + accumulator_da = (grad_r_total * inv_rK) * ct.astype(x_tile, ct.float32) + + weight_tile = ct.load( + WEIGHT, + index=(0, b_tile_idx), + shape=(TILE_N_SIZE, TILE_DA_SIZE_K), + padding_mode=zero_pad, + ) + accumulator_da = ct.mma( + grad_proj_tile.astype(ct.tfloat32), + weight_tile.astype(ct.tfloat32), + acc=accumulator_da, + ) + ct.store( + GRAD_X, + index=(dd_tile_idx, b_tile_idx), + tile=accumulator_da.astype(GRAD_X.dtype), + ) + + def _fused_grad_x_weight_autotune_configs(N): + """Autotune search space for fused grad_x + grad_weight kernel.""" + TILE_N = _next_power_of_2(N) + tile_ms = (32, 64, 128) + tile_ks = (32, 64, 128, 256) + for tile_m in tile_ms: + for tile_k in tile_ks: + yield {"TILE_SIZE_M": tile_m, "TILE_SIZE_N": TILE_N, "TILE_SIZE_K": tile_k} + + _fused_grad_x_weight_best_cfg: dict = {} + + def _cutile_fused_compute_h_proj_rms_bwd( + x: Tensor, + weight: Tensor, + grad_h_pre: Tensor, + grad_h_post: Tensor, + grad_h_res: Tensor, + h_pre: Tensor, + h_post: Tensor, + h_res: Tensor, + proj: Tensor, + r: Tensor, + grad_r_ext: Tensor, + alpha_pre: Tensor, + alpha_post: Tensor, + alpha_res: Tensor, + bias: Tensor, + n: int, + eps: float, + compute_h_eps: float, + ) -> Tuple[Tensor, Tensor, Tensor, Tensor, Tensor, Tensor]: + """Fused compute_h + proj_rms backward. + + Returns: + grad_x: [M, K] + grad_weight: [N, K] + grad_alpha_pre: [1] + grad_alpha_post: [1] + grad_alpha_res: [1] + grad_bias: [N] + """ + M, K = x.shape + N = weight.shape[0] + TILE_N = _next_power_of_2(N) + assert TILE_N <= 256, f"TILE_SIZE_N too large: {TILE_N}" + dev = x.device + stream = torch.cuda.current_stream() + + grad_x = torch.empty_like(x) + grad_weight = torch.empty_like(weight) + has_grad_r_ext = grad_r_ext is not None + has_grad_r_ext_flag = int(has_grad_r_ext) + grad_r_ext_arg = grad_r_ext if has_grad_r_ext else r + has_grad_h_pre = grad_h_pre is not None + has_grad_h_post = grad_h_post is not None + has_grad_h_res = grad_h_res is not None + grad_h_pre_arg = grad_h_pre if has_grad_h_pre else h_pre + grad_h_post_arg = grad_h_post if has_grad_h_post else h_post + grad_h_res_arg = grad_h_res if has_grad_h_res else h_res + + # 0. Precompute grad_h, grad_proj, grad_r_total + grad_h_buf = torch.empty(M, TILE_N, dtype=torch.float32, device=dev) + grad_proj_buf = torch.empty(M, TILE_N, dtype=torch.float32, device=dev) + grad_r_total_buf = torch.empty(M, 1, dtype=torch.float32, device=dev) + + tile_m_precomp = _default_tile_m(M) + ct.launch( + stream, + (math.ceil(M / tile_m_precomp),), + _ct_fused_grad_h_proj_kernel, + ( + grad_h_pre_arg, + grad_h_post_arg, + grad_h_res_arg, + h_pre, + h_post, + proj, + r, + grad_r_ext_arg, + alpha_pre, + alpha_post, + alpha_res, + grad_h_buf, + grad_proj_buf, + grad_r_total_buf, + M, + N, + n, + eps, + compute_h_eps, + tile_m_precomp, + TILE_N, + int(has_grad_h_pre), + int(has_grad_h_post), + int(has_grad_h_res), + has_grad_r_ext_flag, + ), + ) + + if K >= 8192: + # 1. Fused grad_x + grad_weight kernel — 1D grid (K-tiles), loops M + cache_key = ('grad_x_weight', M, N, K) + cached = _fused_grad_x_weight_best_cfg.get(cache_key) + + if cached is not None or not _CUTILE_EXPERIMENTAL_AVAILABLE: + if cached is not None: + tm, tn, tk = cached + else: + tm, tn, tk = 128, TILE_N, 128 + ct.launch( + stream, + (math.ceil(K / tk),), + _ct_fused_grad_x_weight_kernel, + ( + x, + weight, + grad_proj_buf, + grad_r_total_buf, + r, + grad_x, + grad_weight, + M, + N, + K, + tm, + tn, + tk, + ), + ) + else: + from types import SimpleNamespace + + configs = [SimpleNamespace(**c) for c in _fused_grad_x_weight_autotune_configs(N)] + tuned = ct_experimental.autotune_launch( + stream, + grid_fn=lambda cfg: (math.ceil(K / cfg.TILE_SIZE_K),), + kernel=_ct_fused_grad_x_weight_kernel, + args_fn=lambda cfg: ( + x, + weight, + grad_proj_buf, + grad_r_total_buf, + r, + grad_x, + grad_weight, + M, + N, + K, + cfg.TILE_SIZE_M, + cfg.TILE_SIZE_N, + cfg.TILE_SIZE_K, + ), + search_space=configs, + ) + best = tuned.tuned_config + _fused_grad_x_weight_best_cfg[cache_key] = ( + best.TILE_SIZE_M, + best.TILE_SIZE_N, + best.TILE_SIZE_K, + ) + ct.launch( + stream, + (math.ceil(K / best.TILE_SIZE_K),), + _ct_fused_grad_x_weight_kernel, + ( + x, + weight, + grad_proj_buf, + grad_r_total_buf, + r, + grad_x, + grad_weight, + M, + N, + K, + best.TILE_SIZE_M, + best.TILE_SIZE_N, + best.TILE_SIZE_K, + ), + ) + else: + num_sms = torch.cuda.get_device_properties("cuda").multi_processor_count + ct.launch( + stream, + (num_sms, 2, 1), + _ct_fused_compute_h_proj_rms_bwd_small_k_kernel, ( x, weight, - norm, - grad_proj, - grad_r, - da, - db, + grad_proj_buf, + grad_r_total_buf, + r, + grad_x, + grad_weight, M, N, K, - TILE_SIZE_M, - TILE_SIZE_N, - TILE_SIZE_K, + TILE_N, ), ) - else: - grid = (num_sms, 2, 1) - ct.launch( - torch.cuda.current_stream(), - grid, - _ct_proj_rms_bwd_small_k_kernel, - (x, weight, norm, grad_proj, grad_r, da, db, M, N, K, TILE_SIZE_N), - ) - return da, db + + # 2. Separate lightweight kernel for scalar gradients (grad_alpha, grad_bias) + tile_m_scalar = min(128, M) + num_m_blocks = math.ceil(M / tile_m_scalar) + grad_alpha_pre_partials = torch.empty(num_m_blocks, 1, dtype=torch.float32, device=dev) + grad_alpha_post_partials = torch.empty(num_m_blocks, 1, dtype=torch.float32, device=dev) + grad_alpha_res_partials = torch.empty(num_m_blocks, 1, dtype=torch.float32, device=dev) + grad_bias_partials = torch.empty(num_m_blocks, TILE_N, dtype=torch.float32, device=dev) + grad_alpha_pre = torch.empty(1, 1, dtype=alpha_pre.dtype, device=dev) + grad_alpha_post = torch.empty(1, 1, dtype=alpha_post.dtype, device=dev) + grad_alpha_res = torch.empty(1, 1, dtype=alpha_res.dtype, device=dev) + grad_bias = torch.empty(1, TILE_N, dtype=bias.dtype, device=dev) + + ct.launch( + stream, + (num_m_blocks,), + _ct_scalar_grads_partials_kernel, + ( + grad_h_buf, + proj, + r, + grad_alpha_pre_partials, + grad_alpha_post_partials, + grad_alpha_res_partials, + grad_bias_partials, + M, + N, + n, + eps, + tile_m_scalar, + TILE_N, + ), + ) + ct.launch( + stream, + (1,), + _ct_scalar_grads_reduce_kernel, + ( + grad_alpha_pre_partials, + grad_alpha_post_partials, + grad_alpha_res_partials, + grad_bias_partials, + grad_alpha_pre, + grad_alpha_post, + grad_alpha_res, + grad_bias, + num_m_blocks, + TILE_N, + ), + ) + + return ( + grad_x, + grad_weight, + grad_alpha_pre.view_as(alpha_pre), + grad_alpha_post.view_as(alpha_post), + grad_alpha_res.view_as(alpha_res), + grad_bias.view(-1)[:N], + ) # ============================================================================ -# Autograd Functions (cuTile only – guarded by _CUTILE_AVAILABLE) +# Unified public dispatch +# ============================================================================ +# The public fused API chooses the fastest validated backend per operation: +# +# sinkhorn fwd/bwd: Triton -> cuTile -> torch +# h_post_bda fwd/bwd: Triton -> cuTile -> torch +# h_aggregate fwd: Triton -> cuTile -> torch +# h_aggregate bwd: cuTile -> torch +# proj_rms/proj_rms_compute_h: cuTile -> torch +# +# Runtime CUDA launch failures are intentionally not swallowed; after such an +# error the CUDA context may not be safely reusable for fallback work. # ============================================================================ -if not _CUTILE_AVAILABLE: - - def _no_cutile_error(*_args, **_kwargs): - raise RuntimeError( - "Fused mHC kernels require cuda.tile (cuTile) which is not installed. " - "Either install cuTile or set use_fused_mhc=False to use reference " - "implementations." +from megatron.core.transformer.hyper_connection import ( + native_fused_add_3, + native_h_aggregate, + native_h_post_bda, + native_proj_rms, + native_sinkhorn, +) + +_BACKEND_INFO_LOGGED = False + + +def _select_triton_cutile_native(triton_impl) -> str: + if triton_impl is not None: + return "triton" + if is_cutile_available(): + return "cutile" + return "native" + + +def _mhc_backend_status() -> Tuple[str, bool]: + """Return backend description and whether every backend is native.""" + sinkhorn = _select_triton_cutile_native(_get_triton_sinkhorn()) + h_aggregate_fwd = _select_triton_cutile_native(_get_triton_h_aggregate_fwd()) + h_aggregate_bwd = "cutile" if is_cutile_available() else "native" + h_post_bda_fwd = _select_triton_cutile_native(_get_triton_h_post_bda_fwd()) + h_post_bda_bwd = _select_triton_cutile_native(_get_triton_h_post_bda_bwd()) + proj_rms = "cutile" if is_cutile_available() else "native" + selected = ( + sinkhorn, + h_aggregate_fwd, + h_aggregate_bwd, + h_post_bda_fwd, + h_post_bda_bwd, + proj_rms, + ) + message = ( + f"MHC_FORCE_BACKEND={_MHC_FORCED_BACKEND}; " + f"sinkhorn={sinkhorn}; " + f"h_aggregate=fwd:{h_aggregate_fwd},bwd:{h_aggregate_bwd}; " + f"h_post_bda=fwd:{h_post_bda_fwd},bwd:{h_post_bda_bwd}; " + f"proj_rms={proj_rms}; " + f"proj_rms_compute_h={proj_rms}" + ) + return message, all(backend == "native" for backend in selected) + + +def _mhc_backend_selection() -> str: + """Return a concise description of the selected mHC fused backends.""" + message, _ = _mhc_backend_status() + return message + + +def log_fused_mhc_backend_once() -> None: + """Log the fused mHC backend selection once per process.""" + _raise_mhc_backend_validation_error() + global _BACKEND_INFO_LOGGED + if _BACKEND_INFO_LOGGED: + return + _BACKEND_INFO_LOGGED = True + backend_selection, all_native = _mhc_backend_status() + log_single_rank( + logger, + logging.WARNING if all_native else logging.INFO, + f"[mHC] fused backend selection: {backend_selection}", + ) + if all_native and safe_get_rank() == 0: + warnings.warn( + "[mHC] No accelerated mHC backend is available; falling back to native torch " + "implementations. The fallback is functionally equivalent, but may not provide " + "the performance benefits of fused mHC backends.", + UserWarning, + stacklevel=2, ) - fused_sinkhorn = _no_cutile_error - fused_h_aggregate = _no_cutile_error - fused_h_post_bda = _no_cutile_error - fused_proj_rms = _no_cutile_error -else: +def fused_add_3(a: Tensor, b: Tensor, c: Tensor) -> Tensor: + """Add three tensors using the native torch.compile-backed implementation.""" + return native_fused_add_3(a, b, c) + + +def _get_triton_sinkhorn(): + if not _TRITON_AVAILABLE: + return None + return _TRITON_IMPLS["sinkhorn"] + + +def _get_triton_h_aggregate_fwd(): + if not _TRITON_AVAILABLE: + return None + return _TRITON_IMPLS["h_aggregate_fwd"] + + +def _get_triton_h_post_bda_fwd(): + if not _TRITON_AVAILABLE: + return None + return _TRITON_IMPLS["h_post_bda_fwd"] + + +def _get_triton_h_post_bda_bwd(): + if not _TRITON_AVAILABLE: + return None + return _TRITON_IMPLS["h_post_bda_bwd"] + + +def _torch_h_aggregate_bwd(grad_output: Tensor, x: Tensor, h_pre: Tensor) -> Tuple[Tensor, Tensor]: + grad_output_expanded = grad_output.unsqueeze(2) + grad_x = grad_output_expanded * h_pre.unsqueeze(-1) + grad_h = torch.sum(grad_output_expanded * x, dim=-1) + return grad_x.to(dtype=x.dtype), grad_h.to(dtype=h_pre.dtype) + + +@torch.compile +def _torch_h_post_bda_bwd( + grad_output: Tensor, + h_res: Tensor, + original_residual: Tensor, + h_post: Tensor, + x: Tensor, + bias: Optional[Tensor], +) -> Tuple[Tensor, Tensor, Tensor, Tensor, Optional[Tensor]]: + s, b, n, C = original_residual.shape + sb = s * b + go = grad_output.reshape(sb, n, C) + hr = h_res.reshape(sb, n, n) + orig = original_residual.reshape(sb, n, C) + hp = h_post.reshape(sb, n) + x_flat = x.reshape(sb, C) + + g_hr = torch.bmm(orig, go.transpose(1, 2)).view(s, b, n, n) + g_res = torch.bmm(hr, go).view(s, b, n, C) + g_x = torch.sum(go * hp.unsqueeze(-1), dim=1).view(s, b, C) + xb = x_flat if bias is None else x_flat + bias.view(1, C) + g_hp = torch.sum(go * xb.unsqueeze(1), dim=2).view(s, b, n) + g_bias = g_x.reshape(sb, C).sum(dim=0).to(dtype=bias.dtype) if bias is not None else None + return ( + g_hr.to(dtype=h_res.dtype), + g_res.to(dtype=original_residual.dtype), + g_hp.to(dtype=h_post.dtype), + g_x.to(dtype=x.dtype), + g_bias, + ) + + +@torch.compile +def _torch_proj_rms_compute_h( + x: Tensor, + weight: Tensor, + alpha_pre: Tensor, + alpha_post: Tensor, + alpha_res: Tensor, + bias: Tensor, + n: int, + eps: float, + compute_h_eps: float = 1e-6, +) -> Tuple[Tensor, Tensor, Tensor, Tensor]: + proj = torch.matmul(x, weight.t()) + r = x.norm(dim=-1, keepdim=True) / math.sqrt(x.shape[-1]) + alpha = torch.cat( + [alpha_pre.expand(n), alpha_post.expand(n), alpha_res.expand(weight.shape[0] - 2 * n)], + dim=-1, + ) + h = proj * alpha.unsqueeze(0) / (r + eps) + bias.unsqueeze(0) + h_pre = h[..., :n].sigmoid() + compute_h_eps + h_post = h[..., n : 2 * n].sigmoid() * 2 + h_res = h[..., 2 * n :] + return h_pre, h_post, h_res, r + + +if _CUTILE_AVAILABLE: - class FusedSinkhornKnopp(torch.autograd.Function): - """Fused Sinkhorn-Knopp projection to doubly stochastic matrix (cuTile).""" + class CutileSinkhornKnopp(torch.autograd.Function): + """cuTile Sinkhorn-Knopp projection fallback.""" @staticmethod def forward(ctx, input_logits: Tensor, num_iterations: int, eps: float = 1e-6): - """cuTile fused Sinkhorn forward.""" + """Run cuTile Sinkhorn forward and save initial matrix for backward.""" output, M_init = _cutile_sinkhorn_fwd(input_logits, num_iterations, eps) ctx.save_for_backward(M_init) ctx.num_iterations = num_iterations @@ -832,65 +3148,33 @@ def forward(ctx, input_logits: Tensor, num_iterations: int, eps: float = 1e-6): @staticmethod def backward(ctx, grad_output): - """cuTile fused Sinkhorn backward.""" + """Run cuTile Sinkhorn backward.""" (M_init,) = ctx.saved_tensors grad_input = _cutile_sinkhorn_bwd(grad_output, M_init, ctx.num_iterations, ctx.eps) return grad_input, None, None - class FusedHAggregate(torch.autograd.Function): - """Fused n-stream weighted aggregation (cuTile).""" + class CutileHAggregate(torch.autograd.Function): + """cuTile n-stream weighted aggregation.""" @staticmethod def forward(ctx, x: Tensor, h_pre: Tensor): - """cuTile fused h_aggregate forward.""" + """Run cuTile h_aggregate forward.""" output = _cutile_h_aggregate_fwd(x, h_pre) ctx.save_for_backward(x, h_pre) return output @staticmethod def backward(ctx, grad_output): - """cuTile fused h_aggregate backward.""" + """Run cuTile h_aggregate backward.""" x, h_pre = ctx.saved_tensors return _cutile_h_aggregate_bwd(grad_output, x, h_pre) - class FusedHPostBDA(torch.autograd.Function): - """Fused: output = H_res @ orig_res + H_post * (x [+ bias]) (cuTile).""" - - @staticmethod - def forward( - ctx, - h_res: Tensor, - original_residual: Tensor, - h_post: Tensor, - x: Tensor, - bias: Optional[Tensor], - ): - """cuTile fused h_post_bda forward.""" - output = _cutile_h_post_bda_fwd(h_res, original_residual, h_post, x, bias) - if bias is not None: - ctx.save_for_backward(h_res, original_residual, h_post, x, bias) - ctx.has_bias = True - else: - ctx.save_for_backward(h_res, original_residual, h_post, x) - ctx.has_bias = False - return output - - @staticmethod - def backward(ctx, grad_output): - """cuTile fused h_post_bda backward.""" - if ctx.has_bias: - h_res, orig_res, h_post, x, bias = ctx.saved_tensors - else: - h_res, orig_res, h_post, x = ctx.saved_tensors - bias = None - return _cutile_h_post_bda_bwd(grad_output, h_res, orig_res, h_post, x, bias) - - class FusedProjRms(torch.autograd.Function): - """Fused projection + RMS normalization (cuTile).""" + class CutileProjRms(torch.autograd.Function): + """cuTile projection + RMS normalization.""" @staticmethod def forward(ctx, x: Tensor, weight: Tensor, eps: float = 1e-6): - """cuTile fused proj_rms forward.""" + """Run cuTile projection plus RMS normalization forward.""" proj, norm, r = _cutile_proj_rms_fwd(x, weight, eps) ctx.save_for_backward(x, weight, norm) ctx.eps = eps @@ -898,67 +3182,216 @@ def forward(ctx, x: Tensor, weight: Tensor, eps: float = 1e-6): @staticmethod def backward(ctx, grad_proj, grad_r): - """cuTile fused proj_rms backward.""" + """Run cuTile projection plus RMS normalization backward.""" x, weight, norm = ctx.saved_tensors grad_x, grad_weight = _cutile_proj_rms_bwd(grad_proj, grad_r, x, weight, norm, ctx.eps) return grad_x, grad_weight, None - # ======================================================================== - # Public API (only available when cuTile is installed) - # ======================================================================== + class CutileProjRmsComputeH(torch.autograd.Function): + """cuTile projection + RMS norm + compute_h activations.""" - def fused_sinkhorn(input_logits: Tensor, num_iterations: int, eps: float = 1e-6) -> Tensor: - """Project logits to doubly stochastic matrix via Sinkhorn-Knopp. + @staticmethod + def forward( + ctx, + x: Tensor, + weight: Tensor, + alpha_pre: Tensor, + alpha_post: Tensor, + alpha_res: Tensor, + bias: Tensor, + n: int, + eps: float = 1e-6, + compute_h_eps: float = 1e-6, + ): + """Run fused cuTile projection, RMS normalization, and compute_h forward.""" + h_pre, h_post, h_res, r, proj_reduced = _cutile_proj_rms_compute_h_fwd( + x, weight, bias, alpha_pre, alpha_post, alpha_res, n, eps, compute_h_eps + ) + ctx.save_for_backward( + x, + weight, + h_pre, + h_post, + h_res, + proj_reduced, + r, + alpha_pre, + alpha_post, + alpha_res, + bias, + ) + ctx.n = n + ctx.eps = eps + ctx.compute_h_eps = compute_h_eps + return h_pre, h_post, h_res, r - Args: - input_logits: [..., n, n] raw logits - num_iterations: Sinkhorn iterations - eps: numerical stability + @staticmethod + def backward(ctx, grad_h_pre, grad_h_post, grad_h_res, grad_r_ext): + """Run fused cuTile projection, RMS normalization, and compute_h backward.""" + ( + x, + weight, + h_pre, + h_post, + h_res, + proj, + r, + alpha_pre, + alpha_post, + alpha_res, + bias_param, + ) = ctx.saved_tensors + + grad_x, grad_weight, grad_ap, grad_apo, grad_ar, grad_bias = ( + _cutile_fused_compute_h_proj_rms_bwd( + x, + weight, + grad_h_pre, + grad_h_post, + grad_h_res, + h_pre, + h_post, + h_res, + proj, + r, + grad_r_ext, + alpha_pre, + alpha_post, + alpha_res, + bias_param, + ctx.n, + ctx.eps, + ctx.compute_h_eps, + ) + ) - Returns: - [..., n, n] doubly stochastic matrix - """ - return FusedSinkhornKnopp.apply(input_logits, num_iterations, eps) + return (grad_x, grad_weight, grad_ap, grad_apo, grad_ar, grad_bias, None, None, None) - def fused_h_aggregate(x: Tensor, h_pre: Tensor) -> Tensor: - """Weighted n-stream to 1-stream aggregation. - Args: - x: [s, b, n, C] n-stream hidden states - h_pre: [s, b, n] aggregation weights +class FusedHAggregate(torch.autograd.Function): + """H_aggregate with Triton/cuTile/torch forward and cuTile/torch backward.""" - Returns: - [s, b, C] aggregated hidden states - """ - return FusedHAggregate.apply(x, h_pre) + @staticmethod + def forward(ctx, x: Tensor, h_pre: Tensor): + """Run h_aggregate forward using the best available backend.""" + triton_fwd = _get_triton_h_aggregate_fwd() + if triton_fwd is not None: + output = triton_fwd(x, h_pre) + elif is_cutile_available(): + output = _cutile_h_aggregate_fwd(x, h_pre) + else: + output = native_h_aggregate(x, h_pre) + ctx.save_for_backward(x, h_pre) + return output + + @staticmethod + def backward(ctx, grad_output): + """Run h_aggregate backward using the best available backend.""" + x, h_pre = ctx.saved_tensors + if is_cutile_available(): + return _cutile_h_aggregate_bwd(grad_output, x, h_pre) + return _torch_h_aggregate_bwd(grad_output, x, h_pre) - def fused_h_post_bda( - h_res: Tensor, original_residual: Tensor, h_post: Tensor, x: Tensor, bias: Optional[Tensor] - ) -> Tensor: - """Fused H_res @ residual + H_post * (x + bias). - Args: - h_res: [s, b, n, n] residual mixing matrix - original_residual: [s, b, n, C] n-stream residual - h_post: [s, b, n] expansion weights - x: [s, b, C] layer output - bias: [C] or None +class FusedHPostBDA(torch.autograd.Function): + """H_post_bda with Triton/cuTile/torch forward and backward.""" - Returns: - [s, b, n, C] fused output - """ - return FusedHPostBDA.apply(h_res, original_residual, h_post, x, bias) + @staticmethod + def forward( + ctx, + h_res: Tensor, + original_residual: Tensor, + h_post: Tensor, + x: Tensor, + bias: Optional[Tensor], + ): + """Run h_post_bda forward using the best available backend.""" + triton_fwd = _get_triton_h_post_bda_fwd() + if triton_fwd is not None: + output = triton_fwd(h_res, original_residual, h_post, x, bias) + elif is_cutile_available(): + output = _cutile_h_post_bda_fwd(h_res, original_residual, h_post, x, bias) + else: + output = native_h_post_bda(h_res, original_residual, h_post, x, bias) + if bias is not None: + ctx.save_for_backward(h_res, original_residual, h_post, x, bias) + ctx.has_bias = True + else: + ctx.save_for_backward(h_res, original_residual, h_post, x) + ctx.has_bias = False + return output + + @staticmethod + def backward(ctx, grad_output): + """Run h_post_bda backward using the best available backend.""" + if ctx.has_bias: + h_res, orig_res, h_post, x, bias = ctx.saved_tensors + else: + h_res, orig_res, h_post, x = ctx.saved_tensors + bias = None + + triton_bwd = _get_triton_h_post_bda_bwd() + if triton_bwd is not None: + return triton_bwd(grad_output, h_res, orig_res, h_post, x, bias) + if is_cutile_available(): + return _cutile_h_post_bda_bwd(grad_output, h_res, orig_res, h_post, x, bias) + return _torch_h_post_bda_bwd(grad_output, h_res, orig_res, h_post, x, bias) - def fused_proj_rms(x: Tensor, weight: Tensor, eps: float = 1e-6) -> Tuple[Tensor, Tensor]: - """Fused projection + RMS normalization. - Args: - x: [M, K] input - weight: [N, K] projection weight - eps: stability epsilon +def fused_sinkhorn(input_logits: Tensor, num_iterations: int, eps: float = 1e-6) -> Tensor: + """Project logits to a doubly stochastic matrix using Triton, cuTile, then torch.""" + _raise_mhc_backend_validation_error() + triton_sinkhorn = _get_triton_sinkhorn() + if triton_sinkhorn is not None: + return triton_sinkhorn(input_logits, num_iterations, eps) + if is_cutile_available(): + return CutileSinkhornKnopp.apply(input_logits, num_iterations, eps) + return native_sinkhorn(input_logits, num_iterations, eps) - Returns: - proj: [M, N] = x @ weight^T - r: [M, 1] = 1 / (||x|| / sqrt(K) + eps) - """ - return FusedProjRms.apply(x, weight, eps) + +def fused_h_aggregate(x: Tensor, h_pre: Tensor) -> Tensor: + """Weighted n-stream to 1-stream aggregation using Triton/cuTile/torch.""" + _raise_mhc_backend_validation_error() + if _TRITON_AVAILABLE or is_cutile_available(): + return FusedHAggregate.apply(x, h_pre) + return native_h_aggregate(x, h_pre) + + +def fused_h_post_bda( + h_res: Tensor, original_residual: Tensor, h_post: Tensor, x: Tensor, bias: Optional[Tensor] +) -> Tensor: + """Fused H_res.T @ residual + H_post * (x + bias).""" + _raise_mhc_backend_validation_error() + if _TRITON_AVAILABLE or is_cutile_available(): + return FusedHPostBDA.apply(h_res, original_residual, h_post, x, bias) + return native_h_post_bda(h_res, original_residual, h_post, x, bias) + + +def fused_proj_rms(x: Tensor, weight: Tensor, eps: float = 1e-6) -> Tuple[Tensor, Tensor]: + """Projection + RMS normalization using cuTile, then torch.""" + _raise_mhc_backend_validation_error() + if is_cutile_available(): + return CutileProjRms.apply(x, weight, eps) + return native_proj_rms(x, weight, eps) + + +def fused_proj_rms_compute_h( + x: Tensor, + weight: Tensor, + alpha_pre: Tensor, + alpha_post: Tensor, + alpha_res: Tensor, + bias: Tensor, + n: int, + eps: float = 1e-6, + compute_h_eps: float = 1e-6, +) -> Tuple[Tensor, Tensor, Tensor, Tensor]: + """Projection + RMS norm + compute_h split outputs using cuTile, then torch.""" + _raise_mhc_backend_validation_error() + if is_cutile_available(): + return CutileProjRmsComputeH.apply( + x, weight, alpha_pre, alpha_post, alpha_res, bias, n, eps, compute_h_eps + ) + return _torch_proj_rms_compute_h( + x, weight, alpha_pre, alpha_post, alpha_res, bias, n, eps, compute_h_eps + ) diff --git a/megatron/core/fusions/fused_mla_yarn_rope_apply.py b/megatron/core/fusions/fused_mla_yarn_rope_apply.py index 6eed7581d03..df81671397b 100644 --- a/megatron/core/fusions/fused_mla_yarn_rope_apply.py +++ b/megatron/core/fusions/fused_mla_yarn_rope_apply.py @@ -29,17 +29,28 @@ @triton.jit def _get_thd_token_idx(cu_seqlens, pid_m, seq_num, cp_rank, cp_size): - token_idx = -1 - this_seq_len = 0 + # Cast ``pid_m`` and ``cu_seqlens`` loads to a single shared dtype so + # the loop-body reassignments don't surface as + # "initial value is int32 but redefined as int64" in newer Triton + # versions (which promote ``// Python_int`` to int64). + pid_m = pid_m.to(tl.int64) + token_idx = tl.full((), -1, dtype=tl.int64) + this_seq_len = tl.full((), 0, dtype=tl.int64) seq_idx = 0 - last_cum_seqlen = tl.load(cu_seqlens) // cp_size + last_cum_seqlen = tl.load(cu_seqlens).to(tl.int64) // cp_size while seq_idx < seq_num: - cur_cum_seqlen = tl.load(cu_seqlens + seq_idx + 1) // cp_size + cur_cum_seqlen = tl.load(cu_seqlens + seq_idx + 1).to(tl.int64) // cp_size if token_idx == -1 and cur_cum_seqlen > pid_m: token_idx = pid_m - last_cum_seqlen this_seq_len = cur_cum_seqlen - last_cum_seqlen last_cum_seqlen = cur_cum_seqlen seq_idx += 1 + # Padding tokens beyond cu_seqlens[-1] (from THD CUDA-graph padding) + # never match any sequence, leaving token_idx == -1. Clamp to 0 so + # the cos/sin table loads stay in-bounds; the wrong RoPE result is + # harmless because padding positions are excluded by loss_mask. + if token_idx == -1: + token_idx = tl.full((), 0, dtype=tl.int64) if cp_size > 1: if token_idx < this_seq_len // 2: token_idx = token_idx + cp_rank * this_seq_len // 2 @@ -422,6 +433,40 @@ def fused_mla_rope_inplace( ) +def fused_mla_rope_out_of_place( + t: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, + nope_dim: int, + emb_dim: int, + cu_seqlens_q: Optional[torch.Tensor] = None, + cp_rank: int = 0, + cp_size: int = 1, + rotary_interleaved: bool = False, + inverse: bool = False, + remove_interleaving: bool = False, +) -> torch.Tensor: + """Apply the fused RoPE kernel without modifying the input tensor. + + Use this wrapper when an upstream autograd function may have retained its + output for backward. The underlying kernel remains in-place, so a private + copy is required to keep the retained tensor unchanged. + """ + return fused_mla_rope_inplace( + t.clone(), + cos, + sin, + nope_dim, + emb_dim, + cu_seqlens_q=cu_seqlens_q, + cp_rank=cp_rank, + cp_size=cp_size, + rotary_interleaved=rotary_interleaved, + inverse=inverse, + remove_interleaving=remove_interleaving, + ) + + @triton.autotune( configs=[ triton.Config({"BLOCK_H": 1}), diff --git a/megatron/core/fusions/fused_pre_gated_delta_rule.py b/megatron/core/fusions/fused_pre_gated_delta_rule.py new file mode 100644 index 00000000000..b2db6946835 --- /dev/null +++ b/megatron/core/fusions/fused_pre_gated_delta_rule.py @@ -0,0 +1,1778 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +"""Fused pre-gated-delta-rule projection kernels. + +The public entry point consumes the dense ``qkvzba`` projection and returns +``query``, ``key``, ``value``, ``gate``, ``beta``, and ``g`` in the layouts +expected by the gated delta rule. The forward path keeps QK, V, Z, and +G/Beta as separate streamed scopes. The backward mirrors those scopes for +layout/l2norm/g-beta work, then delegates depthwise conv gradients to the +``causal_conv1d`` backend. + +Unsupported cases are rejected at the Python entry point: CPU tensors, +conv bias, and ``use_qk_l2norm=False``. Packed THD sequences use separate +QK/V causal-conv kernels so the dense BSHD kernels stay free of packed +metadata and runtime branches. +""" + +from contextlib import nullcontext +from typing import Optional, Tuple + +import torch +import triton +import triton.language as tl +from torch import Tensor + +try: + from causal_conv1d.cpp_functions import causal_conv1d_bwd_function +except ImportError: + causal_conv1d_bwd_function = None + +_L2NORM_EPS = 1e-6 + +_QK_STREAM_SLOT = 0 +_V_STREAM_SLOT = 2 +_G_BETA_STREAM_SLOT = 3 +_Z_STREAM_SLOT = 4 + +_LAYOUT_BLOCK_S = 64 + + +# --------------------------------------------------------------------------- +# Forward kernels +# --------------------------------------------------------------------------- + + +def _conv_autotune_configs(): + return [ + triton.Config({"BLOCK_S": 16}, num_warps=2, num_stages=2), + triton.Config({"BLOCK_S": 32}, num_warps=2, num_stages=2), + triton.Config({"BLOCK_S": 32}, num_warps=4, num_stages=2), + triton.Config({"BLOCK_S": 32}, num_warps=4, num_stages=3), + triton.Config({"BLOCK_S": 64}, num_warps=2, num_stages=2), + triton.Config({"BLOCK_S": 64}, num_warps=4, num_stages=2), + triton.Config({"BLOCK_S": 64}, num_warps=4, num_stages=3), + triton.Config({"BLOCK_S": 64}, num_warps=4, num_stages=4), + triton.Config({"BLOCK_S": 64}, num_warps=8, num_stages=2), + triton.Config({"BLOCK_S": 128}, num_warps=4, num_stages=2), + triton.Config({"BLOCK_S": 128}, num_warps=4, num_stages=3), + triton.Config({"BLOCK_S": 128}, num_warps=8, num_stages=2), + triton.Config({"BLOCK_S": 128}, num_warps=8, num_stages=3), + triton.Config({"BLOCK_S": 256}, num_warps=4, num_stages=2), + triton.Config({"BLOCK_S": 256}, num_warps=8, num_stages=2), + ] + + +def _g_beta_autotune_configs(): + return [ + triton.Config({"BLOCK_S": 32, "BLOCK_H": 16}, num_warps=2, num_stages=2), + triton.Config({"BLOCK_S": 64, "BLOCK_H": 16}, num_warps=2, num_stages=2), + triton.Config({"BLOCK_S": 64, "BLOCK_H": 32}, num_warps=4, num_stages=2), + triton.Config({"BLOCK_S": 128, "BLOCK_H": 16}, num_warps=4, num_stages=2), + triton.Config({"BLOCK_S": 128, "BLOCK_H": 32}, num_warps=4, num_stages=2), + triton.Config({"BLOCK_S": 128, "BLOCK_H": 64}, num_warps=8, num_stages=2), + triton.Config({"BLOCK_S": 256, "BLOCK_H": 16}, num_warps=8, num_stages=2), + triton.Config({"BLOCK_S": 256, "BLOCK_H": 32}, num_warps=8, num_stages=2), + ] + + +@triton.autotune( + configs=_conv_autotune_configs(), + key=["seq_len", "HEAD_DIM", "K_W", "APPLY_L2", "REPEAT", "NUM_GROUPS"], +) +@triton.jit +def _conv_silu_project_kernel( + qkvzba_ptr, + weight_ptr, + bias_ptr, + out_ptr, + silu_save_ptr, + seq_len, + num_in_heads, + in_channel_offset, + in_group_stride, + silu_save_chan_offset, + silu_save_group_stride, + qkvzba_s_stride, + qkvzba_b_stride, + qkvzba_c_stride, + weight_c_stride, + weight_w_stride, + bias_stride, + out_group_dim_stride, + out_b_stride, + out_s_stride, + out_h_stride, + silu_save_b_stride, + silu_save_c_stride, + silu_save_s_stride, + eps, + HEAD_DIM: tl.constexpr, + K_W: tl.constexpr, + REPEAT: tl.constexpr, + NUM_GROUPS: tl.constexpr, + HAS_BIAS: tl.constexpr, + APPLY_L2: tl.constexpr, + SAVE_SILU: tl.constexpr, + BLOCK_S: tl.constexpr, +): + """Depthwise conv1d + silu + (optional l2norm) + (optional head repeat). + + Grid layout (program_id): + 0: batch * NUM_GROUPS * num_in_heads (flat) + 1: num_seq_blocks + + Args: + in_channel_offset: starting channel index of the first group inside + ``qkvzba``. 0 for QK, ``v_channel_offset`` for V. + in_group_stride: channel distance between logical groups. For QK this + is ``qk_channels`` so group 0 is Q and group 1 is K. For V this is + 0 because ``NUM_GROUPS == 1``. + out_group_dim_stride: output-storage distance between logical groups. QK + passes a grouped output buffer and V passes 0. + """ + + pid_bgh = tl.program_id(0) + pid_s = tl.program_id(1) + + heads_per_batch = num_in_heads * NUM_GROUPS + batch_id = pid_bgh // heads_per_batch + local_bgh = pid_bgh - batch_id * heads_per_batch + group_id = local_bgh // num_in_heads + head_id = local_bgh - group_id * num_in_heads + + chan_off = tl.arange(0, HEAD_DIM) + group_channel_offset = in_channel_offset + group_id * in_group_stride + chan = group_channel_offset + head_id * HEAD_DIM + chan_off + + if HAS_BIAS: + bias = tl.load(bias_ptr + chan * bias_stride).to(tl.float32) + else: + bias = tl.zeros([HEAD_DIM], dtype=tl.float32) + + s_offs = pid_s * BLOCK_S + tl.arange(0, BLOCK_S) + s_mask = s_offs < seq_len + + acc = tl.zeros([BLOCK_S, HEAD_DIM], dtype=tl.float32) + for i in tl.static_range(K_W): + x_s = s_offs - (K_W - 1) + i + x_mask = (x_s >= 0) & (x_s < seq_len) + x_ptr = ( + qkvzba_ptr + + x_s[:, None] * qkvzba_s_stride + + batch_id * qkvzba_b_stride + + chan[None, :] * qkvzba_c_stride + ) + x_val = tl.load(x_ptr, mask=x_mask[:, None], other=0.0).to(tl.float32) + w_tap = tl.load(weight_ptr + chan * weight_c_stride + i * weight_w_stride).to(tl.float32) + acc += w_tap[None, :] * x_val + + acc += bias[None, :] + # Mimic the unfused F.conv1d rounding: the reference path stores the conv + # output in the input dtype (bf16) before silu, so do the same here. This + # keeps the fused output bit-aligned with the reference within one ULP. + acc = acc.to(out_ptr.dtype.element_ty).to(tl.float32) + silu_out = acc * tl.sigmoid(acc) + + if APPLY_L2: + # F.silu rounds to the input dtype before l2norm reads it. Round-trip + # via bf16 to match that precision. + silu_out = silu_out.to(out_ptr.dtype.element_ty).to(tl.float32) + if SAVE_SILU: + # Persist only the QK silu output in the channel-last layout + # consumed by the QK l2norm backward. + silu_save_chan = ( + silu_save_chan_offset + + group_id * silu_save_group_stride + + head_id * HEAD_DIM + + chan_off + ) + silu_save_ptrs = ( + silu_save_ptr + + batch_id * silu_save_b_stride + + silu_save_chan[None, :] * silu_save_c_stride + + s_offs[:, None] * silu_save_s_stride + ) + tl.store( + silu_save_ptrs, silu_out.to(silu_save_ptr.dtype.element_ty), mask=s_mask[:, None] + ) + norm_sq = tl.sum(silu_out * silu_out, axis=1) + rstd = 1.0 / tl.sqrt(norm_sq + eps) + out = silu_out * rstd[:, None] + else: + # No l2norm follows. The final store→bf16 already does the rounding; + # an intermediate bf16 round-trip would be redundant. + out = silu_out + + out_typed = out.to(out_ptr.dtype.element_ty) + + # Write the same data to ``REPEAT`` adjacent value heads. ``REPEAT == 1`` + # is the no-repeat case (V branch is handled by a separate kernel that + # always has REPEAT == 1, but using the same code here is convenient). + for r in tl.static_range(REPEAT): + v_head = head_id * REPEAT + r + write_ptr = ( + out_ptr + + group_id * out_group_dim_stride + + batch_id * out_b_stride + + s_offs[:, None] * out_s_stride + + v_head * out_h_stride + + chan_off[None, :] + ) + tl.store(write_ptr, out_typed, mask=s_mask[:, None]) + + +@triton.jit +def _thd_seq_bounds(cu_seqlens_ptr, token_offsets, total_tokens, num_packed_seqs): + """Return lane-wise packed sequence bounds for flattened THD tokens.""" + + safe_tokens = tl.minimum(token_offsets, total_tokens - 1) + seq_start = token_offsets * 0 + seq_end = token_offsets * 0 + total_tokens + + seq_id = 0 + while seq_id < num_packed_seqs: + start = tl.load(cu_seqlens_ptr + seq_id) + end = tl.load(cu_seqlens_ptr + seq_id + 1) + in_seq = (safe_tokens >= start) & (safe_tokens < end) + seq_start = tl.where(in_seq, start, seq_start) + seq_end = tl.where(in_seq, end, seq_end) + seq_id += 1 + + return seq_start, seq_end + + +@triton.autotune( + configs=_conv_autotune_configs(), + key=["seq_len", "HEAD_DIM", "K_W", "APPLY_L2", "REPEAT", "NUM_GROUPS"], +) +@triton.jit +def _conv_silu_project_thd_kernel( + qkvzba_ptr, + weight_ptr, + bias_ptr, + out_ptr, + silu_save_ptr, + cu_seqlens_ptr, + seq_len, + num_packed_seqs, + num_in_heads, + in_channel_offset, + in_group_stride, + silu_save_chan_offset, + silu_save_group_stride, + qkvzba_s_stride, + qkvzba_b_stride, + qkvzba_c_stride, + weight_c_stride, + weight_w_stride, + bias_stride, + out_group_dim_stride, + out_b_stride, + out_s_stride, + out_h_stride, + silu_save_b_stride, + silu_save_c_stride, + silu_save_s_stride, + eps, + HEAD_DIM: tl.constexpr, + K_W: tl.constexpr, + REPEAT: tl.constexpr, + NUM_GROUPS: tl.constexpr, + HAS_BIAS: tl.constexpr, + APPLY_L2: tl.constexpr, + SAVE_SILU: tl.constexpr, + BLOCK_S: tl.constexpr, +): + """THD depthwise conv1d + silu + optional l2norm/repeat. + + This is intentionally separate from ``_conv_silu_project_kernel`` so + packed sequence boundary metadata never enters the dense BSHD hot path. + Only the causal-conv loads use ``cu_seqlens``; the following per-token + transforms and stores are identical to the dense path. + """ + + pid_bgh = tl.program_id(0) + pid_s = tl.program_id(1) + + heads_per_batch = num_in_heads * NUM_GROUPS + batch_id = pid_bgh // heads_per_batch + local_bgh = pid_bgh - batch_id * heads_per_batch + group_id = local_bgh // num_in_heads + head_id = local_bgh - group_id * num_in_heads + + chan_off = tl.arange(0, HEAD_DIM) + group_channel_offset = in_channel_offset + group_id * in_group_stride + chan = group_channel_offset + head_id * HEAD_DIM + chan_off + + if HAS_BIAS: + bias = tl.load(bias_ptr + chan * bias_stride).to(tl.float32) + else: + bias = tl.zeros([HEAD_DIM], dtype=tl.float32) + + s_offs = pid_s * BLOCK_S + tl.arange(0, BLOCK_S) + s_mask = s_offs < seq_len + seq_start, seq_end = _thd_seq_bounds(cu_seqlens_ptr, s_offs, seq_len, num_packed_seqs) + + acc = tl.zeros([BLOCK_S, HEAD_DIM], dtype=tl.float32) + for i in tl.static_range(K_W): + x_s = s_offs - (K_W - 1) + i + x_mask = s_mask & (x_s >= seq_start) & (x_s < seq_end) + safe_x_s = tl.minimum(tl.maximum(x_s, 0), seq_len - 1) + x_ptr = ( + qkvzba_ptr + + safe_x_s[:, None] * qkvzba_s_stride + + batch_id * qkvzba_b_stride + + chan[None, :] * qkvzba_c_stride + ) + x_val = tl.load(x_ptr, mask=x_mask[:, None], other=0.0).to(tl.float32) + w_tap = tl.load(weight_ptr + chan * weight_c_stride + i * weight_w_stride).to(tl.float32) + acc += w_tap[None, :] * x_val + + acc += bias[None, :] + acc = acc.to(out_ptr.dtype.element_ty).to(tl.float32) + silu_out = acc * tl.sigmoid(acc) + + if APPLY_L2: + silu_out = silu_out.to(out_ptr.dtype.element_ty).to(tl.float32) + if SAVE_SILU: + silu_save_chan = ( + silu_save_chan_offset + + group_id * silu_save_group_stride + + head_id * HEAD_DIM + + chan_off + ) + silu_save_ptrs = ( + silu_save_ptr + + batch_id * silu_save_b_stride + + silu_save_chan[None, :] * silu_save_c_stride + + s_offs[:, None] * silu_save_s_stride + ) + tl.store( + silu_save_ptrs, silu_out.to(silu_save_ptr.dtype.element_ty), mask=s_mask[:, None] + ) + norm_sq = tl.sum(silu_out * silu_out, axis=1) + rstd = 1.0 / tl.sqrt(norm_sq + eps) + out = silu_out * rstd[:, None] + else: + out = silu_out + + out_typed = out.to(out_ptr.dtype.element_ty) + + for r in tl.static_range(REPEAT): + v_head = head_id * REPEAT + r + write_ptr = ( + out_ptr + + group_id * out_group_dim_stride + + batch_id * out_b_stride + + s_offs[:, None] * out_s_stride + + v_head * out_h_stride + + chan_off[None, :] + ) + tl.store(write_ptr, out_typed, mask=s_mask[:, None]) + + +@triton.jit +def _copy_z_kernel( + qkvzba_ptr, + gate_ptr, + seq_len, + num_v_heads, + z_channel_offset, + qkvzba_s_stride, + qkvzba_b_stride, + qkvzba_c_stride, + gate_b_stride, + gate_s_stride, + gate_h_stride, + HEAD_DIM: tl.constexpr, + BLOCK_S: tl.constexpr, +): + """Copy the z slice from qkvzba into the final gate layout.""" + + pid_bh = tl.program_id(0) + pid_s = tl.program_id(1) + + batch_id = pid_bh // num_v_heads + head_id = pid_bh - batch_id * num_v_heads + + chan_off = tl.arange(0, HEAD_DIM) + s_offs = pid_s * BLOCK_S + tl.arange(0, BLOCK_S) + s_mask = s_offs < seq_len + + z_chan = z_channel_offset + head_id * HEAD_DIM + chan_off + z_src_ptr = ( + qkvzba_ptr + + s_offs[:, None] * qkvzba_s_stride + + batch_id * qkvzba_b_stride + + z_chan[None, :] * qkvzba_c_stride + ) + z_val = tl.load(z_src_ptr, mask=s_mask[:, None]) + z_write_ptr = ( + gate_ptr + + batch_id * gate_b_stride + + s_offs[:, None] * gate_s_stride + + head_id * gate_h_stride + + chan_off[None, :] + ) + tl.store(z_write_ptr, z_val, mask=s_mask[:, None]) + + +@triton.autotune(configs=_g_beta_autotune_configs(), key=["seq_len", "num_v_heads"]) +@triton.jit +def _compute_g_and_beta_kernel( + qkvzba_ptr, + A_log_ptr, + dt_bias_ptr, + g_out_ptr, + beta_out_ptr, + seq_len, + num_v_heads, + beta_channel_offset, + alpha_channel_offset, + qkvzba_s_stride, + qkvzba_b_stride, + qkvzba_c_stride, + g_b_stride, + g_s_stride, + g_h_stride, + beta_b_stride, + beta_s_stride, + beta_h_stride, + BLOCK_S: tl.constexpr, + BLOCK_H: tl.constexpr, +): + """Compute ``g = -exp(A_log) * softplus(alpha + dt_bias)`` and ``sigmoid(beta)``.""" + + pid_b = tl.program_id(0) + pid_s = tl.program_id(1) + pid_h = tl.program_id(2) + + s_offs = pid_s * BLOCK_S + tl.arange(0, BLOCK_S) + h_offs = pid_h * BLOCK_H + tl.arange(0, BLOCK_H) + s_mask = s_offs < seq_len + h_mask = h_offs < num_v_heads + mask = s_mask[:, None] & h_mask[None, :] + + alpha_ptr = ( + qkvzba_ptr + + s_offs[:, None] * qkvzba_s_stride + + pid_b * qkvzba_b_stride + + (alpha_channel_offset + h_offs[None, :]) * qkvzba_c_stride + ) + beta_ptr = ( + qkvzba_ptr + + s_offs[:, None] * qkvzba_s_stride + + pid_b * qkvzba_b_stride + + (beta_channel_offset + h_offs[None, :]) * qkvzba_c_stride + ) + + alpha = tl.load(alpha_ptr, mask=mask, other=0.0).to(tl.float32) + beta = tl.load(beta_ptr, mask=mask, other=0.0).to(tl.float32) + + A_log = tl.load(A_log_ptr + h_offs, mask=h_mask, other=0.0).to(tl.float32) + dt_bias = tl.load(dt_bias_ptr + h_offs, mask=h_mask, other=0.0).to(tl.float32) + + pre = alpha + dt_bias[None, :] + # softplus(x) = log(1 + exp(x)); torch's softplus thresholds at x>20 but we + # rely on fp32 evaluation here, which stays well within range for typical + # GDN inputs (the unfused path computes the same expression). + softplus_val = tl.log(1.0 + tl.exp(pre)) + g = -tl.exp(A_log)[None, :] * softplus_val + beta_sig = tl.sigmoid(beta) + + g_ptr = ( + g_out_ptr + pid_b * g_b_stride + s_offs[:, None] * g_s_stride + h_offs[None, :] * g_h_stride + ) + beta_out_ptr_calc = ( + beta_out_ptr + + pid_b * beta_b_stride + + s_offs[:, None] * beta_s_stride + + h_offs[None, :] * beta_h_stride + ) + tl.store(g_ptr, g.to(g_out_ptr.dtype.element_ty), mask=mask) + tl.store(beta_out_ptr_calc, beta_sig.to(beta_out_ptr.dtype.element_ty), mask=mask) + + +# --------------------------------------------------------------------------- +# Backward kernels +# --------------------------------------------------------------------------- + + +@triton.autotune( + configs=[ + triton.Config({"BLOCK_S": 32}, num_warps=2, num_stages=2), + triton.Config({"BLOCK_S": 32}, num_warps=4, num_stages=2), + triton.Config({"BLOCK_S": 64}, num_warps=4, num_stages=2), + triton.Config({"BLOCK_S": 64}, num_warps=4, num_stages=3), + triton.Config({"BLOCK_S": 64}, num_warps=8, num_stages=2), + triton.Config({"BLOCK_S": 128}, num_warps=4, num_stages=2), + triton.Config({"BLOCK_S": 128}, num_warps=8, num_stages=2), + triton.Config({"BLOCK_S": 128}, num_warps=8, num_stages=3), + triton.Config({"BLOCK_S": 256}, num_warps=8, num_stages=2), + ], + key=["seq_len", "HEAD_DIM", "REPEAT"], +) +@triton.jit +def _qk_l2norm_repeat_backward_kernel( + dq_ptr, + dk_ptr, + silu_bf16_ptr, + d_silu_bf16_ptr, + seq_len, + num_qk_heads, + qk_channels, + eps, + dq_b_stride, + dq_s_stride, + dq_h_stride, + dk_b_stride, + dk_s_stride, + dk_h_stride, + silu_b_stride, + silu_c_stride, + silu_s_stride, + d_silu_b_stride, + d_silu_c_stride, + d_silu_s_stride, + HEAD_DIM: tl.constexpr, + REPEAT: tl.constexpr, + BLOCK_S: tl.constexpr, +): + """Merged Q/K l2norm + REPEAT-way head broadcast backward.""" + + pid_bgh = tl.program_id(0) + pid_s = tl.program_id(1) + + heads_per_batch = num_qk_heads * 2 + batch_id = pid_bgh // heads_per_batch + local_bgh = pid_bgh - batch_id * heads_per_batch + group_id = local_bgh // num_qk_heads + head_id = local_bgh - group_id * num_qk_heads + is_query = group_id == 0 + is_key = group_id == 1 + + chan_off = tl.arange(0, HEAD_DIM) + chan = group_id * qk_channels + head_id * HEAD_DIM + chan_off + + s_offs = pid_s * BLOCK_S + tl.arange(0, BLOCK_S) + s_mask = s_offs < seq_len + + d_normed = tl.zeros([BLOCK_S, HEAD_DIM], dtype=tl.float32) + for r in tl.static_range(REPEAT): + v_head = head_id * REPEAT + r + dq_ptrs = ( + dq_ptr + + batch_id * dq_b_stride + + s_offs[:, None] * dq_s_stride + + v_head * dq_h_stride + + chan_off[None, :] + ) + dk_ptrs = ( + dk_ptr + + batch_id * dk_b_stride + + s_offs[:, None] * dk_s_stride + + v_head * dk_h_stride + + chan_off[None, :] + ) + d_normed += tl.load(dq_ptrs, mask=s_mask[:, None] & is_query, other=0.0).to(tl.float32) + d_normed += tl.load(dk_ptrs, mask=s_mask[:, None] & is_key, other=0.0).to(tl.float32) + + silu_ptrs = ( + silu_bf16_ptr + + batch_id * silu_b_stride + + chan[None, :] * silu_c_stride + + s_offs[:, None] * silu_s_stride + ) + silu_bf16 = tl.load(silu_ptrs, mask=s_mask[:, None], other=0.0).to(tl.float32) + + norm_sq = tl.sum(silu_bf16 * silu_bf16, axis=1) + rstd = 1.0 / tl.sqrt(norm_sq + eps) + s_row = tl.sum(d_normed * silu_bf16, axis=1) + rstd3 = rstd * rstd * rstd + d_silu = rstd[:, None] * d_normed - rstd3[:, None] * silu_bf16 * s_row[:, None] + + d_silu_ptrs = ( + d_silu_bf16_ptr + + batch_id * d_silu_b_stride + + chan[None, :] * d_silu_c_stride + + s_offs[:, None] * d_silu_s_stride + ) + tl.store(d_silu_ptrs, d_silu.to(d_silu_bf16_ptr.dtype.element_ty), mask=s_mask[:, None]) + + +@triton.jit +def _v_layout_to_conv_kernel( + dv_ptr, # (b, s, num_v_heads, value_head_dim) + d_silu_conv_ptr, # (b, conv_dim, s) — write into V channel slice + seq_len, + num_v_heads, + v_channel_offset, # = 2 * qk_channels + dv_b_stride, + dv_s_stride, + dv_h_stride, + d_silu_b_stride, + d_silu_c_stride, + d_silu_s_stride, + HEAD_DIM: tl.constexpr, + BLOCK_S: tl.constexpr, +): + """Write V-branch gradients into the conv-backward layout. + + ``dv`` is the gradient of ``value`` (forward layout + ``(b, s, num_v_heads, value_head_dim)``). The conv backward needs + ``d_silu_conv`` in layout ``(b, conv_dim, s)`` for the V channel + slice. + """ + + pid_bh = tl.program_id(0) + pid_s = tl.program_id(1) + + batch_id = pid_bh // num_v_heads + head_id = pid_bh - batch_id * num_v_heads + + chan_off = tl.arange(0, HEAD_DIM) + s_offs = pid_s * BLOCK_S + tl.arange(0, BLOCK_S) + s_mask = s_offs < seq_len + + # Read dv at (batch, s, head, chan). + dv_ptrs = ( + dv_ptr + + batch_id * dv_b_stride + + s_offs[:, None] * dv_s_stride + + head_id * dv_h_stride + + chan_off[None, :] + ) + dv_val = tl.load(dv_ptrs, mask=s_mask[:, None], other=0.0) + + # Write to d_silu_conv at (batch, v_channel_offset + head*HEAD_DIM + chan, s). + d_silu_chan = v_channel_offset + head_id * HEAD_DIM + chan_off + d_silu_ptrs = ( + d_silu_conv_ptr + + batch_id * d_silu_b_stride + + d_silu_chan[None, :] * d_silu_c_stride + + s_offs[:, None] * d_silu_s_stride + ) + tl.store(d_silu_ptrs, dv_val, mask=s_mask[:, None]) + + +@triton.jit +def _z_layout_to_qkvzba_kernel( + dgate_ptr, # (b, s, num_v_heads, value_head_dim) + d_qkvzba_ptr, # (s, b, total_channels) — write into z channel slice + seq_len, + num_v_heads, + z_channel_offset, # = 2 * qk_channels + v_channels + dgate_b_stride, + dgate_s_stride, + dgate_h_stride, + d_qkvzba_s_stride, + d_qkvzba_b_stride, + d_qkvzba_c_stride, + HEAD_DIM: tl.constexpr, + BLOCK_S: tl.constexpr, +): + """Write gate gradients into the z slice of ``d_qkvzba``. + + ``dgate`` is the autograd-supplied gradient of ``gate`` (= the z + slice of qkvzba in forward) with layout + ``(b, s, num_v_heads, value_head_dim)``. We need to write it into + ``d_qkvzba``'s z slice — layout ``(s, b, total_channels)`` with + channels in ``[z_channel_offset, z_channel_offset + v_channels)``. + """ + + pid_bh = tl.program_id(0) + pid_s = tl.program_id(1) + + batch_id = pid_bh // num_v_heads + head_id = pid_bh - batch_id * num_v_heads + + chan_off = tl.arange(0, HEAD_DIM) + s_offs = pid_s * BLOCK_S + tl.arange(0, BLOCK_S) + s_mask = s_offs < seq_len + + # Read dgate at (batch, s, head, chan). + dgate_ptrs = ( + dgate_ptr + + batch_id * dgate_b_stride + + s_offs[:, None] * dgate_s_stride + + head_id * dgate_h_stride + + chan_off[None, :] + ) + dgate_val = tl.load(dgate_ptrs, mask=s_mask[:, None], other=0.0) + + # Write to d_qkvzba at (s, batch, z_channel_offset + head*HEAD_DIM + chan). + d_qkvzba_chan = z_channel_offset + head_id * HEAD_DIM + chan_off + d_qkvzba_ptrs = ( + d_qkvzba_ptr + + s_offs[:, None] * d_qkvzba_s_stride + + batch_id * d_qkvzba_b_stride + + d_qkvzba_chan[None, :] * d_qkvzba_c_stride + ) + tl.store(d_qkvzba_ptrs, dgate_val, mask=s_mask[:, None]) + + +@triton.autotune( + configs=_g_beta_autotune_configs(), + key=["seq_len", "num_v_heads"], + # Each autotune trial atomic-adds partial sums into these accumulators. + # Without reset_to_zero the trials would stack on top of one another and + # produce values that are ``num_trials`` × the correct result. + reset_to_zero=["d_A_log_ptr", "d_dt_bias_ptr"], +) +@triton.jit +def _g_beta_backward_kernel( + qkvzba_ptr, + A_log_ptr, + dt_bias_ptr, + d_g_ptr, + d_beta_out_ptr, + d_qkvzba_ptr, + d_A_log_ptr, + d_dt_bias_ptr, + seq_len, + num_v_heads, + beta_channel_offset, + alpha_channel_offset, + qkvzba_s_stride, + qkvzba_b_stride, + qkvzba_c_stride, + d_g_b_stride, + d_g_s_stride, + d_g_h_stride, + d_beta_b_stride, + d_beta_s_stride, + d_beta_h_stride, + BLOCK_S: tl.constexpr, + BLOCK_H: tl.constexpr, +): + """Backward for ``_compute_g_and_beta_kernel``. + + Forward: + pre = alpha + dt_bias # fp32 + softplus_pre = log(1 + exp(pre)) + g = -exp(A_log) * softplus_pre + beta_sig = sigmoid(beta_raw) + + Backward (given d_g and d_beta_out): + d_alpha = d_g * (-exp(A_log) * sigmoid(pre)) + d_beta_raw = d_beta_out * beta_sig * (1 - beta_sig) + d_dt_bias[h] = Σ_{b,s} d_alpha[b,s,h] + d_A_log[h] = Σ_{b,s} d_g[b,s,h] * g[b,s,h] + + ``d_alpha`` and ``d_beta_raw`` are written into the matching channel slices + of ``d_qkvzba``. ``d_A_log`` and ``d_dt_bias`` are reduced via per-element + atomic_add to fp32 buffers; the caller casts those to the parameter dtype. + """ + + pid_b = tl.program_id(0) + pid_s = tl.program_id(1) + pid_h = tl.program_id(2) + + s_offs = pid_s * BLOCK_S + tl.arange(0, BLOCK_S) + h_offs = pid_h * BLOCK_H + tl.arange(0, BLOCK_H) + s_mask = s_offs < seq_len + h_mask = h_offs < num_v_heads + mask = s_mask[:, None] & h_mask[None, :] + + # ----- Forward recompute ----- + alpha_ptr = ( + qkvzba_ptr + + s_offs[:, None] * qkvzba_s_stride + + pid_b * qkvzba_b_stride + + (alpha_channel_offset + h_offs[None, :]) * qkvzba_c_stride + ) + beta_ptr = ( + qkvzba_ptr + + s_offs[:, None] * qkvzba_s_stride + + pid_b * qkvzba_b_stride + + (beta_channel_offset + h_offs[None, :]) * qkvzba_c_stride + ) + alpha = tl.load(alpha_ptr, mask=mask, other=0.0).to(tl.float32) + beta_raw = tl.load(beta_ptr, mask=mask, other=0.0).to(tl.float32) + A_log = tl.load(A_log_ptr + h_offs, mask=h_mask, other=0.0).to(tl.float32) + dt_bias = tl.load(dt_bias_ptr + h_offs, mask=h_mask, other=0.0).to(tl.float32) + + pre = alpha + dt_bias[None, :] + sigmoid_pre = tl.sigmoid(pre) + softplus_pre = tl.log(1.0 + tl.exp(pre)) + exp_A = tl.exp(A_log)[None, :] + g = -exp_A * softplus_pre + beta_sig = tl.sigmoid(beta_raw) + + # ----- Load upstream gradients ----- + d_g_ptrs = ( + d_g_ptr + + pid_b * d_g_b_stride + + s_offs[:, None] * d_g_s_stride + + h_offs[None, :] * d_g_h_stride + ) + d_beta_out_ptrs = ( + d_beta_out_ptr + + pid_b * d_beta_b_stride + + s_offs[:, None] * d_beta_s_stride + + h_offs[None, :] * d_beta_h_stride + ) + d_g = tl.load(d_g_ptrs, mask=mask, other=0.0).to(tl.float32) + d_beta_out = tl.load(d_beta_out_ptrs, mask=mask, other=0.0).to(tl.float32) + + # ----- Per-element gradients ----- + d_alpha = d_g * (-exp_A * sigmoid_pre) + d_beta_raw = d_beta_out * beta_sig * (1.0 - beta_sig) + + # ----- (b, s) → h reductions ----- + d_g_masked = tl.where(mask, d_g, 0.0) + d_alpha_masked = tl.where(mask, d_alpha, 0.0) + d_A_log_partial = tl.sum(d_g_masked * g, axis=0) + d_dt_bias_partial = tl.sum(d_alpha_masked, axis=0) + + # ----- Store per-element grads back to d_qkvzba ----- + d_alpha_ptrs = ( + d_qkvzba_ptr + + s_offs[:, None] * qkvzba_s_stride + + pid_b * qkvzba_b_stride + + (alpha_channel_offset + h_offs[None, :]) * qkvzba_c_stride + ) + d_beta_ptrs = ( + d_qkvzba_ptr + + s_offs[:, None] * qkvzba_s_stride + + pid_b * qkvzba_b_stride + + (beta_channel_offset + h_offs[None, :]) * qkvzba_c_stride + ) + tl.store(d_alpha_ptrs, d_alpha.to(d_qkvzba_ptr.dtype.element_ty), mask=mask) + tl.store(d_beta_ptrs, d_beta_raw.to(d_qkvzba_ptr.dtype.element_ty), mask=mask) + + # ----- Atomic-add (b, s) partials into per-head accumulators ----- + tl.atomic_add(d_A_log_ptr + h_offs, d_A_log_partial, mask=h_mask) + tl.atomic_add(d_dt_bias_ptr + h_offs, d_dt_bias_partial, mask=h_mask) + + +# --------------------------------------------------------------------------- +# Python entry points +# --------------------------------------------------------------------------- + + +def _is_power_of_two(value: int) -> bool: + return value > 0 and (value & (value - 1)) == 0 + + +_SIDE_STREAMS: dict = {} + + +def _get_side_stream(device: torch.device, slot: int) -> "torch.cuda.Stream": + """Lazily allocate and cache CUDA streams keyed by ``(device, slot)``. + + Reusing streams across calls keeps launches free of stream-creation + overhead, which would otherwise dominate the small kernels. + """ + + key = (device.index if device.index is not None else torch.cuda.current_device(), slot) + stream = _SIDE_STREAMS.get(key) + if stream is None: + stream = torch.cuda.Stream(device=device) + _SIDE_STREAMS[key] = stream + return stream + + +def _triton_qk_l2norm_repeat_backward( + dq: Tensor, + dk: Tensor, + silu_bf16: Tensor, + d_silu_bf16: Tensor, + *, + num_key_heads: int, + num_value_heads: int, + key_head_dim: int, + eps: float = 1e-6, + stream: Optional["torch.cuda.Stream"] = None, +) -> Tensor: + """Merged Q/K l2norm + REPEAT backward launch.""" + + batch = dq.shape[0] + seq_len = dq.shape[1] + qk_channels = num_key_heads * key_head_dim + repeat = num_value_heads // num_key_heads + device = dq.device + + grid = lambda meta: (batch * 2 * num_key_heads, triton.cdiv(seq_len, meta["BLOCK_S"])) + + with _launch_context(device, stream): + _qk_l2norm_repeat_backward_kernel[grid]( + dq, + dk, + silu_bf16, + d_silu_bf16, + seq_len, + num_key_heads, + qk_channels, + eps, + dq.stride(0), + dq.stride(1), + dq.stride(2), + dk.stride(0), + dk.stride(1), + dk.stride(2), + silu_bf16.stride(0), + silu_bf16.stride(1), + silu_bf16.stride(2), + d_silu_bf16.stride(0), + d_silu_bf16.stride(1), + d_silu_bf16.stride(2), + HEAD_DIM=key_head_dim, + REPEAT=repeat, + ) + + return d_silu_bf16 + + +def _triton_v_layout_to_conv( + dv: Tensor, + d_silu_conv: Tensor, + *, + v_channel_offset: int, + num_value_heads: int, + value_head_dim: int, + stream: Optional["torch.cuda.Stream"] = None, +) -> None: + """Write ``dv`` into ``d_silu_conv``'s V channel slice.""" + + batch, seq_len, _, _ = dv.shape + device = dv.device + + BLOCK_S = _LAYOUT_BLOCK_S + num_seq_blocks = triton.cdiv(seq_len, BLOCK_S) + grid = (batch * num_value_heads, num_seq_blocks) + + with _launch_context(device, stream): + _v_layout_to_conv_kernel[grid]( + dv, + d_silu_conv, + seq_len, + num_value_heads, + v_channel_offset, + dv.stride(0), + dv.stride(1), + dv.stride(2), + d_silu_conv.stride(0), + d_silu_conv.stride(1), + d_silu_conv.stride(2), + HEAD_DIM=value_head_dim, + BLOCK_S=BLOCK_S, + num_warps=4, + num_stages=2, + ) + + +def _triton_z_layout_to_qkvzba( + dgate: Tensor, + d_qkvzba: Tensor, + *, + z_channel_offset: int, + num_value_heads: int, + value_head_dim: int, + stream: Optional["torch.cuda.Stream"] = None, +) -> None: + """Write ``dgate`` into ``d_qkvzba``'s z channel slice.""" + + batch, seq_len, _, _ = dgate.shape + device = dgate.device + + BLOCK_S = _LAYOUT_BLOCK_S + num_seq_blocks = triton.cdiv(seq_len, BLOCK_S) + grid = (batch * num_value_heads, num_seq_blocks) + + with _launch_context(device, stream): + _z_layout_to_qkvzba_kernel[grid]( + dgate, + d_qkvzba, + seq_len, + num_value_heads, + z_channel_offset, + dgate.stride(0), + dgate.stride(1), + dgate.stride(2), + d_qkvzba.stride(0), + d_qkvzba.stride(1), + d_qkvzba.stride(2), + HEAD_DIM=value_head_dim, + BLOCK_S=BLOCK_S, + num_warps=4, + num_stages=2, + ) + + +def _triton_g_beta_backward( + qkvzba: Tensor, + A_log: Tensor, + dt_bias: Tensor, + d_g: Tensor, + d_beta_out: Tensor, + *, + num_value_heads: int, + key_head_dim: int, + value_head_dim: int, + num_key_heads: int, + d_qkvzba_out: Optional[Tensor] = None, + stream: Optional["torch.cuda.Stream"] = None, +) -> Tuple[Tensor, Tensor, Tensor]: + """Launch ``_g_beta_backward_kernel`` and return its outputs. + + Returns: + ``(d_qkvzba_out, d_A_log, d_dt_bias)``. ``d_qkvzba_out`` only has its + alpha and beta slices filled in; the caller is expected to allocate + the buffer while the other backward kernels fill the rest. + ``d_A_log`` and ``d_dt_bias`` are fp32 and need to be cast back to + the parameter dtype by the caller. + """ + + seq_len, batch, total_channels = qkvzba.shape + qk_channels = num_key_heads * key_head_dim + v_channels = num_value_heads * value_head_dim + beta_channel_offset = 2 * qk_channels + 2 * v_channels + alpha_channel_offset = beta_channel_offset + num_value_heads + + if d_qkvzba_out is None: + d_qkvzba_out = torch.zeros_like(qkvzba) + + device = qkvzba.device + + g_beta_grid = lambda meta: ( + batch, + triton.cdiv(seq_len, meta["BLOCK_S"]), + triton.cdiv(num_value_heads, meta["BLOCK_H"]), + ) + with _launch_context(device, stream): + d_param_grads = torch.empty((2, num_value_heads), dtype=torch.float32, device=device) + d_param_grads.zero_() + d_A_log = d_param_grads[0] + d_dt_bias = d_param_grads[1] + _g_beta_backward_kernel[g_beta_grid]( + qkvzba, + A_log, + dt_bias, + d_g, + d_beta_out, + d_qkvzba_out, + d_A_log, + d_dt_bias, + seq_len, + num_value_heads, + beta_channel_offset, + alpha_channel_offset, + qkvzba.stride(0), + qkvzba.stride(1), + qkvzba.stride(2), + d_g.stride(0), + d_g.stride(1), + d_g.stride(2), + d_beta_out.stride(0), + d_beta_out.stride(1), + d_beta_out.stride(2), + ) + return d_qkvzba_out, d_A_log, d_dt_bias + + +def _launch_context(device: torch.device, stream: Optional["torch.cuda.Stream"]): + """Return a CUDA launch context after wiring the optional side stream.""" + + if stream is None: + return nullcontext() + stream.wait_stream(torch.cuda.current_stream(device)) + return torch.cuda.stream(stream) + + +def _wait_for_streams(dst_stream: "torch.cuda.Stream", *src_streams: "torch.cuda.Stream") -> None: + for stream in src_streams: + dst_stream.wait_stream(stream) + + +def _resolve_packed_seq_idx( + cu_seqlens: Optional[Tensor], seq_idx: Optional[Tensor], total_tokens: int +) -> Optional[Tensor]: + """Return the token-level sequence-id buffer for causal-conv backward.""" + + if cu_seqlens is None: + assert seq_idx is None, "seq_idx requires cu_seqlens for packed THD mode." + return None + + if seq_idx is None: + seq_lengths = cu_seqlens[1:] - cu_seqlens[:-1] + seq_idx = torch.repeat_interleave( + torch.arange(seq_lengths.numel(), device=cu_seqlens.device, dtype=torch.int32), + seq_lengths, + ) + seq_idx = seq_idx.unsqueeze(0) + elif seq_idx.dim() == 1: + seq_idx = seq_idx.unsqueeze(0) + + assert seq_idx.is_cuda, f"Packed seq_idx must be CUDA, got {seq_idx.device}." + assert seq_idx.dtype == torch.int32, f"Packed seq_idx must be int32, got {seq_idx.dtype}." + assert seq_idx.shape == (1, total_tokens), ( + "Packed seq_idx must have shape [1, total_tokens], " + f"got {seq_idx.shape=} and {total_tokens=}." + ) + return seq_idx.contiguous() + + +def _triton_pre_gated_delta_rule_forward( + qkvzba: Tensor, + conv1d_weight: Tensor, + A_log: Tensor, + dt_bias: Tensor, + *, + num_key_heads: int, + num_value_heads: int, + key_head_dim: int, + value_head_dim: int, + cu_seqlens: Optional[Tensor] = None, +) -> Tuple[Tensor, Tensor, Tensor, Tensor, Tensor, Tensor, Tensor]: + """Triton-backed forward for the pre-gated-delta-rule front-end. + + Returns ``(query, key, value, gate, beta, g, silu_qk_save)``. The last + element is the bf16-rounded ``silu(conv(x))`` for the QK channel range + laid out channel-last so the backward can feed it straight into + ``causal_conv1d_bwd_function`` — see module docstring. + """ + + seq_len, batch, total_channels = qkvzba.shape + is_packed_thd = cu_seqlens is not None + if is_packed_thd: + assert batch == 1, ( + "Packed THD fused_pre_gated_delta_rule expects batch dimension 1; " f"got {batch=}." + ) + num_packed_seqs = cu_seqlens.shape[0] - 1 + else: + num_packed_seqs = 0 + + qk_channels = num_key_heads * key_head_dim + v_channels = num_value_heads * value_head_dim + repeat_factor = num_value_heads // num_key_heads + k_w = conv1d_weight.shape[-1] + assert _is_power_of_two(key_head_dim), ( + "Triton kernel currently expects key_head_dim to be a power of two; " + f"got {key_head_dim=}." + ) + assert _is_power_of_two(value_head_dim), ( + "Triton kernel currently expects value_head_dim to be a power of two; " + f"got {value_head_dim=}." + ) + + expected_channels = 2 * qk_channels + 2 * v_channels + 2 * num_value_heads + assert ( + total_channels == expected_channels + ), f"qkvzba last-dim mismatch: got {total_channels}, expected {expected_channels}." + + out_dtype = qkvzba.dtype + device = qkvzba.device + + # Output buffers: contiguous (b, s, h, d) for q/k/v and (b, s, h) for g/beta. + # Q and K share one allocation so the fused-streamed QK kernel can select + # the logical group by pointer stride instead of branching between two + # unrelated base pointers inside Triton. + qk_out = torch.empty( + 2, batch, seq_len, num_value_heads, key_head_dim, dtype=out_dtype, device=device + ) + query = qk_out[0] + key = qk_out[1] + value = torch.empty( + batch, seq_len, num_value_heads, value_head_dim, dtype=out_dtype, device=device + ) + g = torch.empty(batch, seq_len, num_value_heads, dtype=torch.float32, device=device) + beta = torch.empty(batch, seq_len, num_value_heads, dtype=out_dtype, device=device) + + # Conv weight is (conv_dim, 1, K_W); we treat it as (conv_dim, K_W). + weight_2d = conv1d_weight.view(conv1d_weight.shape[0], k_w) + + # No conv bias support: the entry point asserts this. We still pass a + # dummy ``bias_tensor`` to the kernel so the launch signature stays + # stable; ``HAS_BIAS=False`` ensures the kernel never reads it. + bias_tensor = qkvzba + bias_stride = 0 + + # Allocate the gate (z) output buffer that the independent Z kernel will + # populate. Keeping Z separate makes the forward scopes QK / V / Z / + # G-Beta explicit. + gate = torch.empty( + batch, seq_len, num_value_heads, value_head_dim, dtype=out_dtype, device=device + ) + + # Persist the QK silu(conv(x)) intermediate in channel-last layout so the + # backward can feed it directly into the l2norm backward. + silu_qk_save = torch.empty( + (batch, seq_len, 2 * qk_channels), dtype=out_dtype, device=device + ).permute( + 0, 2, 1 + ) # → (b, 2*qk_c, s) with stride(1)==1 + silu_save_b_stride = silu_qk_save.stride(0) + silu_save_c_stride = silu_qk_save.stride(1) + silu_save_s_stride = silu_qk_save.stride(2) + + # Stream setup. Each side stream handles one of the four sub-computations + # (QK conv+l2norm, V conv, Z copy, g/beta). + main_stream = torch.cuda.current_stream(device=device) + qk_stream = _get_side_stream(device, slot=_QK_STREAM_SLOT) + v_stream = _get_side_stream(device, slot=_V_STREAM_SLOT) + g_beta_stream = _get_side_stream(device, slot=_G_BETA_STREAM_SLOT) + z_stream = _get_side_stream(device, slot=_Z_STREAM_SLOT) + for stream in (qk_stream, v_stream, g_beta_stream, z_stream): + stream.wait_stream(main_stream) + + # --- QK conv + silu + l2norm + repeat --- + qk_grid = lambda meta: (batch * 2 * num_key_heads, triton.cdiv(seq_len, meta["BLOCK_S"])) + with torch.cuda.stream(qk_stream): + if is_packed_thd: + _conv_silu_project_thd_kernel[qk_grid]( + qkvzba, + weight_2d, + bias_tensor, + qk_out, + silu_qk_save, + cu_seqlens, + seq_len, + num_packed_seqs, + num_key_heads, + 0, # QK starts at channel 0; group 1 starts at +qk_channels. + qk_channels, + 0, # silu_save_chan_offset + qk_channels, + qkvzba.stride(0), + qkvzba.stride(1), + qkvzba.stride(2), + weight_2d.stride(0), + weight_2d.stride(1), + bias_stride, + qk_out.stride(0), + qk_out.stride(1), + qk_out.stride(2), + qk_out.stride(3), + silu_save_b_stride, + silu_save_c_stride, + silu_save_s_stride, + _L2NORM_EPS, + HEAD_DIM=key_head_dim, + K_W=k_w, + REPEAT=repeat_factor, + NUM_GROUPS=2, + HAS_BIAS=False, + SAVE_SILU=True, + APPLY_L2=True, + ) + else: + _conv_silu_project_kernel[qk_grid]( + qkvzba, + weight_2d, + bias_tensor, + qk_out, + silu_qk_save, + seq_len, + num_key_heads, + 0, # QK starts at channel 0; group 1 starts at +qk_channels. + qk_channels, + 0, # silu_save_chan_offset + qk_channels, + qkvzba.stride(0), + qkvzba.stride(1), + qkvzba.stride(2), + weight_2d.stride(0), + weight_2d.stride(1), + bias_stride, + qk_out.stride(0), + qk_out.stride(1), + qk_out.stride(2), + qk_out.stride(3), + silu_save_b_stride, + silu_save_c_stride, + silu_save_s_stride, + _L2NORM_EPS, + HEAD_DIM=key_head_dim, + K_W=k_w, + REPEAT=repeat_factor, + NUM_GROUPS=2, + HAS_BIAS=False, + SAVE_SILU=True, + APPLY_L2=True, + ) + + # --- V conv + silu (no l2norm, no repeat) --- + v_channel_offset = 2 * qk_channels + z_channel_offset = 2 * qk_channels + v_channels + v_grid = lambda meta: (batch * num_value_heads, triton.cdiv(seq_len, meta["BLOCK_S"])) + with torch.cuda.stream(v_stream): + if is_packed_thd: + _conv_silu_project_thd_kernel[v_grid]( + qkvzba, + weight_2d, + bias_tensor, + value, + qkvzba, # silu_save unused (SAVE_SILU=False) + cu_seqlens, + seq_len, + num_packed_seqs, + num_value_heads, + v_channel_offset, + 0, # in_group_stride unused for NUM_GROUPS=1 + 0, # silu_save_chan_offset unused + 0, # silu_save_group_stride unused + qkvzba.stride(0), + qkvzba.stride(1), + qkvzba.stride(2), + weight_2d.stride(0), + weight_2d.stride(1), + bias_stride, + 0, # out_group_dim_stride unused for NUM_GROUPS=1 + value.stride(0), + value.stride(1), + value.stride(2), + 0, # silu_save strides unused + 0, + 0, + _L2NORM_EPS, + HEAD_DIM=value_head_dim, + K_W=k_w, + REPEAT=1, + NUM_GROUPS=1, + HAS_BIAS=False, + SAVE_SILU=False, + APPLY_L2=False, + ) + else: + _conv_silu_project_kernel[v_grid]( + qkvzba, + weight_2d, + bias_tensor, + value, + qkvzba, # silu_save unused (SAVE_SILU=False) + seq_len, + num_value_heads, + v_channel_offset, + 0, # in_group_stride unused for NUM_GROUPS=1 + 0, # silu_save_chan_offset unused + 0, # silu_save_group_stride unused + qkvzba.stride(0), + qkvzba.stride(1), + qkvzba.stride(2), + weight_2d.stride(0), + weight_2d.stride(1), + bias_stride, + 0, # out_group_dim_stride unused for NUM_GROUPS=1 + value.stride(0), + value.stride(1), + value.stride(2), + 0, # silu_save strides unused + 0, + 0, + _L2NORM_EPS, + HEAD_DIM=value_head_dim, + K_W=k_w, + REPEAT=1, + NUM_GROUPS=1, + HAS_BIAS=False, + SAVE_SILU=False, + APPLY_L2=False, + ) + + # --- Z copy --- + BLOCK_Z_S = _LAYOUT_BLOCK_S + z_grid = (batch * num_value_heads, triton.cdiv(seq_len, BLOCK_Z_S)) + with torch.cuda.stream(z_stream): + _copy_z_kernel[z_grid]( + qkvzba, + gate, + seq_len, + num_value_heads, + z_channel_offset, + qkvzba.stride(0), + qkvzba.stride(1), + qkvzba.stride(2), + gate.stride(0), + gate.stride(1), + gate.stride(2), + HEAD_DIM=value_head_dim, + BLOCK_S=BLOCK_Z_S, + num_warps=4, + num_stages=2, + ) + + # --- g and beta --- + beta_channel_offset = 2 * qk_channels + 2 * v_channels + alpha_channel_offset = beta_channel_offset + num_value_heads + g_beta_grid = lambda meta: ( + batch, + triton.cdiv(seq_len, meta["BLOCK_S"]), + triton.cdiv(num_value_heads, meta["BLOCK_H"]), + ) + with torch.cuda.stream(g_beta_stream): + _compute_g_and_beta_kernel[g_beta_grid]( + qkvzba, + A_log, + dt_bias, + g, + beta, + seq_len, + num_value_heads, + beta_channel_offset, + alpha_channel_offset, + qkvzba.stride(0), + qkvzba.stride(1), + qkvzba.stride(2), + g.stride(0), + g.stride(1), + g.stride(2), + beta.stride(0), + beta.stride(1), + beta.stride(2), + ) + + # Re-join the side streams so the caller's stream observes the writes. + _wait_for_streams(main_stream, qk_stream, v_stream, z_stream, g_beta_stream) + + return query, key, value, gate, beta, g, silu_qk_save + + +def _triton_pre_gated_delta_rule_backward( + qkvzba: Tensor, + conv1d_weight: Tensor, + silu_qk_save: Tensor, + dq: Tensor, + dk: Tensor, + dv: Tensor, + dgate: Tensor, + dbeta: Tensor, + dg: Tensor, + A_log: Tensor, + dt_bias: Tensor, + *, + num_key_heads: int, + num_value_heads: int, + key_head_dim: int, + value_head_dim: int, + seq_idx: Optional[Tensor] = None, +) -> Tuple[Tensor, Tensor, Tensor, Tensor]: + """Triton-backed backward for the pre-gated-delta-rule front-end. + + Mirror of :func:`_triton_pre_gated_delta_rule_forward`. Takes upstream + gradients (``dq``/``dk``/``dv``/``dgate``/``dbeta``/``dg``) plus the + saved forward intermediates and returns input/parameter gradients + ``(d_qkvzba, d_weight, d_A_log, d_dt_bias)``. + + Five Triton kernels + one C++ ``causal_conv1d_bwd_function`` call, + fanned out on five side streams so memory-bound work overlaps while + the conv backward runs on the default stream. See module docstring + for the overall design. + """ + + seq_len, batch, _ = qkvzba.shape + qk_channels = num_key_heads * key_head_dim + v_channels = num_value_heads * value_head_dim + conv_dim = 2 * qk_channels + v_channels + z_offset = 2 * qk_channels + v_channels + k_w = conv1d_weight.shape[-1] + device = qkvzba.device + + # Rebuild the conv input as a NON-contiguous (b, c, s) view of qkvzba. + # ``causal_conv1d_fn`` / ``_bwd_function`` accept inputs where either + # ``stride(1) == 1`` or ``stride(2) == 1``; the permuted view of qkvzba + # satisfies the former (channel stride is 1 in the original (s, b, c) + # layout), so we can skip a 256 MB ``.contiguous()`` copy. + qkvzba_conv = qkvzba[:, :, :conv_dim].permute(1, 2, 0) + weight_2d = conv1d_weight.view(conv1d_weight.shape[0], k_w) + + # ``silu_qk_save`` is the (b, 2*qk_channels, s) bf16 buffer the + # forward wrote ``silu(conv(x))`` into for QK. Reuse it directly as + # the silu input to the l2norm backward. + silu_conv = silu_qk_save + + # Allocate d_silu_conv channel-last (stride(1)==1) — that's what + # ``causal_conv1d_channellast_bwd_kernel`` consumes natively. + d_silu_conv = torch.empty( + (batch, seq_len, conv_dim), dtype=qkvzba.dtype, device=device + ).permute(0, 2, 1) + + # Use the same stream slots as the forward for the matching scopes. + qk_stream = _get_side_stream(device, slot=_QK_STREAM_SLOT) + v_stream = _get_side_stream(device, slot=_V_STREAM_SLOT) + g_beta_stream = _get_side_stream(device, slot=_G_BETA_STREAM_SLOT) + z_stream = _get_side_stream(device, slot=_Z_STREAM_SLOT) + + # Q + K: l2norm + REPEAT backward writes into d_silu_conv's Q/K slices. + _triton_qk_l2norm_repeat_backward( + dq, + dk, + silu_conv, + d_silu_conv, + num_key_heads=num_key_heads, + num_value_heads=num_value_heads, + key_head_dim=key_head_dim, + stream=qk_stream, + ) + + # V: no l2norm and no REPEAT in forward, so d_silu_conv's V slice is + # just dv re-laid-out from (b, s, num_v_heads, value_head_dim) to + # (b, v_channels, s). + _triton_v_layout_to_conv( + dv, + d_silu_conv, + v_channel_offset=2 * qk_channels, + num_value_heads=num_value_heads, + value_head_dim=value_head_dim, + stream=v_stream, + ) + + # g + beta backward fully stores d_qkvzba's alpha + beta slices, plus + # per-head d_A_log / d_dt_bias. Conv and z slices are filled by the + # causal-conv and z kernels, so d_qkvzba does not need a pre-zero. + d_qkvzba = torch.empty_like(qkvzba) + _, d_A_log_fp32, d_dt_bias_fp32 = _triton_g_beta_backward( + qkvzba, + A_log, + dt_bias, + dg, + dbeta, + num_value_heads=num_value_heads, + key_head_dim=key_head_dim, + value_head_dim=value_head_dim, + num_key_heads=num_key_heads, + d_qkvzba_out=d_qkvzba, + stream=g_beta_stream, + ) + + # Z slice gradient: stream dgate into d_qkvzba's z slice. + _triton_z_layout_to_qkvzba( + dgate, + d_qkvzba, + z_channel_offset=z_offset, + num_value_heads=num_value_heads, + value_head_dim=value_head_dim, + stream=z_stream, + ) + + # Join only streams that wrote into d_silu_conv before causal_conv1d_bwd_function. + # g/beta and z write disjoint outputs and can continue overlapping with conv bwd. + default_stream = torch.cuda.current_stream(device) + _wait_for_streams(default_stream, qk_stream, v_stream) + + # Pre-allocate d_x_conv as a strided view INTO d_qkvzba's conv slice. + # d_qkvzba memory layout is (s, b, total_channels) contiguous, so + # element [s, b, c] sits at offset s*b_stride + b*c_stride + c. + # Re-interpreting that storage as (b, conv_dim, s) lets + # causal_conv1d_bwd_function write d_x directly into the right cells. + seq_stride = qkvzba.stride(0) + batch_stride = qkvzba.stride(1) + d_x_conv_view = d_qkvzba.as_strided((batch, conv_dim, seq_len), (batch_stride, 1, seq_stride)) + + # Hand-tuned C++ conv backward. Internally folds the silu' factor and + # computes both d_x and d_w in fp32; writes d_x directly into the + # view above. + assert causal_conv1d_bwd_function is not None + _, d_weight_fp32, _, _ = causal_conv1d_bwd_function( + qkvzba_conv, + weight_2d, + None, # no bias + d_silu_conv, + seq_idx, + None, # initial_states + None, # dfinal_states + d_x_conv_view, # dx pre-allocated into d_qkvzba's conv slice + False, # return_dinitial_states + True, # activation (silu) + ) + + d_weight = d_weight_fp32.view(*conv1d_weight.shape).to(conv1d_weight.dtype) + default_stream.wait_stream(g_beta_stream) + d_A_log = d_A_log_fp32.to(A_log.dtype) + d_dt_bias = d_dt_bias_fp32.to(dt_bias.dtype) + default_stream.wait_stream(z_stream) + + return d_qkvzba, d_weight, d_A_log, d_dt_bias + + +class FusedPreGatedDeltaRuleFunction(torch.autograd.Function): + """Thin :class:`torch.autograd.Function` wrapper around the fused path. + + Stashes the forward inputs + the saved ``silu_qk_save`` intermediate + in ``ctx`` and dispatches to :func:`_triton_pre_gated_delta_rule_forward` + / :func:`_triton_pre_gated_delta_rule_backward`. The actual kernel + logic lives in those two free functions so it's easy to read and + reuse outside the autograd machinery. + """ + + @staticmethod + def forward( + ctx, + qkvzba, + conv1d_weight, + A_log, + dt_bias, + cu_seqlens, + seq_idx, + num_key_heads, + num_value_heads, + key_head_dim, + value_head_dim, + ): + """Run the fused pre-GDR forward path and stash tensors for backward.""" + + ctx.num_key_heads = num_key_heads + ctx.num_value_heads = num_value_heads + ctx.key_head_dim = key_head_dim + ctx.value_head_dim = value_head_dim + query, key, value, gate, beta, g, silu_qk_save = _triton_pre_gated_delta_rule_forward( + qkvzba, + conv1d_weight, + A_log, + dt_bias, + num_key_heads=num_key_heads, + num_value_heads=num_value_heads, + key_head_dim=key_head_dim, + value_head_dim=value_head_dim, + cu_seqlens=cu_seqlens, + ) + ctx.has_seq_idx = seq_idx is not None + if ctx.has_seq_idx: + ctx.save_for_backward(qkvzba, conv1d_weight, A_log, dt_bias, silu_qk_save, seq_idx) + else: + ctx.save_for_backward(qkvzba, conv1d_weight, A_log, dt_bias, silu_qk_save) + return query, key, value, gate, beta, g + + @staticmethod + def backward(ctx, dq, dk, dv, dgate, dbeta, dg): + """Run the fused pre-GDR backward path from saved forward tensors.""" + + if ctx.has_seq_idx: + qkvzba, conv1d_weight, A_log, dt_bias, silu_qk_save, seq_idx = ctx.saved_tensors + else: + qkvzba, conv1d_weight, A_log, dt_bias, silu_qk_save = ctx.saved_tensors + seq_idx = None + d_qkvzba, d_weight, d_A_log, d_dt_bias = _triton_pre_gated_delta_rule_backward( + qkvzba, + conv1d_weight, + silu_qk_save, + dq, + dk, + dv, + dgate, + dbeta, + dg, + A_log, + dt_bias, + num_key_heads=ctx.num_key_heads, + num_value_heads=ctx.num_value_heads, + key_head_dim=ctx.key_head_dim, + value_head_dim=ctx.value_head_dim, + seq_idx=seq_idx, + ) + # Match forward inputs: (qkvzba, conv1d_weight, A_log, dt_bias, + # cu_seqlens, seq_idx, num_key_heads, num_value_heads, + # key_head_dim, value_head_dim). + # Non-tensor args get None. + return (d_qkvzba, d_weight, d_A_log, d_dt_bias, None, None, None, None, None, None) + + +def fused_streamed_pre_gated_delta_rule( + qkvzba: Tensor, + conv1d_weight: Tensor, + conv1d_bias: Optional[Tensor], + A_log: Tensor, + dt_bias: Tensor, + *, + num_key_heads: int, + num_value_heads: int, + key_head_dim: int, + value_head_dim: int, + use_qk_l2norm: bool = True, + cu_seqlens: Optional[Tensor] = None, + seq_idx: Optional[Tensor] = None, +) -> Tuple[Tensor, Tensor, Tensor, Tensor, Tensor, Tensor]: + """Streamed fused pre-gated-delta-rule entry point. + + Args: + qkvzba: ``[seq_len, batch, in_proj_dim]`` projection output. Must be + on CUDA. + conv1d_weight: ``[conv_dim, 1, k_w]`` depthwise conv weight. + conv1d_bias: Must be ``None`` (conv bias is not supported). + A_log: ``[num_value_heads]`` raw decay parameter. + dt_bias: ``[num_value_heads]`` time-step bias. + num_key_heads / num_value_heads / key_head_dim / value_head_dim: GDN + architecture parameters. ``num_value_heads`` must be an integer + multiple of ``num_key_heads``. + use_qk_l2norm: Must be ``True``; the fused backward closes over the + l2norm path. + cu_seqlens: Optional packed THD cumulative sequence lengths. When set, + ``qkvzba`` must have ``batch == 1`` and ``cu_seqlens[-1] == seq_len``. + seq_idx: Optional precomputed token-to-sequence map with shape + ``[1, seq_len]``. Used by causal-conv backward in packed THD mode. + + Returns: + ``(query, key, value, gate, beta, g)`` matching the unfused + :meth:`GatedDeltaNet.pre_gated_delta_rule` API. + """ + + if causal_conv1d_bwd_function is None: + raise ImportError( + "gdn_pre_gated_delta_rule_fusion requires causal-conv1d. " + "Install causal-conv1d~=1.6 in environments that enable this fusion." + ) + + assert qkvzba.is_cuda, ( + "fused_pre_gated_delta_rule requires CUDA inputs; " f"got qkvzba.device={qkvzba.device}." + ) + assert conv1d_bias is None, ( + "Conv bias is not supported by fused_pre_gated_delta_rule " + "(production GDN config has none)." + ) + assert use_qk_l2norm, ( + "use_qk_l2norm=False is not supported by fused_pre_gated_delta_rule " + "(the backward closes over the l2norm path)." + ) + assert ( + num_value_heads % num_key_heads == 0 + ), f"{num_value_heads=} must be a multiple of {num_key_heads=}." + if cu_seqlens is not None: + assert cu_seqlens.is_cuda, ( + "Packed fused_pre_gated_delta_rule requires CUDA cu_seqlens; " + f"got cu_seqlens.device={cu_seqlens.device}." + ) + assert cu_seqlens.dtype == torch.int32, ( + "Packed fused_pre_gated_delta_rule requires int32 cu_seqlens; " + f"got {cu_seqlens.dtype=}." + ) + assert cu_seqlens.dim() == 1, ( + "Packed fused_pre_gated_delta_rule expects 1-D cu_seqlens; " f"got {cu_seqlens.shape=}." + ) + assert qkvzba.shape[1] == 1, ( + "Packed THD fused_pre_gated_delta_rule expects batch dimension 1; " + f"got qkvzba.shape={qkvzba.shape}." + ) + assert cu_seqlens.shape[0] >= 2, ( + "Packed fused_pre_gated_delta_rule requires at least one packed sequence; " + f"got {cu_seqlens.shape=}." + ) + assert cu_seqlens[0].item() == 0, ( + "Packed fused_pre_gated_delta_rule requires cu_seqlens[0] == 0, " + f"got {cu_seqlens[0].item()}." + ) + assert torch.all(cu_seqlens[1:] >= cu_seqlens[:-1]).item(), ( + "Packed fused_pre_gated_delta_rule requires monotonically non-decreasing " + f"cu_seqlens, got {cu_seqlens}." + ) + assert cu_seqlens[-1].item() == qkvzba.shape[0], ( + "Packed fused_pre_gated_delta_rule requires cu_seqlens[-1] to match " + f"seq_len, got {cu_seqlens[-1].item()} vs {qkvzba.shape[0]}." + ) + cu_seqlens = cu_seqlens.contiguous() + seq_idx = _resolve_packed_seq_idx(cu_seqlens, seq_idx, qkvzba.shape[0]) + else: + assert seq_idx is None, "seq_idx requires cu_seqlens for packed THD mode." + + return FusedPreGatedDeltaRuleFunction.apply( + qkvzba, + conv1d_weight, + A_log, + dt_bias, + cu_seqlens, + seq_idx, + num_key_heads, + num_value_heads, + key_head_dim, + value_head_dim, + ) diff --git a/megatron/core/hyper_comm_grid.py b/megatron/core/hyper_comm_grid.py index 4b860396c4e..ad1546608b5 100644 --- a/megatron/core/hyper_comm_grid.py +++ b/megatron/core/hyper_comm_grid.py @@ -1,19 +1,13 @@ # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +import numbers import os -from operator import itemgetter -from typing import Any, Optional, Tuple, Union +from dataclasses import dataclass +from typing import Any, Optional, Union import numpy as np import torch.distributed as dist -try: - import einops - - HAVE_EINOPS = True -except ImportError: - HAVE_EINOPS = False - try: from absl import logging @@ -30,6 +24,25 @@ HAVE_ABSL = False +def _is_process_group_member(pg: Optional[dist.ProcessGroup]) -> bool: + """Whether the current rank belongs to ``pg`` (not the non-member sentinel).""" + non_member = getattr(getattr(dist, "GroupMember", None), "NON_GROUP_MEMBER", None) + return pg is not None and pg is not non_member + + +_BASE_VIEW_NAME = "base" + + +@dataclass +class _RankViewSpec: + """A named rank factorization over the same rank span as the base grid.""" + + name: str + shape: list[int] + dim_names: list[str] + shared_dims: list[str] + + class HyperCommGrid: r"""N-dimensional communication grid. @@ -41,6 +54,9 @@ class HyperCommGrid: For any combination of dimensions, a process group can only be created once. Creating process groups for the same combination with different options is not supported. + Methods default to the base factorization. Register additional factorizations of the same + rank span with :meth:`register_view` and target them via ``view="..."``. + Note: ``create_pg()`` over specific dims must be explicitly called to create a process group. We don't create a process group in the ``get_pg()`` function because there are many options @@ -102,7 +118,7 @@ def __init__( "initialize torch.distributed before creating HyperCommGrid." ) self.rank_offset = rank_offset - self.size = np.prod(shape) + self.size = int(np.prod(shape)) if rank_offset < 0: raise ValueError(f"rank_offset must be non-negative, got {rank_offset}") if self.size > world_size - rank_offset: @@ -115,9 +131,81 @@ def __init__( self.shape = shape[:] self.dim_names = dim_names[:] self.backend = backend - self._pgs: dict[str, dist.ProcessGroup] = {} + self._views: dict[str, _RankViewSpec] = { + _BASE_VIEW_NAME: _RankViewSpec( + _BASE_VIEW_NAME, self.shape[:], self.dim_names[:], shared_dims=[] + ) + } + # Base-view groups are keyed by their dash-joined dim string (unchanged from the + # single-view design); view-private groups are keyed by ``(view_name, dims_tuple)``. + self._pgs: dict[Union[str, tuple[str, tuple[str, ...]]], dist.ProcessGroup] = {} - def create_pg(self, dims: Union[str, list[str]], **kwargs: Any) -> dist.ProcessGroup | None: + def register_view( + self, + name: str, + shape: list[int], + dim_names: list[str], + shared_dims: Optional[list[str]] = None, + ) -> None: + r"""Register an additional rank factorization over this grid's rank span. + + Shared dims must exist in both the base view and the new view, and must enumerate to the + same rank groups as the base view. + """ + if name in self._views: + raise ValueError(f"View {name!r} is already registered") + if len(shape) != len(dim_names): + raise ValueError(f"len(shape) {shape} != len(dim_names) {dim_names}") + if len(set(dim_names)) != len(dim_names): + raise ValueError(f"View {name!r} has duplicate dim_names: {dim_names}") + if any(not isinstance(s, numbers.Integral) or s <= 0 for s in shape): + raise ValueError(f"View {name!r} shape must be positive ints, got {shape}") + if int(np.prod(shape)) != self.size: + raise ValueError( + f"View {name!r} shape {shape} has size {int(np.prod(shape))}, but the grid " + f"size is {self.size}" + ) + + shared_dims = list(shared_dims) if shared_dims is not None else [] + if len(set(shared_dims)) != len(shared_dims): + raise ValueError(f"View {name!r} has duplicate shared_dims: {shared_dims}") + for dim in shared_dims: + if dim not in self.dim_names: + raise ValueError( + f"Shared dim {dim!r} of view {name!r} is not in the base view " + f"{self.dim_names}" + ) + if dim not in dim_names: + raise ValueError( + f"Shared dim {dim!r} of view {name!r} is not in the view's dim_names " + f"{dim_names}" + ) + base_dims, _ = self._order_dims_for(self.dim_names, dim) + base_enum = self._gen_rank_enum_for(self.shape, self.dim_names, base_dims) + view_dims, _ = self._order_dims_for(dim_names, dim) + view_enum = self._gen_rank_enum_for(shape, dim_names, view_dims) + if base_enum != view_enum: + raise ValueError( + f"Shared dim {dim!r} has different membership across views: base " + f"enumeration {base_enum} != view {name!r} enumeration {view_enum}" + ) + + if len(shared_dims) > 1: + base_dims, _ = self._order_dims_for(self.dim_names, shared_dims) + base_enum = self._gen_rank_enum_for(self.shape, self.dim_names, base_dims) + view_dims, _ = self._order_dims_for(dim_names, shared_dims) + view_enum = self._gen_rank_enum_for(shape, dim_names, view_dims) + if base_enum != view_enum: + raise ValueError( + f"Shared dims {shared_dims!r} have different membership across views: base " + f"enumeration {base_enum} != view {name!r} enumeration {view_enum}" + ) + + self._views[name] = _RankViewSpec(name, shape[:], dim_names[:], shared_dims[:]) + + def create_pg( + self, dims: Union[str, list[str]], *, view: Optional[str] = None, **kwargs: Any + ) -> dist.ProcessGroup | None: r"""Create a process group based on a list of dimension names Note: The unique key used to store the process group internally will follow the reversed @@ -127,6 +215,7 @@ def create_pg(self, dims: Union[str, list[str]], **kwargs: Any) -> dist.ProcessG Args: dims: Name of leading dimensions to create process group + view: Optional registered rank view name. Defaults to the base view. Keyword arguments are directly passed into new_subgroups_by_enumeration(). The docstring is copied from new_subgroups_by_enumeration(). @@ -145,49 +234,85 @@ def create_pg(self, dims: Union[str, list[str]], **kwargs: Any) -> dist.ProcessG Raises: KeyError: If attempting to recreate a process group with an existing key. """ - # ordered_dims and unique_group_key will follow the reversed order of self.dim_names - ordered_dims, unique_group_key = self._order_dims(dims) + view_spec = self._resolve_view(view) + ordered_dims, _ = self._order_dims_for_view(view_spec, dims) + unique_group_key, enum_view, enum_dims = self._canonical_pg_key_and_enum_view( + view_spec, ordered_dims + ) if unique_group_key in self._pgs: + if self._is_base_pg_key(unique_group_key): + raise KeyError( + f"Process group {dims} has already been created. Because there is no way " + f"to check whether options to create process group matches the first, we " + f"error out instead of returning the process group that has already been " + f"created before." + ) raise KeyError( - f"Process group {dims} has already been created. Because there is no way to check " - f"whether options to create process group matches the first, we error out instead " - f"of returning the process group that has already been created before." + f"Process group {dims} for view {view_spec.name!r} has already been created. " + f"Because there is no way to check whether options to create process group " + f"matches the first, we error out instead of returning the process group that " + f"has already been created before." ) - rank_enum = self._gen_rank_enum(ordered_dims) + rank_enum = self._gen_rank_enum_for(enum_view.shape, enum_view.dim_names, enum_dims) pg, _ = dist.new_subgroups_by_enumeration(rank_enum, backend=self.backend, **kwargs) - if dist.get_rank() == 0: - logging.info( - f"Generated process group for {unique_group_key} with enumeration {rank_enum}" - ) + if dist.is_initialized() and dist.get_rank() == 0: + if self._is_base_pg_key(unique_group_key): + logging.info( + f"Generated process group for {unique_group_key} with enumeration {rank_enum}" + ) + else: + logging.info( + f"Generated process group for view {view_spec.name!r} {ordered_dims} with " + f"enumeration {rank_enum}" + ) self._pgs[unique_group_key] = pg return pg def destroy(self) -> None: - """Destroy all process groups created by this grid.""" + """Destroy all process groups created by this grid that the current rank belongs to. + + This includes base-view groups and view-private groups. A base group reused by a + view for a shared dim is stored under a single key, so it is torn down exactly once. + """ + destroyed: set[int] = set() for pg in self._pgs.values(): - if pg is not None: + if _is_process_group_member(pg) and id(pg) not in destroyed: dist.destroy_process_group(pg) + destroyed.add(id(pg)) self._pgs.clear() - def get_pg(self, dims: Union[str, list[str]]) -> dist.ProcessGroup: + def get_pg( + self, dims: Union[str, list[str]], *, view: Optional[str] = None + ) -> dist.ProcessGroup: r"""Get a process group based on a list of dimension names Args: dims: Name of leading dimensions to create process group + view: Optional registered rank view name. Defaults to the base view. """ - _, unique_group_key = self._order_dims(dims) + view_spec = self._resolve_view(view) + ordered_dims, _ = self._order_dims_for_view(view_spec, dims) + unique_group_key, _, _ = self._canonical_pg_key_and_enum_view(view_spec, ordered_dims) if unique_group_key not in self._pgs: + if self._is_base_pg_key(unique_group_key): + raise KeyError( + f"Process group for {unique_group_key} hasn't been created. Call create_pg " + f"first." + ) raise KeyError( - f"Process group for {unique_group_key} hasn't been created. Call create_pg first." + f"Process group {dims} for view {view_spec.name!r} hasn't been created. Call " + f"create_pg first." ) return self._pgs[unique_group_key] - def get_rank_enum(self, dims: Union[str, list[str]]) -> list[list[int]]: + def get_rank_enum( + self, dims: Union[str, list[str]], *, view: Optional[str] = None + ) -> list[list[int]]: r"""Get the rank enumeration for the requested dimension(s). This is the exact enumeration that would be used by create_pg for the same @@ -196,12 +321,14 @@ def get_rank_enum(self, dims: Union[str, list[str]]) -> list[list[int]]: Args: dims: Dimension name or list of dimension names. + view: Optional registered rank view name. Defaults to the base view. Returns: List of rank lists (one per subgroup). """ - ordered_dims, _ = self._order_dims(dims) - return self._gen_rank_enum(ordered_dims) + view_spec = self._resolve_view(view) + ordered_dims, _ = self._order_dims_for_view(view_spec, dims) + return self._gen_rank_enum_for(view_spec.shape, view_spec.dim_names, ordered_dims) def _gen_rank_enum(self, dims: list[str]) -> list[list[int]]: r"""Generate rank enumeration before calling new_subgroups_by_enumeration @@ -224,45 +351,88 @@ def _gen_rank_enum(self, dims: list[str]) -> list[list[int]]: Although the function is lightweight enough to be inlined, a standalone one makes it easier to test against MCore's RankGenerator """ + return self._gen_rank_enum_for(self.shape, self.dim_names, dims) + + def _gen_rank_enum_for( + self, shape: list[int], dim_names: list[str], dims: list[str] + ) -> list[list[int]]: + r"""Generate rank enumeration for ``dims`` under an explicit ``shape``/``dim_names``.""" + # Need to reverse order of dim_names to match MCore convention. + dim_names_reverse = dim_names[::-1] + shape_dict = {d: s for d, s in zip(dim_names, shape)} + size = int(np.prod(shape)) + rank_tensor = np.arange(self.rank_offset, self.rank_offset + size).reshape( + [shape_dict[d] for d in dim_names_reverse] + ) - if not HAVE_EINOPS: - raise RuntimeError( - "einops is not installed. Please install it with `pip install einops`." - ) - - # Need to reverse order of dim_names to match MCore convention - dim_names_reverse = self.dim_names[::-1] - - remaining_dims = [] - for v in dim_names_reverse: - if v not in dims: - remaining_dims.append(v) - - rearrange_str = ( - f"({' '.join(dim_names_reverse)}) -> ({' '.join(remaining_dims)}) ({' '.join(dims)})" + source_axes = [dim_names_reverse.index(d) for d in dims] + target_axes = list(range(len(dim_names_reverse) - len(dims), len(dim_names_reverse))) + logging.debug( + "Moving axes %s to %s for dim_names=%s dims=%s", + source_axes, + target_axes, + dim_names, + dims, ) - logging.debug(rearrange_str) + rank_tensor = np.moveaxis(rank_tensor, source_axes, target_axes) - shape_dict = {d: s for d, s in zip(self.dim_names, self.shape)} - return einops.rearrange( - np.arange(self.rank_offset, self.rank_offset + self.size), rearrange_str, **shape_dict - ).tolist() + group_size = int(np.prod([shape_dict[d] for d in dims])) + return rank_tensor.reshape(-1, group_size).tolist() - def _order_dims(self, dims: Union[str, list[str]]) -> Tuple[list[str], str]: - r"""Reorder dims based on the order of self.dim_names""" + def _order_dims_for( + self, dim_names: list[str], dims: Union[str, list[str]] + ) -> tuple[list[str], str]: + r"""Reorder ``dims`` against an explicit ``dim_names``.""" if not isinstance(dims, list): ordered_dims = [dims] else: - dim_names_reverse = self.dim_names[::-1] + dim_names_reverse = dim_names[::-1] indices = sorted([dim_names_reverse.index(d) for d in dims]) - if len(indices) == 1: - ordered_dims = [dim_names_reverse[indices[0]]] - else: - ordered_dims = list(itemgetter(*indices)(dim_names_reverse)) + ordered_dims = [dim_names_reverse[i] for i in indices] unique_group_key = "-".join(ordered_dims) return ordered_dims, unique_group_key + def _resolve_view(self, view: Optional[str]) -> _RankViewSpec: + r"""Return the requested rank view, defaulting to the base view.""" + view_name = _BASE_VIEW_NAME if view is None else view + if view_name not in self._views: + raise KeyError( + f"View {view_name!r} is not registered. Registered views: {sorted(self._views)}" + ) + return self._views[view_name] + + def _order_dims_for_view( + self, view: _RankViewSpec, dims: Union[str, list[str]] + ) -> tuple[list[str], str]: + r"""Reorder ``dims`` against a registered view and report missing dims clearly.""" + requested_dims = [dims] if not isinstance(dims, list) else dims + missing_dims = [d for d in requested_dims if d not in view.dim_names] + if missing_dims: + raise ValueError( + f"{missing_dims[0]!r} is not in view {view.name!r} with dim_names " + f"{view.dim_names}" + ) + return self._order_dims_for(view.dim_names, dims) + + def _canonical_pg_key_and_enum_view( + self, view: _RankViewSpec, ordered_dims: list[str] + ) -> tuple[Union[str, tuple[str, tuple[str, ...]]], _RankViewSpec, list[str]]: + r"""Return the storage key and rank view used to enumerate a process group.""" + if view.name == _BASE_VIEW_NAME: + return "-".join(ordered_dims), view, ordered_dims + + if all(d in view.shared_dims for d in ordered_dims): + base_view = self._views[_BASE_VIEW_NAME] + base_ordered_dims, base_key = self._order_dims_for_view(base_view, ordered_dims) + return base_key, base_view, base_ordered_dims + + return (view.name, tuple(ordered_dims)), view, ordered_dims + + def _is_base_pg_key(self, key: Union[str, tuple[str, tuple[str, ...]]]) -> bool: + r"""Whether a process-group key belongs to the base view namespace.""" + return isinstance(key, str) + def is_current_rank_in_grid(self) -> bool: """Check if the current rank belongs to this grid. diff --git a/megatron/core/inference/README.md b/megatron/core/inference/README.md new file mode 100644 index 00000000000..1c133349445 --- /dev/null +++ b/megatron/core/inference/README.md @@ -0,0 +1,92 @@ +# Megatron Inference + +Use `MegatronLLM` (sync) or `MegatronAsyncLLM` (async, with HTTP serving via `serve()`) for typical inference workflows. Both classes hide the underlying engine pipeline (`DynamicInferenceContext` + `GPTInferenceWrapper` + `TextGenerationController` + `DynamicInferenceEngine`) and provide a vLLM-style `generate(prompts, sampling_params)` API. Choose **direct mode** (`use_coordinator=False`) when you manage data sharding yourself; **coordinator mode** (`use_coordinator=True`) when you want the engine to route requests across data-parallel replicas (required for HTTP serving). + +## Quickstart + +### Offline batch (sync) + +```python +from megatron.core.inference.apis import MegatronLLM, SamplingParams + +# Caller owns initialize_megatron(...), model construction, and model.eval(). +# See examples/inference/offline_inference.py for a runnable end-to-end script. +with MegatronLLM( + model=model, + tokenizer=tokenizer, + inference_config=inference_config, + use_coordinator=False, +) as llm: + results = llm.generate( + ["Megatron inference is", "Hello, world"], + SamplingParams(num_tokens_to_generate=64), + ) + for r in results: + print(r.generated_text) +``` + +### OpenAI-compatible HTTP server + +```python +import asyncio +from megatron.core.inference.apis import MegatronAsyncLLM, ServeConfig + +async def main(): + async with MegatronAsyncLLM( + model=model, + tokenizer=tokenizer, + inference_config=inference_config, + use_coordinator=True, # serve() requires coordinator mode + ) as llm: + await llm.serve(ServeConfig(host="0.0.0.0", port=5000)) # blocks until shutdown + +asyncio.run(main()) +``` + +## Public API + +| Symbol | Purpose | +|---|---| +| `MegatronLLM` | Sync entry. Methods: `generate`, `pause`/`unpause`/`suspend`/`resume`, `shutdown`/`wait_for_shutdown`. Properties: `engine`, `context`, `controller`, `is_primary_rank`. Context-manager protocol. | +| `MegatronAsyncLLM` | Async-flavored equivalent. Adds `serve(serve_config, blocking=True)` for HTTP. | +| `ServeConfig` | Dataclass for the HTTP frontend. Fields: `host` (`"0.0.0.0"`), `port` (`5000`), `parsers` (`[]`), `verbose` (`False`), `frontend_replicas` (`4`). | +| `SamplingParams`, `DynamicInferenceRequest`, `DynamicInferenceRequestRecord` | Re-exports from `megatron.core.inference`. | + +## Caller responsibilities + +- Call `initialize_megatron(...)` (full Megatron distributed setup) BEFORE construction. +- Call `model.eval()` BEFORE construction. The class does not toggle model state. +- Lifecycle methods (`pause`/`unpause`/`suspend`/`resume`) require `use_coordinator=True`; they raise `RuntimeError` in direct mode. + +## Future roadmap + +Planned new features: + +- **Dynamic streaming.** Offline streaming via `engine.async_step()`; HTTP streaming requires extending the coordinator / `InferenceClient` protocol to carry partial outputs (not just final request records). + +- **Weight update APIs.** `suspend_for_refit()`, `update_weights_from_collective()`, `resume_after_refit()` wrapping the existing resharding/refit primitives for RL workflows where weights swap between rollout steps. + +- **`megatron serve` CLI.** Single-binary launcher reusing `MegatronAsyncLLM.serve(...)`, with single-node and multi-node / headless modes — mirrors `vllm serve`. + +- **Config-based model construction.** `MegatronLLM(model="...")` style with model recipes and checkpoint resolution, removing manual model building from caller responsibilities. + +## Known limitations + +- **`MegatronAsyncLLM` requires `use_coordinator=True`** -- constructing with `use_coordinator=False` raises `ValueError` at `__init__`. The underlying `DynamicInferenceEngine` caches its loop reference at construction time and binds internal asyncio primitives (`_cond`, `_state_events`) to it. Coordinator mode rebinds those to a dedicated daemon-thread loop via `start_listening_to_data_parallel_coordinator`; direct mode has no such rebinding, so the synchronous `engine.generate()` path collides with the caller's running asyncio loop and raises `RuntimeError: This event loop is already running`. Use `MegatronLLM` for sync direct/coordinator workflows. Tracked for an upstream `engine.async_generate(...)` (or engine loop-rebinding) fix that would let `MegatronAsyncLLM` support direct mode. + +- **`llm.engine.reset()` is unsafe in coordinator mode.** Two failure modes, both upstream in `dynamic_engine.py`: + - *Deadlock*: `reset()` *rebinds* (does not mutate in-place) `_cond` / `_state_events`. Any coroutine on the engine-loop task that is `await`ing one of those primitives holds a reference to the OLD object in its suspended frame. Subsequent `notify_all()` / `set()` calls hit the NEW objects, leaving the suspended waiter stranded; the next `generate()` hangs. + - *Silent corruption*: `reset()` also sets `self.use_coordinator = False`, which silently re-routes failed-request handling, scheduling notification, and `suspend()`'s state machine to direct-mode branches. Outcome: not-a-hang but wrong behavior, harder to diagnose. + - The example `offline_inference.py` blocks `--inference-repeat-n > 1` with `--use-coordinator` for these reasons. Direct-mode reset is safe. + +- **HTTP frontend is fixed to global rank 0.** There is no per-rank `role` override on `ServeConfig` to host the HTTP server on a non-rank-0 rank or to opt a rank out of HTTP. Control placement via the launcher (e.g., torchrun rank-0 placement), mirroring how vLLM's `--headless` is invoked today. + +- **Server returns `"model": "EMPTY"`.** The HTTP frontend doesn't expose a `ServeConfig.model_name` to echo in `/v1/completions` / `/v1/chat/completions` responses, doesn't validate the request `model` field against a configured name, and exposes no `GET /v1/models` discovery endpoint. Clients can still pass any `model` in their request body — the dynamic server ignores it. + +## Low-level APIs + +For step-level control, custom forward-step integration, or migration from existing pipelines, drop down to the building blocks in this directory: `DynamicInferenceEngine` (manual `add_request` / `step_modern` stepping), `DynamicInferenceContext`, `TextGenerationController`, and the model inference wrappers under `model_inference_wrappers/`. Runnable examples live in [`examples/inference/advanced/`](../../examples/inference/advanced/): `gpt_dynamic_inference.py` (manual stepping), `gpt_dynamic_inference_with_coordinator.py` (explicit coordinator + `InferenceClient` lifecycle), `gpt_static_inference.py` (static engine), and `simple_t5_batch_inference.py` (T5). + +## See also + +- Examples: [`examples/inference/offline_inference.py`](../../examples/inference/offline_inference.py) (4 modes via `--mode` / `--use-coordinator`), [`examples/inference/launch_inference_server.py`](../../examples/inference/launch_inference_server.py) (HTTP server). diff --git a/megatron/core/inference/apis/__init__.py b/megatron/core/inference/apis/__init__.py new file mode 100644 index 00000000000..19b27250406 --- /dev/null +++ b/megatron/core/inference/apis/__init__.py @@ -0,0 +1,19 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +from megatron.core.inference.apis.async_llm import MegatronAsyncLLM +from megatron.core.inference.apis.llm import MegatronLLM +from megatron.core.inference.apis.serve_config import ServeConfig +from megatron.core.inference.inference_request import ( + DynamicInferenceRequest, + DynamicInferenceRequestRecord, +) +from megatron.core.inference.sampling_params import SamplingParams + +__all__ = [ + "DynamicInferenceRequest", + "DynamicInferenceRequestRecord", + "MegatronAsyncLLM", + "MegatronLLM", + "SamplingParams", + "ServeConfig", +] diff --git a/megatron/core/inference/apis/_llm_base.py b/megatron/core/inference/apis/_llm_base.py new file mode 100644 index 00000000000..0c0f9881b11 --- /dev/null +++ b/megatron/core/inference/apis/_llm_base.py @@ -0,0 +1,462 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +"""Internal building blocks for the Megatron inference high-level API. + +This module hosts private helpers shared by ``MegatronLLM`` and +``MegatronAsyncLLM``: ``_EventLoopManager``, ``_CoordinatorRuntime``, and +``_MegatronLLMBase``. The public sync/async wrappers live on the subclasses; +this base only exposes shared engine state, runtime spawn, validation +helpers, and the private ``__impl`` coroutines. +""" + +import asyncio +import concurrent.futures +import threading +from typing import Coroutine, List, Optional, Tuple, Union + +import torch.distributed as dist + +from megatron.core.inference.config import InferenceConfig +from megatron.core.inference.contexts.dynamic_context import DynamicInferenceContext +from megatron.core.inference.engines.dynamic_engine import DynamicInferenceEngine, EngineState +from megatron.core.inference.inference_request import DynamicInferenceRequest +from megatron.core.inference.model_inference_wrappers.gpt.gpt_inference_wrapper import ( + GPTInferenceWrapper, +) +from megatron.core.inference.sampling_params import SamplingParams +from megatron.core.inference.text_generation_controllers.text_generation_controller import ( + TextGenerationController, +) + + +class _EventLoopManager: + """Per-instance background daemon thread + persistent asyncio event loop. + + Bridges sync and async user-thread callers to coroutines that run on the + background loop via ``asyncio.run_coroutine_threadsafe``. + """ + + def __init__(self) -> None: + self._loop: Optional[asyncio.AbstractEventLoop] = None + self._thread: Optional[threading.Thread] = None + self._started: bool = False + self._stopped: bool = False + + def start(self) -> None: + """Spawn the daemon thread and start the event loop. Idempotent.""" + if self._started: + return + + # PyTorch's CUDA current-device is thread-local and defaults to 0 on + # new threads. Capture the spawning thread's device so NCCL ops + # scheduled on the runtime loop (e.g. inside + # ``start_listening_to_data_parallel_coordinator``) hit the right GPU + # under torchrun, where every process sees all GPUs and rank-to-device + # mapping is set on the main thread only. + import torch + + parent_device = torch.cuda.current_device() if torch.cuda.is_available() else None + + loop_ready = threading.Event() + + def _run_loop() -> None: + if parent_device is not None: + torch.cuda.set_device(parent_device) + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + self._loop = loop + # Fires once run_forever() starts dispatching callbacks, so + # callers blocked on loop_ready.wait() resume only after the + # loop is actually running. + loop.call_soon(loop_ready.set) + loop.run_forever() + + self._thread = threading.Thread(target=_run_loop, daemon=True) + self._thread.start() + loop_ready.wait() + self._started = True + + @property + def loop(self) -> asyncio.AbstractEventLoop: + """The background asyncio loop. Raises if ``start()`` has not been called.""" + if not self._started or self._loop is None: + raise RuntimeError("_EventLoopManager.start() must be called before accessing loop.") + return self._loop + + def submit(self, coro: Coroutine) -> concurrent.futures.Future: + """Schedule ``coro`` on the background loop and return its future. + + The caller decides how to wait on the returned future (e.g. + ``.result()`` for blocking sync, ``asyncio.wrap_future(...)`` for + awaiting from another loop). + """ + if not self._started or self._loop is None: + raise RuntimeError("_EventLoopManager.start() must be called before submit().") + return asyncio.run_coroutine_threadsafe(coro, self._loop) + + def run_sync(self, coro: Coroutine): + """Schedule ``coro`` on the background loop and block on its result. + + Must not be called from a coroutine running on ``self._loop`` itself + -- that would deadlock, since the only loop that could dispatch + ``coro`` would be the one already blocked waiting for the caller. + Calling from a different loop (e.g., the user's main-thread asyncio + loop) is allowed: ``coro`` runs on the background loop while the + caller's loop is stalled until ``.result()`` returns. + """ + try: + running = asyncio.get_running_loop() + except RuntimeError: + running = None # no loop on this thread, safe + if running is self._loop: + raise RuntimeError( + "run_sync called from a coroutine running on the background " + "loop -- would deadlock waiting for the same loop." + ) + return self.submit(coro).result() + + async def run_async(self, coro: Coroutine): + """Schedule ``coro`` on the background loop and await it from any loop.""" + return await asyncio.wrap_future(self.submit(coro)) + + def stop(self) -> None: + """Stop the event loop and join the background thread. Idempotent.""" + if not self._started or self._stopped: + return + assert self._loop is not None + assert self._thread is not None + self._loop.call_soon_threadsafe(self._loop.stop) + self._thread.join() + self._stopped = True + self._started = False + + +class _CoordinatorRuntime: + """Owns the dynamic-inference coordinator and ``InferenceClient`` lifecycle. + + Async-native: :meth:`setup` and :meth:`teardown` are coroutines meant to + run on a background loop owned by :class:`_EventLoopManager`. The primary + rank additionally holds an :class:`InferenceClient` used by the high-level + API to submit requests and send control signals. + """ + + def __init__( + self, + engine: "DynamicInferenceEngine", + *, + is_primary: bool, + coordinator_host: Optional[str], + coordinator_port: Optional[int], + ) -> None: + self._engine = engine + self._is_primary = is_primary + self._coordinator_host = coordinator_host + self._coordinator_port = coordinator_port + self._client: "Optional[InferenceClient]" = None + self._coord_addr: Optional[str] = None + + async def setup(self, *, loop: asyncio.AbstractEventLoop) -> None: + """Bring the coordinator and (on primary) the ``InferenceClient`` up. + + Calls ``engine.start_listening_to_data_parallel_coordinator(loop=loop)`` + on every rank. Only host/port kwargs that the caller actually supplied + are forwarded so the engine can auto-bind when both are ``None``. + """ + kwargs = {"loop": loop} + if self._coordinator_host is not None: + kwargs["hostname"] = self._coordinator_host + if self._coordinator_port is not None: + kwargs["inference_coordinator_port"] = self._coordinator_port + + coord_addr = await self._engine.start_listening_to_data_parallel_coordinator(**kwargs) + self._coord_addr = coord_addr + + if self._is_primary: + # Lazy import: keep this module importable without pyzmq/msgpack + # installed when the user only needs direct mode. + from megatron.core.inference.inference_client import InferenceClient + + # deserialize=True returns DynamicInferenceRequest objects from + # add_request futures, matching the high-level API contract. + client = InferenceClient(coord_addr, deserialize=True) + client.start(loop=loop) + self._client = client + + async def teardown(self) -> None: + """Idempotent best-effort shutdown of the coordinator + client. + + Safe to call from partial-setup state (e.g., when :meth:`setup` raised + after the coordinator subprocess spawned but before the client opened). + Worker ranks are always no-op; their ``engine_loop_task`` is awaited by + :meth:`_MegatronLLMBase._shutdown_impl` after the primary has issued + the STOP signal. + """ + if not self._is_primary: + return + + # Happy path: client open -> graceful protocol shutdown. + if self._client is not None: + try: + self._client.shutdown_coordinator() + self._client.stop() + finally: + self._client = None + return + + # Partial-setup path: client never opened. If the coordinator + # subprocess was spawned, kill it via the engine's process handle. + proc = getattr(self._engine, "inference_coordinator_process", None) + if proc is not None and proc.is_alive(): + proc.terminate() + proc.join(timeout=5) + if proc.is_alive(): + proc.kill() + proc.join(timeout=2) + + @property + def client(self) -> "Optional[InferenceClient]": + """The :class:`InferenceClient` on the primary rank; ``None`` on workers.""" + return self._client + + @property + def coord_addr(self) -> Optional[str]: + """Address returned by ``start_listening_to_data_parallel_coordinator``.""" + return self._coord_addr + + +class _MegatronLLMBase: + """Private base shared by ``MegatronLLM`` and ``MegatronAsyncLLM``. + + This base intentionally exposes no public ``generate`` / lifecycle + methods -- those live on the subclasses, which call into the private + ``__impl`` coroutines defined here. The base owns: + + - the engine pipeline (engine, context, controller), + - the per-instance background runtime (``_loop_manager``, + ``_coord_runtime``) when ``use_coordinator=True``, + - validation helpers (``_assert_primary``, ``_assert_coordinator``) and + the input shape helper (``_normalize_prompts``). + + Two execution modes are supported: + + - **Direct mode** (``use_coordinator=False``): every rank is treated as + primary and ``generate`` runs the engine synchronously (offloaded to a + thread when called from an event loop). Lifecycle methods are invalid + and raise :class:`RuntimeError` via ``_assert_coordinator``. + - **Coordinator mode** (``use_coordinator=True``): a background event loop + hosts the engine pipeline and an :class:`InferenceClient` (on global + rank 0). Only the primary rank may submit requests via ``generate``. + + ``model`` must be in eval mode before construction; this class does not + modify the model state. + """ + + def __init__( + self, + *, + model, + tokenizer, + inference_config: Optional[InferenceConfig] = None, + use_coordinator: bool = False, + coordinator_host: Optional[str] = None, + coordinator_port: Optional[int] = None, + ) -> None: + if (coordinator_host is not None or coordinator_port is not None) and not use_coordinator: + raise ValueError("coordinator_host/port require use_coordinator=True") + + if not use_coordinator: + from megatron.core import parallel_state + + ep_size = parallel_state.get_expert_model_parallel_world_size() + if ep_size > 1: + raise ValueError( + f"use_coordinator=True is required when expert_model_parallel_size > 1 " + f"(got EP={ep_size}). Use coordinator mode to handle EP routing." + ) + + if inference_config is None: + inference_config = InferenceConfig() + + # Build the engine pipeline. Mirrors examples/inference/gpt/gpt_dynamic_inference.py. + context = DynamicInferenceContext(model.config, inference_config) + wrapper = GPTInferenceWrapper(model, context) + controller = TextGenerationController(inference_wrapped_model=wrapper, tokenizer=tokenizer) + engine = DynamicInferenceEngine(controller=controller, context=context) + + if use_coordinator: + is_primary_rank = dist.get_rank() == 0 + else: + is_primary_rank = True + + self._engine = engine + self._context = context + self._controller = controller + self._use_coordinator = use_coordinator + self._is_primary_rank = is_primary_rank + self._loop_manager: "Optional[_EventLoopManager]" = None + self._coord_runtime: "Optional[_CoordinatorRuntime]" = None + self._shutdown_called: bool = False + + if use_coordinator: + loop_manager = _EventLoopManager() + loop_manager.start() + coord_runtime: "Optional[_CoordinatorRuntime]" = None + try: + coord_runtime = _CoordinatorRuntime( + engine, + is_primary=is_primary_rank, + coordinator_host=coordinator_host, + coordinator_port=coordinator_port, + ) + loop_manager.run_sync(coord_runtime.setup(loop=loop_manager.loop)) + except BaseException: + if coord_runtime is not None: + try: + loop_manager.run_sync(coord_runtime.teardown()) + except Exception: + pass # best-effort; don't mask the original failure + loop_manager.stop() + raise + self._loop_manager = loop_manager + self._coord_runtime = coord_runtime + + # ---- properties ---- + + @property + def is_primary_rank(self) -> bool: + """Whether ``generate`` may be called on this rank.""" + return self._is_primary_rank + + @property + def engine(self) -> "DynamicInferenceEngine": + """The underlying :class:`DynamicInferenceEngine`.""" + return self._engine + + @property + def context(self) -> "DynamicInferenceContext": + """The underlying :class:`DynamicInferenceContext`.""" + return self._context + + @property + def controller(self) -> "TextGenerationController": + """The underlying :class:`TextGenerationController`.""" + return self._controller + + # ---- internal helpers ---- + + def _assert_primary(self) -> None: + if not self._is_primary_rank: + raise RuntimeError( + "generate(...) is only valid on the primary rank in coordinator mode" + ) + + def _assert_coordinator(self) -> None: + if not self._use_coordinator: + raise RuntimeError("This method requires use_coordinator=True") + + def _normalize_prompts( + self, prompts: Union[str, List[int], List[str], List[List[int]]] + ) -> Tuple[Union[List[str], List[List[int]]], bool]: + """Return ``(normalized_list, is_batch_input)``. + + - ``"abc"`` -> ``(["abc"], False)`` + - ``[1, 2, 3]`` -> ``([[1, 2, 3]], False)`` (single token-id prompt) + - ``["abc", "def"]`` -> ``(["abc", "def"], True)`` + - ``[[1, 2], [3, 4]]`` -> ``([[1, 2], [3, 4]], True)`` + - ``[]`` -> ``([], True)`` + + Only the first element is inspected to distinguish single vs batch; + per-element type validation is left to the engine. + """ + if isinstance(prompts, str): + return [prompts], False + if isinstance(prompts, list): + if not prompts: + return [], True + first = prompts[0] + if isinstance(first, int): + return [prompts], False + if isinstance(first, (str, list)): + return prompts, True + raise TypeError( + f"Unsupported prompt element type: {type(first)}; " + "expected str, list[int], list[str], or list[list[int]]." + ) + raise TypeError( + f"prompts must be str, list[int], list[str], or list[list[int]]; " + f"got {type(prompts)}" + ) + + # ---- private impl coroutines ---- + # Subclasses' public methods bridge to these via ``_EventLoopManager`` + # (coordinator mode, on the runtime loop) or await them directly + # (direct mode, on the caller's event loop). + # We need this bridge in coordinator mode because the coordinator requires + # a long running event loop, so we need to route the user's event + # loop to our runtime loop + + async def _generate_impl( + self, prompts: Union[List[str], List[List[int]]], sp: SamplingParams + ) -> List["DynamicInferenceRequest"]: + """Run inference for a non-empty list of prompts; returns input-ordered list. + + - Coordinator mode: must run on the runtime loop (via + ``_loop_manager.run_async``); enqueues requests through + ``client.add_request`` and gathers all futures. + - Direct mode: runs on the caller's event loop; offloads the synchronous + ``engine.generate`` to a thread. + """ + if self._use_coordinator: + # ``add_request`` calls ``asyncio.get_running_loop().create_future()`` + # so it must be invoked from a coroutine on the runtime loop. This + # coroutine runs on that same loop, so ``asyncio.gather`` over the + # returned futures is safe. + assert self._coord_runtime is not None and self._coord_runtime.client is not None + futures = [self._coord_runtime.client.add_request(p, sp) for p in prompts] + return list(await asyncio.gather(*futures)) + # TODO: replace with an upstream ``engine.async_generate`` so direct-mode + # async generate doesn't block the caller's event loop. + records = self._engine.generate(prompts, sp) + return [r.merge() for r in records] + + async def _pause_impl(self) -> None: + if self._is_primary_rank: + assert self._coord_runtime is not None and self._coord_runtime.client is not None + self._coord_runtime.client.pause_engines() + await self._engine.wait_until(EngineState.PAUSED) + + async def _unpause_impl(self) -> None: + if self._is_primary_rank: + assert self._coord_runtime is not None and self._coord_runtime.client is not None + self._coord_runtime.client.unpause_engines() + await self._engine.wait_until(EngineState.RUNNING) + + async def _suspend_impl(self) -> None: + if self._is_primary_rank: + assert self._coord_runtime is not None and self._coord_runtime.client is not None + self._coord_runtime.client.suspend_engines() + await self._engine.wait_until(EngineState.SUSPENDED) + + async def _resume_impl(self) -> None: + if self._is_primary_rank: + assert self._coord_runtime is not None and self._coord_runtime.client is not None + self._coord_runtime.client.resume_engines() + await self._engine.wait_until(EngineState.RESUMED) + + async def _shutdown_impl(self) -> None: + if self._is_primary_rank: + assert self._coord_runtime is not None and self._coord_runtime.client is not None + # The coordinator only honors STOP from PAUSED or SUSPENDED. If + # the engine is RUNNING (the typical state at shutdown), pause + # first so the STOP isn't ignored. + if self._engine.state == EngineState.RUNNING: + self._coord_runtime.client.pause_engines() + await self._engine.wait_until(EngineState.PAUSED) + self._coord_runtime.client.stop_engines() + await self._engine.wait_until(EngineState.STOPPED) + await self._coord_runtime.teardown() + else: + await self._engine.engine_loop_task + + async def _wait_for_shutdown_impl(self) -> None: + await self._engine.engine_loop_task diff --git a/megatron/core/inference/apis/async_llm.py b/megatron/core/inference/apis/async_llm.py new file mode 100644 index 00000000000..f2cea47b848 --- /dev/null +++ b/megatron/core/inference/apis/async_llm.py @@ -0,0 +1,231 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +"""Async high-level inference API for Megatron (``MegatronAsyncLLM``).""" + +from typing import List, Optional, Union + +from megatron.core.inference.apis._llm_base import _MegatronLLMBase +from megatron.core.inference.apis.serve_config import ServeConfig +from megatron.core.inference.config import InferenceConfig +from megatron.core.inference.inference_request import DynamicInferenceRequest +from megatron.core.inference.sampling_params import SamplingParams + + +class MegatronAsyncLLM(_MegatronLLMBase): + """Async high-level inference API for Megatron. + + Asyncio-native wrapper over the shared engine + runtime managed by + :class:`_MegatronLLMBase` -- see that class for caller responsibilities + and the ``model.eval()`` contract. Requires ``use_coordinator=True``; + direct mode is rejected at ``__init__`` (see Known Limitations in the + package README). + + On top of the base, this class provides: + + - ``async generate`` accepting single or batched prompts. + - ``async`` lifecycle controls: ``pause`` / ``unpause`` / ``suspend`` / + ``resume`` / ``shutdown`` / ``wait_for_shutdown``. + - :meth:`serve` for OpenAI-compatible HTTP serving on the primary rank. + - ``async with`` context-manager protocol; exit calls :meth:`shutdown`. + """ + + def __init__( + self, + *, + model, + tokenizer, + inference_config: Optional[InferenceConfig] = None, + use_coordinator: bool = False, + coordinator_host: Optional[str] = None, + coordinator_port: Optional[int] = None, + ) -> None: + # MegatronAsyncLLM requires coordinator mode: direct mode invokes the + # synchronous ``engine.generate()`` from inside the caller's asyncio + # loop, which collides with the engine's loop-bound internal state + # (``_cond``, ``_state_events``). Coordinator mode rebinds those to a + # daemon-thread loop via ``start_listening_to_data_parallel_coordinator`` + # and avoids the conflict. + if not use_coordinator: + raise ValueError( + "MegatronAsyncLLM requires use_coordinator=True. Direct mode is " + "not supported in async because the underlying engine's " + "asyncio primitives bind to the caller's loop and collide with " + "the synchronous engine.generate() path. Use MegatronLLM for " + "sync direct/coordinator workflows." + ) + super().__init__( + model=model, + tokenizer=tokenizer, + inference_config=inference_config, + use_coordinator=use_coordinator, + coordinator_host=coordinator_host, + coordinator_port=coordinator_port, + ) + # Set in serve() when this rank starts the HTTP frontend; consulted by shutdown(). + self._serve_started: bool = False + + async def generate( + self, + prompts: Union[str, List[int], List[str], List[List[int]]], + sampling_params: Optional[SamplingParams] = None, + ) -> Union["DynamicInferenceRequest", List["DynamicInferenceRequest"]]: + """Run inference for one prompt or a batch of prompts. + + Single input (``str`` or ``list[int]``) returns a single + ``DynamicInferenceRequest``; batched input (``list[str]`` or + ``list[list[int]]``) returns ``list[DynamicInferenceRequest]`` in + input order. + + Raises: + RuntimeError: if called on a non-primary rank. + """ + self._assert_primary() + if sampling_params is None: + sampling_params = SamplingParams() + + normalized, is_batch = self._normalize_prompts(prompts) + + if not normalized: + # Empty batch: nothing to schedule. ``is_batch`` is always True + # here since single input is wrapped to a one-element list. + return [] + + assert self._loop_manager is not None + results = await self._loop_manager.run_async( + self._generate_impl(normalized, sampling_params) + ) + return results if is_batch else results[0] + + async def pause(self) -> None: + """Transition the engine to ``PAUSED``. + + Raises: + RuntimeError: in direct mode (``use_coordinator=False``). + """ + self._assert_coordinator() + assert self._loop_manager is not None + await self._loop_manager.run_async(self._pause_impl()) + + async def unpause(self) -> None: + """Transition the engine from ``PAUSED`` back to ``RUNNING``. + + Raises: + RuntimeError: in direct mode (``use_coordinator=False``). + """ + self._assert_coordinator() + assert self._loop_manager is not None + await self._loop_manager.run_async(self._unpause_impl()) + + async def suspend(self) -> None: + """Transition the engine to ``SUSPENDED`` (offloads GPU buffers). + + The caller must ``pause()`` first; this method does not enforce that. + + Raises: + RuntimeError: in direct mode (``use_coordinator=False``). + """ + self._assert_coordinator() + assert self._loop_manager is not None + await self._loop_manager.run_async(self._suspend_impl()) + + async def resume(self) -> None: + """Transition the engine from ``SUSPENDED`` to ``RESUMED``. + + Raises: + RuntimeError: in direct mode (``use_coordinator=False``). + """ + self._assert_coordinator() + assert self._loop_manager is not None + await self._loop_manager.run_async(self._resume_impl()) + + async def shutdown(self) -> None: + """Stop the engine, tear down the coordinator, and join the runtime thread. + + Idempotent. No-op in direct mode. + """ + if self._shutdown_called: + return + self._shutdown_called = True + + # If we started an HTTP frontend, stop it first so no new requests + # arrive while we tear down the coordinator. Invariant: + # ``_serve_started`` can only be True when ``use_coordinator=True`` + # because ``serve()`` raises otherwise. + if self._serve_started: + from megatron.core.inference.text_generation_server.dynamic_text_gen_server.text_generation_server import ( # pylint: disable=line-too-long + stop_text_gen_server, + ) + + stop_text_gen_server() + self._serve_started = False + + if not self._use_coordinator: + return + assert self._loop_manager is not None + await self._loop_manager.run_async(self._shutdown_impl()) + self._loop_manager.stop() + + async def serve(self, serve_config: ServeConfig, *, blocking: bool = True) -> None: + """Start the OpenAI-compatible HTTP frontend. + + Coordinator mode only. The HTTP frontend runs only on the primary + rank (global rank 0); other ranks no-op the HTTP setup but still + respect ``blocking`` (so all ranks return together). + + With ``blocking=True`` (default), this awaits the engine loop until + :meth:`shutdown` is called -- suitable for standalone serving scripts. + With ``blocking=False``, this returns once the HTTP frontend is up + (primary) or immediately (workers); the engine loop continues in the + background runtime, and the user can call :meth:`generate` / + :meth:`shutdown` afterward. + + Raises: + ValueError: if ``use_coordinator=False`` (HTTP serving requires + the coordinator path). + """ + if not self._use_coordinator: + raise ValueError("MegatronAsyncLLM.serve() requires use_coordinator=True") + + if self._is_primary_rank: + # Lazy import: keep the module importable in environments where + # the HTTP server backend (Quart/Hypercorn) isn't installed. + import torch.distributed as dist + + from megatron.core.inference.text_generation_server.dynamic_text_gen_server.text_generation_server import ( # pylint: disable=line-too-long + start_text_gen_server, + ) + + assert self._coord_runtime is not None + start_text_gen_server( + coordinator_addr=self._coord_runtime.coord_addr, + tokenizer=self._controller.tokenizer, + rank=dist.get_rank(), + server_port=serve_config.port, + parsers=serve_config.parsers, + verbose=serve_config.verbose, + num_replicas=serve_config.frontend_replicas, + hostname=serve_config.host, + ) + self._serve_started = True + + if blocking: + # Block until the engine loop terminates (shutdown was invoked + # somewhere in this process; for serve(blocking=True) typically by + # SIGINT or out-of-band orchestration). + await self.wait_for_shutdown() + + async def wait_for_shutdown(self) -> None: + """Block until the engine's background loop task terminates. + + No-op in direct mode. + """ + if not self._use_coordinator: + return + assert self._loop_manager is not None + await self._loop_manager.run_async(self._wait_for_shutdown_impl()) + + async def __aenter__(self) -> "MegatronAsyncLLM": + return self + + async def __aexit__(self, exc_type, exc, tb) -> None: + await self.shutdown() diff --git a/megatron/core/inference/apis/llm.py b/megatron/core/inference/apis/llm.py new file mode 100644 index 00000000000..7179bafa427 --- /dev/null +++ b/megatron/core/inference/apis/llm.py @@ -0,0 +1,153 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +"""Sync high-level inference API for Megatron (``MegatronLLM``).""" + +from typing import List, Optional, Union + +from megatron.core.inference.apis._llm_base import _MegatronLLMBase +from megatron.core.inference.config import InferenceConfig +from megatron.core.inference.inference_request import DynamicInferenceRequest +from megatron.core.inference.sampling_params import SamplingParams + + +class MegatronLLM(_MegatronLLMBase): + """Sync high-level inference API for Megatron. + + See :class:`_MegatronLLMBase` for execution modes (direct vs + coordinator), caller responsibilities, and the ``model.eval()`` contract. + + On top of the base, this class provides: + + - :meth:`generate` accepting one prompt or a batch; **always returns a + ``list[DynamicInferenceRequest]``** (single-prompt input returns a + one-element list -- deliberate asymmetry vs the async API). + - Sync lifecycle controls: :meth:`pause` / :meth:`unpause` / + :meth:`suspend` / :meth:`resume` / :meth:`shutdown` / + :meth:`wait_for_shutdown`. + - Context-manager protocol: ``with MegatronLLM(...) as llm:``; exit + calls :meth:`shutdown`. + + Note: + ``serve()`` (online HTTP serving) is async-only by design; use + :class:`MegatronAsyncLLM` for serving. + """ + + def __init__( + self, + *, + model, + tokenizer, + inference_config: Optional[InferenceConfig] = None, + use_coordinator: bool = False, + coordinator_host: Optional[str] = None, + coordinator_port: Optional[int] = None, + ) -> None: + super().__init__( + model=model, + tokenizer=tokenizer, + inference_config=inference_config, + use_coordinator=use_coordinator, + coordinator_host=coordinator_host, + coordinator_port=coordinator_port, + ) + + def generate( + self, + prompts: Union[str, List[int], List[str], List[List[int]]], + sampling_params: Optional[SamplingParams] = None, + ) -> List["DynamicInferenceRequest"]: + """Run inference for one prompt or a batch. + + Returns ``list[DynamicInferenceRequest]`` in input order. Single-prompt + input returns a one-element list -- the always-list shape is the + deliberate sync-vs-async asymmetry. + + No concurrency guard: sync is single-caller by Python's GIL. If you + need to call ``generate`` concurrently from multiple threads, callers + must serialize externally. + + Raises: + RuntimeError: if called on a non-primary rank in coordinator mode. + """ + self._assert_primary() + if sampling_params is None: + sampling_params = SamplingParams() + + normalized, _is_batch = self._normalize_prompts(prompts) + if not normalized: + return [] + + if self._use_coordinator: + assert self._loop_manager is not None + return self._loop_manager.run_sync(self._generate_impl(normalized, sampling_params)) + # Direct mode: bypass _generate_impl (which would use to_thread, + # pointless for sync). Call the engine directly and merge. + records = self._engine.generate(normalized, sampling_params) + return [r.merge() for r in records] + + def pause(self) -> None: + """Transition the engine to ``PAUSED``. Coordinator mode only. + + Raises: + RuntimeError: in direct mode (``use_coordinator=False``). + """ + self._assert_coordinator() + assert self._loop_manager is not None + self._loop_manager.run_sync(self._pause_impl()) + + def unpause(self) -> None: + """Transition the engine from ``PAUSED`` back to ``RUNNING``. + + Raises: + RuntimeError: in direct mode (``use_coordinator=False``). + """ + self._assert_coordinator() + assert self._loop_manager is not None + self._loop_manager.run_sync(self._unpause_impl()) + + def suspend(self) -> None: + """Transition the engine to ``SUSPENDED`` (offloads GPU buffers). + + The caller must ``pause()`` first; this method does not enforce that. + + Raises: + RuntimeError: in direct mode (``use_coordinator=False``). + """ + self._assert_coordinator() + assert self._loop_manager is not None + self._loop_manager.run_sync(self._suspend_impl()) + + def resume(self) -> None: + """Transition the engine from ``SUSPENDED`` to ``RESUMED``. + + Raises: + RuntimeError: in direct mode (``use_coordinator=False``). + """ + self._assert_coordinator() + assert self._loop_manager is not None + self._loop_manager.run_sync(self._resume_impl()) + + def shutdown(self) -> None: + """Tear down the engine and runtime. Idempotent. Direct mode is a no-op.""" + if self._shutdown_called: + return + self._shutdown_called = True + if not self._use_coordinator: + return # direct mode: nothing to tear down + assert self._loop_manager is not None + self._loop_manager.run_sync(self._shutdown_impl()) + # Sync caller already on its own thread; no need for to_thread. + self._loop_manager.stop() + + def wait_for_shutdown(self) -> None: + """Block until the engine loop terminates. Direct mode no-op.""" + if not self._use_coordinator: + return + assert self._loop_manager is not None + self._loop_manager.run_sync(self._wait_for_shutdown_impl()) + + def __enter__(self) -> "MegatronLLM": + return self + + def __exit__(self, exc_type, exc, tb) -> None: + self.shutdown() diff --git a/megatron/core/inference/apis/serve_config.py b/megatron/core/inference/apis/serve_config.py new file mode 100644 index 00000000000..aa7c6afe8fd --- /dev/null +++ b/megatron/core/inference/apis/serve_config.py @@ -0,0 +1,44 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +from dataclasses import dataclass, field + + +@dataclass +class ServeConfig: + """Programmatic configuration for ``MegatronAsyncLLM.serve(...)``. + + This dataclass also serves as the future source of truth for a + ``megatron serve`` CLI. It controls only the HTTP serving surface; engine + construction and coordinator addressing are configured separately via the + ``MegatronLLM`` / ``MegatronAsyncLLM`` constructor. + """ + + host: str = "0.0.0.0" + """HTTP bind host for the OpenAI-compatible frontend. + + Distinct from the ``MegatronLLM`` / ``MegatronAsyncLLM`` constructor's + ``coordinator_host`` argument: ``coordinator_host`` is the internal/routable + address used for coordinator ZMQ traffic, whereas ``host`` is the + externally-visible interface where the HTTP server accepts client + connections. + """ + + port: int = 5000 + """HTTP bind port for the OpenAI-compatible frontend.""" + + parsers: list[str] = field(default_factory=list) + """Response parser names to enable on the HTTP frontend. + + Examples include ``["json", "tool_use"]``. Values are passed through to the + underlying text-generation server unchanged. + """ + + verbose: bool = False + """Whether the HTTP frontend should log per-request detail.""" + + frontend_replicas: int = 4 + """Number of HTTP frontend processes spawned on the primary rank. + + The default of 4 matches the existing ``start_text_gen_server`` default of + ``num_replicas=4``. + """ diff --git a/megatron/core/inference/batch_dimensions_utils.py b/megatron/core/inference/batch_dimensions_utils.py index b9f62e59547..1229d333d0a 100644 --- a/megatron/core/inference/batch_dimensions_utils.py +++ b/megatron/core/inference/batch_dimensions_utils.py @@ -213,57 +213,72 @@ class CUDAGraphBatchDimensionBuilder: """ # Constant for rounding token counts when generating CUDA graph batch dimensions - CUDA_GRAPH_ROUNDER = 8 + CUDA_GRAPH_ROUNDER = 2 @staticmethod def _calculate_cuda_graph_token_counts( - tp_size: int, num_cuda_graphs: int, cuda_graph_max_tokens: int + tp_size: int, + num_cuda_graphs: int, + cuda_graph_max_tokens: int, + sizing_distribution: "CudaGraphSizingDistribution" = None, ) -> List[int]: """ Calculate CUDA graph token counts for a given configuration. - This method computes evenly-spaced token counts from step_size up to - cuda_graph_max_tokens, ensuring proper rounding and TP alignment. + Dispatches on `sizing_distribution`: + - EXPONENTIAL (default): halves from cuda_graph_max_tokens down to tp_size, log-spaced, + creates log2(max_tokens) graphs. + - LINEAR: small graphs [1, 2, 4] + range(8, 256, 8) + range(256, max+1, 16); + explicit-N path uses even 16-stride from 0 to max. Args: tp_size: Tensor parallel size (for alignment) - num_cuda_graphs: Number of CUDA graphs to generate (must be >= 1) + num_cuda_graphs: Number of CUDA graphs to generate (must be >= 1, or -1 to auto-size) cuda_graph_max_tokens: Maximum token count for CUDA graphs (must be > 0) + sizing_distribution: Distribution of cudagraph sizes. Defaults to EXPONENTIAL. Returns: List of token counts in descending order - Example: - >>> _calculate_cuda_graph_token_counts - (tp_size=2, num_cuda_graphs=4, cuda_graph_max_tokens=1000) - [1000, 752, 504, 256] + Example (EXPONENTIAL): + >>> _calculate_cuda_graph_token_counts(tp_size=1, num_cuda_graphs=8, + cuda_graph_max_tokens=128) + [128, 64, 32, 16, 8, 4, 2, 1] """ - if num_cuda_graphs == -1: - # automatically determine the number of CUDA graphs to - # capture based on the `max_requests` value - cuda_graph_token_counts = ( - [1, 2, 4] + list(range(8, 256, 8)) + list(range(256, cuda_graph_max_tokens + 1, 16)) + from megatron.core.inference.config import CudaGraphSizingDistribution + + if sizing_distribution is None: + sizing_distribution = CudaGraphSizingDistribution.EXPONENTIAL + + if sizing_distribution == CudaGraphSizingDistribution.LINEAR: + return CUDAGraphBatchDimensionBuilder._calculate_token_counts_linear( + tp_size, num_cuda_graphs, cuda_graph_max_tokens ) - # Align each entry to TP size - cuda_graph_token_counts = list( - dict.fromkeys( - round_up_to_nearest_multiple(s, tp_size) for s in cuda_graph_token_counts - ) + + # Default path: exponential decay. + if num_cuda_graphs == -1: + # Pick a graph count: we halve from cuda_graph_max_tokens down to 1, so + # log2(max_tokens) halvings are needed. Add a small margin for the two forced endpoints + # (cuda_graph_max_tokens and tp_size) that are unioned into the set after the loop. + # Floor at MIN_GRAPHS so the trim logic always has at least 2 entries to work with. + HEADROOM = 2 + MIN_GRAPHS = 4 + num_halvings = int(math.log2(max(2, cuda_graph_max_tokens))) + auto_n = max(MIN_GRAPHS, num_halvings + HEADROOM) + return CUDAGraphBatchDimensionBuilder._calculate_cuda_graph_token_counts( + tp_size=tp_size, + num_cuda_graphs=auto_n, + cuda_graph_max_tokens=cuda_graph_max_tokens, + sizing_distribution=sizing_distribution, ) - # Clamp to max tokens - cuda_graph_token_counts = [ - s for s in cuda_graph_token_counts if s <= cuda_graph_max_tokens - ] - if not cuda_graph_token_counts or cuda_graph_token_counts[-1] != cuda_graph_max_tokens: - cuda_graph_token_counts.append(cuda_graph_max_tokens) - cuda_graph_token_counts.reverse() - return cuda_graph_token_counts assert num_cuda_graphs >= 1, f"num_cuda_graphs must be >= 1, got {num_cuda_graphs}" assert ( cuda_graph_max_tokens > 0 ), f"cuda_graph_max_tokens must be > 0, got {cuda_graph_max_tokens}" + rounder = CUDAGraphBatchDimensionBuilder.CUDA_GRAPH_ROUNDER + # Cuda graph step size. cuda_graph_step_size = cuda_graph_max_tokens / num_cuda_graphs cuda_graph_step_size = CUDAGraphBatchDimensionBuilder.CUDA_GRAPH_ROUNDER * int( @@ -271,26 +286,90 @@ def _calculate_cuda_graph_token_counts( ) # Make sure divisible by TP size cuda_graph_step_size = round_up_to_nearest_multiple(cuda_graph_step_size, tp_size) + # Ensure non-zero step size (can happen when max_tokens < num_cuda_graphs). + cuda_graph_step_size = max(cuda_graph_step_size, tp_size) - # round down cuda graph max tokens to be multiple of TP size + # Round down cuda graph max tokens to be multiple of TP size cuda_graph_max_tokens = (cuda_graph_max_tokens // tp_size) * tp_size - # Cuda graph token counts. if num_cuda_graphs == 1: - cuda_graph_token_counts = [cuda_graph_max_tokens] - else: - cuda_graph_token_counts = list( - range(cuda_graph_step_size, cuda_graph_max_tokens, cuda_graph_step_size) - ) - if ( - len(cuda_graph_token_counts) == 0 - or cuda_graph_token_counts[-1] != cuda_graph_max_tokens - ): - cuda_graph_token_counts.append(cuda_graph_max_tokens) - cuda_graph_token_counts.reverse() + return [cuda_graph_max_tokens] + + # Exponentially decreasing token counts: halve from max_tokens until below the rounder floor + # or num_cuda_graphs. Dedupe (the rounding/TP-alignment can collide for small values), + # then sort descending. + sizes = set() + val = cuda_graph_max_tokens + for _ in range(num_cuda_graphs): + # Round down to multiple of rounder, then up to multiple of TP size + rounded = max(rounder, (val // rounder) * rounder) + rounded = math.ceil(rounded / tp_size) * tp_size + sizes.add(rounded) + val //= 2 + if val < 1: + break + + # Always include the endpoints: cuda_graph_max_tokens (largest) and tp_size (smallest). + sizes.add(cuda_graph_max_tokens) + sizes.add(tp_size) + + cuda_graph_token_counts = sorted(sizes, reverse=True) + + # Trim from the middle if we exceed num_cuda_graphs requested by the user. + # Since num_cuda_graphs >= 1, this only runs when we have at least 2 elements. + while len(cuda_graph_token_counts) > num_cuda_graphs: + cuda_graph_token_counts.pop(-2) + + assert len(cuda_graph_token_counts) <= num_cuda_graphs + assert cuda_graph_max_tokens in cuda_graph_token_counts return cuda_graph_token_counts + @staticmethod + def _calculate_token_counts_linear( + tp_size: int, num_cuda_graphs: int, cuda_graph_max_tokens: int + ) -> List[int]: + """Linear-stride token count distribution. + + For num_cuda_graphs == -1, returns [1, 2, 4] + range(8, 256, 8) + range(256, max+1, 16) + TP-aligned and deduped. + For positive N, returns evenly-spaced sizes with step ~ max_tokens / N. + """ + rounder = CUDAGraphBatchDimensionBuilder.CUDA_GRAPH_ROUNDER + + if num_cuda_graphs == -1: + sizes = ( + [1, 2, 4] + list(range(8, 256, 8)) + list(range(256, cuda_graph_max_tokens + 1, 16)) + ) + # TP-align and dedupe in order; preserve original ordering for parity. + sizes = list(dict.fromkeys(round_up_to_nearest_multiple(s, tp_size) for s in sizes)) + sizes = [s for s in sizes if s <= cuda_graph_max_tokens] + if not sizes or sizes[-1] != cuda_graph_max_tokens: + sizes.append(cuda_graph_max_tokens) + sizes.reverse() + return sizes + + assert num_cuda_graphs >= 1, f"num_cuda_graphs must be >= 1, got {num_cuda_graphs}" + assert ( + cuda_graph_max_tokens > 0 + ), f"cuda_graph_max_tokens must be > 0, got {cuda_graph_max_tokens}" + + # Even stride: step = round_up_to(max / N, rounder), TP-aligned. + step = cuda_graph_max_tokens / num_cuda_graphs + step = rounder * int(math.ceil(int(step) / rounder)) + step = round_up_to_nearest_multiple(step, tp_size) + step = max(step, tp_size) + cuda_graph_max_tokens = (cuda_graph_max_tokens // tp_size) * tp_size + + if num_cuda_graphs == 1: + return [cuda_graph_max_tokens] + + sizes = list(range(step, cuda_graph_max_tokens, step)) + if not sizes or sizes[-1] != cuda_graph_max_tokens: + sizes.append(cuda_graph_max_tokens) + sizes.reverse() + return sizes + @staticmethod def generate_cuda_graph_batch_dimensions_list( tp_size: int, @@ -302,6 +381,7 @@ def generate_cuda_graph_batch_dimensions_list( max_sequence_length: int, use_cuda_graphs_for_non_decode_steps: bool, num_speculative_tokens: int = 0, + sizing_distribution: "CudaGraphSizingDistribution" = None, ) -> Tuple[List[InferenceBatchDimensions], Optional[List[int]]]: """ Generate CUDA graph batch dimensions. @@ -359,6 +439,12 @@ def add_if_valid(token_count: int, prefill_req_count: int, decode_req_count: int cuda_graph_decode_token_counts = None if num_cuda_graphs is not None: + # Lazy import to avoid a circular dependency with config.py. + from megatron.core.inference.config import CudaGraphSizingDistribution + + if sizing_distribution is None: + sizing_distribution = CudaGraphSizingDistribution.EXPONENTIAL + # Ensure valid num_cuda_graphs. if ( cuda_graph_max_tokens is None @@ -367,11 +453,9 @@ def add_if_valid(token_count: int, prefill_req_count: int, decode_req_count: int ): cuda_graph_max_tokens = max_tokens - assert cuda_graph_max_tokens == max_requests * (num_speculative_tokens + 1), ( - f"cuda_graph_max_tokens ({cuda_graph_max_tokens}) must equal max_requests *" - f"(num_speculative_tokens + 1) ({max_requests * (num_speculative_tokens + 1)}). " - "This is required for correctly syncing EP ranks: " - f"prefill and decode graph pools must have the same token count granularity." + assert cuda_graph_max_tokens >= max_requests * (num_speculative_tokens + 1), ( + f"cuda_graph_max_tokens ({cuda_graph_max_tokens}) must be >= max_requests * " + f"(num_speculative_tokens + 1) ({max_requests * (num_speculative_tokens + 1)})." ) if num_cuda_graphs != -1: @@ -387,6 +471,7 @@ def add_if_valid(token_count: int, prefill_req_count: int, decode_req_count: int tp_size=tp_size, num_cuda_graphs=num_cuda_graphs, cuda_graph_max_tokens=cuda_graph_max_tokens, + sizing_distribution=sizing_distribution, ) ) @@ -399,9 +484,33 @@ def add_if_valid(token_count: int, prefill_req_count: int, decode_req_count: int tp_size=tp_size, num_cuda_graphs=num_cuda_graphs, cuda_graph_max_tokens=cuda_graph_max_tokens_decode, + sizing_distribution=sizing_distribution, ) ) + # Include the smallest decode-only graphs when auto-sizing (num_cuda_graphs == -1). + # Without this, TP alignment and the num_speculative_tokens floor division can drop + # the smallest 1- and 2-request shapes from the captured set. + # + # The minimum valid decode token_count is lcm(spec_unit, tp_size): + # - Ensure divisible by tp_size (required so TP / sequence-parallel never produces a + # single-token graph when tp_size > 1). + # - Ensure a multiple of (spec+1) so it accommodates an integer number of decode + # requests when speculative decoding is enabled. + if num_cuda_graphs == -1: + spec_unit = num_speculative_tokens + 1 + min_decode_tokens = math.lcm(spec_unit, tp_size) + for req_count_multiple in (1, 2): + floor_tokens = min_decode_tokens * req_count_multiple + if ( + floor_tokens <= cuda_graph_max_tokens_decode + and floor_tokens not in cuda_graph_decode_token_counts + ): + cuda_graph_decode_token_counts.append(floor_tokens) + cuda_graph_decode_token_counts = sorted( + set(cuda_graph_decode_token_counts), reverse=True + ) + cuda_graph_batch_dimensions_list = [] if num_cuda_graphs is None: cuda_graph_batch_dimensions_list = [] @@ -419,34 +528,62 @@ def add_if_valid(token_count: int, prefill_req_count: int, decode_req_count: int token_count=token_count, prefill_req_count=0, decode_req_count=decode_req_count ) else: - # Mixed prefill and decode mode + # Mixed prefill and decode mode. + # + # Under EXPONENTIAL distribution (default): generate mixed CGs across a + # geometric P-grid {1, 2, 4, ..., max_requests}. This bounds the relative + # overhead per real batch (~2x P slack worst case) and is the structural fix + # that makes mixed CGs usable for real batches with P != fixed_P. + # + # Under LINEAR distribution: use the legacy fixed P value + # (cuda_graph_mixed_prefill_request_count) — same single-P behavior main has + # today, for apples-to-apples benchmarking against vLLM-style configurations. + if sizing_distribution == CudaGraphSizingDistribution.LINEAR: + p_values = [min(cuda_graph_mixed_prefill_request_count, max_requests)] + # In legacy mode, the prefill-only floor uses the fixed P value to match + # main's behavior exactly. + prefill_only_floor = cuda_graph_mixed_prefill_request_count + else: + p_values = [] + p = 1 + while p < max_requests: + p_values.append(p) + p *= 2 + if not p_values or p_values[-1] != max_requests: + p_values.append(max_requests) + prefill_only_floor = 1 + # Create prefill and mixed dimensions with full token counts for size in cuda_graph_prefill_token_counts: assert size % tp_size == 0 - prefill_req_count = min(cuda_graph_mixed_prefill_request_count, max_requests) - decode_req_count = max( - 0, - min( - (size - prefill_req_count) // (num_speculative_tokens + 1), - max_requests - prefill_req_count, - ), - ) - add_if_valid( - token_count=size, - prefill_req_count=prefill_req_count, - decode_req_count=decode_req_count, - ) + for prefill_req_count in p_values: + decode_req_count = max( + 0, + min( + (size - prefill_req_count) // (num_speculative_tokens + 1), + max_requests - prefill_req_count, + ), + ) + # Skip token_count == 1 with prefill_req == 1: the gather kernel asserts + # on index >= 1 against a 1-element tensor at capture time. Larger + # `(size, size, 0)` shapes (each prefill = 1 token, total batch >= 2) are + # fine because the gather has multiple indices to read. + if size < 2: + continue + add_if_valid( + token_count=size, + prefill_req_count=prefill_req_count, + decode_req_count=decode_req_count, + ) # We need to ensure the prefill requests are shorter than the max sequence length, # considering the one decode token is used for prefill request construction prefill_only_minimal_num = max( - cuda_graph_mixed_prefill_request_count, - math.ceil(size / max(1, max_sequence_length - 1)), + prefill_only_floor, math.ceil(size / max(1, max_sequence_length - 1)) ) - if prefill_only_minimal_num < max_requests: + if prefill_only_minimal_num < max_requests and size >= 2: + prefill_req_count = max(prefill_only_minimal_num, min(max_requests, size)) add_if_valid( - token_count=size, - prefill_req_count=max(prefill_only_minimal_num, min(max_requests, size)), - decode_req_count=0, + token_count=size, prefill_req_count=prefill_req_count, decode_req_count=0 ) # Create decode-only dimensions with optimized token counts diff --git a/megatron/core/inference/config.py b/megatron/core/inference/config.py index a39ec038051..ea4b08e5183 100644 --- a/megatron/core/inference/config.py +++ b/megatron/core/inference/config.py @@ -117,6 +117,22 @@ class KVCacheManagementMode(str, Enum): """Deallocate large tensors and recompute them from scratch during allocation.""" +class CudaGraphSizingDistribution(str, Enum): + """How CUDA graph token-count sizes are spaced when generating the captured graphs. + + EXPONENTIAL (default) — token counts halve from `cuda_graph_max_tokens` down to `tp_size`, + giving a log-spaced distribution. Bounded relative padding (~2x worst case) at every scale and + `log2(max_tokens)` total graphs. + + LINEAR — Include size-1 and size-2 graphs where applicable, linear spacing up until 256, and + sparser linear spacing past 256. e.g. `[1, 2, 4] + range(8, 256, 8) + range(256, max+1, 16)`. + Higher graph density at the top end. + """ + + EXPONENTIAL = "exponential" + LINEAR = "linear" + + @dataclass class InferenceConfig: """ @@ -188,20 +204,42 @@ class InferenceConfig: # ================================= num_cuda_graphs: Optional[int] = None """ - Maximum number of cuda graphs to capture, where the cuda graph batch sizes range from 1 to - `max_requests`. Due to rounding, the actual number of cuda graphs may not equal this argument. + Maximum number of cuda graphs to capture. + Graph token counts are spaced from 1 up to a per-graph-type budget: + - Decode-only graphs are always bounded by `max_requests * (num_speculative_tokens + 1)`. + - Prefill/mixed graphs share that same bound by default, + or extend up to `max_tokens` when `cuda_graph_all_prefills` is set. + Due to rounding, the actual number of cuda graphs may not equal this argument. """ cuda_graph_mixed_prefill_count: Optional[int] = 16 - """ + """ The number of mixed prefill graphs to capture if mixed prefill/decode graphs are enabled. """ + cuda_graph_sizing_distribution: CudaGraphSizingDistribution = ( + CudaGraphSizingDistribution.EXPONENTIAL + ) + """ + How CUDA graph token counts are spaced. EXPONENTIAL (default) halves from + `cuda_graph_max_tokens` down to `tp_size` (log-spaced, ~log2(max_tokens) graphs). + LINEAR uses a range of linear strides (includes small graphs + mid-range linearity + + a bigger step size at the top end). + """ + use_cuda_graphs_for_non_decode_steps: bool = True """ Whether to use CUDA graphs for non-decode steps. """ + cuda_graph_all_prefills: bool = False + """ + Whether prefill/mixed CUDA graphs should span up to `max_tokens`. + When False (default), prefill/mixed graphs are bounded by the same token limit as decode graphs: + `max_requests * (num_speculative_tokens + 1)`. + When True, prefill/mixed graph capture is extended to cover the full `max_tokens` budget. + """ + static_kv_memory_pointers: bool = False """ Whether the KV cache (and Mamba states) will reside at the same memory addresses diff --git a/megatron/core/inference/contexts/attention_context/mamba_metadata.py b/megatron/core/inference/contexts/attention_context/mamba_metadata.py index ff6423be16b..3e98f0324e6 100644 --- a/megatron/core/inference/contexts/attention_context/mamba_metadata.py +++ b/megatron/core/inference/contexts/attention_context/mamba_metadata.py @@ -348,7 +348,10 @@ def update( # This converts per-request token offsets to chunk indices and # absolute positions, padded to fixed size for CUDA graph compat. self._update_intermediate_metadata( - intermediate_offsets_gpu, intermediate_counts_gpu, real_prefill_count + intermediate_offsets_gpu, + intermediate_counts_gpu, + real_prefill_count, + padded_prefill_count, ) if padded_decode_count > 0 and padded_prefill_count > 0: @@ -363,6 +366,7 @@ def _update_intermediate_metadata( intermediate_offsets_gpu: Optional[torch.Tensor], intermediate_counts_gpu: Optional[torch.Tensor], real_prefill_count: int, + padded_prefill_count: int, cu_seqlens_gpu: Optional[torch.Tensor] = None, ) -> None: """Precompute intermediate extraction metadata for CUDA graph compatibility. @@ -383,7 +387,7 @@ def _update_intermediate_metadata( shared ``ContextGPUView.mamba_cu_seqlens`` view. """ chunk_size = self.mamba_chunk_size - max_count = self.max_intermediate_count + max_count = padded_prefill_count * MAX_INTERMEDIATE_OFFSETS_PER_REQUEST if cu_seqlens_gpu is None: cu_seqlens_gpu = self._cu_seqlens_buffer @@ -444,15 +448,15 @@ def _update_intermediate_metadata( # - abs_positions=d_conv: conv gather reads tokens [0..d_conv-1], # which are within bounds and produce a valid but unused state if real_count < max_count: - self._intermediate_chunk_indices_buffer[real_count:].fill_(0) - self._intermediate_abs_positions_buffer[real_count:].fill_(self.d_conv) + self._intermediate_chunk_indices_buffer[real_count:max_count].fill_(0) + self._intermediate_abs_positions_buffer[real_count:max_count].fill_(self.d_conv) self.intermediate_count = real_count self.per_request_intermediate_counts = counts_list else: # All counts are 0 - self._intermediate_chunk_indices_buffer.fill_(0) - self._intermediate_abs_positions_buffer.fill_(self.d_conv) + self._intermediate_chunk_indices_buffer[:max_count] = 0 + self._intermediate_abs_positions_buffer[:max_count] = self.d_conv self.intermediate_count = 0 self.per_request_intermediate_counts = counts_list @@ -461,8 +465,8 @@ def _update_intermediate_metadata( else: # No extraction: fill with safe defaults for CUDA graph warmup # (same rationale as padding comment above) - self._intermediate_chunk_indices_buffer.fill_(0) - self._intermediate_abs_positions_buffer.fill_(self.d_conv) + self._intermediate_chunk_indices_buffer[:max_count] = 0 + self._intermediate_abs_positions_buffer[:max_count] = self.d_conv self.intermediate_count = 0 self.per_request_intermediate_counts = [] self.intermediate_chunk_indices = self._intermediate_chunk_indices_buffer[:max_count] @@ -677,6 +681,7 @@ def load_from_cpu(self, d: dict) -> None: d["intermediate_offsets_gpu"], d["intermediate_counts_gpu"], real_prefill_count, + padded_prefill_count, cu_seqlens_gpu=v.mamba_cu_seqlens, ) diff --git a/megatron/core/inference/contexts/dynamic_context.py b/megatron/core/inference/contexts/dynamic_context.py index e0d341bac70..1aa460f8fab 100644 --- a/megatron/core/inference/contexts/dynamic_context.py +++ b/megatron/core/inference/contexts/dynamic_context.py @@ -41,7 +41,7 @@ ) from megatron.core.utils import deprecate_args from megatron.core.utils import divide as core_divide -from megatron.core.utils import get_pg_size, internal_api +from megatron.core.utils import get_pg_rank, get_pg_size, internal_api from .attention_context.mamba_metadata import MambaMetadata from .attention_context.mha_metadata import GraphedMHAMetadata, NonGraphedMHAMetadata @@ -361,7 +361,25 @@ def __init__(self, model_config: TransformerConfig, inference_config: InferenceC self.layer_map = attention_layer_map | dsa_layer_map | mamba_layer_map else: # The layer map is the identity function for pure Transformer models. - self.num_attention_layers = model_config.num_layers // pp_size + # Use the same per-PP-rank layer count as TransformerBlock (handles + # account_for_embedding_in_pipeline_split, account_for_loss_in_pipeline_split, + # uneven first/last PP stages, and pipeline_model_parallel_layout). Using + # num_layers // pp_size mis-sizes the KV layer_map and can raise KeyError in + # append_key_value_cache. + from megatron.core.transformer.transformer_block import get_num_layers_to_build + + # Interleaved / virtual PP is not used for inference (see + # AbstractModelInferenceWrapper: Iterable models are rejected). Always pass + # vp_stage=None into get_num_layers_to_build, consistent with attention inference + # (e.g. get_transformer_layer_offset(..., vp_stage=None, pp_rank=...)). + # When pg_collection is set, use the PP group's rank (same as attention.py). + if pg_collection is not None: + pp_rank = get_pg_rank(pg_collection.pp) + else: + pp_rank = None + self.num_attention_layers = get_num_layers_to_build( + model_config, vp_stage=None, pp_rank=pp_rank + ) self.num_mamba_layers = 0 (self.mamba_conv_states_shape, self.mamba_ssm_states_shape) = (None, None) self.layer_map = {i: i for i in range(self.num_attention_layers)} @@ -623,18 +641,28 @@ def __init__(self, model_config: TransformerConfig, inference_config: InferenceC and not (force_disable_non_decode_cuda_graphs) ) + # CUDA graph token budget for prefill/mixed graphs. Decode graphs are always + # capped at max_requests * (num_speculative_tokens + 1) inside the helper; this + # only widens the prefill/mixed range when `cuda_graph_all_prefills` is set. + cuda_graph_max_tokens = ( + self.max_tokens + if inference_config.cuda_graph_all_prefills + else self.max_requests * (self.num_speculative_tokens + 1) + ) + # CUDA graph config list. self.cuda_graph_batch_dimensions_list, self.cuda_graph_token_counts = ( CUDAGraphBatchDimensionBuilder.generate_cuda_graph_batch_dimensions_list( tp_size=tp_size, num_cuda_graphs=inference_config.num_cuda_graphs, - cuda_graph_max_tokens=self.max_requests * (self.num_speculative_tokens + 1), + cuda_graph_max_tokens=cuda_graph_max_tokens, cuda_graph_mixed_prefill_request_count=inference_config.cuda_graph_mixed_prefill_count, max_requests=self.max_requests, max_tokens=self.max_tokens, max_sequence_length=self.max_sequence_length, use_cuda_graphs_for_non_decode_steps=self.use_cuda_graphs_for_non_decode_steps, num_speculative_tokens=self.num_speculative_tokens, + sizing_distribution=inference_config.cuda_graph_sizing_distribution, ) ) @@ -939,7 +967,7 @@ def initialize_all_tensors(self) -> None: # then int32 token fields, then int32/float32 request-staging fields. # token_to_input_ids (int64, max_tokens) # token_to_pos_ids (int64, max_tokens) - # token_to_block_idx (int32, max_tokens) + # token_to_block_idx (int64, max_tokens) # token_to_local_position_within_kv_block (int32, max_tokens) # token_to_request_idx (int32, max_tokens) # token_to_position_in_request (int32, max_tokens) @@ -969,11 +997,24 @@ def initialize_all_tensors(self) -> None: _mha_kv_seq_lengths_bytes = self.max_requests * 4 _mha_cu_kv_seq_lengths_bytes = (self.max_requests + 1) * 4 _mha_block_table_bytes = self.max_requests * self.max_kv_block_count * 4 - # Mamba section: 9 int32 fields (hybrid models only). Must match the - # MambaMetadata shapes (mirrors the layout documented in ContextGPUView). + _pre_mamba_bytes = ( + 3 * _tok_int64_bytes + + 3 * _tok_int32_bytes + + 7 * _req_4byte_bytes + + _mha_query_lengths_bytes + + _mha_cu_query_seq_lengths_bytes + + _mha_kv_seq_lengths_bytes + + _mha_cu_kv_seq_lengths_bytes + + _mha_block_table_bytes + ) + # Mamba section (hybrid models only). Must match the MambaMetadata + # shapes (mirrors the layout documented in ContextGPUView). + # batch_indices_decode is int64; all other fields are int32. if self.is_hybrid_model: + # mamba_batch_indices_decode is int64; pad to 8-byte alignment. + _mamba_align_pad = (8 - _pre_mamba_bytes % 8) % 8 self._max_mamba_chunks = self.max_tokens // self.mamba_chunk_size + self.max_requests - _mamba_batch_indices_decode_bytes = self.max_requests * 4 + _mamba_batch_indices_decode_bytes = self.max_requests * 8 _mamba_batch_indices_prefill_bytes = self.max_requests * 4 _mamba_seq_idx_bytes = self.max_tokens * 4 _mamba_cu_seqlens_bytes = (self.max_requests + 1) * 4 @@ -983,6 +1024,7 @@ def initialize_all_tensors(self) -> None: _mamba_conv_seq_idx_bytes = self.max_tokens * 4 _mamba_conv_seq_start_bytes = self.max_tokens * 4 else: + _mamba_align_pad = 0 self._max_mamba_chunks = 0 _mamba_batch_indices_decode_bytes = 0 _mamba_batch_indices_prefill_bytes = 0 @@ -994,14 +1036,8 @@ def initialize_all_tensors(self) -> None: _mamba_conv_seq_idx_bytes = 0 _mamba_conv_seq_start_bytes = 0 _total_bytes = ( - 2 * _tok_int64_bytes - + 4 * _tok_int32_bytes - + 7 * _req_4byte_bytes - + _mha_query_lengths_bytes - + _mha_cu_query_seq_lengths_bytes - + _mha_kv_seq_lengths_bytes - + _mha_cu_kv_seq_lengths_bytes - + _mha_block_table_bytes + _pre_mamba_bytes + + _mamba_align_pad + _mamba_batch_indices_decode_bytes + _mamba_batch_indices_prefill_bytes + _mamba_seq_idx_bytes @@ -1032,10 +1068,10 @@ def initialize_all_tensors(self) -> None: torch.long ) _off += _tok_int64_bytes - self.token_to_block_idx = self._cpu_bookkeeping_buf[_off : _off + _tok_int32_bytes].view( - torch.int32 + self.token_to_block_idx = self._cpu_bookkeeping_buf[_off : _off + _tok_int64_bytes].view( + torch.int64 ) - _off += _tok_int32_bytes + _off += _tok_int64_bytes # i.e For a set of tokens A B C D E F .. and block_size 4: # token_to_position_in_request is [0, 1, 2, 3, 4, 5] # token_to_local_position_within_kv_block is [0 , 1, 2, 3, 0, 1, 2] @@ -1131,9 +1167,10 @@ def initialize_all_tensors(self) -> None: # by MambaMetadata.compute_cpu_metadata(); transferred as part of the # single coalesced H2D in transfer_bookkeeping_to_gpu(). if self.is_hybrid_model: + _off += _mamba_align_pad self._cpu_mamba_batch_indices_decode = self._cpu_bookkeeping_buf[ _off : _off + _mamba_batch_indices_decode_bytes - ].view(torch.int32) + ].view(torch.int64) _off += _mamba_batch_indices_decode_bytes self._cpu_mamba_batch_indices_prefill = self._cpu_bookkeeping_buf[ _off : _off + _mamba_batch_indices_prefill_bytes @@ -1649,6 +1686,7 @@ def apply_rotary_emb_query( cp_group=cp_group, mscale=mscale, mla_rotary_interleaved=config.multi_latent_attention, + max_seqlen=query_emb.size(0), ) return query @@ -2401,10 +2439,6 @@ def reset_metadata(self) -> None: self.kv_block_allocator.reset() self.request_to_kv_block_ids.fill_(-1) - # Reset step counter and LRU clock - self.step_count = 0 - self.prefix_cache_lru_clock = 0 - # Reset chunked prefill state self.chunked_prefill_request_id = -1 self.num_prefill_requests = 0 @@ -2429,6 +2463,11 @@ def reset(self) -> None: self.reset_tensors() self.reset_metadata() + # Reset lifetime counters (not reset in reset_metadata, which is also + # called during suspend/resume where these must persist). + self.step_count = 0 + self.prefix_cache_lru_clock = 0 + # Reset Mamba cache state if self.mamba_slot_allocator is not None: self.mamba_slot_allocator.reset() diff --git a/megatron/core/inference/contexts/gpu_view.py b/megatron/core/inference/contexts/gpu_view.py index 65c401163b0..651d95055da 100644 --- a/megatron/core/inference/contexts/gpu_view.py +++ b/megatron/core/inference/contexts/gpu_view.py @@ -37,8 +37,8 @@ def __init__( # fields, then int32 request fields, then int32 MHA fields, then # int32 Mamba fields (hybrid models only; omitted when # max_mamba_chunks == 0). - tok_int64_bytes = max_tokens * 8 # 2 fields of int64 = 8 bytes/elem - tok_int32_bytes = max_tokens * 4 # 4 fields of int32 = 4 bytes/elem + tok_int64_bytes = max_tokens * 8 # 3 fields of int64 = 8 bytes/elem + tok_int32_bytes = max_tokens * 4 # 3 fields of int32 = 4 bytes/elem # Request-level fields are all 4 bytes wide. 3 int32 (in_prefill_status, # query_lengths, kv_length_offsets) + 1 int32 (top_k) + 2 float32 # (temperature, top_p) + 1 int32 (active_request_last_token_idxs) = 7 fields. @@ -59,8 +59,8 @@ def __init__( mha_cu_kv_seq_lengths_bytes = (max_bs + 1) * 4 mha_block_table_bytes = max_bs * max_kv_blocks * 4 - # Mamba section: 9 int32 fields, only present for hybrid models. - # mamba_batch_indices_decode int32 (max_bs,) + # Mamba section, only present for hybrid models. + # mamba_batch_indices_decode int64 (max_bs,) # mamba_batch_indices_prefill int32 (max_bs,) # mamba_seq_idx int32 (1, max_tokens) # mamba_cu_seqlens int32 (max_bs + 1,) @@ -69,8 +69,22 @@ def __init__( # mamba_seq_idx_for_varlen int32 (max_mamba_chunks,) # mamba_conv_seq_idx int32 (max_tokens,) # mamba_conv_seq_start int32 (max_tokens,) + # Bytes before the Mamba section (needed to compute alignment padding). + pre_mamba_bytes = ( + 3 * tok_int64_bytes + + 3 * tok_int32_bytes + + 7 * req_4byte_bytes + + mha_query_lengths_bytes + + mha_cu_query_seq_lengths_bytes + + mha_kv_seq_lengths_bytes + + mha_cu_kv_seq_lengths_bytes + + mha_block_table_bytes + ) + if max_mamba_chunks > 0: - mamba_batch_indices_decode_bytes = max_bs * 4 + # mamba_batch_indices_decode is int64; pad to 8-byte alignment. + mamba_align_pad = (8 - pre_mamba_bytes % 8) % 8 + mamba_batch_indices_decode_bytes = max_bs * 8 mamba_batch_indices_prefill_bytes = max_bs * 4 mamba_seq_idx_bytes = max_tokens * 4 mamba_cu_seqlens_bytes = (max_bs + 1) * 4 @@ -80,6 +94,7 @@ def __init__( mamba_conv_seq_idx_bytes = max_tokens * 4 mamba_conv_seq_start_bytes = max_tokens * 4 else: + mamba_align_pad = 0 mamba_batch_indices_decode_bytes = 0 mamba_batch_indices_prefill_bytes = 0 mamba_seq_idx_bytes = 0 @@ -91,14 +106,8 @@ def __init__( mamba_conv_seq_start_bytes = 0 total_bytes = ( - 2 * tok_int64_bytes - + 4 * tok_int32_bytes - + 7 * req_4byte_bytes - + mha_query_lengths_bytes - + mha_cu_query_seq_lengths_bytes - + mha_kv_seq_lengths_bytes - + mha_cu_kv_seq_lengths_bytes - + mha_block_table_bytes + pre_mamba_bytes + + mamba_align_pad + mamba_batch_indices_decode_bytes + mamba_batch_indices_prefill_bytes + mamba_seq_idx_bytes @@ -119,8 +128,8 @@ def __init__( off += tok_int64_bytes self.token_to_pos_ids = self._buf[off : off + tok_int64_bytes].view(torch.long) off += tok_int64_bytes - self.token_to_block_idx = self._buf[off : off + tok_int32_bytes].view(torch.int32) - off += tok_int32_bytes + self.token_to_block_idx = self._buf[off : off + tok_int64_bytes].view(torch.int64) + off += tok_int64_bytes self.token_to_local_position_within_kv_block = self._buf[off : off + tok_int32_bytes].view( torch.int32 ) @@ -180,9 +189,10 @@ def __init__( # per-step coalesced H2D copy covers both MHA and Mamba alongside the # token/request bookkeeping. if max_mamba_chunks > 0: + off += mamba_align_pad self.mamba_batch_indices_decode = self._buf[ off : off + mamba_batch_indices_decode_bytes - ].view(torch.int32) + ].view(torch.int64) off += mamba_batch_indices_decode_bytes self.mamba_batch_indices_prefill = self._buf[ off : off + mamba_batch_indices_prefill_bytes diff --git a/megatron/core/inference/engines/dynamic_engine.py b/megatron/core/inference/engines/dynamic_engine.py index c0318c3b10a..33e9d661e48 100644 --- a/megatron/core/inference/engines/dynamic_engine.py +++ b/megatron/core/inference/engines/dynamic_engine.py @@ -42,9 +42,9 @@ from megatron.core.inference.text_generation_controllers.text_generation_controller import ( TextGenerationController, ) -from megatron.core.inference.utils import Counter, await_process_call +from megatron.core.inference.utils import Counter, InferenceMode, await_process_call from megatron.core.process_groups_config import ProcessGroupCollection -from megatron.core.transformer.cuda_graphs import delete_cuda_graphs +from megatron.core.transformer.cuda_graphs import CudaGraphManager, delete_cuda_graphs from megatron.core.transformer.enums import InferenceCudaGraphScope from megatron.core.transformer.moe.router_replay import RouterReplay, RouterReplayAction from megatron.core.utils import ( @@ -133,6 +133,8 @@ class EngineSuspendedError(Exception): def format_mem_bytes(mem_bytes): """Convert a byte count to a human-readable string in tb, gb, mb, kb, or bytes.""" + if mem_bytes < 0: + return "-" + format_mem_bytes(-mem_bytes) for power, suffix in [(4, "tb"), (3, "gb"), (2, "mb"), (1, "kb"), (0, "bytes")]: suffix_bytes = 1024**power if mem_bytes >= suffix_bytes: @@ -140,6 +142,33 @@ def format_mem_bytes(mem_bytes): return "%d bytes" % mem_bytes +def _cuda_graph_mempool_bytes() -> Tuple[int, int]: + """Return (reserved, allocated) bytes belonging to the global CUDA graph mempool. + + PyTorch's `torch.cuda.memory_stats()` reports process-wide totals that mix in + every other allocation (KV cache, NCCL workspaces, layer scratch). To isolate + growth caused by graph capture, we walk `torch.cuda.memory_snapshot()` and + filter segments by their `segment_pool_id` against the graph pool handle. + Returns (0, 0) if the pool hasn't been created yet. + """ + pool_id = CudaGraphManager.global_mempool + if pool_id is None: + return 0, 0 + reserved = 0 + allocated = 0 + for seg in torch.cuda.memory_snapshot(): + seg_pool_id = ( + seg.get("segment_pool_id") + or seg.get("private_pool_id") + or seg.get("pool_id") + or seg.get("pool") + ) + if seg_pool_id == pool_id: + reserved += seg.get("total_size", 0) + allocated += seg.get("allocated_size", 0) + return reserved, allocated + + @dataclass(kw_only=True) class RequestEntry: """Entry in the engine's `self.requests` dict.""" @@ -212,8 +241,8 @@ def __init__(self, controller: TextGenerationController, context: DynamicInferen if self.num_speculative_tokens > 0: assert ( model_config.mtp_use_repeated_layer - or self.num_speculative_tokens <= self.controller.num_mtp_heads - ), f"Number of speculative tokens {self.num_speculative_tokens} must be less than or equal to number of MTP heads {self.controller.num_mtp_heads}" + or self.num_speculative_tokens <= model_config.mtp_num_layers + ), f"Number of speculative tokens {self.num_speculative_tokens} must be less than or equal to number of MTP layers {model_config.mtp_num_layers}" self.track_paused_request_events = inference_config.track_paused_request_events self.track_generated_token_events = inference_config.track_generated_token_events self.enable_chunked_prefill = inference_config.enable_chunked_prefill @@ -259,6 +288,9 @@ def __init__(self, controller: TextGenerationController, context: DynamicInferen max_step = int(val) self.inference_step_offset = int(max_step) + # Mark the inference engine as active. Cleared in `suspend()` and re-set in `resume()`. + InferenceMode.set_active() + # Create cuda graphs. self.create_cuda_graphs() @@ -297,9 +329,15 @@ def reset(self) -> None: self.resume_request_ids = None - # Speculative decoding acceptance tracking. - self._spec_tokens_proposed = 0 - self._spec_tokens_accepted = 0 + # Speculative decoding acceptance tracking (per-position). + # Each tensor has length num_speculative_tokens; index i tracks position i+1 + # (i.e. the i-th draft token proposed by the MTP head). + self._spec_tokens_proposed_per_pos = torch.zeros( + self.num_speculative_tokens, dtype=torch.int64 + ) + self._spec_tokens_accepted_per_pos = torch.zeros( + self.num_speculative_tokens, dtype=torch.int64 + ) self._spec_steps = 0 # Prefix caching tracking. @@ -344,6 +382,13 @@ def create_cuda_graphs(self, reset_context: bool = True): time_start = time.time() mem_stats_start = torch.cuda.memory_stats() + # Snapshot of process-wide stats for the "total memory used by capture" summary. + start_proc_reserved = mem_stats_start["reserved_bytes.all.current"] + start_proc_alloc = mem_stats_start["allocated_bytes.all.current"] + + # Pool-scoped baselines for the per-iteration deltas. + prev_pool_reserved, prev_pool_alloc = _cuda_graph_mempool_bytes() + logging.info("> dynamic_engine.py: building cuda graphs for ") for graph in context.cuda_graph_batch_dimensions_list: logging.info(graph) @@ -355,7 +400,7 @@ def create_cuda_graphs(self, reset_context: bool = True): # decoder graphs within the same loop rather than in a separate pass. unwrapped = unwrap_model(controller.inference_wrapped_model.model) mtp_warmup_enabled = ( - controller.num_mtp_heads > 0 + controller.num_mtp_depths > 0 and (controller.num_speculative_tokens or 0) > 0 and hasattr(unwrapped, 'mtp') ) @@ -363,7 +408,7 @@ def create_cuda_graphs(self, reset_context: bool = True): tp_size = get_pg_size(controller.inference_wrapped_model.tp_group) sp_enabled = model_config.sequence_parallel and tp_size > 1 mtp_pass_depth = not unwrapped.mtp.mtp_use_repeated_layer - mtp_warmup_depths = range(controller._num_mtp_depths) if mtp_pass_depth else [None] + mtp_warmup_depths = range(controller.num_mtp_depths) if mtp_pass_depth else [None] mtp_seen_batch_sizes = set() tbar = enumerate(context.cuda_graph_batch_dimensions_list) @@ -424,27 +469,43 @@ def create_cuda_graphs(self, reset_context: bool = True): context.reset() + # Per-iteration memory accounting, scoped to the CUDA-graph mempool. + # This isolates pool growth from process-wide scratch churn (KV cache, + # NCCL workspaces, etc.) that pollutes `torch.cuda.memory_stats()`. + pool_reserved, pool_alloc = _cuda_graph_mempool_bytes() + logging.info( + " [graph %d/%d] %s | pool reserved=%s (Δiter=%s) " "pool allocated=%s (Δiter=%s)", + tbar_idx + 1, + len(context.cuda_graph_batch_dimensions_list), + cuda_graph_batch_dimension, + format_mem_bytes(pool_reserved), + format_mem_bytes(pool_reserved - prev_pool_reserved), + format_mem_bytes(pool_alloc), + format_mem_bytes(pool_alloc - prev_pool_alloc), + ) + prev_pool_reserved, prev_pool_alloc = pool_reserved, pool_alloc + if mtp_warmup_enabled and mtp_seen_batch_sizes: logging.info("> MTP CUDA graph warmup: %d batch size(s)", len(mtp_seen_batch_sizes)) # Memory usage. time_end = time.time() mem_stats_end = torch.cuda.memory_stats() + final_pool_reserved, final_pool_alloc = _cuda_graph_mempool_bytes() capture_stats = { "time": time_end - time_start, - "allocated_bytes": ( - mem_stats_end["allocated_bytes.all.current"] - - mem_stats_start["allocated_bytes.all.current"] - ), - "reserved_bytes": ( - mem_stats_end["reserved_bytes.all.current"] - - mem_stats_start["reserved_bytes.all.current"] - ), + "allocated_bytes": (mem_stats_end["allocated_bytes.all.current"] - start_proc_alloc), + "reserved_bytes": (mem_stats_end["reserved_bytes.all.current"] - start_proc_reserved), + "pool_reserved_bytes": final_pool_reserved, + "pool_allocated_bytes": final_pool_alloc, } logging.info( - "> built cuda graph(s) in %.2f sec, with total memory usage: " - "allocated %s, reserved %s.", + "> built cuda graph(s) in %.2f sec. " + "Mempool: reserved %s, allocated %s. " + "Process-wide delta: allocated %s, reserved %s.", capture_stats["time"], + format_mem_bytes(capture_stats["pool_reserved_bytes"]), + format_mem_bytes(capture_stats["pool_allocated_bytes"]), format_mem_bytes(capture_stats["allocated_bytes"]), format_mem_bytes(capture_stats["reserved_bytes"]), ) @@ -723,6 +784,8 @@ def suspend(self): if self.state in (EngineState.SUSPENDED, EngineState.SUSPENDING): return + InferenceMode.unset_active() + # Deallocate context tensors. with self.__class__.suspend_resume_ctx( "suspended", unified_memory_level=self.unified_memory_level @@ -772,6 +835,8 @@ def resume(self): if self.state not in (EngineState.SUSPENDED, EngineState.SUSPENDING): return + InferenceMode.set_active() + # Resume. with self.__class__.suspend_resume_ctx( "resumed", unified_memory_level=self.unified_memory_level @@ -964,12 +1029,21 @@ def _add_request( eod = -1 request.sampling_params.termination_id = eod - if ( - len(request.prompt_tokens) + request.sampling_params.num_tokens_to_generate - > self.context.max_sequence_length - ) or (request.sampling_params.num_tokens_to_generate < 0): + # Clamp large `num_tokens_to_generate` instead of rejecting the request. + # This is included for compatibility with other frameworks. + remaining_tokens = self.context.max_sequence_length - len(request.prompt_tokens) + if request.sampling_params.num_tokens_to_generate < 0 or remaining_tokens < 0: request.status = Status.FAILED request.add_event_error_nontransient(MaxSequenceLengthOverflowError(request_id)) + elif request.sampling_params.num_tokens_to_generate > remaining_tokens: + requested_tokens = request.sampling_params.num_tokens_to_generate + request.sampling_params.num_tokens_to_generate = remaining_tokens + if self.rank == 0: + warnings.warn( + f"Request {request_id} requested num_tokens_to_generate={requested_tokens} " + f"which exceeds the maximum sequence length of the engine. " + f"Clamping num_tokens_to_generate to {remaining_tokens}." + ) if len(request.prompt_tokens) > self.context.max_tokens and not self.enable_chunked_prefill: request.status = Status.FAILED @@ -1141,6 +1215,7 @@ def post_process_requests( tokens = accepted_tokens + tokens num_stop_word_trim = 0 + is_prefill = len(request.generated_tokens) == 0 if request_id != self.context.chunked_prefill_request_id: # Skip appending token for requests being finished due to stop words # (they already have their final token from the previous step) @@ -1152,12 +1227,21 @@ def post_process_requests( keep = request.sampling_params.num_tokens_to_generate - len( request.generated_tokens ) + num_tokens_before_trim = len(tokens) tokens = tokens[:keep] - # Trim log probs / top-n to match so the counts stay in sync. - if request_log_probs is not None: - request_log_probs = request_log_probs[:keep] - if top_n_logprobs is not None and req_idx in top_n_logprobs: - top_n_logprobs[req_idx] = top_n_logprobs[req_idx][:keep] + # Drop only the excess *trailing* log probs / top-n so the counts stay + # in sync. We must trim from the end, not the front: on a prefill step + # request_log_probs covers the whole prompt and is laid out as + # [, ], so front-slicing + # (e.g. [:keep] with keep == 0 when num_tokens_to_generate == 0) would + # discard the prompt log probs that echo+logprobs requests need. In a + # decode step all entries are generated, so trailing == front-equivalent. + num_dropped = num_tokens_before_trim - len(tokens) + if num_dropped > 0: + if request_log_probs is not None: + request_log_probs = request_log_probs[:-num_dropped] + if top_n_logprobs is not None and req_idx in top_n_logprobs: + top_n_logprobs[req_idx] = top_n_logprobs[req_idx][:-num_dropped] if request_id not in self.stop_word_being_finished_ids: is_first_token = len(request.generated_tokens) == 0 request.generated_tokens += tokens @@ -1185,7 +1269,7 @@ def post_process_requests( ) if first_token_event is None: first_token_event = event - if is_first_token: + if is_first_token and tokens: if not self.track_generated_token_events: first_token_event = DynamicInferenceEvent( type=DynamicInferenceEventType.GENERATED_TOKEN, @@ -1198,7 +1282,7 @@ def post_process_requests( # non-logging steps (async_forward skips the event sync), # so gate the update to keep the metric a truthful sparse # sample instead of polluting it with zeros. - if step_time > 0: + if step_time > 0 and tokens: per_token_step_time = step_time / len(tokens) request.tpot.extend([per_token_step_time] * len(tokens)) @@ -1211,13 +1295,21 @@ def post_process_requests( request ) - # Track acceptance statistics for logging. - if len(request.generated_tokens) > 0 and self.num_speculative_tokens > 0: + # Track per-position acceptance statistics for logging. + # Skip prefill requests: MTP heads only propose speculative tokens + # for decode requests, so counting prefill requests would inflate + # the denominator and artificially deflate the acceptance rate. + if ( + not is_prefill + and len(request.generated_tokens) > 0 + and self.num_speculative_tokens > 0 + ): actual_proposed = max(0, self.num_speculative_tokens - num_stop_word_trim) - actual_accepted = max(0, len(accepted_tokens) - num_stop_word_trim) - - self._spec_tokens_proposed += actual_proposed - self._spec_tokens_accepted += actual_accepted + self._spec_tokens_proposed_per_pos[:actual_proposed] += 1 + accepted_t = torch.tensor(accepted_tokens_list[:actual_proposed]) + self._spec_tokens_accepted_per_pos[:actual_proposed] += ( + accepted_t != -1 + ).long() if request_id in finished_request_ids: # Reconstruct routing from per-block storage before popping. @@ -1884,13 +1976,24 @@ async def async_bookkeep( else: metrics[f'inference/{key}'] = value - # Add speculative decoding acceptance metrics. - if self.num_speculative_tokens > 0 and self._spec_tokens_proposed > 0: - acceptance_rate = self._spec_tokens_accepted / self._spec_tokens_proposed + # Add speculative decoding acceptance metrics (aggregate + per-position). + total_proposed = sum(self._spec_tokens_proposed_per_pos) + total_accepted = sum(self._spec_tokens_accepted_per_pos) + if self.num_speculative_tokens > 0 and total_proposed > 0: + acceptance_rate = total_accepted / total_proposed metrics['inference/spec_decode_acceptance_rate'] = float(acceptance_rate * 100.0) - metrics['inference/spec_decode_tokens_proposed'] = int(self._spec_tokens_proposed) - metrics['inference/spec_decode_tokens_accepted'] = int(self._spec_tokens_accepted) + metrics['inference/spec_decode_tokens_proposed'] = int(total_proposed) + metrics['inference/spec_decode_tokens_accepted'] = int(total_accepted) metrics['inference/spec_decode_num_steps'] = int(self._spec_steps) + for pos in range(self.num_speculative_tokens): + if self._spec_tokens_proposed_per_pos[pos] > 0: + pos_rate = ( + self._spec_tokens_accepted_per_pos[pos] + / self._spec_tokens_proposed_per_pos[pos] + ) + metrics[f'inference/spec_decode_acceptance_rate_pos{pos + 1}'] = float( + pos_rate * 100.0 + ) # Add prefix caching metrics. if self.context.enable_prefix_caching and self._prefix_cache_hits > 0: @@ -1952,16 +2055,28 @@ async def async_bookkeep( mem["reserved_bytes.all.current"] / (1024**3), ) ) - if self.num_speculative_tokens > 0 and self._spec_tokens_proposed > 0: - spec_rate = self._spec_tokens_accepted / self._spec_tokens_proposed * 100.0 - output_str += " ... spec: accept %.1f%% (%d/%d in %d steps)" % ( + total_proposed = sum(self._spec_tokens_proposed_per_pos) + total_accepted = sum(self._spec_tokens_accepted_per_pos) + if self.num_speculative_tokens > 0 and total_proposed > 0: + spec_rate = total_accepted / total_proposed * 100.0 + per_pos_rates = [] + for pos in range(self.num_speculative_tokens): + if self._spec_tokens_proposed_per_pos[pos] > 0: + pos_rate = ( + self._spec_tokens_accepted_per_pos[pos] + / self._spec_tokens_proposed_per_pos[pos] + * 100.0 + ) + per_pos_rates.append("t%d=%.1f%%" % (pos + 1, pos_rate)) + output_str += " ... spec (cumul): accept %.1f%% (%d/%d in %d steps) [%s]" % ( spec_rate, - self._spec_tokens_accepted, - self._spec_tokens_proposed, + total_accepted, + total_proposed, self._spec_steps, + ", ".join(per_pos_rates), ) if self.context.enable_prefix_caching and self._prefix_cache_hits > 0: - output_str += " ... prefix cache: %d hits, %d blocks matched" % ( + output_str += " ... prefix cache (cumul): %d hits, %d blocks matched" % ( self._prefix_cache_hits, self._prefix_cache_blocks_matched, ) @@ -1969,17 +2084,6 @@ async def async_bookkeep( output_str = f"\033[94m{output_str}\033[0m" logging.info(output_str) - # Reset speculative decoding accumulators after both wandb and console logging. - if self.num_speculative_tokens > 0: - self._spec_tokens_proposed = 0 - self._spec_tokens_accepted = 0 - self._spec_steps = 0 - - # Reset prefix caching accumulators after both wandb and console logging. - if self.context.enable_prefix_caching: - self._prefix_cache_hits = 0 - self._prefix_cache_blocks_matched = 0 - nvtx_range_pop("console_logging") return { diff --git a/megatron/core/inference/engines/static_engine.py b/megatron/core/inference/engines/static_engine.py index 0b3b9c1b856..c079921a271 100644 --- a/megatron/core/inference/engines/static_engine.py +++ b/megatron/core/inference/engines/static_engine.py @@ -18,6 +18,7 @@ from megatron.core.inference.text_generation_controllers.text_generation_controller import ( TextGenerationController, ) +from megatron.core.inference.utils import InferenceMode from megatron.core.utils import get_asyncio_loop try: @@ -129,6 +130,8 @@ def __init__( self.controller.inference_wrapped_model.inference_context = original_context self.legacy = True + InferenceMode.set_active() + def get_new_request_id(self) -> str: """Gets a new request id from the scheduler""" return self.scheduler.get_new_request_id() diff --git a/megatron/core/inference/inference_request.py b/megatron/core/inference/inference_request.py index d6e7c67a959..f9fd60f1033 100644 --- a/megatron/core/inference/inference_request.py +++ b/megatron/core/inference/inference_request.py @@ -373,6 +373,9 @@ class DynamicInferenceRequest(InferenceRequest): routing_indices: Optional[np.ndarray] = None finished_chunk_token_count: int = 0 stop_word_ids: Optional[List[List[int]]] = None # Tokenized stop words (populated internally) + # Consecutive steps this request has been deferred by CG-aware admission gating. + # Reset to 0 on successful admission. Used only for starvation logging. + cg_wait_iters: int = 0 # Prefix caching fields block_size_tokens: Optional[int] = None # Block size for hash computation diff --git a/megatron/core/inference/shards.py b/megatron/core/inference/shards.py new file mode 100644 index 00000000000..8d73e0ad56b --- /dev/null +++ b/megatron/core/inference/shards.py @@ -0,0 +1,248 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + +"""Framework-agnostic primitives for heterogeneous inference sharding: build a +``ProcessGroupCollection`` per shard, each over a contiguous rank window at its +own parallelism.""" + +from dataclasses import dataclass +from typing import List, Optional, Sequence, Union + +import torch.distributed as dist + +from megatron.core import mpu +from megatron.core.hyper_comm_grid import HyperCommGrid +from megatron.core.inference.shards_spec import InferenceShardSpec, normalize_shard_specs +from megatron.core.process_groups_config import ProcessGroupCollection + + +def build_inference_pg_collection( + world_size: int, + tp_size: Optional[int] = None, + pp_size: Optional[int] = None, + cp_size: Optional[int] = None, + ep_size: Optional[int] = None, + expt_tp_size: Optional[int] = None, + use_tp_pp_dp_mapping: bool = False, + rank_offset: int = 0, +) -> ProcessGroupCollection: + """Build a ProcessGroupCollection for one inference model. + + Uses two HyperCommGrids matching mpu: + - decoder_grid for dense/attention layers (tp, cp, dp, pp) + - expert_grid for MoE expert layers (expt_tp, ep, expt_dp, pp) + + Args: + world_size: Number of ranks in this inference window. + tp_size: Tensor model parallel size. Defaults to training's TP size. + pp_size: Pipeline parallel size. Defaults to training's PP size. + cp_size: Context parallel size. Defaults to training's CP size. + ep_size: Expert parallel size. Defaults to training's EP size. + expt_tp_size: Expert tensor parallel size. Defaults to training's + expert TP size. + use_tp_pp_dp_mapping: If True, use 'tp-pp-dp' order; otherwise + 'tp-dp-pp'. + rank_offset: Starting global rank of the window. Use ``0`` for + collocated inference (shares ranks with training); use a non-zero + offset for non-collocated setups where inference ranks are disjoint + from training ranks. + + Returns: + ProcessGroupCollection configured for the inference model. On ranks + outside the ``[rank_offset, rank_offset + world_size)`` window every + process-group field is a non-member sentinel returned by + :func:`torch.distributed.new_subgroups_by_enumeration` — callers should + not use that instance; see + :func:`build_inference_pg_collections_for_shards` for the right way to + filter. + """ + if tp_size is None: + tp_size = mpu.get_tensor_model_parallel_world_size() + if cp_size is None: + cp_size = mpu.get_context_parallel_world_size() + if pp_size is None: + pp_size = mpu.get_pipeline_model_parallel_world_size() + if ep_size is None: + ep_size = mpu.get_expert_model_parallel_world_size() + if expt_tp_size is None: + expt_tp_size = mpu.get_expert_tensor_parallel_world_size() + + # Dense layer DP size (world = tp * cp * dp * pp) + dp_size = world_size // (tp_size * cp_size * pp_size) + assert dp_size >= 1 and (tp_size * cp_size * dp_size * pp_size) == world_size, ( + f"World size ({world_size}) must be divisible by tp*cp*pp " + f"({tp_size * cp_size * pp_size})" + ) + + # Expert DP size (world = expt_tp * ep * expt_dp * pp) + expt_dp_size = world_size // (expt_tp_size * ep_size * pp_size) + assert expt_dp_size >= 1 and (expt_tp_size * ep_size * expt_dp_size * pp_size) == world_size, ( + f"World size ({world_size}) must be divisible by expt_tp*ep*pp " + f"({expt_tp_size * ep_size * pp_size})" + ) + + rank = dist.get_rank() + + if use_tp_pp_dp_mapping: + decoder_grid = HyperCommGrid( + [tp_size, cp_size, pp_size, dp_size], ["tp", "cp", "pp", "dp"], rank_offset=rank_offset + ) + else: + decoder_grid = HyperCommGrid( + [tp_size, cp_size, dp_size, pp_size], ["tp", "cp", "dp", "pp"], rank_offset=rank_offset + ) + + tp_group = decoder_grid.create_pg("tp") + cp_group = decoder_grid.create_pg("cp") + pp_group = decoder_grid.create_pg("pp") + dp_group = decoder_grid.create_pg("dp") + mp_group = decoder_grid.create_pg(["tp", "pp"]) + tp_cp_group = decoder_grid.create_pg(["tp", "cp"]) + dp_cp_group = decoder_grid.create_pg(["cp", "dp"]) + tp_dp_cp_group = decoder_grid.create_pg(["tp", "cp", "dp"]) + + if use_tp_pp_dp_mapping: + expert_grid = HyperCommGrid( + [expt_tp_size, ep_size, pp_size, expt_dp_size], + ["tp", "ep", "pp", "dp"], + rank_offset=rank_offset, + ) + else: + expert_grid = HyperCommGrid( + [expt_tp_size, ep_size, expt_dp_size, pp_size], + ["tp", "ep", "dp", "pp"], + rank_offset=rank_offset, + ) + + decoder_pp_enum = decoder_grid.get_rank_enum("pp") + expert_pp_enum = expert_grid.get_rank_enum("pp") + assert decoder_pp_enum == expert_pp_enum, ( + f"PP groups must match between decoder and expert grids. " + f"Decoder: {decoder_pp_enum}, Expert: {expert_pp_enum}" + ) + + ep_group = expert_grid.create_pg("ep") + expt_tp_group = expert_grid.create_pg("tp") + expt_dp_group = expert_grid.create_pg("dp") + tp_ep_group = expert_grid.create_pg(["tp", "ep"]) + tp_ep_pp_group = expert_grid.create_pg(["tp", "ep", "pp"]) + + embd_group = None + pos_embd_group = None + pp_rank_enum = decoder_grid.get_rank_enum("pp") + for pp_ranks in pp_rank_enum: + if len(pp_ranks) == 1: + embd_ranks = [pp_ranks[0]] + else: + embd_ranks = [pp_ranks[0], pp_ranks[-1]] + group = dist.new_group(ranks=embd_ranks) + if rank in embd_ranks: + embd_group = group + pos_embd_ranks = [pp_ranks[0]] + group = dist.new_group(ranks=pos_embd_ranks) + if rank in pos_embd_ranks: + pos_embd_group = group + + return ProcessGroupCollection( + tp=tp_group, + cp=cp_group, + pp=pp_group, + ep=ep_group, + embd=embd_group, + pos_embd=pos_embd_group, + dp=dp_group, + tp_cp=tp_cp_group, + mp=mp_group, + expt_tp=expt_tp_group, + expt_dp=expt_dp_group, + tp_ep=tp_ep_group, + tp_ep_pp=tp_ep_pp_group, + dp_cp=dp_cp_group, + tp_dp_cp=tp_dp_cp_group, + ) + + +@dataclass +class InferenceShard: + """One shard in a multi-shard inference layout: its identity, its rank + window, and this rank's process groups for it. + + Attributes: + spec: This shard's :class:`~megatron.core.inference.shards_spec.InferenceShardSpec` + (``tp``/``pp``/``ep``/``expt_tp``/``dp`` and optional + ``role`` = ``prefill``/``decode``). + rank_offset: First global rank belonging to this shard. + world_size: Number of ranks in this shard (tp*pp*dp). + pg_collection: The shard's ProcessGroupCollection if the current rank + is a member of this shard, else ``None`` -- the ``is not None`` check + is how a rank finds its own shard. Every rank still participates in + the collective process-group creation for every shard + (``dist.new_group`` is world-collective); only members get a usable + handle. + """ + + spec: InferenceShardSpec + rank_offset: int + world_size: int + pg_collection: Optional[ProcessGroupCollection] + + +def build_inference_pg_collections_for_shards( + total_world_size: int, + shards: Union[str, Sequence[InferenceShardSpec], Sequence[dict]], + use_tp_pp_dp_mapping: bool = False, +) -> List[InferenceShard]: + """Build one ProcessGroupCollection per heterogeneous inference shard. + + Partitions global ranks into contiguous non-overlapping windows, one per + shard. Shard ``i`` owns ranks + ``[offset_i, offset_i + tp_i*pp_i*dp_i)``. + + Every rank must call this function so the collective ``dist.new_group`` + calls inside :func:`build_inference_pg_collection` succeed for every shard. + The returned ``pg_collection`` is populated only on ranks belonging to + that shard; others see ``None``. + + Args: + total_world_size: Full world size across training + all inference + shards. + shards: Shard layout in any form ``normalize_shard_specs`` accepts -- a + spec string, a list of :class:`InferenceShardSpec`, or a list of + raw dicts. Normalized internally to validated specs. + use_tp_pp_dp_mapping: Passed through to ``build_inference_pg_collection``. + + Returns: + List of :class:`InferenceShard`, one per input spec. + """ + specs = normalize_shard_specs(shards, total_world_size) + rank = dist.get_rank() + results: List[InferenceShard] = [] + offset = 0 + for i, spec in enumerate(specs): + tp, pp, ep, expt_tp, dp = spec.tp, spec.pp, spec.ep, spec.expt_tp, spec.dp + shard_world = tp * pp * dp + assert offset + shard_world <= total_world_size, ( + f"Shard {i} ({spec}) runs out of ranks: needs " + f"[{offset}, {offset + shard_world}), total_world_size={total_world_size}." + ) + pgc = build_inference_pg_collection( + world_size=shard_world, + tp_size=tp, + pp_size=pp, + # Inference shards don't context-parallelize; the spec validates cp == 1. + cp_size=spec.cp, + ep_size=ep, + expt_tp_size=expt_tp, + use_tp_pp_dp_mapping=use_tp_pp_dp_mapping, + rank_offset=offset, + ) + in_shard = offset <= rank < offset + shard_world + results.append( + InferenceShard( + spec=spec, + rank_offset=offset, + world_size=shard_world, + pg_collection=pgc if in_shard else None, + ) + ) + offset += shard_world + return results diff --git a/megatron/core/inference/shards_spec.py b/megatron/core/inference/shards_spec.py new file mode 100644 index 00000000000..6b2bd06ac10 --- /dev/null +++ b/megatron/core/inference/shards_spec.py @@ -0,0 +1,177 @@ +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +"""Parser and typed representation (:class:`InferenceShardSpec`) for the +``--inference-shards`` shard-layout string.""" + +from dataclasses import dataclass +from typing import List, Optional, Sequence, Union + +VALID_INT_KEYS = ("tp", "pp", "ep", "expt_tp", "dp", "cp") +VALID_ROLES = ("prefill", "decode") +VALID_KEYS = (*VALID_INT_KEYS, "role") + + +@dataclass(frozen=True) +class InferenceShardSpec: + """One inference shard's parallelism -- the canonical shard-layout type. + + Frozen and self-validating: ``expt_tp`` defaults to ``tp`` (resolved at + construction), the expert decomposition is checked to tile within the shard, + and ``role`` (``"prefill"`` / ``"decode"`` for disaggregation, else ``None``) + is validated. A list of these, produced by :func:`normalize_shard_specs`, is + what every inference-shard consumer (the PG builder, the disaggregation + setup) operates on. + """ + + tp: int = 1 + pp: int = 1 + ep: int = 1 + dp: int = 1 + cp: int = 1 + expt_tp: Optional[int] = None + role: Optional[str] = None + + def __post_init__(self): + if self.role is not None and self.role not in VALID_ROLES: + raise ValueError(f"role must be one of {VALID_ROLES} or None; got {self.role!r}") + # cp is accepted in the spec for forward-compatibility, but inference + # shards don't context-parallelize (the KV reshard models layer x head, + # not the sequence dim), so reject anything but cp=1 with a clear error. + if self.cp != 1: + raise ValueError( + f"context parallelism is not supported for inference shards; cp must be 1, " + f"got cp={self.cp}" + ) + # Resolve expt_tp's default (tp) on this frozen instance. + if self.expt_tp is None: + object.__setattr__(self, "expt_tp", self.tp) + # Expert decomposition must tile cleanly within the shard. + if self.world_size % (self.expt_tp * self.ep * self.pp) != 0: + raise ValueError( + f"shard {self} has tp*pp*dp={self.world_size} but expt_tp*ep*pp=" + f"{self.expt_tp * self.ep * self.pp} does not divide it; " + f"choose compatible sizes." + ) + + @property + def world_size(self) -> int: + """Number of ranks this shard occupies (``tp * pp * dp``).""" + return self.tp * self.pp * self.dp + + def to_dict(self) -> dict: + """Plain-dict form (e.g. for serialization or external consumers).""" + d = {"tp": self.tp, "pp": self.pp, "ep": self.ep, "dp": self.dp, "expt_tp": self.expt_tp} + if self.role is not None: + d["role"] = self.role + return d + + +def normalize_shard_specs( + shards: Union[str, Sequence["InferenceShardSpec"], Sequence[dict]], world_size: int +) -> List["InferenceShardSpec"]: + """Coerce the public shard-spec input (a spec string, a list of + :class:`InferenceShardSpec`, or a list of raw dicts) into the validated + list of :class:`InferenceShardSpec` the shard builders consume.""" + if isinstance(shards, str): + return parse_inference_shards_spec(shards, world_size) + out: List[InferenceShardSpec] = [ + s if isinstance(s, InferenceShardSpec) else InferenceShardSpec(**dict(s)) for s in shards + ] + return _finalize_and_validate(out, world_size) + + +def spec_declares_disaggregation(spec_str: str) -> bool: + """Whether a shard spec tags any shard with a ``role=`` (prefill/decode). + + A role tag is what marks the layout as a prefill->decode handoff rather + than plain multi-shard / data-parallel inference. Cheap and world_size- + free, so it can be checked at arg-validation time; full parsing + + validation is :func:`parse_inference_shards_spec`. + """ + if not spec_str: + return False + return any( + kv.strip().startswith("role=") + for shard in spec_str.replace("+", ";").split(";") + for kv in shard.split(",") + ) + + +def parse_inference_shards_spec(spec_str: str, world_size: int) -> List[dict]: + """Parse + validate the ``--inference-shards`` string. + + Args: + spec_str: Raw CLI value, e.g. ``"tp=2,dp=1+tp=1,dp=2"`` or with + disaggregation roles ``"tp=2,role=prefill+tp=1,role=decode"``. + world_size: Total number of ranks. Specs must partition it + exactly (no idle ranks; see note below). + + Returns: + List of :class:`InferenceShardSpec`, one per shard. Order matches the + input (left-to-right corresponds to ascending ``rank_offset``). + + Raises: + AssertionError: on syntax errors, unknown keys, or a rank-count + mismatch with ``world_size``. Idle ranks are rejected to keep the + partition explicit — any world-collective consumer must be able to + enumerate every rank's shard membership from the parsed list alone. + ValueError: on an expert-grid mismatch within a shard (raised by + :class:`InferenceShardSpec`). + """ + parsed: List[InferenceShardSpec] = [] + # ``+`` is convenient from shell recipes where ``;`` would otherwise + # be treated as a command terminator. Normalize before splitting. + shards_raw = spec_str.replace("+", ";") + for shard_str in shards_raw.split(";"): + shard_str = shard_str.strip() + if not shard_str: + continue + kwargs: dict = {} + for kv in shard_str.split(","): + kv = kv.strip() + if not kv: + continue + if "=" not in kv: + raise AssertionError( + f"Bad --inference-shards spec entry {kv!r}: expected key=value." + ) + k, v = kv.split("=", 1) + k = k.strip() + v = v.strip() + if k not in VALID_KEYS: + raise AssertionError( + f"Unknown key {k!r} in --inference-shards " + f"(allowed: {','.join(VALID_KEYS)})." + ) + if k == "role": + role = v.lower() + assert role in VALID_ROLES, ( + f"Unknown role {v!r} in --inference-shards " + f"(allowed: {','.join(VALID_ROLES)})." + ) + kwargs[k] = role + else: + kwargs[k] = int(v) + parsed.append(InferenceShardSpec(**kwargs)) + + return _finalize_and_validate(parsed, world_size) + + +def _finalize_and_validate( + specs: List["InferenceShardSpec"], world_size: int +) -> List["InferenceShardSpec"]: + """Assert the shards partition the world exactly. + + Shared by the string parser and the object path (:func:`normalize_shard_specs`). + Per-shard defaults and expert-grid validation live in + :class:`InferenceShardSpec`; this only enforces the cross-shard total. Idle + ranks are rejected so any world-collective consumer can enumerate every + rank's shard membership from the list alone. + """ + assert specs, "--inference-shards was empty." + total_ranks = sum(s.world_size for s in specs) + assert total_ranks == world_size, ( + f"--inference-shards consumes {total_ranks} ranks but world size is " + f"{world_size}; specs must partition the full world." + ) + return specs diff --git a/megatron/core/inference/text_generation_controllers/text_generation_controller.py b/megatron/core/inference/text_generation_controllers/text_generation_controller.py index 3e788fec0b1..399da90202d 100644 --- a/megatron/core/inference/text_generation_controllers/text_generation_controller.py +++ b/megatron/core/inference/text_generation_controllers/text_generation_controller.py @@ -104,9 +104,21 @@ def __init__(self, inference_wrapped_model: AbstractModelInferenceWrapper, token self.vocab_size = unwrapped_model.vocab_size self.sampling_rng = torch.Generator(device=torch.cuda.current_device()) - self.num_mtp_heads = self._get_mtp_num_heads() self.sampling_rng.manual_seed(self.model_config.inference_sampling_seed) + if not self.num_speculative_tokens: + self.num_mtp_depths = 0 + else: + assert ( + self.model_config.mtp_num_layers and self.model_config.mtp_num_layers >= 1 + ), "mtp_num_layers must be >= 1 when num_speculative_tokens > 0" + if self.model_config.mtp_use_repeated_layer: + self.num_mtp_depths = self.num_speculative_tokens + else: + self.num_mtp_depths = min( + self.num_speculative_tokens, self.model_config.mtp_num_layers + ) + if ( self.model_config.cuda_graph_impl == "local" and self.model_config.expert_model_parallel_size > 1 @@ -120,13 +132,6 @@ def __init__(self, inference_wrapped_model: AbstractModelInferenceWrapper, token if self.inference_wrapped_model.inference_context.is_dynamic_batching(): self._init_dynamic_sampling_tensors() - def _get_mtp_num_heads(self) -> int: - """Get the number of MTP layers from the model config.""" - model = self.inference_wrapped_model.model - if hasattr(model, 'config') and hasattr(model.config, 'mtp_num_layers'): - return model.config.mtp_num_layers or 0 - return 0 - def set_stop_word_finished_ids_callback(self, callback): """Set a callback to get request IDs that should be marked as finished due to stop words. @@ -222,7 +227,6 @@ def _init_mtp_sampling_tensors(self): max_requests, dtype=torch.int64, device=device ) self._last_accepted_seq_indices = None - self._num_mtp_depths = min(self.num_speculative_tokens, self.num_mtp_heads) self._mtp_token_ids_buf = torch.empty([1, max_requests], dtype=torch.int64, device=device) self._mtp_position_ids_buf = torch.empty( [1, max_requests], dtype=torch.int64, device=device @@ -833,7 +837,7 @@ def _compute_serial_mtp_and_sample(self): position_ids_buf[0, active_request_count:] = 0 nvtx_range_pop("mtp-spec-decoding/serial-mtp-init") - for depth in range(self._num_mtp_depths): + for depth in range(self.num_mtp_depths): nvtx_range_push(f"mtp-spec-decoding/depth-{depth}") token_ids_buf[0, :active_request_count] = next_token_ids @@ -1508,7 +1512,7 @@ def _dummy_serial_mtp_forward(self): - When PP > 1: participate in the ``broadcast_from_last_pipeline_stage`` that the real ranks also perform. """ - if self.num_speculative_tokens == 0 or self.num_mtp_heads == 0: + if self.num_speculative_tokens == 0 or self.num_mtp_depths == 0: return if self.model_config.expert_model_parallel_size <= 1: return @@ -1549,7 +1553,7 @@ def _dummy_serial_mtp_forward(self): context = self.inference_wrapped_model.inference_context - for depth in range(self._num_mtp_depths): + for depth in range(self.num_mtp_depths): nvtx_range_push(f"mtp-spec-decoding/dummy-depth-{depth}") mtp_logits_2d = None if has_mtp: @@ -1806,6 +1810,14 @@ async def async_generate_output_tokens_dynamic_batch( ) range_pop() + # Capture before update_requests (called by _dynamic_step_context_bookkeeping) + # resets num_prefill_requests to 0, which would make num_decode_requests + # always equal to the full active count. + num_decode_requests = context.num_decode_requests + if self.num_speculative_tokens > 0: + # Prefill-only batches must not have any accepted speculative tokens. + assert num_decode_requests > 0 or (self._accepted_tokens_per_request == -1).all() + if skip_bookkeeping: # _transfer_samples_to_cpu wasn't invoked on this path, so do # a one-shot D2H here to keep "sample" as a CPU tensor for @@ -1820,9 +1832,9 @@ async def async_generate_output_tokens_dynamic_batch( ret = { "accepted_tokens": ( - # Clone needed: .fill_(-1) on line 1480 would corrupt the returned value. + # Clone needed: .fill_(-1) below would corrupt the returned value. self._accepted_tokens_per_request.clone() - if self.num_speculative_tokens > 0 + if self.num_speculative_tokens > 0 and num_decode_requests > 0 else None ), "log_probs": log_probs, diff --git a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py index 66102c5e73c..460acf39e9b 100644 --- a/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py +++ b/megatron/core/inference/text_generation_server/dynamic_text_gen_server/endpoints/chat_completions.py @@ -360,6 +360,35 @@ def _replace_prefix_tokens( return previous_turn_token_ids + current_turn_additional_token_ids +def _coerce_to_token_id_list(result): + """Convert the return value of `tokenizer.apply_chat_template` to `list[int]`. + + transformers >= 5.x.x sometimes returns a `BatchEncoding` object instead of a `list[int]`. + """ + # BatchEncoding / dict-like with input_ids + if isinstance(result, dict) or hasattr(result, "input_ids"): + ids = result["input_ids"] + if hasattr(ids, "tolist"): + ids = ids.tolist() + if ids and isinstance(ids[0], list): + ids = ids[0] + return list(ids) + # Fast-tokenizer Encoding object + if hasattr(result, "ids"): + ids = result.ids + if hasattr(ids, "tolist"): + ids = ids.tolist() + return list(ids) + # Raw tensor / ndarray + if hasattr(result, "tolist"): + ids = result.tolist() + if ids and isinstance(ids[0], list): + ids = ids[0] + return ids + # Plain list + return list(result) + + try: import orjson @@ -433,12 +462,14 @@ async def chat_completions(): hasattr(tokenizer, 'apply_chat_template') and getattr(tokenizer, "chat_template", None) is not None ): - prompt_tokens = tokenizer.apply_chat_template( - template_messages, - tokenize=True, - add_generation_prompt=True, - tools=template_tools, - **chat_template_kwargs, + prompt_tokens = _coerce_to_token_id_list( + tokenizer.apply_chat_template( + template_messages, + tokenize=True, + add_generation_prompt=True, + tools=template_tools, + **chat_template_kwargs, + ) ) if req.get("prevent_retokenization", True): @@ -479,12 +510,14 @@ async def chat_completions(): ] # Get the templated tokenization of just the previous generation - retokenized_previous_turn_token_ids = tokenizer.apply_chat_template( - messages_to_last_assistant_message, - tokenize=True, - add_generation_prompt=False, - tools=template_tools, - **chat_template_kwargs, + retokenized_previous_turn_token_ids = _coerce_to_token_id_list( + tokenizer.apply_chat_template( + messages_to_last_assistant_message, + tokenize=True, + add_generation_prompt=False, + tools=template_tools, + **chat_template_kwargs, + ) ) # Replace the prefix tokens with the tokens from the previous generation. @@ -699,6 +732,9 @@ async def chat_completions(): message["prompt_token_ids"] = result["prompt_tokens"] message["generation_token_ids"] = result["generated_tokens"] message["generation_log_probs"] = result.get("generated_log_probs", []) + message["policy_epoch"] = result["policy_epoch"] + message["kv_cache_epoch"] = result["kv_cache_epoch"] + message["num_evictions"] = sum(1 for e in result["events"] if e.get("type") == "EVICT") return_log_probs = sampling_params.return_log_probs # Determine finish_reason following vLLM conventions: @@ -727,11 +763,6 @@ async def chat_completions(): "logprobs": {"content": logprobs_content} if return_log_probs else None, "finish_reason": finish_reason, } - choice_data["policy_epoch"] = result["policy_epoch"] - choice_data["kv_cache_epoch"] = result["kv_cache_epoch"] - choice_data["num_evictions"] = sum( - 1 for e in result["events"] if e.get("type") == "EVICT" - ) if current_app.config['verbose']: logging.info(_redact_token_id_lists_for_logging(result)) diff --git a/megatron/core/inference/unified_memory.py b/megatron/core/inference/unified_memory.py index 53d8d862b92..613fb246f10 100644 --- a/megatron/core/inference/unified_memory.py +++ b/megatron/core/inference/unified_memory.py @@ -277,6 +277,16 @@ def create_unified_mempool() -> "MemPool": + details ) else: + # torch.cuda.MemPool can't coexist with expandable_segments; bail so the + # caller falls back to non-UVM allocation. + alloc_conf = os.environ.get("PYTORCH_CUDA_ALLOC_CONF", "") + if "expandable_segments:true" in alloc_conf.lower(): + raise UnifiedMemoryUnsupportedError( + "UVM mempool is incompatible with the expandable-segments allocator " + "(PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True, which " + "torch.cuda.MemPool does not support). Unset expandable_segments to " + "use UVM." + ) return MemPool(allocator=_alloc) diff --git a/megatron/core/inference/utils.py b/megatron/core/inference/utils.py index 11973551aa2..f20debe2589 100644 --- a/megatron/core/inference/utils.py +++ b/megatron/core/inference/utils.py @@ -1,6 +1,7 @@ # Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. import asyncio +import contextlib import logging import multiprocessing import sys @@ -8,7 +9,6 @@ import torch -from megatron.core.transformer.moe.moe_layer import MoELayer from megatron.core.utils import get_model_config try: @@ -17,6 +17,42 @@ FLASHINFER_JIT_CACHE_VERSION = None +class InferenceMode: + """Process-wide flag indicating whether an inference engine is currently using the model. + + Modules that need to distinguish between inference and non-inference (e.g. training, + RL logprobs) paths should read `InferenceMode.is_active()` rather than relying on + `self.training`, `torch.is_grad_enabled()`, or `inference_context is not None`. + """ + + _is_active: bool = False + + @classmethod + def is_active(cls) -> bool: + """Return True while an inference engine is currently using the model.""" + return cls._is_active + + @classmethod + def set_active(cls) -> None: + """Mark the inference engine as active. Idempotent.""" + cls._is_active = True + + @classmethod + def unset_active(cls) -> None: + """Mark the inference engine as inactive. Idempotent.""" + cls._is_active = False + + @classmethod + @contextlib.contextmanager + def active(cls): + """Context manager: set the flag for the duration of the `with` block.""" + cls.set_active() + try: + yield + finally: + cls.unset_active() + + def device_memory_summary() -> str: """One-line GPU memory summary for torch_memory_saver logging.""" dev = torch.cuda.current_device() @@ -80,6 +116,8 @@ def _init_moe_expert_cache(model): """ Initialize the cache of MoE layers once """ + from megatron.core.transformer.moe.moe_layer import MoELayer + global moe_layer_cache if moe_layer_cache is not None: return # already initialized diff --git a/megatron/core/model_parallel_config.py b/megatron/core/model_parallel_config.py index d56c6282e0c..5c2786285b1 100644 --- a/megatron/core/model_parallel_config.py +++ b/megatron/core/model_parallel_config.py @@ -2,13 +2,28 @@ import warnings from dataclasses import dataclass, field -from typing import Callable, ContextManager, Literal, Optional +from typing import Callable, ContextManager, Literal, Optional, Union import torch from megatron.core.utils import experimental_api +def _parse_pad_packed_seq_alignment(value): + """Parse THD packed-sequence padding alignment. + + Accepts ``"max"`` or a positive integer alignment. + """ + if value == "max": + return value + try: + return int(value) + except (TypeError, ValueError) as exc: + raise ValueError( + "pad_packed_seq_alignment must be 'max' or a positive integer alignment." + ) from exc + + @dataclass @experimental_api class ModelParallelConfig: @@ -87,6 +102,31 @@ class ModelParallelConfig: default_dynamic_cp: Dynamic-CP scheduler for packed sequence balancing. """ + pad_packed_seq_alignment: Optional[Union[int, Literal["max"]]] = field( + default=None, + metadata={ + "argparse_meta": { + "arg_names": ["--pad-packed-seq-alignment"], + "type": _parse_pad_packed_seq_alignment, + } + }, + ) + """Pad THD packed sequence tensors after packing. + + If set to ``max``, token-like tensors are padded to + max_seqlen_per_dp_cp_rank. If set to a positive integer N, token-like + tensors are padded to a multiple of N. + """ + + pad_packed_seq_by_appending_dummy_seq: bool = True + """Represent a THD packed-sequence padding tail by appending a dummy sequence. + + When disabled, token-like tensors are still padded according to + pad_packed_seq_alignment, but cu_seqlens sequence boundaries are not extended + for the padding tail. CUDA Graph static-input padding may still pad the + cu_seqlens tensors to thd_max_packed_sequences + 1 entries. + """ + expert_model_parallel_size: int = 1 """Distributes Moe Experts across sub data parallel dimension.""" @@ -139,6 +179,17 @@ class ModelParallelConfig: None, no function is called on the loss. """ + moe_grad_scale_func: Optional[Callable] = None + """If using loss scaling for MoE auxiliary losses, this function should return the + scale tensor for MoE aux loss. If None, falls back to grad_scale_func. + """ + + mtp_grad_scale_func: Optional[Callable] = None + """If using loss scaling for MTP (Multi-Token Prediction), this function should return the + scalar or size-1 scale value for MTP loss. The value is converted to the output tensor + device. If None, falls back to grad_scale_func with torch.ones(1). + """ + no_sync_func: Optional[Callable] = None """Function that creates a context that suppresses asynchronous data-parallel communication. If the model is an instance of core.distributed.DistributedDataParallel, the default is to use @@ -463,6 +514,28 @@ def __post_init__(self): f"got {self.min_dynamic_context_parallel_size}" ) + if self.pad_packed_seq_alignment is not None: + self.pad_packed_seq_alignment = _parse_pad_packed_seq_alignment( + self.pad_packed_seq_alignment + ) + if self.max_seqlen_per_dp_cp_rank is None: + raise ValueError( + "max_seqlen_per_dp_cp_rank must be set when pad_packed_seq_alignment " + "is enabled." + ) + if self.pad_packed_seq_alignment != "max": + if self.pad_packed_seq_alignment <= 0: + raise ValueError( + "pad_packed_seq_alignment must be 'max' or a positive integer " "alignment." + ) + if self.pad_packed_seq_alignment > self.max_seqlen_per_dp_cp_rank: + raise ValueError( + "pad_packed_seq_alignment must not exceed " + "max_seqlen_per_dp_cp_rank " + f"({self.max_seqlen_per_dp_cp_rank}), got " + f"{self.pad_packed_seq_alignment}." + ) + if self.sequence_parallel: if self.tensor_model_parallel_size <= 1: raise ValueError("Cannot use sequence parallelism without tensor parallelism") @@ -479,6 +552,15 @@ def __post_init__(self): if self.autocast_dtype is None: self.autocast_dtype = self.params_dtype + if self.cross_entropy_loss_fusion and self.cross_entropy_fusion_impl == 'te': + warnings.warn( + "Transformer Engine cross entropy loss fusion has known stability issues. " + "Megatron-LM training args validation rejects this combination by default. " + "Use cross_entropy_fusion_impl='native', or disable cross_entropy_loss_fusion.", + UserWarning, + stacklevel=2, + ) + if self.defer_embedding_wgrad_compute and self.pipeline_model_parallel_size == 1: raise ValueError( "Cannot defer embedding wgrad compute when pipeline model parallel is not used" diff --git a/megatron/core/models/T5/t5_spec.py b/megatron/core/models/T5/t5_spec.py index 9f465df5c21..0b273b8f9e7 100644 --- a/megatron/core/models/T5/t5_spec.py +++ b/megatron/core/models/T5/t5_spec.py @@ -1,4 +1,6 @@ # Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +from functools import partial + from megatron.core.extensions.transformer_engine import HAVE_TE from megatron.core.fusions.fused_bias_dropout import get_bias_dropout_add from megatron.core.tensor_parallel.layers import ColumnParallelLinear, RowParallelLinear @@ -63,14 +65,14 @@ def encoder_model_with_transformer_engine_default_spec() -> ModuleSpec: submodules=SelfAttentionSubmodules( linear_qkv=not_none(TELayerNormColumnParallelLinear), core_attention=not_none(TEDotProductAttention), - linear_proj=TERowParallelLinear, + linear_proj=not_none(TERowParallelLinear), q_layernorm=IdentityOp, k_layernorm=IdentityOp, ), ), self_attn_bda=get_bias_dropout_add, - mlp=ModuleSpec( - module=MLP, + mlp=partial( + MLP.as_mlp_submodule, submodules=MLPSubmodules( linear_fc1=not_none(TELayerNormColumnParallelLinear), linear_fc2=not_none(TERowParallelLinear), @@ -93,7 +95,7 @@ def decoder_model_with_transformer_engine_default_spec() -> ModuleSpec: submodules=SelfAttentionSubmodules( linear_qkv=not_none(TELayerNormColumnParallelLinear), core_attention=not_none(TEDotProductAttention), - linear_proj=TERowParallelLinear, + linear_proj=not_none(TERowParallelLinear), q_layernorm=IdentityOp, k_layernorm=IdentityOp, ), @@ -107,12 +109,12 @@ def decoder_model_with_transformer_engine_default_spec() -> ModuleSpec: linear_q=not_none(TEColumnParallelLinear), linear_kv=not_none(TEColumnParallelLinear), core_attention=not_none(TEDotProductAttention), - linear_proj=TERowParallelLinear, + linear_proj=not_none(TERowParallelLinear), ), ), cross_attn_bda=get_bias_dropout_add, - mlp=ModuleSpec( - module=MLP, + mlp=partial( + MLP.as_mlp_submodule, submodules=MLPSubmodules( linear_fc1=not_none(TELayerNormColumnParallelLinear), linear_fc2=not_none(TERowParallelLinear), @@ -143,8 +145,8 @@ def encoder_model_with_local_spec() -> ModuleSpec: ), self_attn_bda=get_bias_dropout_add, pre_mlp_layernorm=LNImpl, - mlp=ModuleSpec( - module=MLP, + mlp=partial( + MLP.as_mlp_submodule, submodules=MLPSubmodules( linear_fc1=ColumnParallelLinear, linear_fc2=RowParallelLinear ), @@ -190,8 +192,8 @@ def decoder_model_with_local_spec() -> ModuleSpec: ), cross_attn_bda=get_bias_dropout_add, pre_mlp_layernorm=LNImpl, - mlp=ModuleSpec( - module=MLP, + mlp=partial( + MLP.as_mlp_submodule, submodules=MLPSubmodules( linear_fc1=ColumnParallelLinear, linear_fc2=RowParallelLinear ), diff --git a/megatron/core/models/backends.py b/megatron/core/models/backends.py index b019d527342..a270161ddd6 100644 --- a/megatron/core/models/backends.py +++ b/megatron/core/models/backends.py @@ -103,7 +103,7 @@ def column_parallel_linear(self) -> type: """Which column parallel linear module the backend uses""" return ColumnParallelLinear - def row_parallel_linear(self) -> type: + def row_parallel_linear(self) -> type[RowParallelLinear]: """Which row parallel linear module the backend uses""" return RowParallelLinear @@ -157,8 +157,8 @@ def column_parallel_linear(self) -> type: """Which column parallel linear module TE backend uses""" return InferenceColumnParallelLinear - def row_parallel_linear(self) -> type: - """Which row parallel linear module TE backend uses""" + def row_parallel_linear(self) -> type[InferenceRowParallelLinear]: + """Which row parallel linear module Inference backend uses""" return InferenceRowParallelLinear def fuse_layernorm_and_linear(self) -> bool: diff --git a/megatron/core/models/bert/bert_layer_specs.py b/megatron/core/models/bert/bert_layer_specs.py index 53cc0f4280d..dc0099fa66e 100644 --- a/megatron/core/models/bert/bert_layer_specs.py +++ b/megatron/core/models/bert/bert_layer_specs.py @@ -1,5 +1,6 @@ # Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. import warnings +from functools import partial from megatron.core.extensions.transformer_engine import HAVE_TE from megatron.core.fusions.fused_bias_dropout import get_bias_dropout_add @@ -66,8 +67,8 @@ def get_bert_layer_with_transformer_engine_submodules() -> TransformerLayerSubmo ), ), self_attn_bda=get_bias_dropout_add, - mlp=ModuleSpec( - module=MLP, + mlp=partial( + MLP.as_mlp_submodule, submodules=MLPSubmodules( linear_fc1=not_none(TELayerNormColumnParallelLinear), linear_fc2=not_none(TERowParallelLinear), @@ -117,8 +118,8 @@ def __getattr__(name): ), self_attn_bda=get_bias_dropout_add, pre_mlp_layernorm=LNImpl, - mlp=ModuleSpec( - module=MLP, + mlp=partial( + MLP.as_mlp_submodule, submodules=MLPSubmodules(linear_fc1=ColumnParallelLinear, linear_fc2=RowParallelLinear), ), mlp_bda=get_bias_dropout_add, diff --git a/megatron/core/models/common/embeddings/rope_utils.py b/megatron/core/models/common/embeddings/rope_utils.py index c97f738771b..ed9893f2223 100644 --- a/megatron/core/models/common/embeddings/rope_utils.py +++ b/megatron/core/models/common/embeddings/rope_utils.py @@ -216,18 +216,22 @@ def _apply_rotary_pos_emb_thd( mla_output_remove_interleaving: bool = False, cp_group: torch.distributed.ProcessGroup = None, multi_latent_attention: Optional[bool] = None, + max_seqlen: Optional[int] = None, ) -> Tensor: - """A baseline implementation of applying RoPE for `thd` format. + """Apply RoPE for `thd` format using pure CUDA ops (CUDA Graph compatible). + + Replaces the original Python-loop + .tolist() implementation with vectorized + CUDA operations. No GPU->CPU syncs, compatible with CUDA Graph capture. Args: - t (Tensor): Input tensor T is of shape [t, h, d] - cu_seqlens(Tensor): Cumulative sum of sequence lengths in a batch for `t`, - with shape [b + 1] and dtype torch.int32. - freqs (Tensor): Rotary Positional embedding tensor freq is of shape [max_s, 1, 1, d] - cp_group (torch.distributed.ProcessGroup): The context parallel group + t (Tensor): Input tensor of shape [total_tokens, h, d] + cu_seqlens (Tensor): Cumulative sequence lengths, shape [num_seqs + 1], int32. + freqs (Tensor): RoPE frequencies, shape [max_s, 1, 1, d] or [total_tokens, 1, 1, d] + cp_group: Context parallel group + max_seqlen: Global max sequence length for this packed batch when known. Returns: - Tensor: Shape [t, h, d]. The input tensor after applying RoPE. + Tensor: Shape [total_tokens, h, d]. Input with RoPE applied. """ if multi_latent_attention is not None: warnings.warn( @@ -240,54 +244,70 @@ def _apply_rotary_pos_emb_thd( raise ValueError("cp_group must be provided for THD format RoPE") cp_size = cp_group.size() cp_rank = cp_group.rank() - seqlens = ((cu_seqlens[1:] - cu_seqlens[:-1]) // cp_size).tolist() - - # Handle two different frequency tensor formats: - # 1. If freqs.size(0) == cu_seqlens[-1]: freqs contains all positions across all sequences - # -> Use offset-based mapping for exact positional correspondence - # 2. Otherwise: freqs contains only max sequence length positions - # -> Use traditional mapping without offsets (map first :seqlen part) - if freqs.dim() >= 1 and freqs.size(0) == cu_seqlens[-1]: - # CASE 1: Exact mapping with offsets - # Build packed freqs in one pass, then apply once to the whole packed tensor - sequence_splits = torch.split(t, seqlens) - freq_slices = [] - for i, x in enumerate(sequence_splits): - # cu_seqlens[i] is the starting offset of this sequence in the original batch - seq_start_offset = cu_seqlens[i].item() - freq_slices.append( - _get_thd_freqs_on_this_cp_rank(cp_rank, cp_size, x, freqs, seq_start_offset) - ) - freqs_packed = torch.cat(freq_slices, dim=0) + total_tokens = t.shape[0] + device = t.device - return _apply_rotary_pos_emb_bshd( - t.unsqueeze(1), - freqs_packed, - rotary_interleaved=rotary_interleaved, - mla_rotary_interleaved=mla_rotary_interleaved, - mscale=mscale, - inverse=inverse, - mla_output_remove_interleaving=mla_output_remove_interleaving, - ).squeeze(1) - else: - # CASE 2: Traditional mapping without offsets - # Build packed freqs for all sequences using the standard mapping, then apply once - sequence_splits = torch.split(t, seqlens) - freqs_packed = torch.cat( - [_get_thd_freqs_on_this_cp_rank(cp_rank, cp_size, x, freqs) for x in sequence_splits], - dim=0, + token_pos = torch.arange(total_tokens, device=device, dtype=torch.int64) + + # `cu_seqlens` describes the global packed sequence. With CP, `t` is already + # CP-partitioned, so build a local cumulative-length view before assigning + # local tokens to packed sequences. + cu_seqlens_i64 = cu_seqlens.to(torch.int64) + global_seq_lens = cu_seqlens_i64[1:] - cu_seqlens_i64[:-1] + local_seq_lens = global_seq_lens // cp_size if cp_size > 1 else global_seq_lens + local_cu_seqlens = torch.zeros_like(cu_seqlens_i64) + local_cu_seqlens[1:] = torch.cumsum(local_seq_lens, dim=0) + + # `searchsorted(..., right=True) - 1` returns the local sequence index. The + # clamp guards padded tokens that sit beyond the final real local token; they + # get a harmless frequency and are later masked out. + seq_idx = torch.searchsorted(local_cu_seqlens, token_pos, right=True) - 1 + seq_idx = seq_idx.clamp(min=0, max=cu_seqlens.shape[0] - 2) + + local_seq_start = local_cu_seqlens[seq_idx] + local_pos = token_pos - local_seq_start + local_seq_len = local_seq_lens[seq_idx] + global_seq_start = cu_seqlens_i64[seq_idx] + + if cp_size > 1: + cp_seg = local_seq_len // 2 + full_seqlen = local_seq_len * cp_size + is_first_half = local_pos < cp_seg + freq_pos = torch.where( + is_first_half, + cp_rank * cp_seg + local_pos, + full_seqlen - (cp_rank + 1) * cp_seg + (local_pos - cp_seg), ) + else: + freq_pos = local_pos.to(torch.int64) - return _apply_rotary_pos_emb_bshd( - t.unsqueeze(1), - freqs_packed, - rotary_interleaved=rotary_interleaved, - mla_rotary_interleaved=mla_rotary_interleaved, - mscale=mscale, - inverse=inverse, - mla_output_remove_interleaving=mla_output_remove_interleaving, - ).squeeze(1) + assert max_seqlen is not None, ( + "max_seqlen must be provided for THD RoPE so packed-frequency offset " + "detection does not silently depend on tensor shape heuristics." + ) + exact_packed_freqs = freqs.dim() >= 1 and freqs.size(0) > max_seqlen + if exact_packed_freqs: + # `freqs` covers all positions across all sequences (used for non-1D + # RoPE / VLMs); shift by the per-sequence start offset so each token + # samples its absolute position. When `freqs` only spans one max-len + # sequence, no shift is needed. + freq_pos = freq_pos + global_seq_start + + # Padded positions can sit outside the frequency table. Clamp them into + # range; downstream padding masks exclude those positions from the result. + freq_pos = freq_pos.clamp(min=0, max=freqs.shape[0] - 1) + freqs_packed = freqs[freq_pos] + + return _apply_rotary_pos_emb_bshd( + t.unsqueeze(1), + freqs_packed, + rotary_interleaved=rotary_interleaved, + mla_rotary_interleaved=mla_rotary_interleaved, + mscale=mscale, + inverse=inverse, + mla_output_remove_interleaving=mla_output_remove_interleaving, + ).squeeze(1) def apply_rotary_pos_emb( @@ -300,6 +320,7 @@ def apply_rotary_pos_emb( mla_rotary_interleaved: bool = False, inverse: bool = False, mla_output_remove_interleaving: bool = False, + max_seqlen: Optional[int] = None, ): """ Reroute to the appropriate apply_rotary_pos_emb function depending on @@ -310,6 +331,8 @@ def apply_rotary_pos_emb( # Keep for backward compatibility. Will deprecate in the future. if cp_group is None: cp_group = parallel_state.get_context_parallel_group() + if mla_rotary_interleaved is None: + mla_rotary_interleaved = config.multi_latent_attention if config.apply_rope_fusion: if cu_seqlens is None: @@ -375,6 +398,7 @@ def apply_rotary_pos_emb( cp_group=cp_group, inverse=inverse, mla_output_remove_interleaving=mla_output_remove_interleaving, + max_seqlen=max_seqlen, ) diff --git a/megatron/core/models/common/embeddings/rotary_pos_embedding.py b/megatron/core/models/common/embeddings/rotary_pos_embedding.py index 804bdb7c537..77eb94a34bf 100644 --- a/megatron/core/models/common/embeddings/rotary_pos_embedding.py +++ b/megatron/core/models/common/embeddings/rotary_pos_embedding.py @@ -205,6 +205,47 @@ def forward( return emb + def _set_cos_sin_cache(self, seq_len, offset, dtype, packed_seq=False, cp_group=None): + """Materialize cached cos/sin tensors for ``[seq_len, ..., dim]``.""" + self.max_seq_len_cached = seq_len + self.offset_cached = offset + self.dtype_cached = dtype + self.packed_seq_cached = packed_seq + + emb = self.forward(seq_len, offset, packed_seq=packed_seq, cp_group=cp_group) + self.register_buffer("cos_cached", emb.cos().to(dtype).contiguous(), persistent=False) + self.register_buffer("sin_cached", emb.sin().to(dtype).contiguous(), persistent=False) + + def get_cached_cos_sin( + self, + seq_len, + offset=0, + dtype=torch.get_default_dtype(), + packed_seq=False, + cp_group=None, + mscale=None, + ): + """Get cached cos and sin values. + + The cache is rebuilt on first use or whenever ``seq_len`` grows + beyond the cached length, or any of ``offset`` / ``dtype`` / + ``packed_seq`` changes from the previous call. + ``YarnRotaryEmbedding`` overrides this to also bake its + concentration factor into the cached cos/sin (controlled by + ``mscale``); for the base class without a concentration + factor the argument is accepted-and-ignored for API uniformity. + """ + del mscale # base class has no concentration factor + if ( + not hasattr(self, "max_seq_len_cached") + or seq_len > self.max_seq_len_cached + or offset != self.offset_cached + or dtype != self.dtype_cached + or packed_seq != self.packed_seq_cached + ): + self._set_cos_sin_cache(seq_len, offset, dtype, packed_seq, cp_group) + return (self.cos_cached[:seq_len, ...], self.sin_cached[:seq_len, ...]) + def _load_from_state_dict(self, state_dict, prefix, *args, **kwargs): state_dict.pop(f'{prefix}inv_freq', None) return super()._load_from_state_dict(state_dict, prefix, *args, **kwargs) diff --git a/megatron/core/models/common/embeddings/yarn_rotary_pos_embedding.py b/megatron/core/models/common/embeddings/yarn_rotary_pos_embedding.py index bc5a9c5fa3f..cb8a03d0b2b 100644 --- a/megatron/core/models/common/embeddings/yarn_rotary_pos_embedding.py +++ b/megatron/core/models/common/embeddings/yarn_rotary_pos_embedding.py @@ -186,13 +186,18 @@ def forward( emb = get_pos_emb_on_this_cp_rank(emb, 0, cp_group) return emb, _mscale - def _set_cos_sin_cache(self, seq_len, offset, dtype, packed_seq=False, cp_group=None): + def _set_cos_sin_cache( + self, seq_len, offset, dtype, packed_seq=False, cp_group=None, mscale=None + ): self.max_seq_len_cached = seq_len self.offset_cached = offset self.dtype_cached = dtype self.packed_seq_cached = packed_seq + self.mscale_cached = mscale emb, _mscale = self.forward(seq_len, offset, packed_seq=packed_seq, cp_group=cp_group) + if mscale is not None: + _mscale = mscale self.register_buffer( "cos_cached", (emb.cos() * _mscale).to(dtype).contiguous(), persistent=False ) @@ -201,16 +206,34 @@ def _set_cos_sin_cache(self, seq_len, offset, dtype, packed_seq=False, cp_group= ) def get_cached_cos_sin( - self, seq_len, offset=0, dtype=torch.get_default_dtype(), packed_seq=False, cp_group=None + self, + seq_len, + offset=0, + dtype=torch.get_default_dtype(), + packed_seq=False, + cp_group=None, + mscale=None, ): - """Get cached cos and sin values.""" + """Get cached cos and sin values. + + Args: + mscale: when ``None`` (default), the cached cos/sin are + multiplied by yarn's internal concentration factor (the + normal long-context behaviour). When a float is supplied, + that value is used in place of the internal factor — e.g. + the DSv4 hybrid model passes ``mscale=1.0`` to enforce + its "pure rotation" contract and keep the fused / + unfused rope paths bit-equivalent. + """ if ( - seq_len > self.max_seq_len_cached + not hasattr(self, "max_seq_len_cached") + or seq_len > self.max_seq_len_cached or offset != self.offset_cached or dtype != self.dtype_cached or packed_seq != self.packed_seq_cached + or mscale != getattr(self, "mscale_cached", None) ): - self._set_cos_sin_cache(seq_len, offset, dtype, packed_seq, cp_group) + self._set_cos_sin_cache(seq_len, offset, dtype, packed_seq, cp_group, mscale) return (self.cos_cached[:seq_len, ...], self.sin_cached[:seq_len, ...]) diff --git a/megatron/core/models/common/language_module/language_module.py b/megatron/core/models/common/language_module/language_module.py index 9d8eddbdda6..92db84ce0c9 100644 --- a/megatron/core/models/common/language_module/language_module.py +++ b/megatron/core/models/common/language_module/language_module.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. import logging import os from typing import Optional, Tuple @@ -180,7 +180,9 @@ def compute_language_model_loss(self, labels: Tensor, logits: Tensor) -> Tensor: elif self.config.cross_entropy_fusion_impl == 'native': loss = fused_vocab_parallel_cross_entropy(logits, labels, self.pg_collection.tp) else: - loss = tensor_parallel.vocab_parallel_cross_entropy(logits, labels) + loss = tensor_parallel.vocab_parallel_cross_entropy( + logits, labels, tp_group=self.tp_group + ) # [s b] => [b, s] loss = loss.transpose(0, 1).contiguous() @@ -202,7 +204,12 @@ def setup_embeddings_and_output_layer(self) -> None: # Mark embedding and output layer for decoupled_lr and other features. # This is the original Megatron attribute used by decoupled_lr, Muon, FSDP, etc. - if self.pre_process and hasattr(self, 'embedding'): + # Include MTP-stage embedding too: it is a duplicated copy of the pre_process + # embedding (kept in sync via cross-stage all-reduce). Without this tag, the + # LayerWise distributed optimizer routes it to its Muon-managed buffer and + # `_emit_bucket(shared_embedding=True)` replicates the (vocab x hidden) tensor + # across all dp_size shards, blowing up the chunk's buffer by ~8x. + if (self.pre_process or getattr(self, 'mtp_process', False)) and hasattr(self, 'embedding'): self.embedding.word_embeddings.weight.is_embedding_or_output_parameter = True if ( self.post_process diff --git a/megatron/core/models/common/model_chunk_schedule_plan.py b/megatron/core/models/common/model_chunk_schedule_plan.py index ce7fea4870d..8358e05a612 100644 --- a/megatron/core/models/common/model_chunk_schedule_plan.py +++ b/megatron/core/models/common/model_chunk_schedule_plan.py @@ -1,7 +1,7 @@ # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. from contextlib import nullcontext -from typing import Optional +from typing import Any, Callable, Optional import torch from torch import Tensor @@ -325,6 +325,9 @@ def __init__( runtime_gather_output: Optional[bool] = None, loss_mask: Optional[Tensor] = None, padding_mask=None, + *, + output_processor: Optional[Callable[..., Tensor]] = None, + output_processor_context: Optional[Any] = None, ): """Initialize the schedule plan of all Transformer layers' sub-modules. @@ -342,6 +345,10 @@ def __init__( extra_block_kwargs: Additional keyword arguments for blocks. runtime_gather_output: Whether to gather output at runtime. loss_mask (torch.Tensor): Used to mask out some portions of the loss + output_processor (Callable): Custom postprocess hook to run instead of the + default logits/loss path. + output_processor_context (Any): User-defined context object forwarded to + `output_processor`. Returns: The model chunk schedule plan. @@ -367,6 +374,8 @@ def __init__( self._model_chunk_state.padding_mask = padding_mask self._model_chunk_state.extra_block_kwargs = extra_block_kwargs self._model_chunk_state.runtime_gather_output = runtime_gather_output + self._model_chunk_state.output_processor = output_processor + self._model_chunk_state.output_processor_context = output_processor_context self._model_chunk_state.model = model self._model_chunk_state.context = None self._model_chunk_state.context_mask = None diff --git a/megatron/core/models/gpt/experimental_attention_variant_module_specs.py b/megatron/core/models/gpt/experimental_attention_variant_module_specs.py index 329b8f259ea..d2716dfc317 100644 --- a/megatron/core/models/gpt/experimental_attention_variant_module_specs.py +++ b/megatron/core/models/gpt/experimental_attention_variant_module_specs.py @@ -38,10 +38,12 @@ from megatron.core.transformer.transformer_config import TransformerConfig from megatron.core.transformer.transformer_layer import ( HyperConnectionTransformerLayer, + MlpBuilder, TransformerLayer, TransformerLayerSubmodules, get_transformer_layer_offset, ) +from megatron.core.typed_torch import not_none try: import transformer_engine as te # type: ignore[import-untyped] # pylint: disable=unused-import @@ -283,14 +285,18 @@ def get_transformer_layer_with_experimental_attention_variant_spec( moe_layer_pattern = [0] * config.num_layers if 1 in moe_layer_pattern: - moe_layer_spec = _get_moe_module_spec(config=config, backend=backend) + moe_layer_spec, fuse_layernorm_pre_moe = _get_moe_module_spec( + config=config, backend=backend + ) else: - moe_layer_spec = None + moe_layer_spec, fuse_layernorm_pre_moe = None, False if 0 in moe_layer_pattern: - dense_mlp_layer_spec = _get_dense_mlp_module_spec(config=config, backend=backend) + dense_mlp_layer_spec, fuse_layernorm_pre_dense = _get_dense_mlp_module_spec( + config=config, backend=backend + ) else: - dense_mlp_layer_spec = None + dense_mlp_layer_spec, fuse_layernorm_pre_dense = None, False # Get GPT decoder block layer specs rms_norm = config.normalization == "RMSNorm" @@ -306,6 +312,11 @@ def get_transformer_layer_with_experimental_attention_variant_spec( else standard_attention_spec ) mlp = moe_layer_spec if moe_layer_pattern[layer_number] == 1 else dense_mlp_layer_spec + fuse_pre_mlp_layernorm = ( + fuse_layernorm_pre_moe + if moe_layer_pattern[layer_number] == 1 + else fuse_layernorm_pre_dense + ) input_layernorm = ( IdentityOp if attention.metainfo["fuse_input_layernorm"] @@ -313,7 +324,7 @@ def get_transformer_layer_with_experimental_attention_variant_spec( ) pre_mlp_layernorm = ( IdentityOp - if mlp.metainfo["fuse_pre_mlp_layernorm"] + if fuse_pre_mlp_layernorm else backend.layer_norm(rms_norm=rms_norm, for_qk=False) ) @@ -326,7 +337,7 @@ def get_transformer_layer_with_experimental_attention_variant_spec( self_attn_bda=get_bias_dropout_add, self_attention_hyper_connection=hc_module, pre_mlp_layernorm=pre_mlp_layernorm, - mlp=mlp, + mlp=not_none(mlp), mlp_bda=get_bias_dropout_add, mlp_hyper_connection=hc_module, ), @@ -524,41 +535,50 @@ def _get_self_attention_module_spec( def _get_dense_mlp_module_spec( config: TransformerConfig, backend: BackendSpecProvider = None -) -> ModuleSpec: +) -> tuple[MlpBuilder, bool]: """Get dense MLP module spec. For hybrid models that mix dense MLP and experimental attention architectures. - Warning: This function may be deprecated in the future.""" + Warning: This function may be deprecated in the future. + + Returns: + A tuple of (MLP module spec, whether to fuse pre-MLP layernorm) + """ if backend is None: backend = _get_backend_spec_provider(config=config) from megatron.core.models.gpt.gpt_layer_specs import get_mlp_module_spec_for_backend - mlp_spec = get_mlp_module_spec_for_backend(backend=backend, num_experts=None) - mlp_spec.metainfo["fuse_pre_mlp_layernorm"] = backend.fuse_layernorm_and_linear() - - return mlp_spec + return ( + get_mlp_module_spec_for_backend(backend=backend, num_experts=None), + backend.fuse_layernorm_and_linear(), + ) def _get_moe_module_spec( config: TransformerConfig, backend: BackendSpecProvider = None -) -> ModuleSpec: +) -> tuple[MlpBuilder, bool]: """Get MoE module spec. For hybrid models that mix MoE and experimental attention architectures. - Warning: This function may be deprecated in the future.""" + Warning: This function may be deprecated in the future. + + Returns: + A tuple of (MoE module spec, whether to fuse pre-MoE layernorm) + """ if backend is None: backend = _get_backend_spec_provider(config=config) from megatron.core.models.gpt.moe_module_specs import get_moe_module_spec_for_backend - moe_spec = get_moe_module_spec_for_backend( - backend=backend, - num_experts=config.num_moe_experts, - moe_grouped_gemm=config.moe_grouped_gemm, - use_te_activation_func=config.use_te_activation_func, + return ( + get_moe_module_spec_for_backend( + backend=backend, + num_experts=config.num_moe_experts, + moe_grouped_gemm=config.moe_grouped_gemm, + use_te_activation_func=config.use_te_activation_func, + ), + False, ) - moe_spec.metainfo["fuse_pre_mlp_layernorm"] = False - return moe_spec diff --git a/megatron/core/models/gpt/fine_grained_callables.py b/megatron/core/models/gpt/fine_grained_callables.py index b41e1e7afff..191786ddca3 100644 --- a/megatron/core/models/gpt/fine_grained_callables.py +++ b/megatron/core/models/gpt/fine_grained_callables.py @@ -61,7 +61,7 @@ def should_free_input(name, is_moe, config, num_local_experts): return False enable_deepep = ( config.moe_token_dispatcher_type == "flex" - and config.moe_flex_dispatcher_backend == "deepep" + and config.moe_flex_dispatcher_backend in ("deepep", "deepepv2") ) enable_hybridep = ( config.moe_token_dispatcher_type == "flex" @@ -228,6 +228,8 @@ def forward_impl(self, hidden_states): sequence_len_offset=self.chunk_state.sequence_len_offset, runtime_gather_output=self.chunk_state.runtime_gather_output, extra_block_kwargs=self.chunk_state.extra_block_kwargs, + output_processor=self.chunk_state.output_processor, + output_processor_context=self.chunk_state.output_processor_context, ) # For now, 1f1b only supports fp16 module @@ -378,7 +380,6 @@ def backward_dw(self): # Execute TransformerLayer backward hook. if self.is_layer_first_node: self._post_backward_hook() - self.bwd_dw_callables = None def set_post_forward_hook(self, hook): @@ -493,7 +494,7 @@ def build_transformer_layer_callables(layer: TransformerLayer): is_moe = isinstance(layer.mlp, MoELayer) enable_deepep = ( layer.config.moe_token_dispatcher_type == "flex" - and layer.config.moe_flex_dispatcher_backend == "deepep" + and layer.config.moe_flex_dispatcher_backend in ("deepep", "deepepv2") ) enable_hybridep = ( layer.config.moe_token_dispatcher_type == "flex" @@ -726,14 +727,17 @@ def submodule_mtp_attn_forward(node, hidden_states): node.chunk_state.mtp_hidden_states = list(torch.chunk(hidden_states, 1 + offset, dim=0)) hidden_states = node.chunk_state.mtp_hidden_states[offset] - input_ids, position_ids, decoder_input, hidden_states = layer._get_embeddings( + input_ids, position_ids, padding_mask, decoder_input, hidden_states = layer._get_embeddings( input_ids=node.chunk_state.input_ids, position_ids=node.chunk_state.position_ids, embedding=node.chunk_state.model.embedding, hidden_states=hidden_states, + packed_seq_params=node.chunk_state.packed_seq_params, + padding_mask=node.chunk_state.padding_mask, ) node.chunk_state.input_ids = input_ids node.chunk_state.position_ids = position_ids + node.chunk_state.padding_mask = padding_mask # MTP Layer Preprocess # norm, linear projection and transformer diff --git a/megatron/core/models/gpt/gpt_layer_specs.py b/megatron/core/models/gpt/gpt_layer_specs.py index 1a6e37f1faa..5b938cc6364 100755 --- a/megatron/core/models/gpt/gpt_layer_specs.py +++ b/megatron/core/models/gpt/gpt_layer_specs.py @@ -1,6 +1,7 @@ # Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. import copy import warnings +from functools import partial from typing import Optional, Union from megatron.core.extensions.transformer_engine import HAVE_TE @@ -37,18 +38,23 @@ from megatron.core.transformer.transformer_config import TransformerConfig from megatron.core.transformer.transformer_layer import ( HyperConnectionTransformerLayer, + MlpBuilder, TransformerLayer, TransformerLayerSubmodules, get_transformer_layer_offset, ) -from megatron.core.typed_torch import copy_signature +from megatron.core.typed_torch import copy_signature, not_none from megatron.core.utils import is_te_min_version if HAVE_TE: - from megatron.core.extensions.transformer_engine import TEFusedDenseMLP, TEFusedMLP, TENorm + from megatron.core.extensions.transformer_engine import ( + TEFusedMLP, + TEFusedMLPWithGroupedLinear, + TENorm, + ) from megatron.core.extensions.transformer_engine_spec_provider import TESpecProvider else: - TEFusedDenseMLP, TEFusedMLP, TENorm, TESpecProvider = None, None, None, None + TEFusedMLP, TEFusedMLPWithGroupedLinear, TENorm, TESpecProvider = (None, None, None, None) try: from megatron.core.extensions.kitchen import HAVE_KITCHEN, KitchenSpecProvider @@ -188,6 +194,7 @@ def get_gpt_layer_with_transformer_engine_submodules( enable_hyper_connection: bool = False, mla_down_proj_fusion: bool = False, dense_grouped_gemm: bool = False, + use_grouped_gemm_for_dense_mlp: bool = False, ) -> TransformerLayerSubmodules: """Use these submodules to use lower-level Transformer Engine modules (required for fp8 training). @@ -239,6 +246,7 @@ def get_gpt_layer_with_transformer_engine_submodules( use_te_op_fuser=use_te_op_fuser, use_te_activation_func=use_te_activation_func, dense_grouped_gemm=dense_grouped_gemm, + use_grouped_gemm_for_dense_mlp=use_grouped_gemm_for_dense_mlp, ) hc_module = HyperConnectionModule if enable_hyper_connection else IdentityOp @@ -512,7 +520,7 @@ def get_mlp_module_spec( moe_grouped_gemm: Optional[bool] = False, fp8: Optional[str] = None, # pylint: disable=unused-argument use_te_op_fuser: Optional[bool] = False, -) -> ModuleSpec: +) -> MlpBuilder: """Helper function to get module spec for MLP/MoE""" if fp8 is not None: warnings.warn( @@ -544,7 +552,8 @@ def get_mlp_module_spec_for_backend( use_te_op_fuser: Optional[bool] = False, use_te_activation_func: bool = False, dense_grouped_gemm: bool = False, -) -> ModuleSpec: + use_grouped_gemm_for_dense_mlp: bool = False, +) -> MlpBuilder: """Helper function to get module spec for MLP/MoE""" linear_fc2 = backend.row_parallel_linear() @@ -552,19 +561,21 @@ def get_mlp_module_spec_for_backend( if num_experts is None: # Dense MLP w/ or w/o TE modules. - if dense_grouped_gemm and use_te_op_fuser: - module = TEFusedDenseMLP + # `dense_grouped_gemm` (dev) and `use_grouped_gemm_for_dense_mlp` (main) both request + # the grouped-GEMM dense MLP fusion (SM100+ / MXFP8). + if (dense_grouped_gemm or use_grouped_gemm_for_dense_mlp) and use_te_op_fuser: + module = not_none(TEFusedMLPWithGroupedLinear).as_mlp_submodule elif use_te_op_fuser: - module = TEFusedMLP + module = not_none(TEFusedMLP).as_mlp_submodule else: - module = MLP + module = MLP.as_mlp_submodule if backend.fuse_layernorm_and_linear(): linear_fc1 = backend.column_parallel_layer_norm_linear() assert linear_fc1 is not None else: linear_fc1 = backend.column_parallel_linear() - return ModuleSpec( - module=module, + return partial( + module, submodules=MLPSubmodules( linear_fc1=linear_fc1, linear_fc2=linear_fc2, activation_func=activation_func ), diff --git a/megatron/core/models/gpt/gpt_model.py b/megatron/core/models/gpt/gpt_model.py index 7bce9d96d2c..01df346c05c 100644 --- a/megatron/core/models/gpt/gpt_model.py +++ b/megatron/core/models/gpt/gpt_model.py @@ -1,7 +1,7 @@ # Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. from collections import OrderedDict -from typing import Dict, Literal, Optional +from typing import Any, Callable, Dict, Literal, Optional import torch from torch import Tensor @@ -9,7 +9,10 @@ from megatron.core import tensor_parallel from megatron.core.config_logger import has_config_logger_enabled, log_config_to_disk from megatron.core.dist_checkpointing.mapping import ShardedStateDict +from megatron.core.extensions.transformer_engine import TELMHeadColumnParallelLinear +from megatron.core.fp8_utils import is_mxfp8_output_proj_active from megatron.core.inference.contexts import BaseInferenceContext +from megatron.core.inference.utils import InferenceMode from megatron.core.models.common.embeddings import YarnRotaryEmbedding from megatron.core.models.common.embeddings.language_model_embedding import LanguageModelEmbedding from megatron.core.models.common.embeddings.rotary_pos_embedding import ( @@ -17,7 +20,7 @@ RotaryEmbedding, ) from megatron.core.models.common.language_module.language_module import LanguageModule -from megatron.core.packed_seq_params import PackedSeqParams +from megatron.core.packed_seq_params import PackedSeqParams, resolve_cp_group from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( FineGrainedActivationOffloadingInterface as off_interface, ) @@ -145,7 +148,10 @@ def __init__( self.rotary_scaling = rope_scaling self.mtp_block_spec = mtp_block_spec self.mtp_process = mtp_block_spec is not None and mtp_on_this_rank( - self.config, ignore_virtual=False, vp_stage=vp_stage + layout=self.config.pipeline_model_parallel_layout, + mtp_num_layers=self.config.mtp_num_layers, + ignore_virtual=False, + vp_stage=vp_stage, ) self.fuse_linear_cross_entropy = ( @@ -251,7 +257,12 @@ def __init__( self.embedding_activation_buffer = None self.grad_output_buffer = None - self.output_layer = LinearCrossEntropyModule( + output_layer_cls = ( + TELMHeadColumnParallelLinear + if is_mxfp8_output_proj_active(config) + else LinearCrossEntropyModule + ) + self.output_layer = output_layer_cls( config.hidden_size, self.vocab_size, config=config, @@ -316,7 +327,7 @@ def _preprocess( # If decoder_input is provided (not None), then input_ids and position_ids are ignored. # Otherwise, apply embedding layer on input_ids and position_ids to get decoder_input. - in_inference_mode = inference_context is not None and not self.training + in_inference_mode = InferenceMode.is_active() # Decoder embedding. if decoder_input is not None: @@ -353,7 +364,11 @@ def _preprocess( hasattr(inference_context, 'use_flashinfer_fused_rope') and inference_context.use_flashinfer_fused_rope ) - if in_inference_mode and (self.config.flash_decode or use_flash_infer_fused_rope): + if ( + in_inference_mode + and inference_context is not None + and (self.config.flash_decode or use_flash_infer_fused_rope) + ): assert ( not self.config.flash_decode ) or inference_context.is_static_batching(), ( @@ -385,7 +400,7 @@ def _preprocess( cp_group=packed_seq_params.cp_group if packed_seq_params is not None else None, ) elif self.position_embedding_type == 'yarn' and not self.config.multi_latent_attention: - if self.training or not self.config.flash_decode: + if not InferenceMode.is_active() or not self.config.flash_decode: rotary_seq_len = self.rotary_pos_emb.get_rotary_seq_len( inference_context, self.decoder, decoder_input, self.config, packed_seq_params ) @@ -401,7 +416,7 @@ def _preprocess( "YarnRotaryEmbedding yet." ) elif self.position_embedding_type == 'mrope' and not self.config.multi_latent_attention: - if self.training or not self.config.flash_decode: + if not InferenceMode.is_active() or not self.config.flash_decode: rotary_pos_emb = self.rotary_pos_emb( position_ids, self.mrope_section, @@ -416,6 +431,7 @@ def _preprocess( if ( in_inference_mode + and inference_context is not None and (self.config.cuda_graph_impl == "local" or self.config.flash_decode) and inference_context.is_static_batching() ): @@ -431,8 +447,10 @@ def _preprocess( if in_inference_mode: # Clear the outputs for padding tokens when using dynamic batching with # quantization scales to avoid corrupting amax calculations - if inference_context.is_dynamic_batching() and is_using_quantization_scales( - self.config + if ( + inference_context is not None + and inference_context.is_dynamic_batching() + and is_using_quantization_scales(self.config) ): decoder_input[inference_context.padding_slice] = 0.0 @@ -503,6 +521,8 @@ def forward( inference_params: Optional[BaseInferenceContext] = None, loss_mask: Optional[Tensor] = None, padding_mask: Optional[Tensor] = None, + output_processor: Optional[Callable[..., Tensor]] = None, + output_processor_context: Optional[Any] = None, ) -> Tensor: """Forward function of the GPT Model This function passes the input tensors through the embedding layer, and then the decoder and finally into the post @@ -516,6 +536,10 @@ def forward( padding_mask (Tensor, optional): Padding mask for MoE routing. Shape [bsz, seq_length]. True = padding (exclude), False = valid (include). Only used for MoE layers to exclude padding tokens from routing computations. + output_processor (Callable, optional): Custom postprocess hook that receives + decoder hidden states and output-layer helpers, then returns the model output. + output_processor_context (Any, optional): User-defined context object forwarded to + `output_processor`. """ if self.config.fine_grained_activation_offloading: self.preprocess_for_fine_grained_offloading() @@ -584,6 +608,7 @@ def forward( loss_mask=loss_mask, decoder_input=decoder_input, attention_mask=attention_mask, + padding_mask=padding_mask, inference_params=inference_params, packed_seq_params=packed_seq_params, sequence_len_offset=sequence_len_offset, @@ -591,6 +616,8 @@ def forward( extra_block_kwargs=extra_block_kwargs, inference_context=inference_context, mhc_multistream=mhc_multistream, + output_processor=output_processor, + output_processor_context=output_processor_context, ) def _postprocess( @@ -606,6 +633,7 @@ def _postprocess( loss_mask=None, decoder_input=None, attention_mask=None, + padding_mask=None, inference_params=None, packed_seq_params=None, sequence_len_offset=None, @@ -613,13 +641,15 @@ def _postprocess( extra_block_kwargs=None, inference_context=None, mhc_multistream=None, + output_processor=None, + output_processor_context=None, ): """Postprocesses decoder hidden states to generate logits or compute loss. Applies Multi-Token Prediction if enabled, generates output logits through the output layer, and computes language model loss when labels are provided. """ - in_inference_mode = inference_context is not None and not self.training + in_inference_mode = InferenceMode.is_active() if in_inference_mode: assert runtime_gather_output, "Inference must always gather TP logits" @@ -628,6 +658,7 @@ def _postprocess( # tokens rather than stale speculative tokens from the previous step. is_spec_decode = ( in_inference_mode + and inference_context is not None and inference_context.is_dynamic_batching() and inference_context.num_speculative_tokens > 0 ) @@ -649,6 +680,7 @@ def _postprocess( rotary_pos_sin=rotary_pos_sin, packed_seq_params=packed_seq_params, sequence_len_offset=sequence_len_offset, + padding_mask=padding_mask, embedding=self.embedding, **(extra_block_kwargs or {}), ) @@ -664,6 +696,7 @@ def _postprocess( self._decoder_hidden_states_cache = hidden_states else: # In training/eval, use the utility function for processing MTP loss/scaling. + mtp_cp_group = resolve_cp_group(self.pg_collection.cp, packed_seq_params) hidden_states = process_mtp_loss( hidden_states=hidden_states, labels=labels, @@ -674,13 +707,39 @@ def _postprocess( is_training=self.training, compute_language_model_loss=self.compute_language_model_loss, config=self.config, - cp_group=self.pg_collection.cp, + cp_group=mtp_cp_group, + tp_group=self.tp_group, packed_seq_params=packed_seq_params, scale_logits_fn=self._scale_logits if self.config.use_mup else None, + input_ids=input_ids, ) sequence_parallel_override = False - if in_inference_mode and inference_context.config.materialize_only_last_token_logits: + if output_processor is not None: + return output_processor( + hidden_states=hidden_states, + output_layer=self.output_layer, + output_weight=output_weight, + labels=labels, + loss_mask=loss_mask, + input_ids=input_ids, + position_ids=position_ids, + attention_mask=attention_mask, + decoder_input=decoder_input, + inference_context=inference_context, + packed_seq_params=packed_seq_params, + runtime_gather_output=runtime_gather_output, + context=output_processor_context, + compute_language_model_loss=self.compute_language_model_loss, + scale_logits=self._scale_logits, + config=self.config, + ) + + if ( + in_inference_mode + and inference_context is not None + and inference_context.config.materialize_only_last_token_logits + ): if inference_context.is_static_batching(): hidden_states = hidden_states[-1:, :, :] else: @@ -763,6 +822,9 @@ def build_schedule_plan( inference_params: Optional[BaseInferenceContext] = None, loss_mask: Optional[Tensor] = None, padding_mask: Optional[Tensor] = None, + *, + output_processor: Optional[Callable[..., Tensor]] = None, + output_processor_context: Optional[Any] = None, ): """Builds a computation schedule plan for the model. @@ -789,6 +851,10 @@ def build_schedule_plan( Parameters for inference. Defaults to None. loss_mask (Optional[Tensor], optional): Loss mask. Defaults to None. padding_mask (Optional[Tensor], optional): Padding mask. Defaults to None. + output_processor (Callable, optional): Custom postprocess hook to run in the + schedule-plan postprocess node instead of the default logits/loss path. + output_processor_context (Any, optional): User-defined context object forwarded to + `output_processor`. Returns: TransformerModelChunkSchedulePlan: The model chunk schedule plan. @@ -813,6 +879,8 @@ def build_schedule_plan( runtime_gather_output, loss_mask, padding_mask, + output_processor=output_processor, + output_processor_context=output_processor_context, ) def sharded_state_dict( diff --git a/megatron/core/models/gpt/heterogeneous/heterogeneous_layer_specs.py b/megatron/core/models/gpt/heterogeneous/heterogeneous_layer_specs.py index f4385429422..2c2b26f2290 100644 --- a/megatron/core/models/gpt/heterogeneous/heterogeneous_layer_specs.py +++ b/megatron/core/models/gpt/heterogeneous/heterogeneous_layer_specs.py @@ -1,6 +1,7 @@ # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. import warnings +from functools import partial from typing import Optional from megatron.core.extensions.transformer_engine import HAVE_TE @@ -118,7 +119,7 @@ def _get_heterogenous_attention_spec( not_none(TELayerNormColumnParallelLinear) if use_te else ColumnParallelLinear ), core_attention=not_none(TEDotProductAttention) if use_te else DotProductAttention, - linear_proj=TERowParallelLinear if use_te else RowParallelLinear, + linear_proj=not_none(TERowParallelLinear) if use_te else RowParallelLinear, q_layernorm=ln, k_layernorm=ln, ), @@ -128,17 +129,19 @@ def _get_heterogenous_attention_spec( def _get_heterogenous_mlp_spec(mlp_config: MLPConfig, use_te: bool): if mlp_config.no_op: - mlp = ModuleSpec(module=IdentityOp) + return IdentityOp elif mlp_config.replace_with_linear: - mlp = ModuleSpec( - module=( - TELayerNormColumnParallelLinearGathered if use_te else ColumnParallelLinearGathered + return partial( + ( + not_none(TELayerNormColumnParallelLinearGathered) + if use_te + else ColumnParallelLinearGathered ), - params={"tp_comm_buffer_name": "linear_mlp"}, + tp_comm_buffer_name="linear_mlp", ) else: - mlp = ModuleSpec( - module=MLP, + return partial( + MLP.as_mlp_submodule, submodules=MLPSubmodules( linear_fc1=( not_none(TELayerNormColumnParallelLinear) if use_te else ColumnParallelLinear @@ -146,7 +149,6 @@ def _get_heterogenous_mlp_spec(mlp_config: MLPConfig, use_te: bool): linear_fc2=not_none(TERowParallelLinear) if use_te else RowParallelLinear, ), ) - return mlp def _get_sharded_state_dict_keys_map(block_config: TransformerBlockConfig, use_te: bool): diff --git a/megatron/core/models/gpt/moe_module_specs.py b/megatron/core/models/gpt/moe_module_specs.py index 1c19b02f7b5..44c3eac6c72 100755 --- a/megatron/core/models/gpt/moe_module_specs.py +++ b/megatron/core/models/gpt/moe_module_specs.py @@ -13,14 +13,14 @@ from megatron.core.transformer.moe.moe_layer import MoELayer, MoESubmodules from megatron.core.transformer.moe.router import InferenceTopKRouter from megatron.core.transformer.moe.shared_experts import SharedExpertMLP -from megatron.core.transformer.spec_utils import ModuleSpec +from megatron.core.transformer.transformer_layer import MlpBuilder def get_moe_module_spec( use_te: Optional[bool] = True, num_experts: Optional[int] = None, moe_grouped_gemm: Optional[bool] = False, -) -> ModuleSpec: +) -> MlpBuilder: """Helper function to get module spec for MoE. Called by hybrid_layer_specs.py for standard (non-inference) MoE specs. @@ -45,7 +45,7 @@ def get_moe_module_spec_for_backend( num_experts: Optional[int] = None, moe_grouped_gemm: Optional[bool] = False, use_te_activation_func: bool = False, -) -> ModuleSpec: +) -> MlpBuilder: """Helper function to get module spec for MoE""" assert num_experts is not None @@ -62,15 +62,12 @@ def get_moe_module_spec_for_backend( shared_experts = partial(SharedExpertMLP, submodules=mlp) # MoE module spec - moe_module_spec = ModuleSpec( - module=MoELayer, - submodules=MoESubmodules(experts=experts, shared_experts=shared_experts), - metainfo={"fuse_pre_mlp_layernorm": False}, + return partial( + MoELayer, submodules=MoESubmodules(experts=experts, shared_experts=shared_experts) ) - return moe_module_spec -def get_inference_optimized_moe_spec() -> ModuleSpec: +def get_inference_optimized_moe_spec() -> MlpBuilder: """MoE module spec for inference-optimized transformer impl. Uses InferenceSpecProvider to select inference-optimized modules: @@ -92,10 +89,9 @@ def get_inference_optimized_moe_spec() -> ModuleSpec: ), ) - return ModuleSpec( - module=MoELayer, + return partial( + MoELayer, submodules=MoESubmodules( router=InferenceTopKRouter, experts=experts, shared_experts=shared_experts ), - metainfo={"fuse_pre_mlp_layernorm": False}, ) diff --git a/megatron/core/models/huggingface/fastconformer_model.py b/megatron/core/models/huggingface/fastconformer_model.py new file mode 100644 index 00000000000..25265871240 --- /dev/null +++ b/megatron/core/models/huggingface/fastconformer_model.py @@ -0,0 +1,102 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +import torch + +from megatron.core.models.huggingface import HuggingFaceModule + +# NeMo model loading is slow, so cache the (preprocessor, encoder) tuple per +# `sound_model_type`. Keying by model id avoids returning a stale cached encoder +# when the same process constructs more than one Parakeet variant. +_NEMO_SOUND_MODEL_CACHE: dict[str, tuple] = {} + + +def get_nemo_sound_model(sound_model_type): + """Load (and cache) a NeMo ASR encoder + preprocessor for the given ``nemo://`` model id.""" + if sound_model_type not in _NEMO_SOUND_MODEL_CACHE: + import nemo.collections.asr as nemo_asr + + asr_model = nemo_asr.models.ASRModel.from_pretrained( + model_name=sound_model_type.split("nemo://")[1] + ) + # Avoid hangs from an unnecessary max-seq-len NCCL sync in some edge cases. + asr_model.encoder.sync_max_audio_length = False + for layer in asr_model.encoder.layers: + layer.self_attn.use_pytorch_sdpa = True + _NEMO_SOUND_MODEL_CACHE[sound_model_type] = (asr_model.preprocessor, asr_model.encoder) + return _NEMO_SOUND_MODEL_CACHE[sound_model_type] + + +class ParakeetHuggingFaceModel(HuggingFaceModule): + """Wrapper for Parakeet sound encoders. + + Supports two backends, selected by ``config.sound_model_type`` prefix: + + - ``nemo://`` loads a NeMo ASR encoder + preprocessor. + - ``hf://`` loads the upstream Hugging Face FastConformer model + via ``transformers.AutoModel`` / ``AutoFeatureExtractor``. + """ + + def __init__(self, config): + super().__init__(config) + + self.use_nemo = config.sound_model_type.startswith("nemo://") + if self.use_nemo: + self.feature_extractor, self.model = get_nemo_sound_model(config.sound_model_type) + + for module in self.model.modules(): + if module.__class__.__name__.lower() == "dropout": + module.p = config.hidden_dropout + + if config.recompute_granularity is not None: + from torch.distributed.algorithms._checkpoint.checkpoint_wrapper import ( + checkpoint_wrapper, + ) + + self.model = checkpoint_wrapper(self.model) + elif config.sound_model_type.startswith("hf://"): + from transformers import AutoFeatureExtractor, AutoModel + + sound_model_type = config.sound_model_type.split("hf://")[1] + self.feature_extractor = AutoFeatureExtractor.from_pretrained(sound_model_type) + self.model = AutoModel.from_pretrained(sound_model_type) + + if config.recompute_granularity is not None: + self.model.gradient_checkpointing_enable() + else: + raise ValueError(f"Unknown sound model type: {config.sound_model_type}") + + def _model_dtype(self) -> torch.dtype: + """Return the dtype of the encoder's first parameter (defaults to bf16).""" + for param in self.model.parameters(): + return param.dtype + return torch.bfloat16 + + def _sampling_rate(self) -> int: + """Return the sampling rate the feature extractor expects (default 16 kHz).""" + return int(getattr(self.feature_extractor, "sampling_rate", 16000)) + + def forward(self, *args, **kwargs): + """Forward pass returning (hidden_states, lengths). + + Args: + args[0]: Sound clips tensor. + args[1]: Sound length tensor (used by NeMo backend; ignored for HF). + """ + if self.use_nemo: + features = self.feature_extractor(input_signal=args[0], length=args[1]) + y = self.model(audio_signal=features[0], length=features[1]) + # NeMo encoder returns [B, H, T]; LLaVA expects [B, T, H]. + return y[0].permute(0, 2, 1), y[1] + else: + # HF feature extractor expects audio as the first arg only, + # not (audio, length) as in NeMo. + sound_clips = args[0] + features = self.feature_extractor( + sound_clips, + **kwargs, + return_tensors="pt", + sampling_rate=self._sampling_rate(), + return_attention_mask=True, + ) + y = self.model(features.input_features.to(self._model_dtype()), features.attention_mask) + lengths = features.attention_mask.sum(dim=-1).to(y.last_hidden_state.device) + return y.last_hidden_state, lengths diff --git a/megatron/core/models/huggingface/module.py b/megatron/core/models/huggingface/module.py index 5c78fc96708..2d874c7513b 100644 --- a/megatron/core/models/huggingface/module.py +++ b/megatron/core/models/huggingface/module.py @@ -68,6 +68,27 @@ def get_hf_model_type(model_path): "please install it with `pip install transformers`" ) + # Parakeet is a special case: its model id may be `nemo://...`, which + # AutoConfig cannot resolve, so detect it from the prefix. Require the + # `nemo://` or `hf://` scheme so unrelated local paths that happen to + # contain "parakeet" (e.g. a user directory) don't get misrouted. + lowered = model_path.lower() + if lowered.startswith(("nemo://", "hf://")): + model_id = lowered.split("://", 1)[1] + # Match a path segment whose name begins with "parakeet" (e.g. + # `nvidia/parakeet-tdt-0.6b-v2`). Substring-anywhere matches like + # `myparakeet-clone` are intentionally rejected. + if any(seg.startswith("parakeet") for seg in model_id.split("/")): + return "parakeet" + # Any other `nemo://` model can't be resolved by AutoConfig below; + # raise a clear error rather than letting `split("hf://")[1]` raise + # an IndexError with no context. + if lowered.startswith("nemo://"): + raise NotImplementedError( + f"nemo:// scheme is currently only supported for parakeet models, " + f"got {model_path}" + ) + hf_config = AutoConfig.from_pretrained(model_path.split("hf://")[1]) model_type = hf_config.architectures[0].lower() @@ -91,6 +112,10 @@ def build_hf_model(config, model_path): from megatron.core.models.huggingface.clip_model import SiglipHuggingFaceModel model = SiglipHuggingFaceModel(config) + elif "parakeet" in model_type: + from megatron.core.models.huggingface.fastconformer_model import ParakeetHuggingFaceModel + + model = ParakeetHuggingFaceModel(config) else: raise NotImplementedError(f"unsupported huggingface model {config.hf_config}") diff --git a/megatron/core/models/hybrid/hybrid_block.py b/megatron/core/models/hybrid/hybrid_block.py index 93cb56f5297..d462122c1c2 100644 --- a/megatron/core/models/hybrid/hybrid_block.py +++ b/megatron/core/models/hybrid/hybrid_block.py @@ -7,7 +7,7 @@ from contextlib import nullcontext from dataclasses import dataclass -from typing import Optional, Tuple, Union +from typing import List, Optional, Tuple, Union import torch from torch import Tensor, nn @@ -19,15 +19,27 @@ from megatron.core.fp4_utils import get_fp4_context from megatron.core.fp8_utils import get_fp8_context from megatron.core.inference.contexts import BaseInferenceContext +from megatron.core.inference.utils import InferenceMode from megatron.core.models.hybrid.hybrid_layer_allocation import Symbols as LayerSymbols from megatron.core.packed_seq_params import PackedSeqParams from megatron.core.process_groups_config import ProcessGroupCollection +from megatron.core.recompute import checkpointed_forward +from megatron.core.tensor_parallel.random import CheckpointManager from megatron.core.transformer import TransformerConfig +from megatron.core.transformer.enums import CudaGraphModule +from megatron.core.transformer.hyper_connection import ( + HyperConnectionModule, + learned_output_contract, +) from megatron.core.transformer.identity_op import IdentityOp -from megatron.core.transformer.module import MegatronModule +from megatron.core.transformer.module import GraphableMegatronModule, MegatronModule from megatron.core.transformer.spec_utils import ModuleSpec, build_module from megatron.core.transformer.transformer_layer import TransformerLayer -from megatron.core.transformer.utils import sharded_state_dict_default +from megatron.core.transformer.utils import ( + ensure_metadata_has_dp_cp_group, + make_sharded_tensors_for_checkpoint, + sharded_state_dict_default, +) from megatron.core.utils import WrappedTensor, deprecate_inference_params, make_viewless_tensor @@ -41,11 +53,421 @@ class HybridStackSubmodules: gdn_layer: Union[ModuleSpec, type] = IdentityOp attention_layer: Union[ModuleSpec, type] = IdentityOp dsa_layer: Union[ModuleSpec, type] = IdentityOp + csa_layer: Union[ModuleSpec, type] = IdentityOp + hca_layer: Union[ModuleSpec, type] = IdentityOp + window_layer: Union[ModuleSpec, type] = IdentityOp mlp_layer: Union[ModuleSpec, type] = IdentityOp moe_layer: Union[ModuleSpec, type] = IdentityOp mtp_block_spec: Optional[ModuleSpec] = None +class HyperConnectionHybridLayer(GraphableMegatronModule): + """Layer-boundary mHC wrapper for HybridStack layers. + + Hybrid layers already own their local residual paths. For this initial + integration we treat each hybrid layer as a single function by aggregating + n streams to the layer input, running the existing layer, and feeding only + the layer delta back through mHC expansion. The expansion path intentionally + uses zero additional dropout because the wrapped hybrid layer has already + applied its local dropout/residual update before the delta is computed. + + Checkpoint compatibility: this is a *wrapper* (the inner layer is held as + `self.inner_layer`), so wrapped-layer state_dict keys are nested under + `inner_layer.` (e.g. `layers.0.inner_layer.input_layernorm.weight` instead + of `layers.0.input_layernorm.weight`). HybridStack checkpoints saved with + `enable_hyper_connections=False` cannot be loaded into a model with + `enable_hyper_connections=True` (and vice versa) without a key-mapping + migration. Note: this differs from `HyperConnectionTransformerLayer`, + which subclasses `TransformerLayer` and only adds new sibling fields, + keeping all base keys stable. + + CUDA graphs: this wrapper subclasses ``GraphableMegatronModule`` so that, with + ``cuda_graph_impl="transformer_engine"``, wrapped layers are captured per-layer — + mirroring ``HyperConnectionTransformerLayer`` on the GPT path. Without this, the TE + graph discovery (``_layer_is_graphable``) only inspects the top-level layer type and + silently skips every wrapped layer, so an mHC-enabled HybridStack would run entirely + eager. Two capture modes: + + * Non-MoE inner layers (attention variants, Mamba): the whole wrapper forward + (mHC aggregate + inner layer + n-stream BDA) is captured as one graph. The inner + layer's own ``__call__`` graph routing is bypassed during capture (see + ``_call_inner_layer``) to avoid nested capture. + * MoE inner layers, when ``moe_router`` is in ``cuda_graph_modules``: the expert + all-to-all is not graph-safe, so only the deterministic prefix is graphed (mHC + ``compute_mappings``/``aggregate`` + the inner layer's router/preprocess). The graph + outputs the router intermediates, the mHC state (``h_post``, ``h_res``) and the + n-stream residual; on replay the experts run eagerly and the n-stream BDA (eager) + consumes the inner's raw ``mlp_output_with_bias`` as the layer delta. Routing the + residual *through the graph* (not reusing the layer input directly in the eager BDA) + keeps the backward gradient flowing into the captured graph, which is required for + bit-identical training — again mirroring ``HyperConnectionTransformerLayer``. + + ``_get_submodules_under_cudagraphs`` returns the submodules whose params the wrapper + graph's manual hooks must drive: ``[self]`` for whole-wrapper capture, or the mHC module + + the inner router/preprocess submodules for partial MoE capture (experts stay eager). + """ + + def __init__(self, config: TransformerConfig, layer: MegatronModule) -> None: + super().__init__(config=config) + self.inner_layer = layer + self.layer_number = layer.layer_number + self.hyper_connection = HyperConnectionModule(config=config, layer_number=self.layer_number) + if config.params_dtype is not None: + self.hyper_connection.to(dtype=config.params_dtype) + if hasattr(layer, 'tp_group'): + self.tp_group = layer.tp_group + + def get_layer_static_inputs(self, seq_length, micro_batch_size): + """Override to produce n-stream hidden_states of shape [s, b, n*C]. + + CUDA graph capture allocates static buffers sized by this method. The base + returns [s, b, C], but mHC layers carry n-stream hidden states [s, b, n*C]. + Mirrors ``HyperConnectionTransformerLayer.get_layer_static_inputs``. + """ + static_inputs = super().get_layer_static_inputs(seq_length, micro_batch_size) + hs = static_inputs["hidden_states"] + n = self.config.num_residual_streams + static_inputs["hidden_states"] = torch.ones( + (hs.shape[0], hs.shape[1], n * self.config.hidden_size), + dtype=hs.dtype, + requires_grad=hs.requires_grad, + device=hs.device, + ) + return static_inputs + + def _inner_is_moe(self) -> bool: + """True when the inner layer is an MoE ``TransformerLayer``. Such layers use the + GPT-style raw-delta path (feed the inner's ``mlp_output_with_bias`` straight to the + n-stream BDA) in both the eager forward and the CUDA-graph replay.""" + from megatron.core.transformer.moe.moe_layer import MoELayer + + return isinstance(self.inner_layer, TransformerLayer) and isinstance( + getattr(self.inner_layer, 'mlp', None), MoELayer + ) + + def _inner_is_partial_moe_capture(self) -> bool: + """True when the inner layer is MoE and the configured ``cuda_graph_modules`` request + partial MoE capture (``moe_router``). + + In that case the wrapper does NOT capture the whole forward as one graph (the expert + all-to-all is not graph-safe). Instead it graphs the deterministic prefix (mHC aggregate + + the inner layer's router/preprocess) and runs the experts + mHC BDA eagerly — mirroring + how ``HyperConnectionTransformerLayer`` graphs MoE layers on the GPT path. Whole-wrapper + capture is still used for non-MoE inner layers (attention variants, Mamba). + """ + return ( + self._inner_is_moe() + and bool(self.config.cuda_graph_modules) + and CudaGraphModule.moe_router in self.config.cuda_graph_modules + ) + + def _te_cuda_graph_capture(self, *args, **kwargs): + """Capture the graph-safe portion of the wrapper forward. + + For non-MoE inner layers the whole wrapper forward (mHC aggregate + inner layer + + n-stream BDA) is captured as one graph. For MoE inner layers under ``moe_router`` + partial capture, only the deterministic prefix is graphed: the mHC + ``compute_mappings``/``aggregate`` followed by the inner layer's router/preprocess. + The captured outputs are the inner router/preprocess intermediates plus the mHC + state (``h_post``, ``h_res``) and the aggregated single-stream input needed to + reconstruct the layer delta on replay. ``context`` is ``None`` for the graphed + hybrid layer types, so it is dropped (a tuple containing ``None`` cannot be a + CUDA-graph output). + """ + if self._inner_is_partial_moe_capture(): + hidden_states = args[0] if args else kwargs["hidden_states"] + aggregated, h_res, h_post, residual = self.hyper_connection(hidden_states) + inner_out = list(self.inner_layer._te_cuda_graph_capture(aggregated)) + # inner_out = router/preprocess intermediates ending in the inner residual; + # append the mHC state AND the n-stream `residual` returned by the (graphed) + # hyper_connection. Routing `residual` through the graph as an output keeps its + # backward grad flowing into the graph's backward (mirrors + # HyperConnectionTransformerLayer), instead of a second autograd path the captured + # backward does not account for. The experts' raw mlp_output_with_bias (produced + # on replay) is the layer delta, so `aggregated` need not be captured. + return tuple(inner_out) + (h_post, h_res, residual) + + hidden_states, context = self.forward(*args, **kwargs) + cuda_graph_outputs = [hidden_states] + if context is not None: + cuda_graph_outputs.append(context) + return tuple(cuda_graph_outputs) + + def _te_cuda_graph_replay(self, *args, **kwargs): + """Replay the captured graph and restore the (hidden_states, context) contract. + + Non-MoE inner layers: the whole wrapper forward was captured, so the only graph + output is the layer's n-stream hidden_states; re-append ``None`` for context. + + MoE inner layers (partial capture): replay the graphed prefix, then run the + experts eagerly and apply the mHC n-stream BDA — reproducing exactly the eager + wrapper tail (``layer_delta = layer_output - aggregated`` then + ``fused_h_res_h_post_bda``), just with the deterministic prefix graphed. + """ + if self._inner_is_partial_moe_capture(): + out = list(super()._te_cuda_graph_replay(*args, **kwargs)) + residual = out.pop() # n-stream [s, b, n*C] — graph output (see capture) + h_res = out.pop() + h_post = out.pop() + # Resume the inner MoE experts eagerly to the raw delta (mlp_output_with_bias), + # then let the n-stream BDA own the residual — identical to the eager forward + # (`_call_inner_transformer_layer_without_local_bda` fast path → + # fused_h_res_h_post_bda), just with the router/preprocess prefix graphed. + # Mirror the eager fast path's BDA args (it feeds the inner layer's + # `hidden_dropout` / `bias_dropout_fusion`) so replay == eager bit-for-bit. + mlp_output_with_bias = self.inner_layer.resume_moe_experts_after_partial_cudagraph(out) + hidden_states = self.hyper_connection.fused_h_res_h_post_bda( + h_res, + residual, + h_post, + mlp_output_with_bias, + dropout_prob=self.inner_layer.hidden_dropout, + training=self.training, + fused=self.inner_layer.config.bias_dropout_fusion, + manager=None, + ) + if ( + self.config.fp32_residual_connection + and self.config.params_dtype is not None + and hidden_states.dtype != self.config.params_dtype + ): + hidden_states = hidden_states.to(self.config.params_dtype) + return hidden_states, None + + cuda_graph_output = list(super()._te_cuda_graph_replay(*args, **kwargs)) + return cuda_graph_output[0], None + + def _get_submodules_under_cudagraphs(self): + """Submodules whose params are driven by the wrapper graph's manual hooks. + + Whole-wrapper capture covers the entire wrapper (``[self]``, the base default). + For partial MoE capture only the graphed prefix is covered — the mHC module plus + the inner layer's router/preprocess submodules — so the experts (run eagerly) + keep their normal forward hooks. + """ + if self._inner_is_partial_moe_capture(): + return [self.hyper_connection] + self.inner_layer._get_submodules_under_cudagraphs() + return super()._get_submodules_under_cudagraphs() + + def mamba_state_shapes_per_request(self) -> Optional[Tuple[Tuple[int], Tuple[int]]]: + """Delegate Mamba inference state shape requests to the wrapped layer.""" + if not hasattr(self.inner_layer, 'mamba_state_shapes_per_request'): + return None + return self.inner_layer.mamba_state_shapes_per_request() + + def _call_inner_layer( + self, + hidden_states: Tensor, + attention_mask: Tensor, + inference_context: Optional[BaseInferenceContext], + rotary_pos_emb: Optional[Tensor], + sequence_len_offset: Optional[Tensor], + packed_seq_params: Optional[PackedSeqParams], + padding_mask: Optional[Tensor], + input_ids: Optional[Tensor] = None, + ) -> Tuple[Tensor, Optional[Tensor]]: + # When this wrapper is itself being CUDA-graph captured, the inner layer + # must run as a plain forward: routing through its ``__call__`` would + # trigger nested TE graph capture (the inner layer is also a + # GraphableMegatronModule). During eager steps we keep ``__call__`` so the + # inner layer's forward pre-hooks (e.g. param all-gather) fire normally; + # under graph replay these are driven by the wrapper's manual hooks. + from megatron.core.transformer.cuda_graphs import is_graph_capturing + + inner = self.inner_layer.forward if is_graph_capturing() else self.inner_layer + + if isinstance(self.inner_layer, TransformerLayer): + output = inner( + hidden_states=hidden_states, + attention_mask=attention_mask, + inference_context=inference_context, + rotary_pos_emb=rotary_pos_emb, + sequence_len_offset=sequence_len_offset, + packed_seq_params=packed_seq_params, + padding_mask=padding_mask, + input_ids=input_ids, + _called_from_hybrid_mhc_wrapper=True, + ) + else: + # Non-transformer layers (e.g. MambaLayer; GatedDeltaNet which does + # accept `sequence_len_offset` is currently always wrapped inside a + # TransformerLayer spec, so it takes the branch above) do not accept + # rotary_pos_emb / sequence_len_offset / padding_mask — pass only + # the common arguments. New layer types that consume any of these + # must add explicit handling here. + output = inner( + hidden_states=hidden_states, + attention_mask=attention_mask, + inference_context=inference_context, + packed_seq_params=packed_seq_params, + ) + + if isinstance(output, tuple): + context = output[1] if len(output) > 1 else None + return output[0], context + return output, None + + def _call_inner_transformer_layer_without_local_bda( + self, + hidden_states: Tensor, + attention_mask: Tensor, + inference_context: Optional[BaseInferenceContext], + rotary_pos_emb: Optional[Tensor], + sequence_len_offset: Optional[Tensor], + packed_seq_params: Optional[PackedSeqParams], + padding_mask: Optional[Tensor], + input_ids: Optional[Tensor] = None, + ) -> Optional[Tuple[Tuple[Tensor, Optional[Tensor]], Optional[Tensor], float, bool]]: + """Return a raw TransformerLayer branch output when the wrapped layer is split. + + Hybrid DSv4 layers are usually attention-only (`W/C/H/D`) or MLP/MoE-only (`-/E`) + TransformerLayer instances. For those layers, skip the inner layer's local + residual+BDA and feed the raw branch output directly into the mHC BDA, matching the + GPT mHC path and avoiding a residual add followed by `layer_output - aggregated`. + """ + if not isinstance(self.inner_layer, TransformerLayer): + return None + + layer = self.inner_layer + if (not layer.training) and layer.config.inference_fuse_tp_communication: + return None + + has_attention = not isinstance(layer.self_attention, IdentityOp) + has_cross_attention = not isinstance(layer.cross_attention, IdentityOp) + has_mlp = not isinstance(layer.mlp, IdentityOp) + + if has_cross_attention or has_attention == has_mlp: + return None + + if has_attention: + output_with_bias, attn_norm_manager, residual = ( + layer._forward_self_attention_output_with_bias( + hidden_states=hidden_states, + attention_mask=attention_mask, + inference_context=inference_context, + rotary_pos_emb=rotary_pos_emb, + packed_seq_params=packed_seq_params, + sequence_len_offset=sequence_len_offset, + ) + ) + output_with_bias = layer._group_offload_output_with_bias( + output_with_bias, attn_norm_manager, forced_released_tensors=[residual] + ) + return output_with_bias, None, layer.hidden_dropout, layer.config.bias_dropout_fusion + + output_with_bias, residual = layer._forward_mlp_output_with_bias( + hidden_states, + inference_context=inference_context, + padding_mask=padding_mask, + input_ids=input_ids, + ) + if layer.mlp_norm_manager is not None: + output_with_bias = layer._group_offload_output_with_bias( + output_with_bias, layer.mlp_norm_manager, forced_released_tensors=[residual] + ) + layer.mlp_norm_manager = None + return output_with_bias, None, layer.hidden_dropout, layer.config.bias_dropout_fusion + + def forward( + self, + hidden_states: Tensor, + attention_mask: Optional[Tensor] = None, + inference_context: Optional[BaseInferenceContext] = None, + rotary_pos_emb: Optional[Tensor] = None, + sequence_len_offset: Optional[Tensor] = None, + packed_seq_params: Optional[PackedSeqParams] = None, + padding_mask: Optional[Tensor] = None, + input_ids: Optional[Tensor] = None, + mhc_recompute_manager=None, + ) -> Tuple[Tensor, Optional[Tensor]]: + """Run the wrapped hybrid layer through one layer-boundary mHC update. + + ``attention_mask`` defaults to ``None`` so that CUDA-graph capture, which + calls this forward with only the static ``hidden_states`` input, does not + fail on a missing positional argument (causal masking is inferred by the + attention backend when the mask is ``None``). + """ + + """Run the wrapped hybrid layer through one layer-boundary mHC update.""" + aggregated, h_res, h_post, residual = self.hyper_connection( + hidden_states, mhc_recompute_manager=mhc_recompute_manager + ) + fast_path_result = self._call_inner_transformer_layer_without_local_bda( + aggregated, + attention_mask, + inference_context, + rotary_pos_emb, + sequence_len_offset, + packed_seq_params, + padding_mask, + input_ids, + ) + + if fast_path_result is None: + layer_output, context = self._call_inner_layer( + aggregated, + attention_mask, + inference_context, + rotary_pos_emb, + sequence_len_offset, + packed_seq_params, + padding_mask, + input_ids, + ) + # The inner hybrid layer already applied its own local residual/dropout, so + # it returns `aggregated + f(aggregated)`. We feed only the function + # delta `f(aggregated)` into the n-stream BDA so it does not double-count + # the residual that mHC owns. + if self.config.fp32_residual_connection and aggregated.dtype != layer_output.dtype: + aggregated = aggregated.to(layer_output.dtype) + layer_output_with_bias = (layer_output - aggregated, None) + dropout_prob = 0.0 + bias_dropout_fusion = False + else: + layer_output_with_bias, context, dropout_prob, bias_dropout_fusion = fast_path_result + + layer_output = layer_output_with_bias[0] + # Sanity check: this contract requires the branch output to preserve shape; + # any mismatch indicates a future layer type is breaking the residual assumption + # and would silently corrupt the n-stream state. + if layer_output.shape != aggregated.shape: + raise RuntimeError( + "HyperConnectionHybridLayer requires wrapped branches to preserve " + f"hidden-state shape. Got {tuple(layer_output.shape)} from wrapped branch " + f"vs {tuple(aggregated.shape)} input." + ) + is_last_in_recompute_block = bool( + mhc_recompute_manager is not None + and getattr(mhc_recompute_manager, "is_last_layer_in_recompute_block", False) + ) + mhc_bda_manager = None if is_last_in_recompute_block else mhc_recompute_manager + + hidden_states = self.hyper_connection.fused_h_res_h_post_bda( + h_res, + residual, + h_post, + layer_output_with_bias, + dropout_prob=dropout_prob, + training=self.training, + fused=bias_dropout_fusion, + manager=mhc_bda_manager, + ) + # In `HyperConnectionTransformerLayer` the n-stream output stays in compute + # dtype because the post-attention `x` is in compute dtype. In the hybrid + # wrapper, `layer_delta` may be fp32 (when `fp32_residual_connection=True` + # or an inner layer upcasts), so `fused_h_res_h_post_bda`'s `output.to(x.dtype)` + # would leave the result in fp32 and silently propagate fp32 n-stream + # hidden states to every subsequent layer (~2x activation memory). Restore + # the compute-dtype contract here. + if ( + self.config.fp32_residual_connection + and self.config.params_dtype is not None + and hidden_states.dtype != self.config.params_dtype + ): + hidden_states = hidden_states.to(self.config.params_dtype) + return hidden_states, context + + class HybridStack(MegatronModule): """ Constructor for the HybridStack class. @@ -69,6 +491,7 @@ class HybridStack(MegatronModule): pg_collection (ProcessGroupCollection): the required model communication process groups to use. is_mtp_layer (bool, optional): whether this is an MTP layer. Defaults to False. + mtp_layer_number (int, optional): enclosing MTP depth for logging nested MTP metrics. """ def __init__( @@ -84,12 +507,19 @@ def __init__( dtype=None, pg_collection: ProcessGroupCollection = None, is_mtp_layer: bool = False, + mtp_layer_number: Optional[int] = None, + name: str | None = None, ) -> None: + """ + Args: + name (str | None): module instance name passed top-down from its paranet module + """ super().__init__(config=config) self.pre_process = pre_process self.post_layer_norm = post_layer_norm self.post_process = post_process self.is_mtp_layer = is_mtp_layer + self.mtp_layer_number = mtp_layer_number assert pg_collection is not None, "pg_collection must be provided for HybridStack" @@ -100,6 +530,10 @@ def __init__( self.input_tensor = None self.pg_collection = pg_collection + # Lazily populated mHC recompute layout cache (deterministic from config + # and num_layers); see `_build_mhc_recompute_layer_plan`. + self._mhc_block_end_plan: Optional[List[bool]] = None + assert layer_type_list is not None, ( "layer_type_list must be provided. It should be pre-computed from " "--hybrid-layer-pattern by HybridModel." @@ -124,6 +558,7 @@ def __init__( layer_number=layer_number, pp_layer_offset=pp_layer_offset, pg_collection=pg_collection, + name=(name + f".layers.{i}") if name is not None else None, ) elif layer_type == LayerSymbols.ATTENTION: layer = build_module( @@ -134,6 +569,7 @@ def __init__( is_mtp_layer=is_mtp_layer, add_layer_offset=False, pp_layer_offset=pp_layer_offset, + name=(name + f".layers.{i}") if name is not None else None, ) elif layer_type == LayerSymbols.DS_ATTENTION: layer = build_module( @@ -144,6 +580,41 @@ def __init__( is_mtp_layer=is_mtp_layer, add_layer_offset=False, pp_layer_offset=pp_layer_offset, + name=(name + f".layers.{i}") if name is not None else None, + ) + elif layer_type == LayerSymbols.CSA: + # DSv4 Compressed Sparse Attention (compress_ratio fixed by the spec). + layer = build_module( + submodules.csa_layer, + config=self.config, + layer_number=layer_number, + pg_collection=pg_collection, + is_mtp_layer=is_mtp_layer, + add_layer_offset=False, + pp_layer_offset=pp_layer_offset, + ) + elif layer_type == LayerSymbols.HCA: + # DSv4 Heavily Compressed Attention (compress_ratio fixed by the spec). + layer = build_module( + submodules.hca_layer, + config=self.config, + layer_number=layer_number, + pg_collection=pg_collection, + is_mtp_layer=is_mtp_layer, + add_layer_offset=False, + pp_layer_offset=pp_layer_offset, + ) + elif layer_type == LayerSymbols.WINDOW: + # DSv4 sliding-window-only attention (compress_ratio=0 fixed by the spec; + # no compressor / no top-k indexer — attends only within csa_window_size). + layer = build_module( + submodules.window_layer, + config=self.config, + layer_number=layer_number, + pg_collection=pg_collection, + is_mtp_layer=is_mtp_layer, + add_layer_offset=False, + pp_layer_offset=pp_layer_offset, ) elif layer_type == LayerSymbols.MLP: layer = build_module( @@ -152,6 +623,7 @@ def __init__( layer_number=layer_number, pg_collection=pg_collection, add_layer_offset=False, + name=(name + f".layers.{i}") if name is not None else None, ) elif layer_type == LayerSymbols.MOE: layer = build_module( @@ -159,7 +631,9 @@ def __init__( config=self.config, layer_number=layer_number, pg_collection=pg_collection, + is_mtp_layer=is_mtp_layer, add_layer_offset=False, + name=(name + f".layers.{i}") if name is not None else None, ) elif layer_type == LayerSymbols.GDN: layer = build_module( @@ -169,9 +643,14 @@ def __init__( pg_collection=pg_collection, # Set to False as we do not want to change offset. add_layer_offset=False, + name=(name + f".layers.{i}") if name is not None else None, ) else: raise ValueError("unexpected layer_type") + if self.is_mtp_layer and self.mtp_layer_number is not None: + self._set_mtp_layer_number_for_moe_metrics(layer, self.mtp_layer_number) + if self.config.enable_hyper_connections: + layer = HyperConnectionHybridLayer(config=self.config, layer=layer) self.layers.append(layer) # Required for activation recomputation @@ -185,6 +664,32 @@ def __init__( eps=self.config.layernorm_epsilon, ) + # Skip hc_head_* params inside the nested MTP HybridStack — `forward()` + # no longer calls `learned_output_contract` there (MTP owns that), so these + # params would be orphaned and break DDP's per-param grad-ready accounting + # with a `len(per_param_grad_ready_counts) != len(params)` AssertionError. + if self.config.enable_hyper_connections and self.post_process and not self.is_mtp_layer: + hc_mult = self.config.num_residual_streams + hc_dim = self.config.hidden_size * hc_mult + self.hc_head_fn = nn.Parameter(torch.randn(hc_mult, hc_dim)) + self.hc_head_base = nn.Parameter(torch.zeros(hc_mult)) + self.hc_head_scale = nn.Parameter(torch.ones(1)) + nn.init.xavier_uniform_(self.hc_head_fn) + if self.config.sequence_parallel: + setattr(self.hc_head_fn, 'sequence_parallel', True) + setattr(self.hc_head_base, 'sequence_parallel', True) + setattr(self.hc_head_scale, 'sequence_parallel', True) + + @staticmethod + def _set_mtp_layer_number_for_moe_metrics( + layer: torch.nn.Module, mtp_layer_number: int + ) -> None: + """Tell nested MTP MoE routers which MTP depth they belong to for logging.""" + for module in layer.modules(): + router = getattr(module, "router", None) + if router is not None and getattr(router, "is_mtp_layer", False): + router.mtp_layer_number = mtp_layer_number + def set_input_tensor(self, input_tensor: Tensor): """Set input tensor to be used instead of forward()'s input. @@ -205,6 +710,59 @@ def mamba_state_shapes_per_request(self) -> Optional[Tuple[Tuple[int], Tuple[int return layer.mamba_state_shapes_per_request() return None + def _compute_mhc_block_end_plan(self) -> List[bool]: + """Compute per-layer block-end markers (deterministic from config).""" + num_layers = len(self.layers) + is_recompute_block_end: List[bool] = [False] * num_layers + if num_layers == 0: + return is_recompute_block_end + mhc_recompute_layer_num = self.config.mhc_recompute_layer_num + for l_no in range(num_layers): + is_last_in_stack = l_no == num_layers - 1 + is_last_in_recompute_block = is_last_in_stack + if mhc_recompute_layer_num is not None: + is_last_in_recompute_block = is_last_in_stack or ( + (l_no + 1) % mhc_recompute_layer_num == 0 + ) + is_recompute_block_end[l_no] = is_last_in_recompute_block + return is_recompute_block_end + + def _build_mhc_recompute_layer_plan( + self, use_mhc_recompute: bool + ) -> Tuple[List[Optional[CheckpointManager]], List[bool]]: + """Pre-build per-layer MHC recompute managers and block-end markers. + + The block-end plan is deterministic from config and cached on the + instance; only the per-block ``CheckpointManager`` instances are + allocated fresh per forward pass (managers are single-use). Mirrors + the caching scheme used by ``TransformerBlock``. + """ + num_layers = len(self.layers) + if not use_mhc_recompute or num_layers == 0: + return [None] * num_layers, [False] * num_layers + + if self._mhc_block_end_plan is None: + self._mhc_block_end_plan = self._compute_mhc_block_end_plan() + is_recompute_block_end = self._mhc_block_end_plan + + layer_managers: List[Optional[CheckpointManager]] = [None] * num_layers + mhc_manager = CheckpointManager() + for l_no in range(num_layers): + layer_managers[l_no] = mhc_manager + if is_recompute_block_end[l_no] and l_no != num_layers - 1: + mhc_manager = CheckpointManager() + return layer_managers, is_recompute_block_end + + @staticmethod + def _finalize_mhc_recompute_layer( + mhc_manager: Optional[CheckpointManager], + hidden_states: Tensor, + is_last_in_recompute_block: bool, + ) -> None: + """Finalize MHC recompute state for the current layer when a block ends.""" + if mhc_manager is not None and is_last_in_recompute_block: + mhc_manager.discard_all_outputs_and_register_unified_recompute(hidden_states) + def forward( self, hidden_states: Union[Tensor, WrappedTensor], @@ -215,7 +773,8 @@ def forward( inference_params: Optional[BaseInferenceContext] = None, packed_seq_params: Optional[PackedSeqParams] = None, padding_mask=None, - ): + input_ids: Optional[Tensor] = None, + ) -> Union[Tensor, Tuple[Tensor, Tensor]]: """ Forward function of the HybridStack class. @@ -231,7 +790,11 @@ def forward( rotary_pos_emb (Tensor, optional): the rotary positional embeddings. Defaults to None. Returns: - Tensor: the output tensor. + Tensor in the common case. A 2-tuple ``(hidden_states, mhc_multistream)`` ONLY when + ``enable_hyper_connections and post_process and mtp_num_layers > 0 and not + is_mtp_layer`` — the extra element is the pre-contraction multi-stream tensor that + MTP's ``_concat_embeddings`` consumes. Callers (e.g. ``HybridModel.forward``) must + handle both; pipeline send/recv only ever transfers the contracted ``hidden_states``. """ inference_context = deprecate_inference_params(inference_context, inference_params) @@ -244,6 +807,15 @@ def forward( if isinstance(hidden_states, WrappedTensor): hidden_states = hidden_states.unwrap() + # Skip input_expand inside MTP nested HybridStack: when mHC + MTP, the outer + # decoder hands in already-multi-stream hidden_states via mhc_multistream + # (see multi_token_prediction.py _concat_embeddings), so expanding again would + # produce [s, b, n*(n*h)] instead of [s, b, n*h] and break HC mapping_proj. + if self.config.enable_hyper_connections and self.pre_process and not self.is_mtp_layer: + hidden_states = HyperConnectionModule.input_expand( + hidden_states, self.config.num_residual_streams + ) + if inference_context and inference_context.is_static_batching(): # NOTE(bnorick): match BaseInferenceContext attributes for # mamba_ssm.utils.generation.BaseInferenceContext, @@ -255,7 +827,7 @@ def forward( (self.config.cuda_graph_impl == "local" or self.config.flash_decode) and inference_context and inference_context.is_static_batching() - and not self.training + and InferenceMode.is_active() ): current_batch_size = hidden_states.shape[1] sequence_len_offset = torch.tensor( @@ -291,34 +863,106 @@ def get_inner_quant_context(config, layer_number): def get_inner_quant_context(config, layer_number): return nullcontext() + use_mhc_recompute = ( + self.training + and self.config.enable_hyper_connections + and self.config.recompute_granularity == 'selective' + and "mhc" in self.config.recompute_modules + ) + mhc_layer_managers, mhc_is_last_in_recompute_block = self._build_mhc_recompute_layer_plan( + use_mhc_recompute + ) + with outer_fp8_context: - for layer in self.layers: - # Layers have 1-indexed layer numbers attribute. - inner_quant_context = get_inner_quant_context(self.config, layer.layer_number - 1) - with inner_quant_context: - if isinstance(layer, TransformerLayer): - hidden_states, _ = layer( - hidden_states=hidden_states, - attention_mask=attention_mask, - inference_context=inference_context, - rotary_pos_emb=rotary_pos_emb, - sequence_len_offset=sequence_len_offset, - packed_seq_params=packed_seq_params, - padding_mask=padding_mask, - ) - else: # MambaLayer, Expert, or MLP - hidden_states = layer( - hidden_states=hidden_states, - attention_mask=attention_mask, - inference_context=inference_context, - packed_seq_params=packed_seq_params, + if self.config.recompute_granularity == 'full' and self.training: + hidden_states = checkpointed_forward( + self, + hidden_states=hidden_states, + attention_mask=attention_mask, + context=None, + context_mask=None, + rotary_pos_emb=rotary_pos_emb, + attention_bias=None, + packed_seq_params=packed_seq_params, + padding_mask=padding_mask, + input_ids=input_ids, + use_inner_quantization_context=(use_inner_fp8_context or use_fp4_context), + ) + else: + for l_no, layer in enumerate(self.layers): + # Layers have 1-indexed layer numbers attribute. + inner_quant_context = get_inner_quant_context( + self.config, layer.layer_number - 1 + ) + + mhc_manager = mhc_layer_managers[l_no] + if mhc_manager is not None: + mhc_manager.is_last_layer_in_recompute_block = ( + mhc_is_last_in_recompute_block[l_no] ) - # The attention layer (currently a simplified transformer layer) - # outputs a tuple of (hidden_states, context). Context is intended - # for cross-attention, and is not needed in our model. - if isinstance(hidden_states, tuple): - hidden_states = hidden_states[0] + with inner_quant_context: + if isinstance(layer, (TransformerLayer, HyperConnectionHybridLayer)): + layer_kwargs = dict( + hidden_states=hidden_states, + attention_mask=attention_mask, + inference_context=inference_context, + rotary_pos_emb=rotary_pos_emb, + sequence_len_offset=sequence_len_offset, + packed_seq_params=packed_seq_params, + padding_mask=padding_mask, + ) + if input_ids is not None: + layer_kwargs["input_ids"] = input_ids + if mhc_manager is not None and isinstance( + layer, HyperConnectionHybridLayer + ): + layer_kwargs["mhc_recompute_manager"] = mhc_manager + hidden_states, _ = layer(**layer_kwargs) + else: # MambaLayer, Expert, or MLP + hidden_states = layer( + hidden_states=hidden_states, + attention_mask=attention_mask, + inference_context=inference_context, + packed_seq_params=packed_seq_params, + ) + + # The attention layer (currently a simplified transformer layer) + # outputs a tuple of (hidden_states, context). Context is intended + # for cross-attention, and is not needed in our model. + if isinstance(hidden_states, tuple): + hidden_states = hidden_states[0] + + self._finalize_mhc_recompute_layer( + mhc_manager=mhc_manager, + hidden_states=hidden_states, + is_last_in_recompute_block=mhc_is_last_in_recompute_block[l_no], + ) + + # When mHC + MTP, save the pre-contraction multi-stream tensor for MTP input. + # MTP's _concat_embeddings mHC branch expects [s, b, n*h] (multi-stream), while + # the contracted hidden_states is [s, b, h]. Mirrors transformer_block.py:948-988. + # Only the OUTER decoder stack does this; nested MTP stacks (is_mtp_layer=True) + # must keep returning a single Tensor so MTP's _postprocess receives the right + # type for learned_output_contract. + # On the final stage of a (non-MTP) stack with mHC active, capture the pre-contraction + # multi-stream tensor for MTP's `_concat_embeddings` (only meaningful when MTP layers + # exist, i.e. mtp_num_layers > 0), THEN contract the streams. Combining capture and + # contraction avoids repeating the condition. Nested MTP HybridStacks (is_mtp_layer=True) + # must NOT contract here — MTP's own `_postprocess` calls learned_output_contract + + # final_layernorm itself, so doing it here would double-collapse the multi-stream tensor. + mhc_multistream = None + if self.config.enable_hyper_connections and self.post_process and not self.is_mtp_layer: + if (self.config.mtp_num_layers or 0) > 0: + mhc_multistream = hidden_states + hidden_states = learned_output_contract( + hidden_states, + self.hc_head_fn, + self.hc_head_base, + self.hc_head_scale, + self.config.num_residual_streams, + self.config.layernorm_epsilon, + ) # Final layer norm. if self.post_process and self.post_layer_norm: @@ -330,6 +974,8 @@ def get_inner_quant_context(config, layer_number): inp=hidden_states, requires_grad=hidden_states.requires_grad, keep_graph=True ) + if mhc_multistream is not None: + return hidden_states, mhc_multistream return hidden_states def sharded_state_dict( @@ -354,6 +1000,7 @@ def sharded_state_dict( dict: The sharded state dictionary for the current object. """ + sharded_offsets = sharded_offsets or () sharded_state_dict = {} layer_prefix = f'{prefix}layers.' @@ -388,6 +1035,20 @@ def sharded_state_dict( ) ) + local_state_dict: dict = {} + self._save_to_state_dict(local_state_dict, '', keep_vars=True) + if local_state_dict: + metadata = ensure_metadata_has_dp_cp_group(metadata) + sharded_state_dict.update( + make_sharded_tensors_for_checkpoint( + local_state_dict, + prefix, + sharded_offsets=sharded_offsets or (), + tp_group=self.tp_group, + dp_cp_group=metadata['dp_cp_group'], + ) + ) + return sharded_state_dict diff --git a/megatron/core/models/hybrid/hybrid_layer_allocation.py b/megatron/core/models/hybrid/hybrid_layer_allocation.py index f1ba94ef7fa..a8d2006c3b0 100644 --- a/megatron/core/models/hybrid/hybrid_layer_allocation.py +++ b/megatron/core/models/hybrid/hybrid_layer_allocation.py @@ -18,11 +18,16 @@ class Symbols: GDN = 'G' ATTENTION = "*" DS_ATTENTION = "D" + CSA = "C" # DSv4 Compressed Sparse Attention (compress_ratio=4) + HCA = "H" # DSv4 Heavily Compressed Attention (compress_ratio=128) + WINDOW = "W" # DSv4 sliding-window-only attention (compress_ratio=0; no compressor/indexer) MLP = "-" MOE = 'E' PIPE = '|' MTP_SEPARATOR = "/" - VALID_LAYERS = {MAMBA, GDN, ATTENTION, DS_ATTENTION, MLP, MOE} + VALID_LAYERS = {MAMBA, GDN, ATTENTION, DS_ATTENTION, CSA, HCA, WINDOW, MLP, MOE} + # MLA-based attention layers (incompatible with standard '*' attention in one model). + MLA_ATTENTION = {DS_ATTENTION, CSA, HCA, WINDOW} @classmethod def name_sorted_valid_layer_symbols(cls) -> list[str]: @@ -173,10 +178,10 @@ def get_hybrid_layer_counts(pattern: str) -> Dict[str, int]: Examples: >>> get_hybrid_layer_counts("M*M*") - {'*': 2, 'G': 0, 'D': 0, 'M': 2, '-': 0, 'E': 0} + {'*': 2, 'C': 0, 'D': 0, 'G': 0, 'H': 0, 'M': 2, '-': 0, 'E': 0, 'W': 0} >>> get_hybrid_layer_counts("M-M-|M-M*-/MM/MM") - {'*': 1, 'G': 0, 'D': 0, 'M': 8, '-': 4, 'E': 0} + {'*': 1, 'C': 0, 'D': 0, 'G': 0, 'H': 0, 'M': 8, '-': 4, 'E': 0, 'W': 0} """ parsed = parse_hybrid_pattern(pattern) counts = {symbol: 0 for symbol in Symbols.name_sorted_valid_layer_symbols()} @@ -292,9 +297,12 @@ def _validate_pattern(pattern: str, pattern_name: str, allow_pipe: bool = False) f"Valid symbols are: {valid_chars}" ) - # Disallow Attention + MLA/DSA hybridity. - if Symbols.ATTENTION in pattern and Symbols.DS_ATTENTION in pattern: - raise ValueError("Not supported to have both Attention and MLA/DSA in one model") + # Disallow standard Attention ('*') mixed with any MLA-based attention (D/C/H/W). + # MLA variants (DSA / CSA / HCA / Window) may freely coexist with each other. + if Symbols.ATTENTION in pattern and any(s in pattern for s in Symbols.MLA_ATTENTION): + raise ValueError( + "Not supported to have both Attention and MLA/DSA/CSA/HCA/Window in one model" + ) def validate_segment_layers(segment: str) -> List[str]: @@ -320,9 +328,11 @@ def validate_segment_layers(segment: str) -> List[str]: f"one of {Symbols.VALID_LAYERS}" ) - # Disallow Attention + MLA/DSA hybridity. - if Symbols.ATTENTION in segment and Symbols.DS_ATTENTION in segment: - raise ValueError("Not supported to have both Attention and MLA/DSA in one model") + # Disallow standard Attention ('*') mixed with any MLA-based attention (D/C/H/W). + if Symbols.ATTENTION in segment and any(s in segment for s in Symbols.MLA_ATTENTION): + raise ValueError( + "Not supported to have both Attention and MLA/DSA/CSA/HCA/Window in one model" + ) return layer_type_list @@ -333,6 +343,8 @@ def select_pipeline_segment( vp_stage: Optional[int], first_stage_layers: Optional[int] = None, last_stage_layers: Optional[int] = None, + tp_group: Optional[torch.distributed.ProcessGroup] = None, + dp_cp_group: Optional[torch.distributed.ProcessGroup] = None, ) -> Tuple[List[str], int]: """Select and validate the pipeline segment for the given PP rank and VP stage. @@ -352,6 +364,8 @@ def select_pipeline_segment( uneven PP. Only valid when the pattern has no pipe separators. last_stage_layers: Number of layers on the last pipeline stage for uneven PP. Only valid when the pattern has no pipe separators. + tp_group: Optional tensor-parallel process group used for per-stage logging. + dp_cp_group: Optional data/context-parallel process group used for per-stage logging. Returns: Tuple of (layer_type_list, layer_offset) where layer_type_list is @@ -445,6 +459,8 @@ def select_pipeline_segment( f"HybridModel: pp_rank={pp_rank}/{pp_size}, vp_stage={vp_stage}, " f"layers='{''.join(selected)}' ({len(selected)} layers), " f"layer_offset={offset} (auto-split)", + tp_group=tp_group, + dp_cp_group=dp_cp_group, ) return selected, offset @@ -479,6 +495,8 @@ def select_pipeline_segment( f"segment_index={segment_index}/{len(segments)}, " f"layers='{my_segment}' ({len(layer_type_list)} layers), " f"layer_offset={layer_offset}", + tp_group=tp_group, + dp_cp_group=dp_cp_group, ) return layer_type_list, layer_offset diff --git a/megatron/core/models/hybrid/hybrid_layer_specs.py b/megatron/core/models/hybrid/hybrid_layer_specs.py index a34a45a32ba..a18da8b5452 100755 --- a/megatron/core/models/hybrid/hybrid_layer_specs.py +++ b/megatron/core/models/hybrid/hybrid_layer_specs.py @@ -1,4 +1,5 @@ # Copyright (c) 2023-2026, NVIDIA CORPORATION. All rights reserved. +from functools import partial from megatron.core.extensions.transformer_engine import ( TEColumnParallelLinear, @@ -72,7 +73,12 @@ submodules=MultiTokenPredictionLayerSubmodules( enorm=TENorm, hnorm=TENorm, + # Both projection forms are populated so the layer can switch on + # `config.enable_hyper_connections` at runtime: mHC=True uses + # `e_proj`+`h_proj` per-stream, mHC=False uses fused `eh_proj`. eh_proj=TEColumnParallelLinear, + e_proj=TEColumnParallelLinear, + h_proj=TEColumnParallelLinear, mtp_model_layer=None, # Built via pattern + hybrid_submodules layer_norm=TENorm, ), @@ -170,8 +176,8 @@ mlp_layer=ModuleSpec( module=MLPLayer, submodules=TransformerLayerSubmodules( - mlp=ModuleSpec( - module=MLP, + mlp=partial( + MLP.as_mlp_submodule, submodules=MLPSubmodules( linear_fc1=TELayerNormColumnParallelLinear, linear_fc2=TERowParallelLinear ), @@ -265,8 +271,8 @@ mlp_layer=ModuleSpec( module=MLPLayer, submodules=TransformerLayerSubmodules( - mlp=ModuleSpec( - module=MLP, + mlp=partial( + MLP.as_mlp_submodule, submodules=MLPSubmodules( linear_fc1=InferenceLayerNormColumnParallelLinear, linear_fc2=InferenceRowParallelLinear, @@ -291,7 +297,11 @@ submodules=MultiTokenPredictionLayerSubmodules( enorm=TENorm, hnorm=TENorm, + # Populate both projection forms so the layer can switch on + # `config.enable_hyper_connections` at runtime. eh_proj=InferenceColumnParallelLinear, + e_proj=InferenceColumnParallelLinear, + h_proj=InferenceColumnParallelLinear, mtp_model_layer=None, # Built via pattern + hybrid_submodules layer_norm=TENorm, ), @@ -306,3 +316,56 @@ # Backward-compatible aliases mamba_stack_spec = hybrid_stack_spec mamba_inference_stack_spec = hybrid_inference_stack_spec + + +def hybrid_dsv4_stack_spec(config): + """Config-aware hybrid stack spec whose ``D`` (DS_ATTENTION) layer runs the DSv4 + ``CompressedSparseAttention`` (CSA/HCA + ``CSAIndexer``), identical to the GPT + ``dsv4_hybrid`` path, instead of the legacy ``DSAttention``. + + The default ``hybrid_stack_spec`` wires the ``D`` layer to ``MLASelfAttention + + DSAttention`` (DSv3-style sparse attention with no CSA/HCA compression). To run real + DSv4 on HybridModel — and to be numerically equivalent to a GPT ``dsv4_hybrid`` + attention layer — we reuse GPT's own ``get_dsv4_hybrid_module_spec_for_backend`` + (which is config-aware, e.g. picks the qk-layernorm form from ``config``) so the two + model paths build the *same* attention module. Selected via + ``--spec megatron.core.models.hybrid.hybrid_layer_specs hybrid_dsv4_stack_spec``; + ``hybrid_builder`` invokes this function with ``config`` when the spec is callable. + """ + import dataclasses + + from megatron.core.models.gpt.experimental_attention_variant_module_specs import ( + _get_backend_spec_provider, + get_dsv4_hybrid_module_spec_for_backend, + ) + + backend = _get_backend_spec_provider(config) + dsv4_attention = get_dsv4_hybrid_module_spec_for_backend(config, backend) + + def _wrap_dsv4_layer(compress_ratio=None): + # Wrap the DSv4 attention in a hybrid TransformerLayer. When compress_ratio is given + # (the 'C'/'H'/'W' layer symbols), bake it into the attention params so the layer uses a + # fixed CSA(4)/HCA(128)/window-only(0) ratio regardless of csa_compress_ratios; otherwise + # the layer reads its ratio from config.csa_compress_ratios (array-driven 'D' / GPT-parity + # path). compress_ratio=0 builds neither the compressor nor the top-k indexer, so the + # 'W' layer reuses the entire CSA/HCA code path as pure sliding-window attention. + attn = dsv4_attention + if compress_ratio is not None: + attn = dataclasses.replace( + dsv4_attention, params={**dsv4_attention.params, "compress_ratio": compress_ratio} + ) + return ModuleSpec( + module=TransformerLayer, + submodules=TransformerLayerSubmodules( + input_layernorm=TENorm, self_attention=attn, self_attn_bda=get_bias_dropout_add + ), + ) + + submodules = dataclasses.replace( + hybrid_stack_spec.submodules, + dsa_layer=_wrap_dsv4_layer(), # 'D': array-driven (or window) DSv4 attention + csa_layer=_wrap_dsv4_layer(compress_ratio=4), # 'C': CSA + hca_layer=_wrap_dsv4_layer(compress_ratio=128), # 'H': HCA + window_layer=_wrap_dsv4_layer(compress_ratio=0), # 'W': sliding-window-only + ) + return ModuleSpec(module=HybridStack, submodules=submodules) diff --git a/megatron/core/models/hybrid/hybrid_model.py b/megatron/core/models/hybrid/hybrid_model.py index e89c1c07c67..ed4f3643100 100644 --- a/megatron/core/models/hybrid/hybrid_model.py +++ b/megatron/core/models/hybrid/hybrid_model.py @@ -8,6 +8,7 @@ from megatron.core import tensor_parallel from megatron.core.config_logger import has_config_logger_enabled, log_config_to_disk from megatron.core.inference.contexts import BaseInferenceContext +from megatron.core.inference.utils import InferenceMode from megatron.core.models.common.embeddings.language_model_embedding import LanguageModelEmbedding from megatron.core.models.common.embeddings.rotary_pos_embedding import RotaryEmbedding from megatron.core.models.common.embeddings.yarn_rotary_pos_embedding import YarnRotaryEmbedding @@ -39,6 +40,18 @@ logger = logging.getLogger(__name__) +def _hybrid_logging_pg_kwargs(pg_collection: ProcessGroupCollection) -> dict: + tp_group = getattr(pg_collection, 'tp', None) + dp_cp_group = getattr(pg_collection, 'dp_cp', None) + if (tp_group is None) != (dp_cp_group is None): + raise ValueError( + "pg_collection.tp and pg_collection.dp_cp must both be set or both be unset." + ) + if tp_group is None: + return {} + return {'tp_group': tp_group, 'dp_cp_group': dp_cp_group} + + class HybridModel(LanguageModule, GraphableMegatronModule): """Hybrid language model. @@ -187,12 +200,15 @@ def __init__( self.mtp_pattern = parsed.mtp_pattern self.mtp_num_depths = parsed.mtp_num_depths + logging_pg_kwargs = _hybrid_logging_pg_kwargs(self.pg_collection) + layer_type_list, layer_offset = select_pipeline_segment( parsed.main_pattern or '', self.pg_collection.pp, vp_stage, first_stage_layers=self.config.num_layers_in_first_pipeline_stage, last_stage_layers=self.config.num_layers_in_last_pipeline_stage, + **logging_pg_kwargs, ) # Determine if MTP is needed (based on pattern parsing) @@ -203,7 +219,12 @@ def __init__( # to split the hybrid layer pattern into pipeline stages before parsing the pattern for # the current pipeline stage. This could also enable MTP standalone (MTP in a pipeline # stage separate from loss) to be supported in the hybrid model. - and mtp_on_this_rank(self.config, ignore_virtual=False, vp_stage=self.vp_stage) + and mtp_on_this_rank( + layout=self.config.pipeline_model_parallel_layout, + mtp_num_layers=self.config.mtp_num_layers, + ignore_virtual=False, + vp_stage=self.vp_stage, + ) ) # megatron core pipelining currently depends on model type @@ -260,6 +281,7 @@ def __init__( post_process=self.post_process, dtype=config.params_dtype, pg_collection=self.pg_collection, + name="decoder", ) # MTP block - uses mtp_block_spec from hybrid_stack_spec.submodules @@ -279,6 +301,7 @@ def __init__( mtp_layer_pattern=self.mtp_pattern, mtp_num_depths=self.mtp_num_depths, hybrid_submodules=hybrid_submodules, + name="mtp", ) self._setup_mtp_cuda_graphs() @@ -358,7 +381,7 @@ def _should_call_local_cudagraph(self, *args, **kwargs): Check if we should call the local cudagraph path. """ if ( - not self.training + InferenceMode.is_active() and hasattr(self, 'cudagraph_manager') and ( kwargs.get('inference_context') is not None @@ -421,7 +444,7 @@ def forward( inference_context = deprecate_inference_params(inference_context, inference_params) - in_inference_mode = inference_context is not None and not self.training + in_inference_mode = InferenceMode.is_active() if in_inference_mode: assert runtime_gather_output, "Inference must always gather TP logits" @@ -436,6 +459,7 @@ def forward( # quantization scales to avoid corrupting amax calculations if ( in_inference_mode + and inference_context is not None and inference_context.is_dynamic_batching() and is_using_quantization_scales(self.config) ): @@ -480,15 +504,32 @@ def forward( # be None, so this assert will succeed. # assert attention_mask is None, "The attention mask is ignored and should be set to None" + # Pass input_ids to decoder for hash-based MoE routing. + decoder_extra_block_kwargs = {} + if self.config.moe_n_hash_layers > 0 and input_ids is not None: + decoder_extra_block_kwargs['input_ids'] = input_ids + # Run decoder. - hidden_states = self.decoder( + decoder_output = self.decoder( hidden_states=decoder_input, attention_mask=attention_mask, inference_context=inference_context, rotary_pos_emb=rotary_pos_emb, packed_seq_params=packed_seq_params, padding_mask=padding_mask, + **decoder_extra_block_kwargs, ) + # HybridStack.forward returns a single Tensor in the common case, but a 2-tuple + # (hidden_states, mhc_multistream) in exactly one case: enable_hyper_connections and + # post_process and mtp_num_layers > 0 and not is_mtp_layer — where MTP's mHC branch + # needs the pre-contraction multi-stream tensor for `_concat_embeddings`. Any other + # tuple return would be misinterpreted here, so keep that contract in sync with + # HybridStack.forward (see hybrid_block.py). + if isinstance(decoder_output, tuple): + hidden_states, mhc_multistream = decoder_output + else: + hidden_states = decoder_output + mhc_multistream = None output_weight = None if self.share_embeddings_and_output_weights: @@ -499,6 +540,7 @@ def forward( # tokens rather than stale speculative tokens from the previous step. is_spec_decode = ( in_inference_mode + and inference_context is not None and inference_context.is_dynamic_batching() and inference_context.num_speculative_tokens > 0 ) @@ -509,11 +551,13 @@ def forward( input_ids=input_ids, position_ids=position_ids, hidden_states=hidden_states, + mhc_multistream=mhc_multistream, attention_mask=attention_mask, inference_params=inference_params, rotary_pos_emb=rotary_pos_emb, packed_seq_params=packed_seq_params, embedding=self.embedding, + padding_mask=padding_mask, ) if not self.post_process: @@ -524,6 +568,8 @@ def forward( if in_inference_mode or is_spec_decode: self._decoder_hidden_states_cache = hidden_states else: + # For RL (labels is None), process_mtp_loss derives labels from + # input_ids to match the SFT label format. hidden_states = process_mtp_loss( hidden_states=hidden_states, labels=labels, @@ -535,11 +581,17 @@ def forward( compute_language_model_loss=self.compute_language_model_loss, config=self.config, cp_group=self.pg_collection.cp, + tp_group=self.tp_group, packed_seq_params=packed_seq_params, scale_logits_fn=self._scale_logits if self.config.use_mup else None, + input_ids=input_ids, ) sequence_parallel_override = False - if in_inference_mode and inference_context.config.materialize_only_last_token_logits: + if ( + in_inference_mode + and inference_context is not None + and inference_context.config.materialize_only_last_token_logits + ): if inference_context.is_static_batching(): hidden_states = hidden_states[-1:, :, :] else: diff --git a/megatron/core/models/mimo/model/base.py b/megatron/core/models/mimo/model/base.py index bdfe4289dd0..e52e9ab8258 100644 --- a/megatron/core/models/mimo/model/base.py +++ b/megatron/core/models/mimo/model/base.py @@ -2,7 +2,7 @@ import logging import warnings -from typing import Any, Dict, Optional +from typing import Any, Dict, Optional, Tuple import torch @@ -77,8 +77,11 @@ def __init__(self, mimo_config: MimoModelConfig, cp_group=None, tp_group=None) - max_seq_len = mimo_config.language_model_spec.params.get('max_sequence_length', 4096) self.partition_adapter: Optional[PartitionAdapter] = None - # Create partition adapter only if parallelism is enabled - if language_config.context_parallel_size > 1 or language_config.sequence_parallel: + # Only on language-module ranks: encoder-only ranks never shard and would read + # process groups they do not own. + if self.role.has_language_module and ( + language_config.context_parallel_size > 1 or language_config.sequence_parallel + ): partition_config = PartitionConfig.from_mp_config( mp=language_config, max_seq_len=max_seq_len, @@ -275,6 +278,19 @@ def set_input_tensor(self, input_tensor): if self.language_model is not None and hasattr(self.language_model, 'set_input_tensor'): self.language_model.set_input_tensor(input_tensor) + def _active_submodules(self): + """Yield this rank's present submodules.""" + if self.language_model is not None: + yield self.language_model + for submodule in self.modality_submodules.values(): + if submodule is not None: + yield submodule + + def zero_grad_buffer(self): + """Zero each active submodule's DDP grad buffer.""" + for module in self._active_submodules(): + module.zero_grad_buffer() + def get_text_embeddings( self, input_ids: torch.Tensor, position_ids: torch.Tensor, special_token_ids: Dict[str, int] ) -> torch.Tensor: @@ -298,14 +314,33 @@ def get_text_embeddings( batch_idx, seq_idx = text_mask.nonzero(as_tuple=True) input_ids_text = input_ids[batch_idx, seq_idx].unsqueeze(0) - position_ids_text = ( - position_ids[batch_idx, seq_idx].unsqueeze(0) if position_ids is not None else None - ) + if position_ids is None: + position_ids_text = None + elif position_ids.dim() == 3: + # Multimodal RoPE can carry [rope_dim, batch, seq] ids. Text + # embedding lookup only needs a single absolute position channel. + position_ids_text = position_ids[0, batch_idx, seq_idx].unsqueeze(0) + else: + position_ids_text = position_ids[batch_idx, seq_idx].unsqueeze(0) + + embedding_layer = unwrap_model(self.language_model).embedding + # Combined embeddings are SP-scattered later in PartitionAdapter; a second scatter + # here would split the flat text tokens across TP ranks before alignment. + if ( + self.partition_adapter is not None + and self.partition_adapter.cfg.seq_parallel + and getattr(embedding_layer, 'scatter_to_sequence_parallel', False) + ): + raise RuntimeError( + "MIMO sequence parallelism requires the language embedding scatter to be " + "disabled; pass scatter_embedding_sequence_parallel=False when constructing " + "the language model." + ) - text_embeddings = ( - unwrap_model(self.language_model) - .embedding(input_ids=input_ids_text, position_ids=position_ids_text) - .squeeze(1) + text_embeddings = embedding_layer( + input_ids=input_ids_text, position_ids=position_ids_text + ).squeeze( + 1 ) # Shape: [num_text_tokens, hidden_dim] return text_embeddings @@ -374,14 +409,17 @@ def forward( if self.role.mode == ModuleLayout.NON_COLOCATED: if self.role.has_modality_modules: - return self._forward_encoders(modality_inputs, input_tensors), loss_mask + return self._forward_encoders(input_ids, modality_inputs, input_tensors), loss_mask if self.role.has_language_module: - return ( - self._forward_language_module( - input_ids, position_ids, attention_mask, labels, input_tensors - ), + return self._forward_language_module( + input_ids, + position_ids, + attention_mask, loss_mask, + labels, + input_tensors, + packing_kwargs, ) raise RuntimeError(f"Rank has no modules assigned in role: {self.role}") @@ -390,6 +428,7 @@ def forward( def _forward_encoders( self, + input_ids: Optional[torch.Tensor], modality_inputs: Optional[Dict[str, Dict[str, Any]]], input_tensors: Optional[Dict[str, torch.Tensor]], ) -> Dict[str, torch.Tensor]: @@ -409,38 +448,168 @@ def _forward_encoders( continue submodule = self.modality_submodules[encoder_name] - output = submodule.forward( - encoder_inputs=modality_inputs.get(encoder_name) if modality_inputs else None, - hidden_states=input_tensors.get(encoder_name) if input_tensors else None, - ) + encoder_inputs = modality_inputs.get(encoder_name) if modality_inputs else None + hidden_states = input_tensors.get(encoder_name) if input_tensors else None + output = submodule.forward(encoder_inputs=encoder_inputs, hidden_states=hidden_states) + if output is None and encoder_inputs is None and hidden_states is None: + if self._has_encoder_tokens(input_ids, encoder_name): + raise RuntimeError( + f"{encoder_name} inputs are missing, but matching special tokens exist" + ) + output = self._empty_encoder_output(encoder_name) if output is not None: + self._attach_modality_split_sizes(output, input_ids, encoder_name) outputs[encoder_name] = output return outputs + def _attach_modality_split_sizes( + self, output: torch.Tensor, input_ids: Optional[torch.Tensor], encoder_name: str + ) -> None: + """Annotate flat modality outputs with per-sample split sizes for bridge fan-out. + + Only attaches when per-sample token counts are non-uniform. Uniform counts + give equal splits, which the bridge's ``torch.tensor_split`` fallback + already produces, so the metadata would be a no-op. + + TODO(mimo): non-uniform per-sample counts in fan-in (encoder DP > LM DP) + are not supported. Multiple encoder ranks contribute slices to a single + LM peer, and the receiver-side ``torch.cat`` path in BridgeCommunicator + has no metadata channel today, so per-sample boundaries are lost on the + LM rank. Lift this by routing per-sample sizes through the bridge + alongside the activations and adding a sample-aligned concat path. + """ + token_id = self.special_token_ids.get(encoder_name) + if token_id is None or input_ids is None or output.ndim != 2 or input_ids.size(0) <= 1: + return + + split_sizes = (input_ids == token_id).sum(dim=1).to(torch.long).tolist() + if sum(split_sizes) != output.size(0): + return + if len(set(split_sizes)) <= 1: + # Uniform counts — tensor_split fallback gives the same result. + return + + if self.role.mode is ModuleLayout.NON_COLOCATED: + grid_map = self.mimo_config.module_to_grid_map + encoder_grid = grid_map[encoder_name] + language_grid = grid_map[MIMO_LANGUAGE_MODULE_KEY] + encoder_dp = encoder_grid.shape[encoder_grid.dim_names.index("dp")] + language_dp = language_grid.shape[language_grid.dim_names.index("dp")] + assert encoder_dp <= language_dp, ( + f"Bridge fan-out split metadata with non-uniform per-sample sizes " + f"requires encoder DP <= LM DP (got encoder='{encoder_name}' " + f"DP={encoder_dp}, LM DP={language_dp}). Fan-in with variable " + f"modality token counts is not supported yet — see TODO in " + f"_attach_modality_split_sizes." + ) + + output._mimo_bridge_split_sizes = split_sizes + + def _has_encoder_tokens(self, input_ids: Optional[torch.Tensor], encoder_name: str) -> bool: + """Return whether the batch contains tokens for an encoder module.""" + if input_ids is None or encoder_name not in self.special_token_ids: + return False + return bool((input_ids == self.special_token_ids[encoder_name]).any().item()) + + def _empty_encoder_output(self, encoder_name: str) -> torch.Tensor: + """Return the bridge payload for text-only non-colocated batches.""" + language_config = self.mimo_config.language_model_spec.params['config'] + hidden_size = getattr(language_config, 'hidden_size', None) + if hidden_size is None: + raise ValueError( + "Language model config must define hidden_size for empty modality output" + ) + + output_dtype = getattr(language_config, 'params_dtype', None) or torch.float32 + return torch.empty( + (0, hidden_size), + device=torch.cuda.current_device(), + dtype=output_dtype, + requires_grad=True, + ) + + def _build_packed_seq_params(self, packing_kwargs: Optional[dict]) -> Optional[PackedSeqParams]: + """Build THD ``PackedSeqParams`` from ``packing_kwargs`` (None if not packing).""" + if packing_kwargs is None: + return None + for key in packing_kwargs: + if 'cu_seqlens' in key and packing_kwargs[key] is not None: + packing_kwargs[key] = packing_kwargs[key].to(dtype=torch.int32) + packed_seq_params = PackedSeqParams(**packing_kwargs) + packed_seq_params.qkv_format = 'thd' + return packed_seq_params + + def _shard_language_inputs( + self, + embeddings: Optional[torch.Tensor], + labels: Optional[torch.Tensor], + loss_mask: Optional[torch.Tensor], + packed_seq_params: Optional[PackedSeqParams] = None, + ) -> Tuple[ + Optional[torch.Tensor], + Optional[torch.Tensor], + Optional[torch.Tensor], + Optional[PackedSeqParams], + ]: + """Apply CP/SP sharding via the partition adapter, or pass through if inactive. + + ``embeddings`` are sequence-first ``(S, B, H)`` (``None`` on non-first PP stages) + and come back in ``(S/(cp*tp), B, H)``; labels/loss_mask are ``(B, S)``. + """ + if self.partition_adapter is None: + return embeddings, labels, loss_mask, packed_seq_params + + return self.partition_adapter.shard( + embeddings=embeddings, + labels=labels, + loss_mask=loss_mask, + packed_seq_params=packed_seq_params, + ) + def _forward_language_module( self, input_ids: torch.Tensor, position_ids: Optional[torch.Tensor], attention_mask: Optional[torch.Tensor], + loss_mask: Optional[torch.Tensor], labels: Optional[torch.Tensor], input_tensors: Optional[Dict[str, torch.Tensor]], - ) -> torch.Tensor: + packing_kwargs: Optional[dict] = None, + ) -> Tuple[Any, Optional[torch.Tensor]]: """Forward pass for language module on this rank. Args: input_ids: Token IDs position_ids: Position IDs - attention_mask: Attention mask + attention_mask: Attention mask. Must be ``None`` under context parallelism + (CP-local hidden states cannot line up with a dense mask); mask via a + causal ``attn_mask_type`` or ``packed_seq_params`` instead. + loss_mask: Loss mask for per-token loss normalization labels: Labels for loss computation input_tensors: Hidden states or embeddings from previous stage + packing_kwargs: Optional kwargs to construct packed (THD) sequence params. Returns: - Language model output (hidden states, logits, or loss depending on stage) + Tuple of (language model output, possibly CP-sharded loss mask). The + output is hidden states, logits, or loss depending on the stage. """ lang_name = MIMO_LANGUAGE_MODULE_KEY + if ( + self.partition_adapter is not None + and self.partition_adapter.cfg.use_cp + and attention_mask is not None + ): + raise RuntimeError( + "MIMO context parallelism requires attention_mask=None; mask via a causal " + "attn_mask_type or packed_seq_params (a dense mask cannot line up with the " + "CP-sharded sequence)." + ) + + packed_seq_params = self._build_packed_seq_params(packing_kwargs) + if self.role.is_first_stage(lang_name): # First stage: receive encoder embeddings, combine with text, pass to LM # Build modality embeddings dict from encoder outputs @@ -456,22 +625,43 @@ def _forward_language_module( ) modality_embeddings["text"] = text_embeddings - # Combine all embeddings + # Combine all embeddings ([S, B, H]) combined_embeddings = self.align_embeddings_by_token_positions( modality_embeddings=modality_embeddings, input_ids=input_ids, special_token_ids=self.special_token_ids, ) + # Apply CP/SP sharding; combined_embeddings returns in [S/(cp*tp), B, H]. + combined_embeddings, labels, loss_mask, packed_seq_params = self._shard_language_inputs( + embeddings=combined_embeddings, + labels=labels, + loss_mask=loss_mask, + packed_seq_params=packed_seq_params, + ) + lm_output = self.language_model( + # decoder_input replaces the embedding lookup, so input_ids is + # unused here; position_ids is still consumed by mRoPE in models + # such as Qwen3-VL. input_ids=None, - position_ids=None, + position_ids=position_ids, decoder_input=combined_embeddings, labels=labels, attention_mask=attention_mask, + packed_seq_params=packed_seq_params, ) else: - # Non-first stage: receive hidden states from previous LM stage + # Non-first stage: receive hidden states from previous LM stage. + # Labels/loss_mask still need CP sharding so the loss on the last stage + # lines up with the CP-local hidden states. + _, labels, loss_mask, packed_seq_params = self._shard_language_inputs( + embeddings=None, + labels=labels, + loss_mask=loss_mask, + packed_seq_params=packed_seq_params, + ) + hidden_states = input_tensors.get(lang_name) if input_tensors else None # Set input tensor on language model for PP (unwrap DDP to reach GPTModel) @@ -481,18 +671,21 @@ def _forward_language_module( underlying_lm.set_input_tensor(hidden_states) lm_output = self.language_model( + # Hidden states arrive via set_input_tensor; position_ids is + # still consumed by mRoPE on non-first PP stages. input_ids=None, - position_ids=None, + position_ids=position_ids, decoder_input=None, labels=labels, attention_mask=attention_mask, + packed_seq_params=packed_seq_params, ) # Key output for non-last stages so schedule can route to next LM stage if not self.role.is_last_stage(lang_name): - return {lang_name: lm_output} + return {lang_name: lm_output}, loss_mask - return lm_output + return lm_output, loss_mask def _build_colocated_communicators(self): grid_map = self.mimo_config.module_to_grid_map @@ -549,16 +742,7 @@ def _forward_all_modules( This is the original behavior, preserved for backward compatibility. """ - # If packing_kwargs is provided, construct PackedSeqParams - packed_seq_params = None - if packing_kwargs is not None: - # Ensure correct dtype for seqlens tensors - for key in packing_kwargs: - if 'cu_seqlens' in key and packing_kwargs[key] is not None: - packing_kwargs[key] = packing_kwargs[key].to(dtype=torch.int32) - packed_seq_params = PackedSeqParams(**packing_kwargs) - packed_seq_params.qkv_format = 'thd' - logger.debug(f"Packed sequence parameters: {packed_seq_params}") + packed_seq_params = self._build_packed_seq_params(packing_kwargs) # 1. Process each modality to get embeddings modality_embeddings = {} @@ -596,30 +780,22 @@ def _forward_all_modules( ) logger.debug(f"Combined embeddings shape: {combined_embeddings.shape}") - # 3. If sharding is needed, apply PartitionAdapter. - # combined_embeddings is [S, B, H]; transpose to [B, S, H] for shard() which expects - # batch-first layout (required by get_batch_on_this_cp_rank). After CP sharding each - # rank holds [B, S/cp, H]; transpose back to [S/cp, B, H] for the language model. - if self.partition_adapter is not None: - combined_embeddings = combined_embeddings.transpose(0, 1).contiguous() # [B, S, H] - combined_embeddings, labels, loss_mask, _, packed_seq_params = ( - self.partition_adapter.shard( - embeddings=combined_embeddings, - labels=labels, - loss_mask=loss_mask, - attention_mask=attention_mask, - packed_seq_params=packed_seq_params, - ) - ) - # shard() returns embeddings in [B, S/cp, H]; transpose to [S/cp, B, H] - # which is what the language model expects. - if combined_embeddings is not None: - combined_embeddings = combined_embeddings.transpose(0, 1).contiguous() + # 3. Apply CP/SP sharding. combined_embeddings is [S, B, H] and returns + # [S/(cp*tp), B, H] for the LM (the adapter handles the CP batch-first transpose). + combined_embeddings, labels, loss_mask, packed_seq_params = self._shard_language_inputs( + embeddings=combined_embeddings, + labels=labels, + loss_mask=loss_mask, + packed_seq_params=packed_seq_params, + ) # 5. Forward pass through language model lm_output = self.language_model( + # decoder_input replaces the embedding lookup, so input_ids is + # unused here; position_ids is still consumed by mRoPE in models + # such as Qwen3-VL. input_ids=None, - position_ids=None, + position_ids=position_ids, decoder_input=combined_embeddings, labels=labels, attention_mask=None, diff --git a/megatron/core/models/mimo/optimizer.py b/megatron/core/models/mimo/optimizer.py index 1a79c1f91ff..6d23998490d 100644 --- a/megatron/core/models/mimo/optimizer.py +++ b/megatron/core/models/mimo/optimizer.py @@ -153,6 +153,7 @@ def load_state_dict(self, state_dict: Dict): for sub_sd, inner_opt in _iter_optimizer_sub_dicts(module_sd, info.optimizer): _restore_param_groups(sub_sd, inner_opt, name) + _restore_param_state_sharding_type(sub_sd) _restore_grad_scaler(sub_sd) info.optimizer.load_state_dict(module_sd) @@ -175,6 +176,7 @@ def sharded_state_dict(self, model_sharded_state_dict, is_loading: bool = False, ): suffix = f'.{idx}' if idx > 0 else '' _extract_param_groups(sub_sd, name, suffix, replica_id) + _extract_param_state_sharding_type(sub_sd, name, suffix, replica_id) _extract_grad_scaler(sub_sd, name, suffix, replica_id) sharded_state[name] = module_sd @@ -218,6 +220,8 @@ def _extract_param_groups(sub_sd, module_name, suffix, replica_id): replica_id=replica_id, ) del opt_sub['param_groups'] + if not opt_sub: + del sub_sd['optimizer'] def _extract_grad_scaler(sub_sd, module_name, suffix, replica_id): @@ -232,6 +236,18 @@ def _extract_grad_scaler(sub_sd, module_name, suffix, replica_id): ) +def _extract_param_state_sharding_type(sub_sd, module_name, suffix, replica_id): + """Save: extract param_state_sharding_type into a ShardedObject.""" + if 'param_state_sharding_type' in sub_sd: + sub_sd[f'_mimo_param_state_sharding_type{suffix}'] = ShardedObject( + f'optimizer.mimo.{module_name}{suffix}.param_state_sharding_type', + sub_sd.pop('param_state_sharding_type'), + (1,), + (0,), + replica_id=replica_id, + ) + + def _restore_param_groups(sub_sd, inner_optimizer, module_name): """Load: restore param_groups with current param IDs from the inner optimizer.""" # Find the _mimo_param_groups key (may have a suffix for chained optimizers) @@ -253,7 +269,21 @@ def _restore_param_groups(sub_sd, inner_optimizer, module_name): ) for loaded_g, current_g in zip(loaded_pg, current_pg): loaded_g['params'] = current_g['params'] - sub_sd['optimizer']['param_groups'] = loaded_pg + # `sub_sd['optimizer']` may be absent on load: when the per-module state_dict + # produced by DistributedOptimizer.state_dict() only contains `param_groups` + # under the 'optimizer' key, `_extract_param_groups` removes it at save time + # and the resulting empty dict can be dropped during dist_checkpointing + # common-state save/load. Use setdefault so the restored param_groups land + # in the right place regardless. + sub_sd.setdefault('optimizer', {})['param_groups'] = loaded_pg + + +def _restore_param_state_sharding_type(sub_sd): + """Load: restore param_state_sharding_type from ShardedObject key.""" + for k in list(sub_sd.keys()): + if k.startswith('_mimo_param_state_sharding_type'): + sub_sd['param_state_sharding_type'] = sub_sd.pop(k) + break def _restore_grad_scaler(sub_sd): @@ -267,17 +297,21 @@ def _restore_grad_scaler(sub_sd): def _get_replica_id(pg_collection: Optional[ProcessGroupCollection]) -> tuple: """Build replica_id tuple for ShardedObject deduplication. - Includes pp_rank so only one PP stage writes the metadata, - and dp_rank so only dp_rank=0 writes (others are replicas). + Returns (tp_rank, pp_rank, dp_rank) so only (0, 0, 0) within each + module's parallelism group is the main replica; all other ranks + in the same module are non-main replicas of the same object. """ assert pg_collection is not None, "pg_collection required for checkpoint replica_id" + assert ( + hasattr(pg_collection, 'tp') and pg_collection.tp is not None + ), "pg_collection.tp must be set for checkpoint deduplication" assert ( hasattr(pg_collection, 'pp') and pg_collection.pp is not None ), "pg_collection.pp must be set for checkpoint deduplication" assert ( hasattr(pg_collection, 'dp') and pg_collection.dp is not None ), "pg_collection.dp must be set for checkpoint deduplication" - return (0, pg_collection.pp.rank(), pg_collection.dp.rank()) + return (pg_collection.tp.rank(), pg_collection.pp.rank(), pg_collection.dp.rank()) def _get_pg_collection_for_optimizer(grid) -> ProcessGroupCollection: diff --git a/megatron/core/models/mimo/partition/utils.py b/megatron/core/models/mimo/partition/utils.py index 0b43e5548ff..70e1753e500 100644 --- a/megatron/core/models/mimo/partition/utils.py +++ b/megatron/core/models/mimo/partition/utils.py @@ -50,11 +50,6 @@ class PartitionConfig: cp_group: Optional[ProcessGroup] = None tp_group: Optional[ProcessGroup] = None - @property - def is_partitioning_enabled(self) -> bool: - """Returns True if context parallelism or sequence parallelism is active.""" - return self.use_cp or self.seq_parallel - @classmethod def from_mp_config( cls, @@ -89,7 +84,7 @@ def from_mp_config( class PartitionAdapter: - """Shard batch-first embeddings & label tensors for Context and Sequence Parallelism.""" + """Shard MIMO sequence inputs (CP/SP) and return language-model-ready embeddings.""" def __init__(self, cfg: PartitionConfig): """Initialize the partition adapter. @@ -100,100 +95,86 @@ def __init__(self, cfg: PartitionConfig): def shard( self, - embeddings: torch.Tensor, - labels: torch.Tensor, - loss_mask: torch.Tensor, - attention_mask: torch.Tensor, + embeddings: Optional[torch.Tensor], + labels: Optional[torch.Tensor], + loss_mask: Optional[torch.Tensor], packed_seq_params: Optional[PackedSeqParams] = None, - ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, Optional[PackedSeqParams]]: - """ - Apply context parallel (CP) and sequence parallel (SP) sharding to input tensors. - - All input tensors must be in batch-first layout: - - embeddings: (B, S, H) - - labels / loss_mask / attention_mask: (B, S) - - After this call embeddings are still in (B, S/cp, H) batch-first layout. - The caller is responsible for transposing to (S/cp, B, H) if the language model - requires sequence-first tensors. - - Args: - embeddings (torch.Tensor): - Input embeddings tensor. Shape: (B, S, H) - labels (torch.Tensor): - Labels tensor. Shape: (B, S) - loss_mask (torch.Tensor): - Loss mask tensor. Shape: (B, S) - attention_mask (torch.Tensor): - Attention mask tensor. Shape: (B, S) - packed_seq_params (PackedSeqParams, optional): - Packed sequence parameters. Defaults to None. - - Returns: - Tuple containing: - - embeddings (torch.Tensor): Sharded embeddings. Shape: (B, S/cp, H) - - labels (torch.Tensor): Possibly sharded labels. Shape: (B, S/cp) - - loss_mask (torch.Tensor): Possibly sharded loss mask. Shape: (B, S/cp) - - attention_mask (torch.Tensor): Possibly sharded attention mask. Shape: (B, S/cp) - - packed_seq_params (PackedSeqParams, optional): Updated packed sequence parameters. + ) -> Tuple[ + Optional[torch.Tensor], + Optional[torch.Tensor], + Optional[torch.Tensor], + Optional[PackedSeqParams], + ]: + """Apply context- (CP) and sequence-parallel (SP) sharding along the sequence. + + Inputs: + - embeddings: sequence-first ``(S, B, H)``, or ``None`` on non-first PP stages. + - labels / loss_mask: batch-first ``(B, S)``. + + Returns ``(embeddings, labels, loss_mask, packed_seq_params)`` with embeddings in + ``(S/(cp*tp), B, H)`` language-model layout. CP shards every tensor along the + sequence; SP additionally scatters only the embeddings -- labels/loss_mask are not + SP-scattered because the loss runs after the TP all-gather on the full CP-local + sequence. A dense ``[B, S]`` attention mask is intentionally not handled: under CP + it cannot line up with the sharded sequence, so MIMO masks via a causal + ``attn_mask_type`` or ``packed_seq_params`` (THD) instead. """ - if not (self.cfg.use_cp or self.cfg.seq_parallel): - return embeddings, labels, loss_mask, attention_mask, packed_seq_params - - # Sanity-check the sequence length before any sharding happens. + # Sanity-check the sequence length before sharding. Embeddings are sequence-first, + # so the token sequence is dim 0. if embeddings is not None: shard_factor = None - seq_dim = None # which dimension holds the token sequence if self.cfg.use_cp and self.cfg.seq_parallel: shard_factor = get_pg_size(self.cfg.tp_group) * get_pg_size(self.cfg.cp_group) * 2 - seq_dim = 1 # embeddings shape: [B, S, H] elif self.cfg.use_cp: shard_factor = get_pg_size(self.cfg.cp_group) * 2 - seq_dim = 1 elif self.cfg.seq_parallel: shard_factor = get_pg_size(self.cfg.tp_group) - seq_dim = 0 # embeddings shape: [S, B, H] if shard_factor is not None and ( packed_seq_params is None or getattr(packed_seq_params, 'qkv_format', 'sbhd') == 'sbhd' ): - assert embeddings.shape[seq_dim] % shard_factor == 0, ( + assert embeddings.shape[0] % shard_factor == 0, ( f"Sequence length should be divisible by {shard_factor} " "for Sequence/Context parallelism" ) if self.cfg.seq_parallel and self.cfg.tp_comm_overlap: - assert embeddings.shape[seq_dim] == self.cfg.max_seq_len, ( + assert embeddings.shape[0] == self.cfg.max_seq_len, ( "TP Comm overlap requires Vision+Text token length " "== language_max_sequence_length" ) if self.cfg.use_cp: - embeddings, labels, loss_mask, attention_mask, packed_seq_params = ( - self._apply_context_parallel( - embeddings, labels, loss_mask, attention_mask, packed_seq_params - ) + # CP shards batch-first (get_batch_on_this_cp_rank requirement): transpose + # (S, B, H) -> (B, S, H) in, then the CP-local result back to (S/cp, B, H). + if embeddings is not None: + embeddings = embeddings.transpose(0, 1).contiguous() + embeddings, labels, loss_mask, packed_seq_params = self._apply_context_parallel( + embeddings, labels, loss_mask, packed_seq_params ) + if embeddings is not None: + embeddings = embeddings.transpose(0, 1).contiguous() + # SP scatters the sequence (dim 0); the SP-only path needs no transpose. if self.cfg.seq_parallel and embeddings is not None: - embeddings = tensor_parallel.scatter_to_sequence_parallel_region(embeddings) + embeddings = tensor_parallel.scatter_to_sequence_parallel_region( + embeddings, group=self.cfg.tp_group + ) - return embeddings, labels, loss_mask, attention_mask, packed_seq_params + return embeddings, labels, loss_mask, packed_seq_params def _apply_context_parallel( self, embeddings: Optional[torch.Tensor], labels: Optional[torch.Tensor], loss_mask: Optional[torch.Tensor], - attention_mask: Optional[torch.Tensor], packed_seq_params: Optional[PackedSeqParams], ) -> Tuple[ Optional[torch.Tensor], Optional[torch.Tensor], Optional[torch.Tensor], - Optional[torch.Tensor], Optional[PackedSeqParams], ]: """ @@ -206,8 +187,6 @@ def _apply_context_parallel( Labels tensor. Shape: (B, S) loss_mask (Optional[torch.Tensor]): Loss mask tensor. Shape: (B, S) - attention_mask (Optional[torch.Tensor]): - Attention mask tensor. Shape: (B, S) packed_seq_params (PackedSeqParams, optional): Packed sequence parameters. Defaults to None. @@ -216,12 +195,10 @@ def _apply_context_parallel( - embeddings (Optional[torch.Tensor]): Sharded embeddings. Shape: (B, S/cp, H) - labels (Optional[torch.Tensor]): Possibly sharded labels. Shape: (B, S/cp) - loss_mask (Optional[torch.Tensor]): Possibly sharded loss mask. Shape: (B, S/cp) - - attention_mask (Optional[torch.Tensor]): Possibly sharded attention mask. - Shape: (B, S/cp) - packed_seq_params (PackedSeqParams, optional): Updated packed sequence parameters. """ if not self.cfg.use_cp: - return embeddings, labels, loss_mask, attention_mask, packed_seq_params + return embeddings, labels, loss_mask, packed_seq_params # Distribute sequence across CP ranks batch = dict() @@ -231,11 +208,9 @@ def _apply_context_parallel( batch["labels"] = labels if loss_mask is not None: batch["loss_mask"] = loss_mask - if attention_mask is not None: - batch["attention_mask"] = attention_mask if packed_seq_params is None or getattr(packed_seq_params, 'qkv_format', 'sbhd') == 'sbhd': - batch = get_batch_on_this_cp_rank(batch) + batch = get_batch_on_this_cp_rank(batch, is_hybrid_cp=False, cp_group=self.cfg.cp_group) else: assert _HAVE_TEX and is_te_min_version("1.10.0"), ( "Please update Transformer Engine to >= 1.10 " @@ -250,11 +225,10 @@ def _apply_context_parallel( ) batch[key] = data.index_select(1, index) - # Extract sharded tensors; embeddings remain in [B, S/cp, H] — the caller - # is responsible for transposing to [S/cp, B, H] for the language model. + # Extract sharded tensors; embeddings stay in [B, S/cp, H]. shard() transposes + # them to language-model layout [S/cp, B, H] after CP and before optional SP. embeddings = batch.get("embeddings", None) labels = batch.get("labels", None) loss_mask = batch.get("loss_mask", None) - attention_mask = batch.get("attention_mask", None) - return embeddings, labels, loss_mask, attention_mask, packed_seq_params + return embeddings, labels, loss_mask, packed_seq_params diff --git a/megatron/core/models/mimo/submodules/base.py b/megatron/core/models/mimo/submodules/base.py index 3b54fd737f2..f05ecc6b15c 100644 --- a/megatron/core/models/mimo/submodules/base.py +++ b/megatron/core/models/mimo/submodules/base.py @@ -234,6 +234,15 @@ def encode(self, encoders_data_batch: Dict) -> List[torch.Tensor]: encoder_inputs = encoders_data_batch[name] encoder_outputs = encoder(**encoder_inputs) + # Some encoders return (embeddings, aux_state). MIMO consumes the + # primary embedding tensor here; model-specific aux handling should + # live in a modality-specific submodule. + if ( + isinstance(encoder_outputs, tuple) + and encoder_outputs + and torch.is_tensor(encoder_outputs[0]) + ): + encoder_outputs = encoder_outputs[0] logger.debug(f"Encoder '{name}' output shape: {encoder_outputs.shape}") if encoder_outputs.ndim == 3: diff --git a/megatron/core/models/multimodal/context_parallel.py b/megatron/core/models/multimodal/context_parallel.py index 6a3cb8bdf48..ceee2d0af7d 100644 --- a/megatron/core/models/multimodal/context_parallel.py +++ b/megatron/core/models/multimodal/context_parallel.py @@ -1,9 +1,16 @@ # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. """Multimodal Sequence Parallel (SP) and Context Parallel (CP) functionality.""" +import math + import torch from megatron.core.packed_seq_params import PackedSeqParams +from megatron.core.parallel_state import ( + get_context_parallel_group, + get_context_parallel_rank, + get_context_parallel_world_size, +) def get_padding( @@ -109,3 +116,421 @@ def get_packed_seq_params(tokens, img_seq_len, padding_needed, cp_size, use_pack ) return packed_seq_params + + +def split_to_context_parallel_ranks(global_t, pad_value=0): + """Split the tensor global_t into context parallel world size parts. + + Args: + global_t: [batch, ...] + pad_value: Value to pad the last rank with. + + Returns: + local_t: [samples_per_rank, ...]. samples_per_rank is the # of samples per CP rank. + global_pad: Total padding to have equal samples_per_rank across context parallel ranks. + """ + cp_size = get_context_parallel_world_size() + cp_rank = get_context_parallel_rank() + + samples_per_rank = (global_t.shape[0] + cp_size - 1) // cp_size + local_t = global_t[cp_rank * samples_per_rank : (cp_rank + 1) * samples_per_rank] + global_pad = samples_per_rank * cp_size - global_t.shape[0] + + if local_t.shape[0] < samples_per_rank: + local_pad = samples_per_rank - local_t.shape[0] + zeros = torch.full( + (local_pad, *local_t.shape[1:]), pad_value, device=local_t.device, dtype=local_t.dtype + ) + local_t = torch.cat([local_t, zeros], dim=0) + + return local_t, global_pad + + +def _gather_along_second_dim(local_t): + group = get_context_parallel_group() + cp_size = get_context_parallel_world_size() + if cp_size == 1: + return local_t + + tensor_list = [ + torch.empty(local_t.shape, device=local_t.device, dtype=local_t.dtype) + for _ in range(cp_size) + ] + torch.distributed.all_gather(tensor_list, local_t, group=group) + return torch.cat(tensor_list, dim=1) + + +def _reduce_scatter_along_second_dim(global_t): + cp_size = get_context_parallel_world_size() + if cp_size == 1: + return global_t + + assert global_t.shape[1] % cp_size == 0 + samples_per_rank = global_t.shape[1] // cp_size + + tensor_list = [ + global_t[:, cp_rank * samples_per_rank : (cp_rank + 1) * samples_per_rank] + for cp_rank in range(cp_size) + ] + + local_t = torch.zeros( + global_t.shape[0], + samples_per_rank, + *global_t.shape[2:], + device=global_t.device, + dtype=global_t.dtype, + ) + + torch.distributed.reduce_scatter(local_t, tensor_list, group=get_context_parallel_group()) + return local_t + + +class GatherFromContextParallelRanks(torch.autograd.Function): + """Gather the input from context parallel ranks.""" + + @staticmethod + def symbolic(graph, input_): + """Symbolic forward used during ``torch.jit`` tracing.""" + return _gather_along_second_dim(input_) + + @staticmethod + def forward(ctx, input_): + """All-gather ``input_`` along its second dimension across CP ranks.""" + return _gather_along_second_dim(input_) + + @staticmethod + def backward(ctx, grad_output): + """Reduce-scatter the gradient along the second dimension.""" + return _reduce_scatter_along_second_dim(grad_output) + + +def gather_from_context_parallel_ranks(local_t, global_pad): + """Gather ``local_t`` across CP ranks, removing ``global_pad`` trailing pad tokens.""" + global_t = GatherFromContextParallelRanks.apply(local_t) + if global_pad > 0: + global_t = global_t[:, :-global_pad] + return global_t + + +def gather_from_context_parallel_ranks_dynamic_res(local_t, num_padded_imgs=0): + """Gather dynamic-resolution tensors (variable seq per rank) from CP ranks.""" + cp_size = get_context_parallel_world_size() + shape = torch.as_tensor(local_t.shape, device=local_t.device) + shapes = [torch.empty_like(shape) for _ in range(cp_size)] + + torch.distributed.all_gather(shapes, shape, group=get_context_parallel_group()) + + inputs = [local_t] * cp_size + outputs = [torch.empty(*s, dtype=local_t.dtype, device=local_t.device) for s in shapes] + torch.distributed.nn.functional.all_to_all(outputs, inputs, group=get_context_parallel_group()) + + if num_padded_imgs > 0: + outputs = outputs[:-num_padded_imgs] + + return torch.cat(outputs, dim=0) + + +def _compute_tubelet_aware_split_points(num_frames, temporal_patch_size, cp_size, total_frames): + """Compute frame-space split points that respect tubelet boundaries within videos. + + Returns ``cp_size + 1`` split points in **frame** indices (not tubelet indices), + since callers slice per-frame ``cu_seqlens`` and ``imgs_sizes`` with these bounds. + Splits land on either media boundaries or tubelet boundaries inside a media so + that no rank receives a partial tubelet. + """ + T = temporal_patch_size + target_per_rank = total_frames / cp_size + + media_boundaries = [0] + for nf in num_frames: + media_boundaries.append(media_boundaries[-1] + nf) + boundary_set = set(media_boundaries) + + split_points = [0] + for rank in range(1, cp_size): + target_split = int(rank * target_per_rank) + + # If the target lands exactly on a media boundary, split there cleanly + # without forcing a cut into the next media. + if target_split in boundary_set: + split_point = target_split + else: + media_idx = 0 + for i, boundary in enumerate(media_boundaries[1:], 1): + if boundary > target_split: + media_idx = i - 1 + break + else: + media_idx = len(num_frames) - 1 + + media_start = media_boundaries[media_idx] + media_end = media_boundaries[media_idx + 1] + nf = num_frames[media_idx] + num_tubelets = math.ceil(nf / T) + + if num_tubelets <= 1: + if target_split - media_start < media_end - target_split: + split_point = media_start + else: + split_point = media_end + else: + offset_in_media = target_split - media_start + tubelet_idx = round(offset_in_media / T) + tubelet_idx = max(1, min(tubelet_idx, num_tubelets - 1)) + split_point = media_start + tubelet_idx * T + split_point = min(split_point, media_end) + + split_point = max(split_point, split_points[-1]) + split_points.append(split_point) + + split_points.append(total_frames) + return split_points + + +def _split_num_frames(num_frames, lb, ub): + """Return per-media frame counts clipped to the frame range ``[lb, ub)``. + + ``lb`` and ``ub`` are frame indices (the same coordinate system used by + :func:`_compute_tubelet_aware_split_points` and the per-frame ``seqlens`` + array in :func:`split_to_context_parallel_ranks_dynamic_res`). The returned + list has one entry per media that contributes at least one frame to the + range, with the value being the number of frames of that media in the + range. + """ + new_num_frames = [] + frame_idx = 0 + for nf in num_frames: + media_start = frame_idx + media_end = frame_idx + nf + overlap_start = max(media_start, lb) + overlap_end = min(media_end, ub) + if overlap_start < overlap_end: + new_num_frames.append(overlap_end - overlap_start) + frame_idx = media_end + return new_num_frames + + +def split_to_context_parallel_ranks_dynamic_res( + global_t, + global_imgs_sizes, + global_packed_seq_params, + *, + patch_dim, + fp8_enabled=False, + fp8_recipe=None, + num_frames=None, + temporal_patch_size=1, +): + """Split patched vision input across CP ranks. + + ``global_packed_seq_params`` provides per-image seqlens; the split respects them + so each rank owns an integer number of images. When ``temporal_patch_size > 1``, + splits also respect tubelet boundaries and ``num_frames`` is required. + + Args: + global_t: ``[1, total_patches, C * patch_dim * patch_dim]`` patched tokens + (pre-embedder). The last dim must equal ``3 * patch_dim * patch_dim``. + global_imgs_sizes: ``[num_imgs, 2]`` per-image (H, W) in pixels. + global_packed_seq_params: ``PackedSeqParams`` with per-image ``cu_seqlens_q``. + patch_dim: Patch size of the vision backbone (e.g. 14 for SigLIP, 16 for + many ViTs). Required because dummy padding tensors are sized in patch + units and the default would silently mismatch some backbones. + fp8_enabled: If True, pad each rank's local sequence to the FP8 multiple + (16 by default; 32 for ``mxfp8``). + fp8_recipe: Forwarded to :func:`get_padding` so the FP8 padding multiple + matches the active recipe. + num_frames: Per-media frame count, required when ``temporal_patch_size > 1``. + temporal_patch_size: Tubelet size for temporal compression. + + Returns: + (local_t, local_imgs_sizes, local_packed_seq_params, has_padding, + num_padded_ranks, local_num_frames) + """ + cp_size = get_context_parallel_world_size() + cp_rank = get_context_parallel_rank() + + use_tubelet_aware_split = temporal_patch_size > 1 + if use_tubelet_aware_split: + assert num_frames is not None, ( + f"num_frames must be provided when using temporal compression " + f"(temporal_patch_size={temporal_patch_size})" + ) + num_frames_list = num_frames.tolist() if hasattr(num_frames, "tolist") else list(num_frames) + + cu_seqlens = global_packed_seq_params.cu_seqlens_q + + num_imgs = len(global_imgs_sizes) + if use_tubelet_aware_split: + T = temporal_patch_size + total_tubelets = sum(math.ceil(nf / T) for nf in num_frames_list) + num_padded_imgs = max(0, cp_size - total_tubelets) + else: + num_padded_imgs = max(0, cp_size - num_imgs) + + # This function operates on pre-embedder patches, so the hidden dim is + # exactly ``3 * patch_dim * patch_dim``. Both the dummy padding image and + # the FP8 right-pad tensor below assume this layout. + expected_hidden = 3 * patch_dim * patch_dim + assert int(global_t.shape[2]) == expected_hidden, ( + f"split_to_context_parallel_ranks_dynamic_res expects pre-embedder patches " + f"with hidden dim 3*patch_dim*patch_dim={expected_hidden}, got " + f"{int(global_t.shape[2])} (patch_dim={patch_dim})." + ) + + dummy_img_size = torch.tensor( + [[patch_dim, patch_dim]], device=global_imgs_sizes.device, dtype=global_imgs_sizes.dtype + ) + hidden_dim = expected_hidden + dummy_seqlen = 1 + dummy_img = torch.zeros( + [1, dummy_seqlen, hidden_dim], device=global_t.device, dtype=global_t.dtype + ) + + def _add_dummies(n, global_t, global_imgs_sizes, cu_seqlens, num_frames_list): + seqlens = cu_seqlens[1:] - cu_seqlens[:-1] + for _ in range(n): + global_imgs_sizes = torch.cat([global_imgs_sizes, dummy_img_size], dim=0) + global_t = torch.cat([global_t, dummy_img], dim=1) + seqlens = torch.cat( + [seqlens, torch.tensor([dummy_seqlen], device=seqlens.device, dtype=seqlens.dtype)] + ) + if use_tubelet_aware_split: + num_frames_list = num_frames_list + [1] * n + cu_seqlens = torch.cat( + [ + torch.tensor([0], device=cu_seqlens.device, dtype=cu_seqlens.dtype), + torch.cumsum(seqlens, dim=0), + ] + ) + return global_t, global_imgs_sizes, cu_seqlens, num_frames_list + + if num_padded_imgs > 0: + global_t, global_imgs_sizes, cu_seqlens, num_frames_list = _add_dummies( + num_padded_imgs, + global_t, + global_imgs_sizes, + cu_seqlens, + num_frames_list if use_tubelet_aware_split else None, + ) + + seqlens = cu_seqlens[1:] - cu_seqlens[:-1] + total_frames = len(global_imgs_sizes) + num_padded_ranks = num_padded_imgs + + if use_tubelet_aware_split: + for _retry in range(cp_size): + total_frames = len(global_imgs_sizes) + split_points = _compute_tubelet_aware_split_points( + num_frames_list, temporal_patch_size, cp_size, total_frames + ) + num_empty = sum(1 for k in range(cp_size) if split_points[k] == split_points[k + 1]) + if num_empty == 0: + break + global_t, global_imgs_sizes, cu_seqlens, num_frames_list = _add_dummies( + num_empty, global_t, global_imgs_sizes, cu_seqlens, num_frames_list + ) + num_padded_imgs += num_empty + seqlens = cu_seqlens[1:] - cu_seqlens[:-1] + + original_total_frames = total_frames - num_padded_imgs + if num_padded_imgs > 0 and original_total_frames not in split_points: + for k in range(cp_size): + if split_points[k] < original_total_frames < split_points[k + 1]: + split_points[k + 1] = original_total_frames + break + + num_padded_ranks = 0 + if num_padded_imgs > 0: + for i in range(cp_size - 1, -1, -1): + if split_points[i] >= original_total_frames: + num_padded_ranks += 1 + else: + break + + lb = split_points[cp_rank] + ub = split_points[cp_rank + 1] + local_num_frames = _split_num_frames(num_frames_list, lb, ub) + else: + seq_per_rank = total_frames // cp_size + lb = cp_rank * seq_per_rank + # The last rank absorbs the remainder so the union of [lb, ub) ranges + # exactly covers the [0, total_frames) image set. + ub = (cp_rank + 1) * seq_per_rank if cp_rank < cp_size - 1 else total_frames + local_num_frames = None + + seqlens_local = torch.cat([torch.tensor([0], device=seqlens.device), seqlens[lb:ub]]) + cu_seqlens_local = torch.cumsum(seqlens_local, dim=0).to(torch.int32) + + final_seqlen = cu_seqlens_local[-1] + + pad_img = None + if fp8_enabled: + padding_needed = get_padding( + final_seqlen, 1, 1, False, fp8_enabled=True, fp8_recipe=fp8_recipe + ) + if padding_needed > 0: + pad_img = torch.zeros( + [1, padding_needed, patch_dim * patch_dim * 3], + device=global_t.device, + dtype=global_t.dtype, + ) + cu_seqlens_local = torch.cat( + [ + cu_seqlens_local, + torch.tensor( + [final_seqlen + padding_needed], + device=cu_seqlens_local.device, + dtype=cu_seqlens_local.dtype, + ), + ] + ) + + has_padding = pad_img is not None + + local_packed_seq_params = PackedSeqParams( + qkv_format="thd", + cu_seqlens_q=cu_seqlens_local, + cu_seqlens_kv=cu_seqlens_local, + cu_seqlens_q_padded=None, + cu_seqlens_kv_padded=None, + ) + max_seqlen_local = max(seqlens_local).to(torch.int32) + local_packed_seq_params.max_seqlen_q = max_seqlen_local + local_packed_seq_params.max_seqlen_kv = max_seqlen_local + + local_imgs_sizes = global_imgs_sizes[lb:ub] + if has_padding: + local_imgs_sizes = torch.cat( + [ + local_imgs_sizes, + torch.tensor( + [[patch_dim, patch_dim * padding_needed]], + device=local_imgs_sizes.device, + dtype=local_imgs_sizes.dtype, + ), + ] + ) + + offset = torch.cumsum(seqlens[:lb], dim=0)[-1] if lb > 0 else 0 + + if not has_padding: + local_t = global_t[:, offset + cu_seqlens_local[0] : offset + cu_seqlens_local[-1]] + else: + local_t = torch.cat( + [global_t[:, offset + cu_seqlens_local[0] : offset + cu_seqlens_local[-2]], pad_img], + dim=1, + ) + + if local_num_frames is not None: + local_num_frames = torch.tensor( + local_num_frames, dtype=torch.int32, device=global_imgs_sizes.device + ) + + return ( + local_t, + local_imgs_sizes, + local_packed_seq_params, + has_padding, + num_padded_ranks, + local_num_frames, + ) diff --git a/megatron/core/models/multimodal/llava_model.py b/megatron/core/models/multimodal/llava_model.py index a2e0eed68b2..f714afeca64 100644 --- a/megatron/core/models/multimodal/llava_model.py +++ b/megatron/core/models/multimodal/llava_model.py @@ -12,6 +12,10 @@ from megatron.core.inference.contexts import BaseInferenceContext from megatron.core.models.gpt import GPTModel from megatron.core.models.hybrid.hybrid_model import HybridModel +from megatron.core.models.multimodal.context_parallel import ( + gather_from_context_parallel_ranks_dynamic_res, + split_to_context_parallel_ranks_dynamic_res, +) from megatron.core.models.vision.clip_vit_model import CLIPViTModel, get_num_image_embeddings from megatron.core.models.vision.multimodal_projector import MultimodalProjector from megatron.core.models.vision.radio import RADIOViTModel @@ -43,8 +47,10 @@ IGNORE_INDEX = -100 # ID for labels that should be ignored. # Image token index can be tokenizer dependent so the default value does not work in all cases. DEFAULT_IMAGE_TOKEN_INDEX = -200 +DEFAULT_SOUND_TOKEN_INDEX = -300 IMAGE_TOKEN = "" VIDEO_TOKEN = "