From f247bf4412364f46b95f7617aeb40f4a6ef480ff Mon Sep 17 00:00:00 2001 From: Ashwin Das Date: Fri, 12 Jun 2026 17:03:26 -0400 Subject: [PATCH] Use GITHUB_TOKEN (PAT) for branch detection on url_pull repos When url_pull points to a personal fork, the GitHub App is not installed there and branch detection fails silently (returning no results from git ls-remote). This causes the build to error with "branch does not exist" even though the branch is present. Use PAT-based auth (GITHUB_TOKEN) for url_pull repositories, and fall back to checking the push url (where the App is installed) if the PAT check also fails. Co-authored-by: Cursor --- doozer/doozerlib/source_resolver.py | 32 +++++++++++++---- doozer/tests/test_source_resolver.py | 53 ++++++++++++++++++++++++---- 2 files changed, 72 insertions(+), 13 deletions(-) diff --git a/doozer/doozerlib/source_resolver.py b/doozer/doozerlib/source_resolver.py index 4526671a7a..4aa48ce72c 100644 --- a/doozer/doozerlib/source_resolver.py +++ b/doozer/doozerlib/source_resolver.py @@ -10,7 +10,7 @@ from artcommonlib import assertion, constants, exectools from artcommonlib import util as art_util from artcommonlib.git_helper import git_clone -from artcommonlib.github_auth import get_github_client_for_org, get_github_git_auth_env +from artcommonlib.github_auth import get_github_client_for_org, get_github_git_auth_env, get_github_git_pat_env from artcommonlib.lock import get_named_semaphore from artcommonlib.model import ListModel, Missing, Model from github import GithubException, UnknownObjectException @@ -306,13 +306,22 @@ def detect_remote_source_branch( if use_source_fallback_branch not in ["yes", "always", "never"]: raise ValueError(f"Invalid value for use_source_fallback_branch: {use_source_fallback_branch}") - git_url = source_details.get("url_pull", source_details["url"]) # Use pull URL if available + pull_url = source_details.get("url_pull") + push_url = source_details["url"] + git_url = pull_url or push_url + # When url_pull points to a personal fork, the GitHub App won't have + # access. Use GITHUB_TOKEN (PAT) for branch detection against url_pull + # and fall back to the push url (where the App is installed) if needed. + use_pat = bool(pull_url) branches = source_details["branch"] stage_branch = branches.get("stage", None) if stage else None if stage_branch: LOGGER.info('Normal branch overridden by --stage option, using "{}"'.format(stage_branch)) - result = SourceResolver._get_remote_branch_ref(git_url, stage_branch) + result = SourceResolver._get_remote_branch_ref(git_url, stage_branch, use_pat=use_pat) + if not result and pull_url: + LOGGER.info("Branch check against url_pull %s failed, retrying with push url %s", pull_url, push_url) + result = SourceResolver._get_remote_branch_ref(push_url, stage_branch) if result: return stage_branch, result raise IOError( @@ -331,7 +340,10 @@ def detect_remote_source_branch( if SourceResolver.is_branch_commit_hash(branch): return branch, branch - result = SourceResolver._get_remote_branch_ref(git_url, branch) + result = SourceResolver._get_remote_branch_ref(git_url, branch, use_pat=use_pat) + if not result and pull_url: + LOGGER.info("Branch check against url_pull %s failed, retrying with push url %s", pull_url, push_url) + result = SourceResolver._get_remote_branch_ref(push_url, branch) if result: return branch, result @@ -339,7 +351,10 @@ def detect_remote_source_branch( raise IOError('Requested target branch {} does not exist and no fallback provided'.format(branch)) LOGGER.info('Target branch does not exist in {}, checking fallback branch {}'.format(git_url, fallback_branch)) - result = SourceResolver._get_remote_branch_ref(git_url, fallback_branch) + result = SourceResolver._get_remote_branch_ref(git_url, fallback_branch, use_pat=use_pat) + if not result and pull_url: + LOGGER.info("Branch check against url_pull %s failed, retrying with push url %s", pull_url, push_url) + result = SourceResolver._get_remote_branch_ref(push_url, fallback_branch) if result: return fallback_branch, result raise IOError('Requested fallback branch {} does not exist'.format(branch)) @@ -363,15 +378,18 @@ def is_branch_commit_hash(branch): return False @staticmethod - def _get_remote_branch_ref(git_url, branch): + def _get_remote_branch_ref(git_url, branch, use_pat=False): """ Detect whether a single branch exists on a remote repo; returns git hash if found :param git_url: The URL to the git repo to check. :param branch: The name of the branch. If the name is not a branch and appears to be a commit hash, the hash will be returned without modification. + :param use_pat: If True, use GITHUB_TOKEN (PAT) instead of GitHub App auth. + This is needed for url_pull repos (personal forks) where the + GitHub App is not installed. """ git_url = art_util.ensure_github_https_url(git_url) - auth_env = get_github_git_auth_env(url=git_url) + auth_env = get_github_git_pat_env() if use_pat else get_github_git_auth_env(url=git_url) LOGGER.info('Checking if target branch {} exists in {}'.format(branch, git_url)) try: diff --git a/doozer/tests/test_source_resolver.py b/doozer/tests/test_source_resolver.py index 6714f2bff0..ce2ac96a84 100644 --- a/doozer/tests/test_source_resolver.py +++ b/doozer/tests/test_source_resolver.py @@ -32,6 +32,18 @@ def test_get_remote_branch_ref(self, mock_ensure, mock_auth): flexmock(exectools).should_receive("cmd_assert").once().and_raise(Exception("whatever")) self.assertIsNone(sr._get_remote_branch_ref("giturl", "branch")) + @patch("doozerlib.source_resolver.get_github_git_pat_env", return_value={"GIT_ASKPASS": "/tmp/askpass.sh", "GIT_PASSWORD": "pat-token", "GIT_TERMINAL_PROMPT": "0"}) + @patch("doozerlib.source_resolver.get_github_git_auth_env", return_value={"GIT_ASKPASS": "/tmp/askpass.sh"}) + @patch("doozerlib.source_resolver.art_util.ensure_github_https_url", side_effect=lambda u: u) + def test_get_remote_branch_ref_use_pat(self, mock_ensure, mock_auth, mock_pat): + """When use_pat=True, get_github_git_pat_env should be used instead of get_github_git_auth_env.""" + sr = self.create_source_resolver() + flexmock(exectools).should_receive("cmd_assert").once().and_return("abc123", "") + res = sr._get_remote_branch_ref("giturl", "branch", use_pat=True) + self.assertEqual(res, "abc123") + mock_pat.assert_called_once() + mock_auth.assert_not_called() + def test_detect_remote_source_branch(self): sr = self.create_source_resolver() source_details = dict( @@ -155,8 +167,8 @@ def test_https_pull_url_property_without_pull_url(self): result = resolution.https_pull_url self.assertEqual(result, "https://push.example.com/repo.git") - def test_detect_remote_source_branch_uses_pull_url(self): - """Test that detect_remote_source_branch uses url_pull when available.""" + def test_detect_remote_source_branch_uses_pull_url_with_pat(self): + """Test that detect_remote_source_branch uses url_pull with PAT auth when available.""" sr = self.create_source_resolver() source_details = dict( url='https://push.example.com/repo.git', @@ -167,14 +179,44 @@ def test_detect_remote_source_branch_uses_pull_url(self): ), ) - # Mock the _get_remote_branch_ref to expect pull URL flexmock(SourceResolver).should_receive("_get_remote_branch_ref").with_args( - 'https://pull.example.com/repo.git', 'main_branch' + 'https://pull.example.com/repo.git', 'main_branch', use_pat=True ).once().and_return("commit_hash") result = sr.detect_remote_source_branch(source_details, stage=False) self.assertEqual(result, ("main_branch", "commit_hash")) + def test_detect_remote_source_branch_pull_url_fallback_to_push_url(self): + """When url_pull branch check fails (e.g. GitHub App has no access to + a personal fork), fall back to checking the push url.""" + sr = self.create_source_resolver() + source_details = dict( + url='https://push.example.com/repo.git', + url_pull='https://pull.example.com/repo.git', + branch=dict( + target='main_branch', + fallback='fallback_branch', + ), + ) + + ( + flexmock(SourceResolver) + .should_receive("_get_remote_branch_ref") + .with_args('https://pull.example.com/repo.git', 'main_branch', use_pat=True) + .once() + .and_return(None) + ) + ( + flexmock(SourceResolver) + .should_receive("_get_remote_branch_ref") + .with_args('https://push.example.com/repo.git', 'main_branch') + .once() + .and_return("commit_hash_from_push") + ) + + result = sr.detect_remote_source_branch(source_details, stage=False) + self.assertEqual(result, ("main_branch", "commit_hash_from_push")) + def test_detect_remote_source_branch_falls_back_to_url(self): """Test that detect_remote_source_branch falls back to url when url_pull is not available.""" sr = self.create_source_resolver() @@ -186,9 +228,8 @@ def test_detect_remote_source_branch_falls_back_to_url(self): ), ) - # Mock the _get_remote_branch_ref to expect main URL flexmock(SourceResolver).should_receive("_get_remote_branch_ref").with_args( - 'https://push.example.com/repo.git', 'main_branch' + 'https://push.example.com/repo.git', 'main_branch', use_pat=False ).once().and_return("commit_hash") result = sr.detect_remote_source_branch(source_details, stage=False)