diff --git a/README.md b/README.md index e1793b1..218382d 100644 --- a/README.md +++ b/README.md @@ -171,9 +171,10 @@ Or run a specific suite: ```bash CONFIG_PATH="$(pwd)/configs/" go test ./internal/test/integration/ -run TestPythonSuite -v CONFIG_PATH="$(pwd)/configs/" go test ./internal/test/integration/ -run TestRpmSuite -v +CONFIG_PATH="$(pwd)/configs/" go test ./internal/test/integration/ -run TestMavenSuite -v ``` -The Python integration test syncs `shelf-reader` from PyPI into a random domain via the Pulp API, then asserts tangy can read it from the database. Test data is left in the database after a run; use `make compose-clean` to wipe volumes and start fresh. +The Python integration test syncs `shelf-reader` from PyPI into a random domain via the Pulp API, then asserts tangy can read it from the database. The Maven integration test pull-through caches `junit:junit:4.13.2` from a local nginx fixture in the compose stack (avoids Maven Central rate limits in CI), adds the cached content to a repository, then asserts tangy can read it from the database. Test data is left in the database after a run; use `make compose-clean` to wipe volumes and start fresh. ### Mocking Tangy also exports a mock interface you can regenerate using the [mockery](https://github.com/vektra/mockery) tool. diff --git a/compose_files/pulp/assets/maven-fixture/junit/junit/4.13.2/junit-4.13.2.pom b/compose_files/pulp/assets/maven-fixture/junit/junit/4.13.2/junit-4.13.2.pom new file mode 100644 index 0000000..38be960 --- /dev/null +++ b/compose_files/pulp/assets/maven-fixture/junit/junit/4.13.2/junit-4.13.2.pom @@ -0,0 +1,633 @@ + + + 4.0.0 + + junit + junit + 4.13.2 + + JUnit + JUnit is a unit testing framework for Java, created by Erich Gamma and Kent Beck. + http://junit.org + 2002 + + JUnit + http://www.junit.org + + + + Eclipse Public License 1.0 + http://www.eclipse.org/legal/epl-v10.html + repo + + + + + + dsaff + David Saff + david@saff.net + + + kcooney + Kevin Cooney + kcooney@google.com + + + stefanbirkner + Stefan Birkner + mail@stefan-birkner.de + + + marcphilipp + Marc Philipp + mail@marcphilipp.de + + + + + JUnit contributors + JUnit + team@junit.org + https://github.com/junit-team/junit4/graphs/contributors + + developers + + + + + + 3.0.4 + + + + scm:git:git://github.com/junit-team/junit4.git + scm:git:git@github.com:junit-team/junit4.git + https://github.com/junit-team/junit4 + r4.13.2 + + + github + https://github.com/junit-team/junit4/issues + + + github + https://github.com/junit-team/junit4/actions + + + https://github.com/junit-team/junit4/wiki/Download-and-Install + + junit-snapshot-repo + Nexus Snapshot Repository + https://oss.sonatype.org/content/repositories/snapshots/ + + + junit-releases-repo + Nexus Release Repository + https://oss.sonatype.org/service/local/staging/deploy/maven2/ + + + junit.github.io + gitsite:git@github.com/junit-team/junit4.git + + + + + 1.5 + 2.19.1 + 1.3 + 1.4 + 2.6 + 2.10.3 + ISO-8859-1 + + 67893CC4 + + + + + org.hamcrest + hamcrest-core + ${hamcrestVersion} + + + + org.hamcrest + hamcrest-library + ${hamcrestVersion} + test + + + + + + + ${project.basedir}/src/main/resources + + + ${project.basedir} + + LICENSE-junit.txt + + + + + + + + maven-enforcer-plugin + ${enforcerPluginVersion} + + + enforce-versions + initialize + + enforce + + + true + + + + Current version of Maven ${maven.version} required to build the project + should be ${project.prerequisites.maven}, or higher! + + [${project.prerequisites.maven},) + + + Current JDK version ${java.version} should be ${jdkVersion}, or higher! + + ${jdkVersion} + + + Best Practice is to never define repositories in pom.xml (use a repository + manager instead). + + + + No Snapshots Dependencies Allowed! + + + + + + + + + com.google.code.maven-replacer-plugin + replacer + 1.5.3 + + + process-sources + + replace + + + + + false + ${project.build.sourceDirectory}/junit/runner/Version.java.template + ${project.build.sourceDirectory}/junit/runner/Version.java + false + @version@ + ${project.version} + + + + + maven-compiler-plugin + 3.3 + + ${project.build.sourceEncoding} + ${jdkVersion} + ${jdkVersion} + ${jdkVersion} + ${jdkVersion} + 1.5 + true + true + true + true + + -Xlint:unchecked + + 128m + + + + org.codehaus.mojo + animal-sniffer-maven-plugin + 1.14 + + + signature-check + test + + check + + + + org.codehaus.mojo.signature + java15 + 1.0 + + + + + + + + maven-surefire-plugin + ${surefireVersion} + + org/junit/tests/AllTests.java + true + false + + + + org.apache.maven.surefire + surefire-junit47 + ${surefireVersion} + + + + + + maven-source-plugin + 2.4 + + + + maven-javadoc-plugin + ${javadocPluginVersion} + + ${basedir}/src/main/javadoc/stylesheet.css + protected + false + false + false + true + true + true + JUnit API + UTF-8 + en + ${jdkVersion} + + + api_${jdkVersion} + http://docs.oracle.com/javase/${jdkVersion}.0/docs/api/ + + + *.internal.* + true + 32m + 128m + true + true + + org.hamcrest:hamcrest-core:* + + + + + maven-release-plugin + 2.5.2 + + forked-path + false + -Pgenerate-docs,junit-release ${arguments} + r@{project.version} + + + + maven-site-plugin + 3.4 + + + com.github.stephenc.wagon + wagon-gitsite + 0.4.1 + + + org.apache.maven.doxia + doxia-module-markdown + 1.5 + + + + + maven-jar-plugin + ${jarPluginVersion} + + + false + + true + + + junit + + + + + + maven-clean-plugin + 2.6.1 + + + maven-deploy-plugin + 2.8.2 + + + maven-install-plugin + 2.5.2 + + + maven-resources-plugin + 2.7 + + + + + + + + maven-project-info-reports-plugin + 2.8 + + false + + + + + + index + dependency-info + modules + license + project-team + scm + issue-tracking + mailing-list + dependency-management + dependencies + dependency-convergence + cim + distribution-management + + + + + + maven-javadoc-plugin + ${javadocPluginVersion} + + javadoc/latest + ${basedir}/src/main/javadoc/stylesheet.css + protected + false + false + false + true + true + true + JUnit API + UTF-8 + en + ${jdkVersion} + + + api_${jdkVersion} + http://docs.oracle.com/javase/${jdkVersion}.0/docs/api/ + + + junit.*,*.internal.* + true + 32m + 128m + true + true + + org.hamcrest:hamcrest-core:* + + + + + + javadoc + + + + + + + + + + junit-release + + + + + + maven-gpg-plugin + 1.6 + + + gpg-sign + verify + + sign + + + + + + + + + generate-docs + + + + + maven-source-plugin + + + attach-sources + prepare-package + + jar-no-fork + + + + + + maven-javadoc-plugin + + + attach-javadoc + package + + jar + + + + + + + + + restrict-doclint + + + [1.8,) + + + + + maven-compiler-plugin + + + -Xlint:unchecked + -Xdoclint:accessibility,reference,syntax + + + + + maven-javadoc-plugin + + -Xdoclint:accessibility -Xdoclint:reference + + + + + + + + maven-javadoc-plugin + + -Xdoclint:accessibility -Xdoclint:reference + + + + + + + java9 + + [1.9,12) + + + + 1.6 + + + + + maven-javadoc-plugin + + 1.6 + + + + + + + + maven-javadoc-plugin + + 1.6 + + + + + + + java12 + + [12,) + + + + 1.7 + 3.0.0-M3 + 3.2.0 + 3.2.0 + + + + + maven-javadoc-plugin + + 1.7 + false + + + + maven-compiler-plugin + + + -Xdoclint:none + + + + + + + + + maven-javadoc-plugin + + 1.7 + false + + + + + + + diff --git a/compose_files/pulp/docker-compose.yml b/compose_files/pulp/docker-compose.yml index 64d5536..d755f0c 100644 --- a/compose_files/pulp/docker-compose.yml +++ b/compose_files/pulp/docker-compose.yml @@ -54,6 +54,12 @@ services: healthcheck: test: ["CMD", "redis-cli", "--raw", "incr", "ping"] + maven_fixture: + image: "docker.io/library/nginx:alpine" + volumes: + - "./assets/maven-fixture:/usr/share/nginx/html:ro" + restart: always + pulp_api: image: "quay.io/redhat-services-prod/pulp-services-tenant/pulp:latest" platform: linux/amd64 diff --git a/go.mod b/go.mod index bef7193..c1ef2ec 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/content-services/tang go 1.24.0 require ( - github.com/content-services/zest/release/v2024 v2024.12.1734541842 + github.com/content-services/zest/release/v2026 v2026.7.1783082453 github.com/google/uuid v1.6.0 github.com/jackc/pgx-zerolog v0.0.0-20230315001418-f978528409eb github.com/jackc/pgx/v5 v5.7.6 diff --git a/go.sum b/go.sum index edc8579..0d3fd3e 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -github.com/content-services/zest/release/v2024 v2024.12.1734541842 h1:vIIWFZ5j76MVg2VuxkXAR56NAvkst7LtAeAugbfJmFU= -github.com/content-services/zest/release/v2024 v2024.12.1734541842/go.mod h1:fUWkjhvNTiS9UaaAWNLYkshl37V9IVmmRaprp9eDmrU= +github.com/content-services/zest/release/v2026 v2026.7.1783082453 h1:mZ8LExJlcMcoIUCv7eW+qQwZ9in18byKtuA99WWG0+M= +github.com/content-services/zest/release/v2026 v2026.7.1783082453/go.mod h1:pThh4a5Qm53BZ5V3WqRkWx7WBrIaeDdAnTmmqi6j8Vw= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= diff --git a/internal/test/integration/maven_test.go b/internal/test/integration/maven_test.go new file mode 100644 index 0000000..cd0830e --- /dev/null +++ b/internal/test/integration/maven_test.go @@ -0,0 +1,266 @@ +package integration + +import ( + "context" + "strings" + "testing" + + "github.com/content-services/tang/internal/config" + "github.com/content-services/tang/internal/zestwrapper" + "github.com/content-services/tang/pkg/tangy" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +const ( + testMavenRepoName = "junit-fixture" + testMavenDistributionBasePath = "junit-fixture" + testMavenRemoteURL = "http://maven_fixture/" + testMavenContentOrigin = "http://localhost:8088" + testMavenGroupID = "junit" + testMavenArtifactID = "junit" + testMavenVersion = "4.13.2" + testMavenPomPath = "junit/junit/4.13.2/junit-4.13.2.pom" +) + +type MavenSuite struct { + suite.Suite + client *zestwrapper.MavenZest + tangy tangy.Tangy + domainName string + repositoryHref string +} + +func (m *MavenSuite) createTestRepository(t *testing.T) { + _, err := m.client.LookupOrCreateDomain(m.domainName) + require.NoError(t, err) + + repoHref, remoteHref, err := m.client.CreateRepository( + m.domainName, + testMavenRepoName, + testMavenRemoteURL, + testMavenDistributionBasePath, + ) + require.NoError(t, err) + + err = m.client.FetchArtifact(testMavenContentOrigin, m.domainName, testMavenDistributionBasePath, testMavenPomPath) + require.NoError(t, err) + + addTask, err := m.client.AddCachedContent(repoHref, remoteHref) + require.NoError(t, err) + + _, err = m.client.PollTask(addTask) + require.NoError(t, err) + + m.repositoryHref = repoHref +} + +func TestMavenSuite(t *testing.T) { + s := config.Get().Server + mavenZest := zestwrapper.NewMavenZest(context.Background(), s) + + dbConfig := config.Get().Database + ta, err := tangy.New(tangy.Database{ + Name: dbConfig.Name, + Host: dbConfig.Host, + Port: dbConfig.Port, + User: dbConfig.User, + Password: dbConfig.Password, + }, tangy.Logger{Enabled: true, Logger: &log.Logger, LogLevel: zerolog.LevelDebugValue}) + require.NoError(t, err) + t.Cleanup(ta.Close) + + m := MavenSuite{} + m.client = &mavenZest + m.tangy = ta + m.domainName = RandStringBytes(10) + + m.createTestRepository(t) + + suite.Run(t, &m) +} + +func (m *MavenSuite) TestMavenPackageList() { + response, err := m.tangy.MavenPackageList(context.Background(), m.repositoryHref, tangy.MavenPackageListFilters{}, tangy.PageOptions{ + Offset: 0, + Limit: 10, + }) + require.NoError(m.T(), err) + require.NotEmpty(m.T(), response.Results) + assert.Equal(m.T(), 1, response.Total) + + pkg := response.Results[0] + assert.Equal(m.T(), testMavenGroupID, pkg.GroupID) + assert.Equal(m.T(), testMavenArtifactID, pkg.ArtifactID) + assert.Contains(m.T(), pkg.Versions, testMavenVersion) + require.NotEmpty(m.T(), pkg.LatestReleases) + + foundVersion := false + for _, latest := range pkg.LatestReleases { + if latest.Version == testMavenVersion { + foundVersion = true + assert.NotEmpty(m.T(), latest.CreatedAt) + } + } + assert.True(m.T(), foundVersion) +} + +func (m *MavenSuite) TestMavenPackageListSearchFilter() { + response, err := m.tangy.MavenPackageList(context.Background(), m.repositoryHref, tangy.MavenPackageListFilters{Search: "junit"}, tangy.PageOptions{ + Offset: 0, + Limit: 10, + }) + require.NoError(m.T(), err) + require.NotEmpty(m.T(), response.Results) + assert.Equal(m.T(), 1, response.Total) + assert.Equal(m.T(), testMavenArtifactID, response.Results[0].ArtifactID) + + response, err = m.tangy.MavenPackageList(context.Background(), m.repositoryHref, tangy.MavenPackageListFilters{Search: "nonexistent-artifact"}, tangy.PageOptions{ + Offset: 0, + Limit: 10, + }) + require.NoError(m.T(), err) + assert.Empty(m.T(), response.Results) + assert.Zero(m.T(), response.Total) +} + +func (m *MavenSuite) TestMavenPackageListPagination() { + response, err := m.tangy.MavenPackageList(context.Background(), m.repositoryHref, tangy.MavenPackageListFilters{}, tangy.PageOptions{ + Offset: 0, + Limit: 1, + }) + require.NoError(m.T(), err) + assert.Len(m.T(), response.Results, 1) + assert.Equal(m.T(), 1, response.Total) + assert.Equal(m.T(), 1, response.Limit) + + response, err = m.tangy.MavenPackageList(context.Background(), m.repositoryHref, tangy.MavenPackageListFilters{}, tangy.PageOptions{ + Offset: 1, + Limit: 1, + }) + require.NoError(m.T(), err) + assert.Empty(m.T(), response.Results) + assert.Equal(m.T(), 1, response.Total) +} + +func (m *MavenSuite) TestMavenPackageListEmptyHref() { + response, err := m.tangy.MavenPackageList(context.Background(), "", tangy.MavenPackageListFilters{}, tangy.PageOptions{Limit: 10}) + require.NoError(m.T(), err) + assert.Empty(m.T(), response.Results) + assert.Zero(m.T(), response.Total) +} + +func (m *MavenSuite) TestMavenBuildList() { + response, err := m.tangy.MavenBuildList( + context.Background(), + m.repositoryHref, + testMavenGroupID, + testMavenArtifactID, + testMavenVersion, + tangy.PageOptions{Offset: 0, Limit: 10}, + ) + require.NoError(m.T(), err) + require.NotEmpty(m.T(), response.Results) + assert.Equal(m.T(), 1, response.Total) + assert.Equal(m.T(), 10, response.Limit) + + build := response.Results[0] + assert.Equal(m.T(), testMavenGroupID, build.GroupID) + assert.Equal(m.T(), testMavenArtifactID, build.ArtifactID) + assert.Equal(m.T(), testMavenVersion, build.Version) + assert.True(m.T(), strings.HasSuffix(build.Filename, ".pom"), build.Filename) + assert.NotEmpty(m.T(), build.CreatedAt) +} + +func (m *MavenSuite) TestMavenBuildListPagination() { + response, err := m.tangy.MavenBuildList( + context.Background(), + m.repositoryHref, + testMavenGroupID, + testMavenArtifactID, + testMavenVersion, + tangy.PageOptions{Offset: 0, Limit: 1}, + ) + require.NoError(m.T(), err) + assert.Len(m.T(), response.Results, 1) + assert.Equal(m.T(), 1, response.Total) + + response, err = m.tangy.MavenBuildList( + context.Background(), + m.repositoryHref, + testMavenGroupID, + testMavenArtifactID, + testMavenVersion, + tangy.PageOptions{Offset: 1, Limit: 10}, + ) + require.NoError(m.T(), err) + assert.Empty(m.T(), response.Results) + assert.Equal(m.T(), 1, response.Total) +} + +func (m *MavenSuite) TestMavenBuildListEmptyHref() { + response, err := m.tangy.MavenBuildList(context.Background(), "", testMavenGroupID, testMavenArtifactID, testMavenVersion, tangy.PageOptions{Limit: 10}) + require.NoError(m.T(), err) + assert.Empty(m.T(), response.Results) + assert.Zero(m.T(), response.Total) +} + +func (m *MavenSuite) TestMavenBuildListOptionalFilters() { + response, err := m.tangy.MavenBuildList( + context.Background(), + m.repositoryHref, + "", + "", + "", + tangy.PageOptions{Offset: 0, Limit: 10}, + ) + require.NoError(m.T(), err) + require.NotEmpty(m.T(), response.Results) + assert.Equal(m.T(), 1, response.Total) + + build := response.Results[0] + assert.Equal(m.T(), testMavenGroupID, build.GroupID) + assert.Equal(m.T(), testMavenArtifactID, build.ArtifactID) + assert.Equal(m.T(), testMavenVersion, build.Version) + + filtered, err := m.tangy.MavenBuildList( + context.Background(), + m.repositoryHref, + testMavenGroupID, + "", + "", + tangy.PageOptions{Offset: 0, Limit: 10}, + ) + require.NoError(m.T(), err) + require.Len(m.T(), filtered.Results, 1) + assert.Equal(m.T(), 1, filtered.Total) + + filtered, err = m.tangy.MavenBuildList( + context.Background(), + m.repositoryHref, + "", + testMavenArtifactID, + "", + tangy.PageOptions{Offset: 0, Limit: 10}, + ) + require.NoError(m.T(), err) + require.Len(m.T(), filtered.Results, 1) + assert.Equal(m.T(), 1, filtered.Total) +} + +func (m *MavenSuite) TestMavenRepositoryMetrics() { + metrics, err := m.tangy.MavenRepositoryMetrics(context.Background(), m.repositoryHref) + require.NoError(m.T(), err) + assert.Equal(m.T(), 1, metrics.PackageCount) + assert.Equal(m.T(), 1, metrics.BuildCount) +} + +func (m *MavenSuite) TestMavenRepositoryMetricsEmptyHref() { + metrics, err := m.tangy.MavenRepositoryMetrics(context.Background(), "") + require.NoError(m.T(), err) + assert.Zero(m.T(), metrics.PackageCount) + assert.Zero(m.T(), metrics.BuildCount) +} diff --git a/internal/zestwrapper/href.go b/internal/zestwrapper/href.go new file mode 100644 index 0000000..1f7f35f --- /dev/null +++ b/internal/zestwrapper/href.go @@ -0,0 +1,11 @@ +package zestwrapper + +import "strings" + +// normalizePulpHref strips a leading slash from Pulp resource hrefs before they are +// embedded in zest URL paths. Zest v2026 builds paths as baseURL + "/" + href, so hrefs +// returned by Pulp (e.g. "/api/pulp/default/api/v3/...") would otherwise produce a +// double slash and 404 responses. +func normalizePulpHref(href string) string { + return strings.TrimPrefix(href, "/") +} diff --git a/internal/zestwrapper/href_test.go b/internal/zestwrapper/href_test.go new file mode 100644 index 0000000..3afbef5 --- /dev/null +++ b/internal/zestwrapper/href_test.go @@ -0,0 +1,33 @@ +package zestwrapper + +import "testing" + +func TestNormalizePulpHref(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + href string + want string + }{ + { + name: "leading slash", + href: "/api/pulp/default/api/v3/tasks/abc/", + want: "api/pulp/default/api/v3/tasks/abc/", + }, + { + name: "no leading slash", + href: "api/pulp/default/api/v3/tasks/abc/", + want: "api/pulp/default/api/v3/tasks/abc/", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + if got := normalizePulpHref(tt.href); got != tt.want { + t.Fatalf("normalizePulpHref() = %q, want %q", got, tt.want) + } + }) + } +} diff --git a/internal/zestwrapper/maven.go b/internal/zestwrapper/maven.go new file mode 100644 index 0000000..6742981 --- /dev/null +++ b/internal/zestwrapper/maven.go @@ -0,0 +1,175 @@ +package zestwrapper + +import ( + "context" + "fmt" + "net/http" + "strings" + "time" + + "github.com/content-services/tang/internal/config" + zest "github.com/content-services/zest/release/v2026" +) + +func NewMavenZest(ctx context.Context, server config.Server) MavenZest { + ctx2 := context.WithValue(ctx, zest.ContextServerIndex, 0) + timeout := 120 * time.Second + transport := &http.Transport{ResponseHeaderTimeout: timeout} + httpClient := http.Client{Transport: transport, Timeout: timeout} + + pulpConfig := zest.NewConfiguration() + pulpConfig.HTTPClient = &httpClient + pulpConfig.Servers = zest.ServerConfigurations{zest.ServerConfiguration{ + URL: server.Url, + }} + ctx2 = context.WithValue(ctx2, zest.ContextBasicAuth, zest.BasicAuth{ + UserName: server.Username, + Password: server.Password, + }) + + return MavenZest{ + client: zest.NewAPIClient(pulpConfig), + ctx: ctx2, + httpClient: &httpClient, + } +} + +type MavenZest struct { + client *zest.APIClient + ctx context.Context + httpClient *http.Client +} + +func (m *MavenZest) LookupOrCreateDomain(name string) (string, error) { + d, err := m.LookupDomain(name) + if err != nil { + return "", err + } + if d != "" { + return d, nil + } + + localStorage := zest.STORAGECLASSENUM_PULPCORE_APP_MODELS_STORAGE_FILE_SYSTEM + domain := *zest.NewDomain(name, localStorage, map[string]interface{}{ + "location": fmt.Sprintf("/var/lib/pulp/%v/", name), + }) + domainResp, resp, err := m.client.DomainsAPI.DomainsCreate(m.ctx, DefaultDomain).Domain(domain).Execute() + if resp != nil && resp.Body != nil { + defer resp.Body.Close() + } + if err != nil { + return "", err + } + return *domainResp.PulpHref, nil +} + +func (m *MavenZest) LookupDomain(name string) (string, error) { + list, resp, err := m.client.DomainsAPI.DomainsList(m.ctx, DefaultDomain).Name(name).Execute() + if err != nil { + return "", err + } + defer resp.Body.Close() + + if len(list.Results) == 0 { + return "", nil + } + if list.Results[0].PulpHref == nil { + return "", fmt.Errorf("unexpectedly got a nil href for domain %v", name) + } + return *list.Results[0].PulpHref, nil +} + +// CreateRepository creates a Maven remote, repository, and distribution for pull-through caching. +func (m *MavenZest) CreateRepository(domain, name, remoteURL, distributionBasePath string) (repoHref string, remoteHref string, err error) { + mavenRemote := zest.NewMavenMavenRemote(name, remoteURL) + + remoteResponse, httpResp, err := m.client.RemotesMavenAPI.RemotesMavenMavenCreate(m.ctx, domain). + MavenMavenRemote(*mavenRemote).Execute() + if err != nil { + return "", "", err + } + defer httpResp.Body.Close() + + mavenRepository := zest.NewMavenMavenRepository(name) + if remoteResponse.PulpHref != nil { + mavenRepository.SetRemote(*remoteResponse.PulpHref) + } + + repoResponse, httpResp, err := m.client.RepositoriesMavenAPI.RepositoriesMavenMavenCreate(m.ctx, domain). + MavenMavenRepository(*mavenRepository).Execute() + if err != nil { + return "", "", err + } + defer httpResp.Body.Close() + + distribution := zest.NewMavenMavenDistribution(distributionBasePath, name) + distribution.SetRemote(*remoteResponse.PulpHref) + distribution.SetRepository(*repoResponse.PulpHref) + + distResp, httpResp, err := m.client.DistributionsMavenAPI.DistributionsMavenMavenCreate(m.ctx, domain). + MavenMavenDistribution(*distribution).Execute() + if httpResp != nil { + defer httpResp.Body.Close() + } + if err != nil { + return "", "", err + } + + if _, err := m.PollTask(distResp.Task); err != nil { + return "", "", err + } + + return *repoResponse.PulpHref, *remoteResponse.PulpHref, nil +} + +// FetchArtifact triggers pull-through caching for an artifact path via the content app. +func (m *MavenZest) FetchArtifact(contentOrigin, domain, distributionBasePath, artifactPath string) error { + artifactPath = strings.TrimPrefix(artifactPath, "/") + url := fmt.Sprintf("%s/pulp/content/%s/%s/%s", strings.TrimRight(contentOrigin, "/"), domain, distributionBasePath, artifactPath) + + resp, err := m.httpClient.Get(url) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return fmt.Errorf("fetch artifact %s: %s", url, resp.Status) + } + return nil +} + +// AddCachedContent adds pull-through cached Maven content into the repository. +func (m *MavenZest) AddCachedContent(repoHref, remoteHref string) (string, error) { + addCached := zest.NewRepositoryAddCachedContent() + addCached.SetRemote(remoteHref) + + resp, httpResp, err := m.client.RepositoriesMavenAPI.RepositoriesMavenMavenAddCachedContent(m.ctx, normalizePulpHref(repoHref)). + RepositoryAddCachedContent(*addCached).Execute() + if httpResp != nil { + defer httpResp.Body.Close() + } + if err != nil { + return "", err + } + return resp.Task, nil +} + +func (m *MavenZest) PollTask(taskHref string) (*zest.TaskResponse, error) { + rpmZest := RpmZest{client: m.client, ctx: m.ctx} + return rpmZest.PollTask(taskHref) +} + +func (m *MavenZest) GetMavenRepositoryByName(domain, name string) (*zest.MavenMavenRepositoryResponse, error) { + resp, httpResp, err := m.client.RepositoriesMavenAPI.RepositoriesMavenMavenList(m.ctx, domain).Name(name).Execute() + if err != nil { + return nil, err + } + defer httpResp.Body.Close() + + results := resp.GetResults() + if len(results) > 0 { + return &results[0], nil + } + return nil, nil +} diff --git a/internal/zestwrapper/python.go b/internal/zestwrapper/python.go index 40b6c36..4f3587a 100644 --- a/internal/zestwrapper/python.go +++ b/internal/zestwrapper/python.go @@ -7,7 +7,7 @@ import ( "time" "github.com/content-services/tang/internal/config" - zest "github.com/content-services/zest/release/v2024" + zest "github.com/content-services/zest/release/v2026" ) func NewPythonZest(ctx context.Context, server config.Server) PythonZest { @@ -113,7 +113,7 @@ func (p *PythonZest) SyncPythonRepository(repoHref, remoteHref string) (string, mirror := true syncURL.SetMirror(mirror) - resp, httpResp, err := p.client.RepositoriesPythonAPI.RepositoriesPythonPythonSync(p.ctx, repoHref). + resp, httpResp, err := p.client.RepositoriesPythonAPI.RepositoriesPythonPythonSync(p.ctx, normalizePulpHref(repoHref)). RepositorySyncURL(*syncURL).Execute() if httpResp != nil { defer httpResp.Body.Close() diff --git a/internal/zestwrapper/rpm.go b/internal/zestwrapper/rpm.go index 0673c9e..4ca7349 100644 --- a/internal/zestwrapper/rpm.go +++ b/internal/zestwrapper/rpm.go @@ -8,7 +8,7 @@ import ( "time" "github.com/content-services/tang/internal/config" - zest "github.com/content-services/zest/release/v2024" + zest "github.com/content-services/zest/release/v2026" "golang.org/x/exp/slices" ) @@ -102,7 +102,6 @@ func (r *RpmZest) CreateRepository(domain, name, url string) (repoHref string, r defer httpResp.Body.Close() rpmRpmRepository := *zest.NewRpmRpmRepository(name) - rpmRpmRepository.PackageSigningFingerprint = nil if remoteResponse.PulpHref != nil { rpmRpmRepository.SetRemote(*remoteResponse.PulpHref) } @@ -118,7 +117,7 @@ func (r *RpmZest) CreateRepository(domain, name, url string) (repoHref string, r } func (r *RpmZest) UpdateRemote(remoteHref string, url string) error { - _, httpResp, err := r.client.RemotesRpmAPI.RemotesRpmRpmPartialUpdate(r.ctx, remoteHref).PatchedrpmRpmRemote(zest.PatchedrpmRpmRemote{Url: &url}).Execute() + _, httpResp, err := r.client.RemotesRpmAPI.RemotesRpmRpmPartialUpdate(r.ctx, normalizePulpHref(remoteHref)).PatchedrpmRpmRemote(zest.PatchedrpmRpmRemote{Url: &url}).Execute() if httpResp != nil { defer httpResp.Body.Close() } @@ -133,7 +132,7 @@ func (r *RpmZest) SyncRpmRepository(rpmRpmRepositoryHref string, remoteHref stri rpmRepositoryHref.SetRemote(remoteHref) rpmRepositoryHref.SetSyncPolicy(zest.SYNCPOLICYENUM_MIRROR_CONTENT_ONLY) - resp, httpResp, err := r.client.RepositoriesRpmAPI.RepositoriesRpmRpmSync(r.ctx, rpmRpmRepositoryHref). + resp, httpResp, err := r.client.RepositoriesRpmAPI.RepositoriesRpmRpmSync(r.ctx, normalizePulpHref(rpmRpmRepositoryHref)). RpmRepositorySyncURL(rpmRepositoryHref).Execute() defer httpResp.Body.Close() if err != nil { @@ -144,7 +143,7 @@ func (r *RpmZest) SyncRpmRepository(rpmRpmRepositoryHref string, remoteHref stri // GetTask Fetch a pulp task func (r *RpmZest) GetTask(taskHref string) (zest.TaskResponse, error) { - task, httpResp, err := r.client.TasksAPI.TasksRead(r.ctx, taskHref).Execute() + task, httpResp, err := r.client.TasksAPI.TasksRead(r.ctx, normalizePulpHref(taskHref)).Execute() if err != nil { return zest.TaskResponse{}, err diff --git a/pkg/tangy/maven_test.go b/pkg/tangy/maven_test.go new file mode 100644 index 0000000..5b6c690 --- /dev/null +++ b/pkg/tangy/maven_test.go @@ -0,0 +1,166 @@ +package tangy + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseRepositoryHref(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + href string + want string + wantErr bool + }{ + { + name: "valid maven repository href", + href: "/api/pulp/default/api/v3/repositories/maven/maven/018c1c95-4281-76eb-b277-842cbad524f4/", + want: "018c1c95-4281-76eb-b277-842cbad524f4", + }, + { + name: "valid href without trailing slash", + href: "/api/pulp/default/api/v3/repositories/maven/maven/018c1c95-4281-76eb-b277-842cbad524f4", + want: "018c1c95-4281-76eb-b277-842cbad524f4", + }, + { + name: "invalid href", + href: "/api/pulp/default/api/v3/repositories/maven/", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got, err := parseRepositoryHref(tt.href) + if tt.wantErr { + require.Error(t, err) + return + } + + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestExtractRelease(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + filename string + want string + }{ + { + name: "release suffix in pom filename", + filename: "smallrye-mutiny-vertx-core-3.16.0.rhlw-3002.pom", + want: "rhlw-3002", + }, + { + name: "plain pom filename", + filename: "junit-4.13.2.pom", + want: "", + }, + { + name: "empty filename", + filename: "", + want: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + assert.Equal(t, tt.want, extractRelease(tt.filename)) + }) + } +} + +func TestMockTangyMavenPackageList(t *testing.T) { + t.Parallel() + + mockTangy := NewMockTangy(t) + ctx := context.Background() + repoHref := "/api/pulp/default/api/v3/repositories/maven/maven/018c1c95-4281-76eb-b277-842cbad524f4/" + pageOpts := PageOptions{Offset: 0, Limit: 10} + filterOpts := MavenPackageListFilters{Search: "junit"} + + expected := MavenPackageListResponse{ + Results: []MavenPackageListItem{ + { + GroupID: "junit", + ArtifactID: "junit", + Versions: []string{"4.13.2"}, + LatestReleases: []MavenReleaseInfo{ + {Version: "4.13.2", Release: "", CreatedAt: "2024-01-01T12:00:00Z"}, + }, + }, + }, + Total: 1, + Limit: 10, + Offset: 0, + } + + mockTangy.On("MavenPackageList", ctx, repoHref, filterOpts, pageOpts).Return(expected, nil) + + got, err := mockTangy.MavenPackageList(ctx, repoHref, filterOpts, pageOpts) + require.NoError(t, err) + assert.Equal(t, expected, got) +} + +func TestMockTangyMavenBuildList(t *testing.T) { + t.Parallel() + + mockTangy := NewMockTangy(t) + ctx := context.Background() + repoHref := "/api/pulp/default/api/v3/repositories/maven/maven/018c1c95-4281-76eb-b277-842cbad524f4/" + pageOpts := PageOptions{Offset: 0, Limit: 10} + + expected := MavenBuildListResponse{ + Results: []MavenBuildListItem{ + { + GroupID: "junit", + ArtifactID: "junit", + Version: "4.13.2", + Release: "", + Filename: "junit-4.13.2.pom", + CreatedAt: "2024-01-01T12:00:00Z", + }, + }, + Total: 1, + Limit: 10, + Offset: 0, + } + + mockTangy.On("MavenBuildList", ctx, repoHref, "junit", "junit", "4.13.2", pageOpts).Return(expected, nil) + + got, err := mockTangy.MavenBuildList(ctx, repoHref, "junit", "junit", "4.13.2", pageOpts) + require.NoError(t, err) + assert.Equal(t, expected, got) +} + +func TestMockTangyMavenRepositoryMetrics(t *testing.T) { + t.Parallel() + + mockTangy := NewMockTangy(t) + ctx := context.Background() + repoHref := "/api/pulp/default/api/v3/repositories/maven/maven/018c1c95-4281-76eb-b277-842cbad524f4/" + + expected := MavenRepositoryMetrics{ + PackageCount: 1, + BuildCount: 1, + } + + mockTangy.On("MavenRepositoryMetrics", ctx, repoHref).Return(expected, nil) + + got, err := mockTangy.MavenRepositoryMetrics(ctx, repoHref) + require.NoError(t, err) + assert.Equal(t, expected, got) +} diff --git a/pkg/tangy/queries_test.go b/pkg/tangy/queries_test.go new file mode 100644 index 0000000..da46820 --- /dev/null +++ b/pkg/tangy/queries_test.go @@ -0,0 +1,84 @@ +package tangy + +import ( + "testing" + + "github.com/jackc/pgx/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestContentIdsInVersionNew(t *testing.T) { + t.Parallel() + + args := pgx.NamedArgs{} + query := contentIdsInVersionNew(testRepoVersionUUID, 3, &args) + + assert.Contains(t, query, "crv.repository_id = @repoName") + assert.Contains(t, query, "crv.number = @versionNum") + assert.Contains(t, query, "crv.content_ids IS NOT NULL") + require.Len(t, args, 2) + assert.Contains(t, namedArgValues(args), testRepoVersionUUID) + assert.Contains(t, namedArgValues(args), 3) +} + +func TestContentIdsInVersionOld(t *testing.T) { + t.Parallel() + + args := pgx.NamedArgs{} + query := contentIdsInVersionOld(testRepoVersionUUID, 3, &args) + + assert.Contains(t, query, "crv.repository_id = @repoName") + assert.Contains(t, query, "crv.number <= @versionNum") + assert.Contains(t, query, "crv2.number IS NOT NULL") + require.Len(t, args, 2) + assert.Contains(t, namedArgValues(args), testRepoVersionUUID) + assert.Contains(t, namedArgValues(args), 3) +} + +func TestContentIdsInVersionsNew(t *testing.T) { + t.Parallel() + + args := pgx.NamedArgs{} + secondUUID := "019f3808-fcc2-716e-a7d3-e5a7ef1522a0" + repoVerMap := []ParsedRepoVersion{ + {RepositoryUUID: testRepoVersionUUID, Version: 1}, + {RepositoryUUID: secondUUID, Version: 2}, + } + + query := contentIdsInVersionsNew(repoVerMap, &args) + + assert.Contains(t, query, "INNER JOIN core_repositoryversion crv ON (rp.content_ptr_id = ANY(crv.content_ids))") + assert.Contains(t, query, " OR ") + require.Len(t, args, 4) + values := namedArgValues(args) + assert.Contains(t, values, testRepoVersionUUID) + assert.Contains(t, values, secondUUID) + assert.Contains(t, values, 1) + assert.Contains(t, values, 2) +} + +func TestContentIdsInVersionsOld(t *testing.T) { + t.Parallel() + + args := pgx.NamedArgs{} + repoVerMap := []ParsedRepoVersion{ + {RepositoryUUID: testRepoVersionUUID, Version: 1}, + } + + query := contentIdsInVersionsOld(repoVerMap, &args) + + assert.Contains(t, query, "INNER JOIN core_repositorycontent crc on rp.content_ptr_id = crc.content_id") + assert.Contains(t, query, "LEFT OUTER JOIN core_repositoryversion crv2") + require.Len(t, args, 2) + assert.Contains(t, namedArgValues(args), testRepoVersionUUID) + assert.Contains(t, namedArgValues(args), 1) +} + +func namedArgValues(args pgx.NamedArgs) []any { + values := make([]any, 0, len(args)) + for _, value := range args { + values = append(values, value) + } + return values +} diff --git a/pkg/tangy/rpm.go b/pkg/tangy/rpm.go index 054cb2b..3a82b59 100644 --- a/pkg/tangy/rpm.go +++ b/pkg/tangy/rpm.go @@ -505,7 +505,7 @@ func parseRepositoryVersionHrefsMap(hrefs []string) (mapping []ParsedRepoVersion // /api/pulp/e1c6bee3/api/v3/repositories/rpm/rpm/018c1c95-4281-76eb-b277-842cbad524f4/versions/1/ for _, href := range hrefs { splitHref := strings.Split(href, "/") - if len(splitHref) < 10 { + if len(splitHref) < 12 { return mapping, fmt.Errorf("%v is not a valid href", splitHref) } id := splitHref[9] diff --git a/pkg/tangy/rpm_test.go b/pkg/tangy/rpm_test.go new file mode 100644 index 0000000..cc1a6eb --- /dev/null +++ b/pkg/tangy/rpm_test.go @@ -0,0 +1,153 @@ +package tangy + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testRepoVersionUUID = "018c1c95-4281-76eb-b277-842cbad524f4" + +func TestParseRepositoryVersionHrefsMap(t *testing.T) { + t.Parallel() + + validHref := "/api/pulp/default/api/v3/repositories/rpm/rpm/" + testRepoVersionUUID + "/versions/1/" + + tests := []struct { + name string + hrefs []string + want []ParsedRepoVersion + wantErr bool + }{ + { + name: "single valid href", + hrefs: []string{validHref}, + want: []ParsedRepoVersion{{ + RepositoryUUID: testRepoVersionUUID, + Version: 1, + }}, + }, + { + name: "multiple valid hrefs", + hrefs: []string{ + validHref, + "/api/pulp/default/api/v3/repositories/rpm/rpm/" + testRepoVersionUUID + "/versions/2/", + }, + want: []ParsedRepoVersion{ + {RepositoryUUID: testRepoVersionUUID, Version: 1}, + {RepositoryUUID: testRepoVersionUUID, Version: 2}, + }, + }, + { + name: "href too short", + hrefs: []string{"/api/pulp/default/api/v3/repositories/rpm/rpm/"}, + wantErr: true, + }, + { + name: "invalid uuid", + hrefs: []string{"/api/pulp/default/api/v3/repositories/rpm/rpm/not-a-uuid/versions/1/"}, + wantErr: true, + }, + { + name: "invalid version number", + hrefs: []string{"/api/pulp/default/api/v3/repositories/rpm/rpm/" + testRepoVersionUUID + "/versions/x/"}, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got, err := parseRepositoryVersionHrefsMap(tt.hrefs) + if tt.wantErr { + require.Error(t, err) + return + } + + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestParsePackages(t *testing.T) { + t.Parallel() + + got, err := parsePackages([]map[string]any{ + {"name": "kernel"}, + {"name": "bash"}, + }) + require.NoError(t, err) + assert.Equal(t, []string{"kernel", "bash"}, got) + + _, err = parsePackages([]map[string]any{ + {"name": 123}, + }) + require.Error(t, err) +} + +func TestUnionSlices(t *testing.T) { + t.Parallel() + + assert.Equal(t, []string{"a", "b", "c"}, unionSlices([]string{"a", "b"}, []string{"b", "c"})) + assert.Equal(t, []int{1, 2}, unionSlices([]int{1}, []int{2})) + assert.Empty(t, unionSlices([]string{}, []string{})) +} + +func TestContainsString(t *testing.T) { + t.Parallel() + + assert.True(t, containsString([]string{"kernel", "bash"}, "kernel")) + assert.False(t, containsString([]string{"kernel", "bash"}, "vim")) + assert.False(t, containsString(nil, "kernel")) +} + +func TestMockTangyRpmRepositoryVersionPackageList(t *testing.T) { + t.Parallel() + + mockTangy := NewMockTangy(t) + ctx := context.Background() + href := "/api/pulp/default/api/v3/repositories/rpm/rpm/" + testRepoVersionUUID + "/versions/1/" + filterOpts := RpmListFilters{Name: "kernel"} + pageOpts := PageOptions{Offset: 0, Limit: 20} + + expected := []RpmListItem{ + { + Id: "pkg-1", + Name: "kernel", + Arch: "x86_64", + Version: "6.1.0", + Release: "1", + Epoch: "0", + Summary: "The Linux kernel", + }, + } + + mockTangy.On("RpmRepositoryVersionPackageList", ctx, []string{href}, filterOpts, pageOpts).Return(expected, 1, nil) + + got, total, err := mockTangy.RpmRepositoryVersionPackageList(ctx, []string{href}, filterOpts, pageOpts) + require.NoError(t, err) + assert.Equal(t, expected, got) + assert.Equal(t, 1, total) +} + +func TestMockTangyRpmRepositoryVersionPackageSearch(t *testing.T) { + t.Parallel() + + mockTangy := NewMockTangy(t) + ctx := context.Background() + href := "/api/pulp/default/api/v3/repositories/rpm/rpm/" + testRepoVersionUUID + "/versions/1/" + + expected := []RpmPackageSearch{ + {Name: "kernel", Summary: "The Linux kernel"}, + } + + mockTangy.On("RpmRepositoryVersionPackageSearch", ctx, []string{href}, "kern", 100).Return(expected, nil) + + got, err := mockTangy.RpmRepositoryVersionPackageSearch(ctx, []string{href}, "kern", 100) + require.NoError(t, err) + assert.Equal(t, expected, got) +}