From 3c60ec2a4df6823c39b032d72959a546c43ebc03 Mon Sep 17 00:00:00 2001 From: Jan Harasym Date: Tue, 30 Jun 2026 17:34:03 +0200 Subject: [PATCH 1/6] Sync internal perforce stream bidi sync module --- MODULE.bazel | 1 + java/com/google/copybara/BUILD | 1 + java/com/google/copybara/ModuleSupplier.java | 4 + java/com/google/copybara/perforce/BUILD | 55 +++ .../perforce/PerforceDestination.java | 225 +++++++++ .../copybara/perforce/PerforceModule.java | 98 ++++ .../copybara/perforce/PerforceOptions.java | 123 +++++ .../copybara/perforce/PerforceOrigin.java | 198 ++++++++ .../copybara/perforce/PerforceRevision.java | 84 ++++ .../copybara/perforce/PerforceServer.java | 457 ++++++++++++++++++ javatests/com/google/copybara/perforce/BUILD | 39 ++ .../perforce/PerforceDestinationTest.java | 142 ++++++ .../copybara/perforce/PerforceOriginTest.java | 208 ++++++++ third_party/BUILD | 7 + 14 files changed, 1642 insertions(+) create mode 100644 java/com/google/copybara/perforce/BUILD create mode 100644 java/com/google/copybara/perforce/PerforceDestination.java create mode 100644 java/com/google/copybara/perforce/PerforceModule.java create mode 100644 java/com/google/copybara/perforce/PerforceOptions.java create mode 100644 java/com/google/copybara/perforce/PerforceOrigin.java create mode 100644 java/com/google/copybara/perforce/PerforceRevision.java create mode 100644 java/com/google/copybara/perforce/PerforceServer.java create mode 100644 javatests/com/google/copybara/perforce/BUILD create mode 100644 javatests/com/google/copybara/perforce/PerforceDestinationTest.java create mode 100644 javatests/com/google/copybara/perforce/PerforceOriginTest.java diff --git a/MODULE.bazel b/MODULE.bazel index e87e5ab8e..7b908a5f1 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -54,6 +54,7 @@ maven.install( "com.google.truth.extensions:truth-java8-extension:1.3.0", "com.google.truth:truth:1.3.0", "com.googlecode.java-diff-utils:diffutils:1.3.0", + "com.perforce:p4java:2026.1.2989454", "org.jcommander:jcommander:2.0", "com.ryanharter.auto.value:auto-value-gson-extension:1.3.1", "com.ryanharter.auto.value:auto-value-gson-factory:1.3.1", diff --git a/java/com/google/copybara/BUILD b/java/com/google/copybara/BUILD index ef7f86f5e..86d487778 100644 --- a/java/com/google/copybara/BUILD +++ b/java/com/google/copybara/BUILD @@ -258,6 +258,7 @@ java_library( "//java/com/google/copybara/http", "//java/com/google/copybara/monitor", "//java/com/google/copybara/onboard:options", + "//java/com/google/copybara/perforce", "//java/com/google/copybara/profiler", "//java/com/google/copybara/python", "//java/com/google/copybara/re2", diff --git a/java/com/google/copybara/ModuleSupplier.java b/java/com/google/copybara/ModuleSupplier.java index 421c540b9..27f22ff96 100644 --- a/java/com/google/copybara/ModuleSupplier.java +++ b/java/com/google/copybara/ModuleSupplier.java @@ -46,6 +46,8 @@ import com.google.copybara.hg.HgModule; import com.google.copybara.hg.HgOptions; import com.google.copybara.hg.HgOriginOptions; +import com.google.copybara.perforce.PerforceModule; +import com.google.copybara.perforce.PerforceOptions; import com.google.copybara.html.HtmlModule; import com.google.copybara.http.HttpModule; import com.google.copybara.http.HttpOptions; @@ -113,6 +115,7 @@ public ImmutableSet getModules(Options options) { folderModule), new GitModule(options), new HgModule(options), + new PerforceModule(options), folderModule, new FormatModule( options.get(WorkflowOptions.class), options.get(BuildifierOptions.class), general), @@ -167,6 +170,7 @@ protected Options newOptions() { new GitLabOptions(), new HgOptions(generalOptions), new HgOriginOptions(), + new PerforceOptions(generalOptions), new PatchingOptions(generalOptions), workflowOptions, new RemoteFileOptions(), diff --git a/java/com/google/copybara/perforce/BUILD b/java/com/google/copybara/perforce/BUILD new file mode 100644 index 000000000..ccfecaa10 --- /dev/null +++ b/java/com/google/copybara/perforce/BUILD @@ -0,0 +1,55 @@ +# Copyright 2026 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@rules_java//java:defs.bzl", "java_library") + +licenses(["notice"]) + +package(default_visibility = ["//visibility:public"]) + +JAVACOPTS = [ + "-Xlint:unchecked", +] + +java_library( + name = "perforce", + srcs = [ + "PerforceDestination.java", + "PerforceModule.java", + "PerforceOptions.java", + "PerforceOrigin.java", + "PerforceRevision.java", + "PerforceServer.java", + ], + javacopts = JAVACOPTS, + deps = [ + "//java/com/google/copybara:base", + "//java/com/google/copybara:general_options", + "//java/com/google/copybara:labels", + "//java/com/google/copybara/authoring", + "//java/com/google/copybara/config:base", + "//java/com/google/copybara/doc:annotations", + "//java/com/google/copybara/effect", + "//java/com/google/copybara/exception", + "//java/com/google/copybara/revision", + "//java/com/google/copybara/util", + "//java/com/google/copybara/util/console", + "//third_party:flogger", + "//third_party:guava", + "//third_party:jcommander", + "//third_party:jsr305", + "//third_party:p4java", + "//third_party:starlark", + ], +) diff --git a/java/com/google/copybara/perforce/PerforceDestination.java b/java/com/google/copybara/perforce/PerforceDestination.java new file mode 100644 index 000000000..0b192a186 --- /dev/null +++ b/java/com/google/copybara/perforce/PerforceDestination.java @@ -0,0 +1,225 @@ +/* + * Copyright (C) 2026 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara.perforce; + +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.common.base.CharMatcher; +import com.google.common.collect.ImmutableList; +import com.google.copybara.ChangeMessage; +import com.google.copybara.Destination; +import com.google.copybara.GeneralOptions; +import com.google.copybara.Options; +import com.google.copybara.TransformResult; +import com.google.copybara.WriterContext; +import com.google.copybara.effect.DestinationEffect; +import com.google.copybara.exception.RepoException; +import com.google.copybara.exception.ValidationException; +import com.google.copybara.util.Glob; +import com.google.copybara.util.console.Console; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.PathMatcher; +import java.nio.file.StandardCopyOption; +import java.util.stream.Stream; + +/** A Perforce (Helix Core) destination that submits each migrated change as a changelist. */ +public class PerforceDestination implements Destination { + + private static final String ORIGIN_LABEL_SEPARATOR = ": "; + + private final GeneralOptions generalOptions; + private final PerforceOptions perforceOptions; + private final String stream; + private final boolean submitAsAuthor; + + private PerforceDestination( + GeneralOptions generalOptions, + PerforceOptions perforceOptions, + String stream, + boolean submitAsAuthor) { + this.generalOptions = checkNotNull(generalOptions); + this.perforceOptions = checkNotNull(perforceOptions); + this.stream = CharMatcher.is('/').trimTrailingFrom(checkNotNull(stream)); + this.submitAsAuthor = submitAsAuthor; + } + + @Override + public Writer newWriter(WriterContext writerContext) { + return new WriterImpl(writerContext.getWorkflowName()); + } + + @Override + public String getLabelNameWhenOrigin() { + return PerforceOrigin.PERFORCE_ORIGIN_REV_ID; + } + + @Override + public String getType() { + return "perforce.destination"; + } + + @Override + public com.google.common.collect.ImmutableSetMultimap describe(Glob originFiles) { + return new com.google.common.collect.ImmutableSetMultimap.Builder() + .put("type", getType()) + .put("stream", stream) + .build(); + } + + class WriterImpl implements Writer { + + private final String workflowName; + // Lazily created on first write: a stream client and its on-disk workspace under the cache. + private String clientName; + private Path workspaceRoot; + + WriterImpl(String workflowName) { + this.workflowName = checkNotNull(workflowName); + } + + @Override + public DestinationStatus getDestinationStatus(Glob destinationFiles, String labelName) + throws RepoException, ValidationException { + String baseline = perforceOptions.server().findOriginLabel(stream, labelName); + return baseline == null ? null : new DestinationStatus(baseline, ImmutableList.of()); + } + + @Override + public boolean supportsHistory() { + return true; + } + + @Override + public ImmutableList write( + TransformResult transformResult, Glob destinationFiles, Console console) + throws ValidationException, RepoException, IOException { + ensureWorkspace(); + PerforceServer server = perforceOptions.server(); + + // cleanWorkspace() reverts any files left open by a prior aborted run, so this attempt starts + // from a clean, head-aligned state. We do NOT retry a failed submit: reconcileAndSubmit() + // verifies the submit landed and otherwise throws with the real Perforce error, failing loud + // rather than silently drifting or masking the cause. A failed run is safe to re-run. + console.progress("Perforce Destination: preparing workspace"); + server.cleanWorkspace(clientName, stream); + + console.progress("Perforce Destination: staging transformed files"); + mirror(transformResult.getPath(), workspaceRoot, destinationFiles); + + console.progress("Perforce Destination: submitting changelist"); + int changelist = + server.reconcileAndSubmit( + clientName, + stream, + changeDescription(transformResult), + transformResult.getAuthor(), + submitAsAuthor); + console.info(String.format("Perforce Destination: submitted changelist %d", changelist)); + return ImmutableList.of( + new DestinationEffect( + DestinationEffect.Type.CREATED, + String.format("Submitted changelist %d", changelist), + transformResult.getChanges().getCurrent(), + new DestinationEffect.DestinationRef( + Integer.toString(changelist), "changelist", /* url= */ null))); + } + + @Override + public void visitChanges(PerforceRevision start, ChangesVisitor visitor) { + throw new UnsupportedOperationException("History traversal of a Perforce destination is not" + + " supported"); + } + + private void ensureWorkspace() throws RepoException, ValidationException { + if (clientName != null) { + return; + } + // Stable per (workflow, stream) so the workspace and its have-list survive across runs, + // avoiding a full re-sync each time. + String suffix = Integer.toHexString((stream + "\0" + workflowName).hashCode()); + clientName = "copybara_" + sanitize(workflowName) + "_" + suffix; + try { + workspaceRoot = generalOptions.getDirFactory().getCacheDir("perforce_ws").resolve(clientName); + Files.createDirectories(workspaceRoot); + } catch (IOException e) { + throw new RepoException("Could not create Perforce workspace directory", e); + } + perforceOptions.server().ensureClient(clientName, workspaceRoot, stream); + } + } + + /** + * Makes the managed (glob-matched) files in {@code to} exactly mirror those in {@code from}: + * existing managed files are removed first, then the desired set is copied in. Whatever the depot + * still has but {@code from} no longer contains is thereby left missing on disk, which `p4 + * reconcile -d` turns into a delete. Files outside the glob are never touched. + */ + private static void mirror(Path from, Path to, Glob destinationFiles) throws RepoException { + PathMatcher fromMatcher = destinationFiles.relativeTo(from); + PathMatcher toMatcher = destinationFiles.relativeTo(to); + try { + if (Files.exists(to)) { + try (Stream walk = Files.walk(to)) { + for (Path file : (Iterable) walk.filter(Files::isRegularFile) + .filter(toMatcher::matches)::iterator) { + Files.delete(file); + } + } + } + try (Stream walk = Files.walk(from)) { + for (Path file : (Iterable) walk.filter(Files::isRegularFile) + .filter(fromMatcher::matches)::iterator) { + Path dest = to.resolve(from.relativize(file)); + Files.createDirectories(dest.getParent()); + Files.copy( + file, dest, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES); + } + } + } catch (IOException e) { + throw new RepoException("Error staging files into the Perforce workspace", e); + } + } + + /** Builds the changelist description, stamping the origin revision label for incremental syncs. */ + private static String changeDescription(TransformResult transformResult) + throws ValidationException { + ChangeMessage message = ChangeMessage.parseMessage(transformResult.getSummary()); + if (transformResult.isSetRevId()) { + message = + message.withNewOrReplacedLabel( + transformResult.getRevIdLabel(), + ORIGIN_LABEL_SEPARATOR, + transformResult.getCurrentRevision().asString()); + } + return message.toString(); + } + + private static String sanitize(String value) { + return value.replaceAll("[^A-Za-z0-9_]", "_"); + } + + static PerforceDestination newPerforceDestination( + Options options, String stream, boolean submitAsAuthor) { + return new PerforceDestination( + options.get(GeneralOptions.class), + options.get(PerforceOptions.class), + stream, + submitAsAuthor); + } +} diff --git a/java/com/google/copybara/perforce/PerforceModule.java b/java/com/google/copybara/perforce/PerforceModule.java new file mode 100644 index 000000000..b7471d1c1 --- /dev/null +++ b/java/com/google/copybara/perforce/PerforceModule.java @@ -0,0 +1,98 @@ +/* + * Copyright (C) 2026 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara.perforce; + +import static com.google.copybara.config.SkylarkUtil.checkNotEmpty; + +import com.google.common.base.Preconditions; +import com.google.copybara.Options; +import com.google.copybara.config.LabelsAwareModule; +import com.google.copybara.doc.annotations.UsesFlags; +import net.starlark.java.annot.Param; +import net.starlark.java.annot.StarlarkBuiltin; +import net.starlark.java.annot.StarlarkMethod; +import net.starlark.java.eval.EvalException; +import net.starlark.java.eval.StarlarkValue; + +/** Main module for Perforce (Helix Core) origins and destinations. */ +@StarlarkBuiltin( + name = "perforce", + doc = "Set of functions to define Perforce (Helix Core) origins and destinations.") +@UsesFlags(PerforceOptions.class) +public class PerforceModule implements LabelsAwareModule, StarlarkValue { + + protected final Options options; + + public PerforceModule(Options options) { + this.options = Preconditions.checkNotNull(options); + } + + @StarlarkMethod( + name = "origin", + doc = + "EXPERIMENTAL: Defines a Perforce origin that reads a stream at a submitted" + + " changelist.", + parameters = { + @Param( + name = "stream", + named = true, + doc = + "The Perforce stream to read from, e.g. //stream/main. The connection" + + " details (server, user, ticket) come from the --perforce-*" + + " flags or the standard P4PORT/P4USER/P4PASSWD environment variables."), + @Param( + name = "ref", + named = true, + defaultValue = "\"head\"", + doc = + "The default reference used to read a revision. Either a submitted changelist" + + " number (e.g. \"12345\") or the literal \"head\"" + + " for the most recent submitted changelist on the stream."), + }) + public PerforceOrigin origin(String stream, String ref) throws EvalException { + return PerforceOrigin.newPerforceOrigin(options, checkNotEmpty(stream, "stream"), ref); + } + + @StarlarkMethod( + name = "destination", + doc = + "EXPERIMENTAL: Defines a Perforce destination that submits each migrated change as" + + " a changelist on a stream.", + parameters = { + @Param( + name = "stream", + named = true, + doc = + "The Perforce stream to submit to, e.g. //stream/main. Connection" + + " details come from the --perforce-* flags or the standard" + + " P4PORT/P4USER/P4PASSWD environment variables."), + @Param( + name = "submit_as_author", + named = true, + defaultValue = "False", + doc = + "If true, each changelist is attributed to the original change author (a Perforce" + + " user is created on demand). Requires the connecting user to have permission" + + " to create users and submit on their behalf. If false, changelists are" + + " submitted as the connecting user and the author is kept in the description."), + }) + public PerforceDestination destination(String stream, boolean submitAsAuthor) + throws EvalException { + return PerforceDestination.newPerforceDestination( + options, checkNotEmpty(stream, "stream"), submitAsAuthor); + } +} diff --git a/java/com/google/copybara/perforce/PerforceOptions.java b/java/com/google/copybara/perforce/PerforceOptions.java new file mode 100644 index 000000000..fbb5ceecb --- /dev/null +++ b/java/com/google/copybara/perforce/PerforceOptions.java @@ -0,0 +1,123 @@ +/* + * Copyright (C) 2026 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara.perforce; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; +import com.google.copybara.GeneralOptions; +import com.google.copybara.Option; +import com.google.copybara.exception.RepoException; +import com.google.copybara.exception.ValidationException; +import com.perforce.p4java.exception.P4JavaException; +import com.perforce.p4java.server.IOptionsServer; +import com.perforce.p4java.server.ServerFactory; +import java.net.URISyntaxException; +import java.util.Properties; +import javax.annotation.Nullable; + +/** Connection arguments for Perforce, resolving from flags and falling back to P4 env vars. */ +@Parameters(separators = "=") +public class PerforceOptions implements Option { + + private final GeneralOptions generalOptions; + + @Parameter( + names = "--perforce-port", + description = + "Perforce server address (P4PORT), e.g. 'ssl:helix.example.com:1666'. Defaults to the" + + " P4PORT environment variable.") + String port = null; + + @Parameter( + names = "--perforce-user", + description = "Perforce user (P4USER). Defaults to the P4USER environment variable.") + String user = null; + + @Parameter( + names = "--perforce-password", + description = + "Perforce password or login ticket. Defaults to the P4PASSWD environment variable. If" + + " empty, an existing ticket from 'p4 login' is used.") + String password = null; + + // Lazily created and cached: a migration only ever talks to one server. + @Nullable private PerforceServer cachedServer; + + public PerforceOptions(GeneralOptions generalOptions) { + this.generalOptions = generalOptions; + } + + /** Returns a connected {@link PerforceServer}, building and caching it on first use. */ + public PerforceServer server() throws RepoException, ValidationException { + if (cachedServer == null) { + cachedServer = new PerforceServer(connect()); + } + return cachedServer; + } + + private IOptionsServer connect() throws RepoException, ValidationException { + String resolvedPort = firstNonEmpty(port, env("P4PORT")); + String resolvedUser = firstNonEmpty(user, env("P4USER")); + String resolvedPassword = firstNonEmpty(password, env("P4PASSWD")); + + if (Strings.isNullOrEmpty(resolvedPort)) { + throw new ValidationException( + "No Perforce server address: set --perforce-port or the P4PORT environment variable"); + } + + try { + IOptionsServer server = ServerFactory.getOptionsServer(toUri(resolvedPort), new Properties()); + server.connect(); + if (!Strings.isNullOrEmpty(resolvedUser)) { + server.setUserName(resolvedUser); + } + if (!Strings.isNullOrEmpty(resolvedPassword)) { + server.login(resolvedPassword); + } + return server; + } catch (URISyntaxException e) { + throw new ValidationException("Invalid Perforce server address: " + resolvedPort, e); + } catch (P4JavaException e) { + throw new RepoException("Could not connect to Perforce server " + resolvedPort, e); + } + } + + /** Maps a P4PORT value onto a P4Java connection URI, honouring the 'ssl:' prefix. */ + private static String toUri(String p4port) { + if (p4port.startsWith("ssl:")) { + return "p4javassl://" + p4port.substring("ssl:".length()); + } + return "p4java://" + p4port; + } + + @Nullable + private String env(String name) { + return generalOptions.getEnvironment().get(name); + } + + @Nullable + private static String firstNonEmpty(@Nullable String a, @Nullable String b) { + return Strings.isNullOrEmpty(a) ? b : a; + } + + @VisibleForTesting + public void setServerForTest(PerforceServer server) { + this.cachedServer = server; + } +} diff --git a/java/com/google/copybara/perforce/PerforceOrigin.java b/java/com/google/copybara/perforce/PerforceOrigin.java new file mode 100644 index 000000000..08028db75 --- /dev/null +++ b/java/com/google/copybara/perforce/PerforceOrigin.java @@ -0,0 +1,198 @@ +/* + * Copyright (C) 2026 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara.perforce; + +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.copybara.util.Glob.affectsRoots; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.CharMatcher; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSetMultimap; +import com.google.copybara.GeneralOptions; +import com.google.copybara.Options; +import com.google.copybara.Origin; +import com.google.copybara.Origin.Reader.ChangesResponse.EmptyReason; +import com.google.copybara.authoring.Authoring; +import com.google.copybara.exception.CannotResolveRevisionException; +import com.google.copybara.exception.RepoException; +import com.google.copybara.exception.ValidationException; +import com.google.copybara.revision.Change; +import com.google.copybara.util.FileUtil; +import com.google.copybara.util.Glob; +import java.io.IOException; +import java.nio.file.Path; +import javax.annotation.Nullable; + +/** A Perforce (Helix Core) origin that reads a stream at a submitted changelist. */ +public class PerforceOrigin implements Origin { + + /** Label used to persist the migrated changelist into destination commit messages. */ + public static final String PERFORCE_ORIGIN_REV_ID = "PerforceOrigin-RevId"; + + private final GeneralOptions generalOptions; + private final PerforceOptions perforceOptions; + private final String stream; + @Nullable private final String configRef; + + private PerforceOrigin( + GeneralOptions generalOptions, + PerforceOptions perforceOptions, + String stream, + @Nullable String configRef) { + this.generalOptions = checkNotNull(generalOptions); + this.perforceOptions = checkNotNull(perforceOptions); + // Streams are referenced without a trailing slash, e.g. "//stream/main". + this.stream = CharMatcher.is('/').trimTrailingFrom(checkNotNull(stream)); + this.configRef = configRef; + } + + @VisibleForTesting + public PerforceServer getServer() throws RepoException, ValidationException { + return perforceOptions.server(); + } + + @Override + public PerforceRevision resolve(@Nullable String reference) + throws RepoException, ValidationException { + String ref = Strings.isNullOrEmpty(reference) ? configRef : reference; + PerforceServer server = getServer(); + + // No explicit reference and no configured default: take the tip of the stream. + if (Strings.isNullOrEmpty(ref) || ref.equals("head") || ref.equals("now")) { + return new PerforceRevision(server.latestChange(stream), stream, /* timestamp= */ null); + } + + int changelist = parseChangelist(ref); + if (!server.changeExists(stream, changelist)) { + throw new CannotResolveRevisionException( + String.format("Changelist %d does not affect stream '%s'", changelist, stream)); + } + return new PerforceRevision(changelist, stream, /* timestamp= */ null); + } + + private int parseChangelist(String ref) throws ValidationException { + // Accept both "12345" and the common "@12345" notation. + String trimmed = ref.startsWith("@") ? ref.substring(1) : ref; + try { + return Integer.parseInt(trimmed); + } catch (NumberFormatException e) { + throw new ValidationException( + String.format("Invalid Perforce reference '%s': expected a changelist number", ref)); + } + } + + @Override + public Reader newReader(Glob originFiles, Authoring authoring) { + return new ReaderImpl(originFiles); + } + + class ReaderImpl implements Reader { + + private final Glob originFiles; + + ReaderImpl(Glob originFiles) { + this.originFiles = checkNotNull(originFiles); + } + + @Override + public void checkout(PerforceRevision revision, Path checkoutDir) + throws RepoException, ValidationException { + try { + FileUtil.deleteRecursively(checkoutDir); + } catch (IOException e) { + throw new RepoException("Error preparing checkout directory " + checkoutDir, e); + } + getServer().syncStreamTo(stream, revision.getChangelist(), checkoutDir); + } + + @Override + public ChangesResponse changes( + @Nullable PerforceRevision fromRef, PerforceRevision toRef) + throws RepoException, ValidationException { + int from = fromRef == null ? 0 : fromRef.getChangelist(); + ImmutableList> changes = + getServer().changes(stream, from, toRef.getChangelist()); + + if (!changes.isEmpty()) { + return ChangesResponse.forChanges(changes); + } + if (fromRef != null && toRef.getChangelist() <= fromRef.getChangelist()) { + return ChangesResponse.noChanges(EmptyReason.TO_IS_ANCESTOR); + } + return ChangesResponse.noChanges(EmptyReason.NO_CHANGES); + } + + @Override + public Change change(PerforceRevision ref) + throws RepoException, ValidationException { + return getServer().describe(stream, ref.getChangelist()); + } + + @Override + public void visitChanges(PerforceRevision start, ChangesVisitor visitor) + throws RepoException, ValidationException { + ImmutableSet roots = originFiles.roots(); + // Perforce stream history is linear, so walk every change up to 'start' newest first and + // stop as soon as the visitor is satisfied. + ImmutableList> changes = + getServer().changes(stream, 0, start.getChangelist()); + for (Change change : changes.reverse()) { + if (!affectsRoots(roots, change.getChangeFiles())) { + continue; + } + if (visitor.visit(change) == VisitResult.TERMINATE) { + return; + } + } + } + } + + @Override + public String getLabelName() { + return PERFORCE_ORIGIN_REV_ID; + } + + @Override + public String getType() { + return "perforce.origin"; + } + + @Override + public ImmutableSetMultimap describe(Glob originFiles) { + ImmutableSetMultimap.Builder builder = + new ImmutableSetMultimap.Builder() + .put("type", getType()) + .put("stream", stream); + if (configRef != null) { + builder.put("ref", configRef); + } + return builder.build(); + } + + @Override + public String toString() { + return String.format("PerforceOrigin{stream = %s, ref = %s}", stream, configRef); + } + + static PerforceOrigin newPerforceOrigin(Options options, String stream, @Nullable String ref) { + return new PerforceOrigin( + options.get(GeneralOptions.class), options.get(PerforceOptions.class), stream, ref); + } +} diff --git a/java/com/google/copybara/perforce/PerforceRevision.java b/java/com/google/copybara/perforce/PerforceRevision.java new file mode 100644 index 000000000..41082a03f --- /dev/null +++ b/java/com/google/copybara/perforce/PerforceRevision.java @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2026 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara.perforce; + +import com.google.common.base.MoreObjects; +import com.google.copybara.revision.Revision; +import java.time.ZonedDateTime; +import javax.annotation.Nullable; + +/** + * A Perforce revision, identified by a submitted changelist number. + * + *

Changelist numbers are monotonically increasing across the whole depot, so a single integer + * fully pins the state of any stream — it is both the {@link #asString()} identifier and the + * {@link #fixedReference()}. + */ +public class PerforceRevision implements Revision { + + private final int changelist; + // The stream this revision was resolved against, e.g. "//stream/main". Acts as the context + // reference, the equivalent of a git branch name. + @Nullable private final String stream; + @Nullable private final ZonedDateTime timestamp; + + public PerforceRevision(int changelist) { + this(changelist, /* stream= */ null, /* timestamp= */ null); + } + + public PerforceRevision(int changelist, @Nullable String stream, + @Nullable ZonedDateTime timestamp) { + this.changelist = changelist; + this.stream = stream; + this.timestamp = timestamp; + } + + int getChangelist() { + return changelist; + } + + @Override + public String asString() { + return Integer.toString(changelist); + } + + @Nullable + @Override + public String contextReference() { + return stream; + } + + @Override + public String fixedReference() { + return asString(); + } + + @Nullable + @Override + public ZonedDateTime readTimestamp() { + return timestamp; + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .omitNullValues() + .add("changelist", changelist) + .add("stream", stream) + .toString(); + } +} diff --git a/java/com/google/copybara/perforce/PerforceServer.java b/java/com/google/copybara/perforce/PerforceServer.java new file mode 100644 index 000000000..8e5aaeba2 --- /dev/null +++ b/java/com/google/copybara/perforce/PerforceServer.java @@ -0,0 +1,457 @@ +/* + * Copyright (C) 2026 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara.perforce; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.flogger.FluentLogger; +import com.google.copybara.ChangeMessage; +import com.google.copybara.authoring.Author; +import com.google.copybara.exception.CannotResolveRevisionException; +import com.google.copybara.exception.EmptyChangeException; +import com.google.copybara.exception.RepoException; +import com.google.copybara.exception.ValidationException; +import com.google.copybara.revision.Change; +import com.perforce.p4java.client.IClient; +import com.perforce.p4java.core.ChangelistStatus; +import com.perforce.p4java.core.IChangelist; +import com.perforce.p4java.core.IChangelistSummary; +import com.perforce.p4java.core.IUser; +import com.perforce.p4java.core.file.FileSpecBuilder; +import com.perforce.p4java.core.file.FileSpecOpStatus; +import com.perforce.p4java.core.file.IFileSpec; +import com.perforce.p4java.exception.P4JavaException; +import com.perforce.p4java.impl.generic.core.Changelist; +import com.perforce.p4java.impl.generic.core.User; +import com.perforce.p4java.impl.mapbased.client.Client; +import com.perforce.p4java.option.client.ReconcileFilesOptions; +import com.perforce.p4java.option.client.RevertFilesOptions; +import com.perforce.p4java.option.client.SyncOptions; +import com.perforce.p4java.option.server.GetChangelistsOptions; +import com.perforce.p4java.server.IOptionsServer; +import java.nio.file.Path; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import javax.annotation.Nullable; + +/** + * The single boundary between Copybara and the P4Java SDK. + * + *

Every {@code com.perforce.*} call lives here. The rest of the Perforce module deals only in + * Copybara types ({@link Change}, {@link PerforceRevision}), which keeps the SDK contained and lets + * the origin be unit-tested by injecting a mocked {@link IOptionsServer}. + */ +public class PerforceServer { + + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + + // How far back to scan submitted changelists when looking for the origin baseline label. + private static final int LABEL_SCAN_LIMIT = 200; + + private final IOptionsServer server; + // Perforce changelists only carry a username; full name and email come from the user spec, which + // we look up at most once per author. + private final Map authorCache = new HashMap<>(); + // Authors we've already created/verified as Perforce users this session (write side). + private final Set ensuredUsers = new HashSet<>(); + + PerforceServer(IOptionsServer server) { + this.server = server; + } + + /** Returns the most recent submitted changelist affecting {@code stream}. */ + int latestChange(String stream) throws RepoException, ValidationException { + List ids = listChangelistIds(rangeSpec(stream, 1, -1), /* max= */ 1); + if (ids.isEmpty()) { + throw new CannotResolveRevisionException( + String.format("No submitted changelists found for stream '%s'", stream)); + } + return ids.get(0); + } + + /** Returns true if {@code changelist} is a submitted changelist on {@code stream}. */ + boolean changeExists(String stream, int changelist) throws RepoException { + return !listChangelistIds(rangeSpec(stream, changelist, changelist), /* max= */ 1).isEmpty(); + } + + /** + * Returns the changes in the half-open interval {@code (from, to]} affecting {@code stream}, + * ordered oldest first. + */ + ImmutableList> changes(String stream, int fromExclusive, int to) + throws RepoException, ValidationException { + int low = fromExclusive + 1; + if (low > to) { + return ImmutableList.of(); + } + // 'p4 changes' lists newest first; walk it backwards so callers get chronological order. The + // summary only carries the id, so each change is fully described (which also fetches its files). + List ids = listChangelistIds(rangeSpec(stream, low, to), /* max= */ -1); + ImmutableList.Builder> changes = ImmutableList.builder(); + for (int i = ids.size() - 1; i >= 0; i--) { + changes.add(describe(stream, ids.get(i))); + } + return changes.build(); + } + + /** Returns the single change identified by {@code changelist}, scoped to {@code stream}. */ + Change describe(String stream, int changelist) + throws RepoException, ValidationException { + try { + IChangelist cl = server.getChangelist(changelist); + if (cl == null) { + throw new CannotResolveRevisionException( + String.format("Changelist %d cannot be found", changelist)); + } + return toChange(stream, cl); + } catch (P4JavaException e) { + throw new RepoException(String.format("Error describing changelist %d", changelist), e); + } + } + + /** + * Syncs {@code stream} at {@code changelist} into {@code checkoutDir} via an ephemeral + * stream-bound client, which the server uses to materialise the stream's view. The client is + * removed afterwards so no workspace state leaks between runs. + */ + void syncStreamTo(String stream, int changelist, Path checkoutDir) throws RepoException { + String clientName = + String.format("copybara_%d_%d", ProcessHandle.current().pid(), System.nanoTime()); + boolean created = false; + try { + Client client = new Client(server); + client.setName(clientName); + client.setRoot(checkoutDir.toString()); + client.setStream(stream); + client.setOwnerName(server.getUserName()); + server.createClient(client); + created = true; + + IClient bound = server.getClient(clientName); + server.setCurrentClient(bound); + List synced = + bound.sync( + FileSpecBuilder.makeFileSpecList(stream + "/...@" + changelist), + new SyncOptions().setForceUpdate(true)); + logFileSpecErrors(synced); + } catch (P4JavaException e) { + throw new RepoException( + String.format("Error syncing stream '%s' at changelist %d", stream, changelist), e); + } finally { + if (created) { + try { + server.deleteClient(clientName, /* force= */ true); + } catch (P4JavaException e) { + logger.atWarning().withCause(e).log("Could not delete temporary client %s", clientName); + } + } + } + } + + // =========================================================================================== + // Write side (perforce.destination) + // =========================================================================================== + + /** + * Returns the most recent value of {@code labelName} stamped into a submitted changelist + * description on {@code stream}, or null if none of the recent changelists carry it. This is how + * the destination rediscovers the last migrated origin revision for incremental syncs. + */ + @Nullable + String findOriginLabel(String stream, String labelName) throws RepoException { + Pattern pattern = + Pattern.compile("^" + Pattern.quote(labelName) + ": *(.+)$", Pattern.MULTILINE); + GetChangelistsOptions options = + new GetChangelistsOptions() + .setType(IChangelist.Type.SUBMITTED) + .setLongDesc(true) + .setMaxMostRecent(LABEL_SCAN_LIMIT); + try { + for (IChangelistSummary summary : + server.getChangelists(FileSpecBuilder.makeFileSpecList(stream + "/..."), options)) { + String description = summary.getDescription(); + if (description == null) { + continue; + } + Matcher matcher = pattern.matcher(description); + if (matcher.find()) { + return matcher.group(1).trim(); + } + } + return null; + } catch (P4JavaException e) { + throw new RepoException("Error scanning destination changelists for " + stream, e); + } + } + + /** Creates a stream-bound client named {@code clientName} rooted at {@code root} if absent. */ + void ensureClient(String clientName, Path root, String stream) throws RepoException { + try { + IClient existing = server.getClient(clientName); + if (existing == null || existing.getRoot() == null) { + Client client = new Client(server); + client.setName(clientName); + client.setRoot(root.toString()); + client.setStream(stream); + client.setOwnerName(server.getUserName()); + server.createClient(client); + } + } catch (P4JavaException e) { + throw new RepoException("Error creating Perforce client " + clientName, e); + } + } + + /** + * Reverts any files left open in {@code clientName} (e.g. from a previously interrupted submit) + * and syncs it to the head of {@code stream}, giving each write a clean, head-aligned workspace. + */ + void cleanWorkspace(String clientName, String stream) throws RepoException { + try { + IClient client = server.getClient(clientName); + server.setCurrentClient(client); + client.revertFiles( + FileSpecBuilder.makeFileSpecList(stream + "/..."), new RevertFilesOptions()); + logFileSpecErrors( + client.sync(FileSpecBuilder.makeFileSpecList(stream + "/...#head"), new SyncOptions())); + } catch (P4JavaException e) { + throw new RepoException("Error preparing client " + clientName, e); + } + } + + /** + * Reconciles the workspace of {@code clientName} against the depot (open adds/edits/deletes) and + * submits them as a single changelist with {@code description}. When {@code author} is provided + * and {@code submitAsAuthor} is set, the changelist is attributed to that author (a Perforce user + * is created on demand) instead of the connected user. Returns the submitted changelist number. + * + * @throws EmptyChangeException if the workspace matches the depot, i.e. nothing to submit. + */ + int reconcileAndSubmit( + String clientName, + String stream, + String description, + @Nullable Author author, + boolean submitAsAuthor) + throws RepoException, ValidationException { + String previousUser = null; + try { + if (submitAsAuthor && author != null) { + previousUser = server.getUserName(); + server.setUserName(ensureUser(author)); + } + IClient client = server.getClient(clientName); + server.setCurrentClient(client); + + IChangelist pending = client.createChangelist(Changelist.newChangelist(client, description)); + client.reconcileFiles( + FileSpecBuilder.makeFileSpecList(stream + "/..."), + new ReconcileFilesOptions() + .setOutsideAdd(true) + .setOutsideEdit(true) + .setRemoved(true) + .setChangelistId(pending.getId())); + + pending.refresh(); + if (pending.getFiles(/* refresh= */ true).isEmpty()) { + tryDeletePending(pending.getId()); + throw new EmptyChangeException( + "No changes to submit: the destination already matches the transformed tree"); + } + + int changelist = pending.getId(); + String submitErrors = errorMessages(pending.submit(/* reOpen= */ false)); + + // A 'p4 submit' can return without throwing yet leave the changelist pending (e.g. the + // unlicensed-server file/user cap, or a server-side error). Verify it actually landed and + // surface the real Perforce message so the failure is loud rather than a silent drift. + IChangelist submitted = server.getChangelist(changelist); + if (submitted == null || submitted.getStatus() != ChangelistStatus.SUBMITTED) { + client.revertFiles( + FileSpecBuilder.makeFileSpecList(stream + "/..."), new RevertFilesOptions()); + tryDeletePending(changelist); + throw new RepoException( + String.format( + "Perforce submit of changelist %d did not complete%s", + changelist, submitErrors.isEmpty() ? "" : ": " + submitErrors)); + } + return changelist; + } catch (P4JavaException e) { + throw new RepoException("Error submitting changelist to " + stream, e); + } finally { + if (previousUser != null) { + server.setUserName(previousUser); + } + } + } + + /** Ensures a Perforce user exists for {@code author}, returning the (sanitised) login name. */ + private String ensureUser(Author author) throws RepoException { + String login = sanitizeUser(author.getEmail()); + if (ensuredUsers.add(login)) { + try { + server.createUser(User.newUser(login, author.getEmail(), author.getName(), ""), true); + } catch (P4JavaException e) { + // Most likely the user already exists from a previous run; tolerate and reuse it. + logger.atFine().withCause(e).log("createUser(%s) failed; assuming it exists", login); + } + } + return login; + } + + private void tryDeletePending(int changelist) { + try { + server.deletePendingChangelist(changelist); + } catch (P4JavaException e) { + logger.atWarning().withCause(e).log("Could not delete empty changelist %d", changelist); + } + } + + private static String sanitizeUser(String email) { + if (Strings.isNullOrEmpty(email)) { + return "unknown"; + } + String sanitized = email.trim().replaceAll("[^A-Za-z0-9._-]", "_"); + return sanitized.isEmpty() ? "unknown" : sanitized; + } + + private List listChangelistIds(String fileSpec, int max) throws RepoException { + GetChangelistsOptions options = + new GetChangelistsOptions().setType(IChangelist.Type.SUBMITTED).setLongDesc(true); + if (max > 0) { + options.setMaxMostRecent(max); + } + try { + List summaries = + server.getChangelists(FileSpecBuilder.makeFileSpecList(fileSpec), options); + List ids = new ArrayList<>(summaries.size()); + for (IChangelistSummary summary : summaries) { + ids.add(summary.getId()); + } + return ids; + } catch (P4JavaException e) { + throw new RepoException(String.format("Error querying changelists for '%s'", fileSpec), e); + } + } + + private Change toChange(String stream, IChangelist cl) { + ZonedDateTime date = toZonedDateTime(cl.getDate()); + PerforceRevision revision = new PerforceRevision(cl.getId(), stream, date); + String description = Strings.nullToEmpty(cl.getDescription()); + return new Change<>( + revision, + resolveAuthor(cl.getUsername()), + description, + date, + ChangeMessage.parseAllAsLabels(description).labelsAsMultimap(), + changeFiles(stream, cl)); + } + + private ImmutableSet changeFiles(String stream, IChangelist cl) { + String prefix = stream + "/"; + try { + ImmutableSet.Builder files = ImmutableSet.builder(); + for (IFileSpec spec : cl.getFiles(/* refresh= */ true)) { + String depotPath = spec.getDepotPathString(); + if (depotPath == null) { + continue; + } + files.add(depotPath.startsWith(prefix) ? depotPath.substring(prefix.length()) : depotPath); + } + return files.build(); + } catch (P4JavaException e) { + logger.atWarning().withCause(e).log("Could not list files for changelist %d", cl.getId()); + return ImmutableSet.of(); + } + } + + private Author resolveAuthor(String username) { + return authorCache.computeIfAbsent(username, this::lookupAuthor); + } + + private Author lookupAuthor(String username) { + try { + IUser user = server.getUser(username); + if (user != null && !Strings.isNullOrEmpty(user.getEmail())) { + String name = Strings.isNullOrEmpty(user.getFullName()) ? username : user.getFullName(); + return new Author(name, user.getEmail()); + } + } catch (P4JavaException e) { + logger.atWarning().withCause(e).log("Could not resolve Perforce user %s", username); + } + // No email on record: fall back to the username on both sides so authoring rules still have + // something stable to match against. + return new Author(username, username); + } + + private static String rangeSpec(String stream, int low, int high) { + // high < 0 means "no upper bound": rely on maxMostRecent to cap the result to the newest + // change(s). Mixing a changelist lower bound with a revision upper bound (e.g. @1,#head) is + // rejected by the server, so leave the path unqualified in that case. + if (high < 0) { + return stream + "/..."; + } + return stream + "/...@" + low + "," + high; + } + + private static ZonedDateTime toZonedDateTime(Date date) { + return date == null ? null : ZonedDateTime.ofInstant(date.toInstant(), ZoneOffset.UTC); + } + + private static void logFileSpecErrors(List specs) { + String errors = errorMessages(specs); + if (!errors.isEmpty()) { + logger.atWarning().log("Perforce reported: %s", errors); + } + } + + /** Joins the messages of any error/non-info file specs, for surfacing in exceptions and logs. */ + private static String errorMessages(List specs) { + StringBuilder result = new StringBuilder(); + for (IFileSpec spec : specs) { + if (spec == null || spec.getOpStatus() == null) { + continue; + } + FileSpecOpStatus status = spec.getOpStatus(); + if (status != FileSpecOpStatus.VALID && status != FileSpecOpStatus.INFO) { + String message = spec.getStatusMessage(); + if (!Strings.isNullOrEmpty(message)) { + if (result.length() > 0) { + result.append("; "); + } + result.append(message); + } + } + } + return result.toString(); + } + + @VisibleForTesting + IOptionsServer getUnderlyingServer() { + return server; + } +} diff --git a/javatests/com/google/copybara/perforce/BUILD b/javatests/com/google/copybara/perforce/BUILD new file mode 100644 index 000000000..737e02dbf --- /dev/null +++ b/javatests/com/google/copybara/perforce/BUILD @@ -0,0 +1,39 @@ +# Copyright 2026 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("//javatests/com/google/copybara:test.bzl", "all_tests") + +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +all_tests( + name = "all_tests", + tests = [ + "PerforceDestinationTest.java", + "PerforceOriginTest.java", + ], + deps = [ + "//java/com/google/copybara:base", + "//java/com/google/copybara/authoring", + "//java/com/google/copybara/exception", + "//java/com/google/copybara/perforce", + "//java/com/google/copybara/revision", + "//java/com/google/copybara/testing", + "//java/com/google/copybara/util", + "//third_party:mockito", + "//third_party:p4java", + "//third_party:truth", + ], +) diff --git a/javatests/com/google/copybara/perforce/PerforceDestinationTest.java b/javatests/com/google/copybara/perforce/PerforceDestinationTest.java new file mode 100644 index 000000000..c57a9a9bc --- /dev/null +++ b/javatests/com/google/copybara/perforce/PerforceDestinationTest.java @@ -0,0 +1,142 @@ +/* + * Copyright (C) 2026 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara.perforce; + +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.copybara.Destination.DestinationStatus; +import com.google.copybara.Destination.Writer; +import com.google.copybara.Options; +import com.google.copybara.WriterContext; +import com.google.copybara.testing.OptionsBuilder; +import com.google.copybara.util.Glob; +import com.perforce.p4java.core.IChangelistSummary; +import com.perforce.p4java.server.IOptionsServer; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class PerforceDestinationTest { + + private static final String STREAM = "//copybara/main"; + private static final String LABEL = "GitOrigin-RevId"; + + private IOptionsServer server; + private Options options; + + @Before + public void setUp() { + server = mock(IOptionsServer.class); + OptionsBuilder optionsBuilder = new OptionsBuilder(); + PerforceOptions perforceOptions = new PerforceOptions(optionsBuilder.general); + perforceOptions.setServerForTest(new PerforceServer(server)); + options = new Options(ImmutableList.of(optionsBuilder.general, perforceOptions)); + } + + private PerforceDestination destination(boolean submitAsAuthor) { + return PerforceDestination.newPerforceDestination(options, STREAM, submitAsAuthor); + } + + private Writer writer() { + WriterContext context = + new WriterContext( + "workflow", + /* workflowIdentityUser= */ null, + /* dryRun= */ false, + new PerforceRevision(1), + ImmutableSet.of()); + return destination(/* submitAsAuthor= */ false).newWriter(context); + } + + @Test + public void destinationMetadata() { + PerforceDestination destination = destination(/* submitAsAuthor= */ false); + assertThat(destination.getType()).isEqualTo("perforce.destination"); + assertThat(destination.getLabelNameWhenOrigin()) + .isEqualTo(PerforceOrigin.PERFORCE_ORIGIN_REV_ID); + assertThat(destination.describe(Glob.ALL_FILES)).containsEntry("stream", STREAM); + } + + @Test + public void writerSupportsHistory() { + assertThat(writer().supportsHistory()).isTrue(); + } + + @Test + public void destinationStatusReadsBaselineFromChangelistLabel() throws Exception { + stubChangelists(summary("Some migrated change\n\n" + LABEL + ": abcdef123")); + + DestinationStatus status = writer().getDestinationStatus(Glob.ALL_FILES, LABEL); + + assertThat(status).isNotNull(); + assertThat(status.getBaseline()).isEqualTo("abcdef123"); + } + + @Test + public void destinationStatusPicksLabelAmongOtherLabels() throws Exception { + stubChangelists( + summary("Title line\n\nChange-Id: I123\nNO_BUG: cleanup\n" + LABEL + ": deadbeef")); + + DestinationStatus status = writer().getDestinationStatus(Glob.ALL_FILES, LABEL); + + assertThat(status.getBaseline()).isEqualTo("deadbeef"); + } + + @Test + public void destinationStatusUsesMostRecentChangelistWithLabel() throws Exception { + // getChangelists returns newest first; the first match wins. + stubChangelists( + summary("Newest\n\n" + LABEL + ": newsha"), + summary("Older\n\n" + LABEL + ": oldsha")); + + DestinationStatus status = writer().getDestinationStatus(Glob.ALL_FILES, LABEL); + + assertThat(status.getBaseline()).isEqualTo("newsha"); + } + + @Test + public void destinationStatusNullWhenNoLabelPresent() throws Exception { + stubChangelists(summary("A change with no origin label")); + + assertThat(writer().getDestinationStatus(Glob.ALL_FILES, LABEL)).isNull(); + } + + @Test + public void destinationStatusNullWhenStreamEmpty() throws Exception { + stubChangelists(); // no submitted changelists at all + + assertThat(writer().getDestinationStatus(Glob.ALL_FILES, LABEL)).isNull(); + } + + private void stubChangelists(IChangelistSummary... summaries) throws Exception { + when(server.getChangelists(anyList(), any())).thenReturn(ImmutableList.copyOf(summaries)); + } + + private static IChangelistSummary summary(String description) { + IChangelistSummary summary = mock(IChangelistSummary.class); + when(summary.getDescription()).thenReturn(description); + return summary; + } +} diff --git a/javatests/com/google/copybara/perforce/PerforceOriginTest.java b/javatests/com/google/copybara/perforce/PerforceOriginTest.java new file mode 100644 index 000000000..eded4306c --- /dev/null +++ b/javatests/com/google/copybara/perforce/PerforceOriginTest.java @@ -0,0 +1,208 @@ +/* + * Copyright (C) 2026 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara.perforce; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.copybara.ChangeVisitable.VisitResult; +import com.google.copybara.Options; +import com.google.copybara.Origin.Reader; +import com.google.copybara.Origin.Reader.ChangesResponse; +import com.google.copybara.Origin.Reader.ChangesResponse.EmptyReason; +import com.google.copybara.authoring.Author; +import com.google.copybara.authoring.Authoring; +import com.google.copybara.authoring.Authoring.AuthoringMappingMode; +import com.google.copybara.exception.ValidationException; +import com.google.copybara.revision.Change; +import com.google.copybara.testing.OptionsBuilder; +import com.google.copybara.util.Glob; +import com.perforce.p4java.core.IChangelist; +import com.perforce.p4java.core.IChangelistSummary; +import com.perforce.p4java.core.file.IFileSpec; +import com.perforce.p4java.server.IOptionsServer; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class PerforceOriginTest { + + private static final String STREAM = "//stream/main"; + + private final Authoring authoring = + new Authoring( + new Author("Copy", "copy@bara.com"), AuthoringMappingMode.PASS_THRU, ImmutableSet.of()); + + private IOptionsServer server; + private Options options; + + @Before + public void setUp() { + server = mock(IOptionsServer.class); + OptionsBuilder optionsBuilder = new OptionsBuilder(); + PerforceOptions perforceOptions = new PerforceOptions(optionsBuilder.general); + perforceOptions.setServerForTest(new PerforceServer(server)); + options = new Options(ImmutableList.of(optionsBuilder.general, perforceOptions)); + } + + private PerforceOrigin origin() { + return PerforceOrigin.newPerforceOrigin(options, STREAM, "head"); + } + + private Reader reader() { + return origin().newReader(Glob.ALL_FILES, authoring); + } + + @Test + public void resolveHead() throws Exception { + stubChangelistsReturn(summary(100)); + + assertThat(origin().resolve("head").asString()).isEqualTo("100"); + } + + @Test + public void resolveExplicitChangelist() throws Exception { + stubChangelistsReturn(summary(42)); + + PerforceRevision rev = origin().resolve("42"); + + assertThat(rev.asString()).isEqualTo("42"); + assertThat(rev.contextReference()).isEqualTo(STREAM); + } + + @Test + public void resolveUnknownChangelistFails() throws Exception { + stubChangelistsReturn(); // empty: the changelist does not affect the stream + + assertThrows(ValidationException.class, () -> origin().resolve("42")); + } + + @Test + public void resolveNonNumericFails() { + assertThrows(ValidationException.class, () -> origin().resolve("not-a-number")); + } + + @Test + public void resolveEmptyStreamFails() throws Exception { + stubChangelistsReturn(); // no submitted changelists at all + + assertThrows(ValidationException.class, () -> origin().resolve("head")); + } + + @Test + public void changeMapsChangelistMetadata() throws Exception { + IChangelist cl = changelist(42, "alice", "Fix bug\n", STREAM + "/src/a.txt"); + when(server.getChangelist(42)).thenReturn(cl); + when(server.getUser("alice")).thenReturn(null); // no email on record -> username fallback + + Change change = reader().change(new PerforceRevision(42)); + + assertThat(change.getRevision().asString()).isEqualTo("42"); + assertThat(change.getAuthor().getName()).isEqualTo("alice"); + assertThat(change.getMessage()).isEqualTo("Fix bug\n"); + // The stream prefix is stripped so paths are relative to the checkout root. + assertThat(change.getChangeFiles()).containsExactly("src/a.txt"); + } + + @Test + public void changesReturnedOldestFirst() throws Exception { + // 'p4 changes' lists newest first; the origin must flip this to chronological order. + stubChangelistsReturn(summary(102), summary(101), summary(100)); + stubDescribe(100, 101, 102); + + ImmutableList> changes = + reader().changes(/* fromRef= */ null, new PerforceRevision(102)).getChanges(); + + assertThat(changes.stream().map(c -> c.getRevision().asString())) + .containsExactly("100", "101", "102") + .inOrder(); + } + + @Test + public void changesEmptyWhenToIsAncestorOfFrom() throws Exception { + ChangesResponse response = + reader().changes(new PerforceRevision(5), new PerforceRevision(5)); + + assertThat(response.isEmpty()).isTrue(); + assertThat(response.getEmptyReason()).isEqualTo(EmptyReason.TO_IS_ANCESTOR); + } + + @Test + public void visitChangesWalksNewestFirstAndTerminates() throws Exception { + stubChangelistsReturn(summary(102), summary(101), summary(100)); + stubDescribe(100, 101, 102); + + List visited = new ArrayList<>(); + reader() + .visitChanges( + new PerforceRevision(102), + change -> { + visited.add(change.getRevision().asString()); + return VisitResult.TERMINATE; + }); + + assertThat(visited).containsExactly("102"); + } + + private void stubChangelistsReturn(IChangelistSummary... summaries) throws Exception { + when(server.getChangelists(anyList(), any())).thenReturn(ImmutableList.copyOf(summaries)); + } + + private void stubDescribe(int... ids) throws Exception { + // Build each mock fully before handing it to thenReturn: stubbing one mock while another's + // stubbing is still open trips Mockito's UnfinishedStubbingException. + for (int id : ids) { + IChangelist cl = changelist(id, "a", "c" + id, STREAM + "/f" + id); + when(server.getChangelist(id)).thenReturn(cl); + } + } + + private static IChangelistSummary summary(int id) { + IChangelistSummary summary = mock(IChangelistSummary.class); + when(summary.getId()).thenReturn(id); + return summary; + } + + private static IChangelist changelist(int id, String user, String description, String... files) + throws Exception { + IChangelist changelist = mock(IChangelist.class); + when(changelist.getId()).thenReturn(id); + when(changelist.getUsername()).thenReturn(user); + when(changelist.getDescription()).thenReturn(description); + when(changelist.getDate()).thenReturn(new Date(0L)); + List specs = new ArrayList<>(); + for (String depotPath : files) { + IFileSpec spec = mock(IFileSpec.class); + when(spec.getDepotPathString()).thenReturn(depotPath); + specs.add(spec); + } + when(changelist.getFiles(anyBoolean())).thenReturn(specs); + return changelist; + } +} diff --git a/third_party/BUILD b/third_party/BUILD index ff0e08f7a..430b8b895 100644 --- a/third_party/BUILD +++ b/third_party/BUILD @@ -121,6 +121,13 @@ java_library( ], ) +java_library( + name = "p4java", + exports = [ + "@copybara_maven//:com_perforce_p4java", + ], +) + java_library( name = "truth", testonly = 1, From 2ff5f4873befcbafa4262cdf4d69e8fbcb4c5fa4 Mon Sep 17 00:00:00 2001 From: Jan Harasym Date: Tue, 30 Jun 2026 17:53:28 +0200 Subject: [PATCH 2/6] Add p4token support for non-rednotice users --- .../copybara/perforce/PerforceModule.java | 4 ++-- .../copybara/perforce/PerforceOptions.java | 18 +++++++++++++++--- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/java/com/google/copybara/perforce/PerforceModule.java b/java/com/google/copybara/perforce/PerforceModule.java index b7471d1c1..873fc6173 100644 --- a/java/com/google/copybara/perforce/PerforceModule.java +++ b/java/com/google/copybara/perforce/PerforceModule.java @@ -53,7 +53,7 @@ public PerforceModule(Options options) { doc = "The Perforce stream to read from, e.g. //stream/main. The connection" + " details (server, user, ticket) come from the --perforce-*" - + " flags or the standard P4PORT/P4USER/P4PASSWD environment variables."), + + " flags or the standard P4PORT/P4USER/P4PASSWD/P4TICKET environment variables."), @Param( name = "ref", named = true, @@ -79,7 +79,7 @@ public PerforceOrigin origin(String stream, String ref) throws EvalException { doc = "The Perforce stream to submit to, e.g. //stream/main. Connection" + " details come from the --perforce-* flags or the standard" - + " P4PORT/P4USER/P4PASSWD environment variables."), + + " P4PORT/P4USER/P4PASSWD/P4TICKET environment variables."), @Param( name = "submit_as_author", named = true, diff --git a/java/com/google/copybara/perforce/PerforceOptions.java b/java/com/google/copybara/perforce/PerforceOptions.java index fbb5ceecb..66032b1b4 100644 --- a/java/com/google/copybara/perforce/PerforceOptions.java +++ b/java/com/google/copybara/perforce/PerforceOptions.java @@ -52,10 +52,18 @@ public class PerforceOptions implements Option { @Parameter( names = "--perforce-password", description = - "Perforce password or login ticket. Defaults to the P4PASSWD environment variable. If" - + " empty, an existing ticket from 'p4 login' is used.") + "Perforce password. Exchanged for a ticket via login. Defaults to the P4PASSWD" + + " environment variable. Ignored if a token is provided.") String password = null; + @Parameter( + names = "--perforce-token", + description = + "Perforce login ticket to authenticate with directly (as issued by 'p4 login -p')," + + " instead of exchanging a password. Defaults to the P4TICKET environment variable." + + " Takes precedence over --perforce-password.") + String token = null; + // Lazily created and cached: a migration only ever talks to one server. @Nullable private PerforceServer cachedServer; @@ -74,6 +82,7 @@ public PerforceServer server() throws RepoException, ValidationException { private IOptionsServer connect() throws RepoException, ValidationException { String resolvedPort = firstNonEmpty(port, env("P4PORT")); String resolvedUser = firstNonEmpty(user, env("P4USER")); + String resolvedToken = firstNonEmpty(token, env("P4TICKET")); String resolvedPassword = firstNonEmpty(password, env("P4PASSWD")); if (Strings.isNullOrEmpty(resolvedPort)) { @@ -87,7 +96,10 @@ private IOptionsServer connect() throws RepoException, ValidationException { if (!Strings.isNullOrEmpty(resolvedUser)) { server.setUserName(resolvedUser); } - if (!Strings.isNullOrEmpty(resolvedPassword)) { + if (!Strings.isNullOrEmpty(resolvedToken)) { + // A pre-issued ticket: use it directly, no password-for-ticket exchange. + server.setAuthTicket(resolvedToken); + } else if (!Strings.isNullOrEmpty(resolvedPassword)) { server.login(resolvedPassword); } return server; From 1e7fe060b92ac3b50f0c2207c31fefdf599a1f76 Mon Sep 17 00:00:00 2001 From: Jan Harasym Date: Tue, 30 Jun 2026 20:09:18 +0200 Subject: [PATCH 3/6] Add symlinks/special-chars support and a --dry-run mode --- .../perforce/PerforceDestination.java | 88 +++++++++++++++++-- .../copybara/perforce/PerforceServer.java | 47 +++++++++- 2 files changed, 125 insertions(+), 10 deletions(-) diff --git a/java/com/google/copybara/perforce/PerforceDestination.java b/java/com/google/copybara/perforce/PerforceDestination.java index 0b192a186..f6531a019 100644 --- a/java/com/google/copybara/perforce/PerforceDestination.java +++ b/java/com/google/copybara/perforce/PerforceDestination.java @@ -27,6 +27,7 @@ import com.google.copybara.TransformResult; import com.google.copybara.WriterContext; import com.google.copybara.effect.DestinationEffect; +import com.google.copybara.exception.EmptyChangeException; import com.google.copybara.exception.RepoException; import com.google.copybara.exception.ValidationException; import com.google.copybara.util.Glob; @@ -61,7 +62,7 @@ private PerforceDestination( @Override public Writer newWriter(WriterContext writerContext) { - return new WriterImpl(writerContext.getWorkflowName()); + return new WriterImpl(writerContext.getWorkflowName(), writerContext.isDryRun()); } @Override @@ -85,12 +86,14 @@ public com.google.common.collect.ImmutableSetMultimap describe(G class WriterImpl implements Writer { private final String workflowName; + private final boolean dryRun; // Lazily created on first write: a stream client and its on-disk workspace under the cache. private String clientName; private Path workspaceRoot; - WriterImpl(String workflowName) { + WriterImpl(String workflowName, boolean dryRun) { this.workflowName = checkNotNull(workflowName); + this.dryRun = dryRun; } @Override @@ -109,19 +112,43 @@ public boolean supportsHistory() { public ImmutableList write( TransformResult transformResult, Glob destinationFiles, Console console) throws ValidationException, RepoException, IOException { + // p4 reconcile silently ignores local files whose names contain Perforce wildcards, which + // would drop them from the changelist without error. Refuse loudly instead. (Full support + // would require per-file encoded add/edit/delete; deferred.) + rejectUnsupportedPaths(transformResult.getPath(), destinationFiles); + ensureWorkspace(); PerforceServer server = perforceOptions.server(); // cleanWorkspace() reverts any files left open by a prior aborted run, so this attempt starts - // from a clean, head-aligned state. We do NOT retry a failed submit: reconcileAndSubmit() - // verifies the submit landed and otherwise throws with the real Perforce error, failing loud - // rather than silently drifting or masking the cause. A failed run is safe to re-run. + // from a clean, head-aligned state. console.progress("Perforce Destination: preparing workspace"); server.cleanWorkspace(clientName, stream); console.progress("Perforce Destination: staging transformed files"); mirror(transformResult.getPath(), workspaceRoot, destinationFiles); + if (dryRun) { + // Compute the diff but never create or submit a changelist. + int changed = server.previewReconcile(clientName, stream); + if (changed == 0) { + throw new EmptyChangeException( + "No changes to submit: the destination already matches the transformed tree"); + } + String summary = + String.format("(dry run) would submit %d file(s) to %s; not submitted", changed, stream); + console.info("Perforce Destination: " + summary); + return ImmutableList.of( + new DestinationEffect( + DestinationEffect.Type.CREATED, + summary, + transformResult.getChanges().getCurrent(), + new DestinationEffect.DestinationRef("(dry run)", "changelist", /* url= */ null))); + } + + // We do NOT retry a failed submit: reconcileAndSubmit() verifies the submit landed and + // otherwise throws with the real Perforce error, failing loud rather than silently drifting. + // A failed run is safe to re-run. console.progress("Perforce Destination: submitting changelist"); int changelist = server.reconcileAndSubmit( @@ -176,19 +203,28 @@ private static void mirror(Path from, Path to, Glob destinationFiles) throws Rep try { if (Files.exists(to)) { try (Stream walk = Files.walk(to)) { - for (Path file : (Iterable) walk.filter(Files::isRegularFile) + for (Path file : (Iterable) walk.filter(PerforceDestination::isManaged) .filter(toMatcher::matches)::iterator) { Files.delete(file); } } } try (Stream walk = Files.walk(from)) { - for (Path file : (Iterable) walk.filter(Files::isRegularFile) + for (Path file : (Iterable) walk.filter(PerforceDestination::isManaged) .filter(fromMatcher::matches)::iterator) { Path dest = to.resolve(from.relativize(file)); Files.createDirectories(dest.getParent()); - Files.copy( - file, dest, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES); + if (Files.isSymbolicLink(file)) { + // Recreate the link itself (not its target) so p4 reconcile stores it as a symlink. + Files.deleteIfExists(dest); + Files.createSymbolicLink(dest, Files.readSymbolicLink(file)); + } else { + Files.copy( + file, + dest, + StandardCopyOption.REPLACE_EXISTING, + StandardCopyOption.COPY_ATTRIBUTES); + } } } } catch (IOException e) { @@ -196,6 +232,40 @@ private static void mirror(Path from, Path to, Glob destinationFiles) throws Rep } } + /** The content kinds Perforce tracks: regular files and symlinks (directories are implicit). */ + private static boolean isManaged(Path path) { + return Files.isSymbolicLink(path) || Files.isRegularFile(path); + } + + /** Fails loudly if any staged file's name contains a Perforce wildcard character (@ # % *). */ + private static void rejectUnsupportedPaths(Path from, Glob destinationFiles) + throws ValidationException, RepoException { + PathMatcher matcher = destinationFiles.relativeTo(from); + ImmutableList.Builder offending = ImmutableList.builder(); + int count = 0; + try (Stream walk = Files.walk(from)) { + for (Path file : + (Iterable) walk.filter(PerforceDestination::isManaged).filter(matcher::matches) + ::iterator) { + String relative = from.relativize(file).toString(); + if (CharMatcher.anyOf("@#%*").matchesAnyOf(relative)) { + offending.add(relative); + if (++count >= 10) { + break; + } + } + } + } catch (IOException e) { + throw new RepoException("Error scanning files for Perforce-incompatible names", e); + } + ImmutableList bad = offending.build(); + if (!bad.isEmpty()) { + throw new ValidationException( + "Perforce cannot store filenames containing '@', '#', '%' or '*'; offending file(s): " + + bad); + } + } + /** Builds the changelist description, stamping the origin revision label for incremental syncs. */ private static String changeDescription(TransformResult transformResult) throws ValidationException { diff --git a/java/com/google/copybara/perforce/PerforceServer.java b/java/com/google/copybara/perforce/PerforceServer.java index 8e5aaeba2..40b302776 100644 --- a/java/com/google/copybara/perforce/PerforceServer.java +++ b/java/com/google/copybara/perforce/PerforceServer.java @@ -226,6 +226,36 @@ void ensureClient(String clientName, Path root, String stream) throws RepoExcept } } + /** + * Reconciles the workspace against the depot to count what *would* be submitted, then reverts so + * nothing is left open. Used for dry-run: it never creates or submits a changelist. Returns the + * number of files that would be added/edited/deleted. + */ + int previewReconcile(String clientName, String stream) throws RepoException { + try { + IClient client = server.getClient(clientName); + server.setCurrentClient(client); + List opened = + client.reconcileFiles( + FileSpecBuilder.makeFileSpecList(stream + "/..."), + new ReconcileFilesOptions() + .setOutsideAdd(true) + .setOutsideEdit(true) + .setRemoved(true)); + int changed = 0; + for (IFileSpec spec : opened) { + if (spec != null && spec.getOpStatus() == FileSpecOpStatus.VALID) { + changed++; + } + } + client.revertFiles( + FileSpecBuilder.makeFileSpecList(stream + "/..."), new RevertFilesOptions()); + return changed; + } catch (P4JavaException e) { + throw new RepoException("Error previewing changes for " + stream, e); + } + } + /** * Reverts any files left open in {@code clientName} (e.g. from a previously interrupted submit) * and syncs it to the head of {@code stream}, giving each write a clean, head-aligned workspace. @@ -331,6 +361,19 @@ private void tryDeletePending(int changelist) { } } + /** + * Decodes the %-escapes Perforce uses for the four special characters in depot paths. (On the + * write side these are encoded automatically by the server when reconciling the `//stream/...` + * wildcard, so only reads need decoding.) %25 must be decoded last. + */ + private static String decodeP4Path(String path) { + return path + .replace("%40", "@") + .replace("%23", "#") + .replace("%2A", "*") + .replace("%25", "%"); + } + private static String sanitizeUser(String email) { if (Strings.isNullOrEmpty(email)) { return "unknown"; @@ -380,7 +423,9 @@ private ImmutableSet changeFiles(String stream, IChangelist cl) { if (depotPath == null) { continue; } - files.add(depotPath.startsWith(prefix) ? depotPath.substring(prefix.length()) : depotPath); + String relative = + depotPath.startsWith(prefix) ? depotPath.substring(prefix.length()) : depotPath; + files.add(decodeP4Path(relative)); } return files.build(); } catch (P4JavaException e) { From d419f2782498073c9a1fdc819951e3a5734fa6e4 Mon Sep 17 00:00:00 2001 From: Jan Harasym Date: Tue, 30 Jun 2026 20:19:11 +0200 Subject: [PATCH 4/6] Add TLS fingerprint for trust & unicode/charset support --- .../copybara/perforce/PerforceOptions.java | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/java/com/google/copybara/perforce/PerforceOptions.java b/java/com/google/copybara/perforce/PerforceOptions.java index 66032b1b4..f7110383a 100644 --- a/java/com/google/copybara/perforce/PerforceOptions.java +++ b/java/com/google/copybara/perforce/PerforceOptions.java @@ -20,11 +20,13 @@ import com.beust.jcommander.Parameters; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; +import com.google.common.flogger.FluentLogger; import com.google.copybara.GeneralOptions; import com.google.copybara.Option; import com.google.copybara.exception.RepoException; import com.google.copybara.exception.ValidationException; import com.perforce.p4java.exception.P4JavaException; +import com.perforce.p4java.option.server.TrustOptions; import com.perforce.p4java.server.IOptionsServer; import com.perforce.p4java.server.ServerFactory; import java.net.URISyntaxException; @@ -35,6 +37,8 @@ @Parameters(separators = "=") public class PerforceOptions implements Option { + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + private final GeneralOptions generalOptions; @Parameter( @@ -64,6 +68,21 @@ public class PerforceOptions implements Option { + " Takes precedence over --perforce-password.") String token = null; + @Parameter( + names = "--perforce-charset", + description = + "Charset for a Unicode-mode Perforce server (P4CHARSET), e.g. 'utf8'. Defaults to the" + + " P4CHARSET environment variable, or 'utf8' when the server is Unicode-enabled.") + String charset = null; + + @Parameter( + names = "--perforce-ssl-fingerprint", + description = + "Expected SSL fingerprint of the Perforce server to pin (for 'ssl:' ports). If unset, the" + + " server's fingerprint is trusted on first use. Defaults to the P4FINGERPRINT" + + " environment variable.") + String sslFingerprint = null; + // Lazily created and cached: a migration only ever talks to one server. @Nullable private PerforceServer cachedServer; @@ -84,6 +103,8 @@ private IOptionsServer connect() throws RepoException, ValidationException { String resolvedUser = firstNonEmpty(user, env("P4USER")); String resolvedToken = firstNonEmpty(token, env("P4TICKET")); String resolvedPassword = firstNonEmpty(password, env("P4PASSWD")); + String resolvedCharset = firstNonEmpty(charset, env("P4CHARSET")); + String resolvedFingerprint = firstNonEmpty(sslFingerprint, env("P4FINGERPRINT")); if (Strings.isNullOrEmpty(resolvedPort)) { throw new ValidationException( @@ -92,10 +113,37 @@ private IOptionsServer connect() throws RepoException, ValidationException { try { IOptionsServer server = ServerFactory.getOptionsServer(toUri(resolvedPort), new Properties()); + + // SSL servers require their certificate fingerprint to be trusted before connecting. + if (resolvedPort.startsWith("ssl:")) { + if (!Strings.isNullOrEmpty(resolvedFingerprint)) { + server.addTrust(resolvedFingerprint, new TrustOptions()); + } else { + logger.atWarning().log( + "Trusting the Perforce SSL fingerprint of %s on first use; pin it with" + + " --perforce-ssl-fingerprint to guard against man-in-the-middle.", + resolvedPort); + server.addTrust(new TrustOptions().setAutoAccept(true)); + } + } + server.connect(); + if (!Strings.isNullOrEmpty(resolvedUser)) { server.setUserName(resolvedUser); } + + // Unicode-mode servers require a charset before any content is exchanged. + if (server.supportsUnicode()) { + String unicodeCharset = Strings.isNullOrEmpty(resolvedCharset) ? "utf8" : resolvedCharset; + if (!server.setCharsetName(unicodeCharset)) { + logger.atWarning().log( + "Perforce charset '%s' was not accepted by the client", unicodeCharset); + } + } else if (!Strings.isNullOrEmpty(resolvedCharset)) { + server.setCharsetName(resolvedCharset); + } + if (!Strings.isNullOrEmpty(resolvedToken)) { // A pre-issued ticket: use it directly, no password-for-ticket exchange. server.setAuthTicket(resolvedToken); From b10a738696d1c543db17973988f22a3e7d1a84fb Mon Sep 17 00:00:00 2001 From: Jan Harasym Date: Tue, 30 Jun 2026 21:50:38 +0200 Subject: [PATCH 5/6] Add credentials, checker and destination reader support --- java/com/google/copybara/perforce/BUILD | 3 + .../perforce/PerforceDestination.java | 77 +++++++++-- .../perforce/PerforceDestinationReader.java | 120 ++++++++++++++++++ .../copybara/perforce/PerforceModule.java | 56 +++++++- .../copybara/perforce/PerforceOptions.java | 23 +++- .../copybara/perforce/PerforceOrigin.java | 18 ++- .../copybara/perforce/PerforceServer.java | 33 ++++- javatests/com/google/copybara/perforce/BUILD | 2 + .../perforce/PerforceDestinationTest.java | 55 +++++++- .../copybara/perforce/PerforceOriginTest.java | 2 +- 10 files changed, 364 insertions(+), 25 deletions(-) create mode 100644 java/com/google/copybara/perforce/PerforceDestinationReader.java diff --git a/java/com/google/copybara/perforce/BUILD b/java/com/google/copybara/perforce/BUILD index ccfecaa10..ec11bf478 100644 --- a/java/com/google/copybara/perforce/BUILD +++ b/java/com/google/copybara/perforce/BUILD @@ -26,6 +26,7 @@ java_library( name = "perforce", srcs = [ "PerforceDestination.java", + "PerforceDestinationReader.java", "PerforceModule.java", "PerforceOptions.java", "PerforceOrigin.java", @@ -38,7 +39,9 @@ java_library( "//java/com/google/copybara:general_options", "//java/com/google/copybara:labels", "//java/com/google/copybara/authoring", + "//java/com/google/copybara/checks", "//java/com/google/copybara/config:base", + "//java/com/google/copybara/credentials", "//java/com/google/copybara/doc:annotations", "//java/com/google/copybara/effect", "//java/com/google/copybara/exception", diff --git a/java/com/google/copybara/perforce/PerforceDestination.java b/java/com/google/copybara/perforce/PerforceDestination.java index f6531a019..8e5ed5601 100644 --- a/java/com/google/copybara/perforce/PerforceDestination.java +++ b/java/com/google/copybara/perforce/PerforceDestination.java @@ -22,10 +22,15 @@ import com.google.common.collect.ImmutableList; import com.google.copybara.ChangeMessage; import com.google.copybara.Destination; +import com.google.copybara.DestinationReader; import com.google.copybara.GeneralOptions; import com.google.copybara.Options; +import com.google.copybara.Origin; import com.google.copybara.TransformResult; import com.google.copybara.WriterContext; +import com.google.copybara.checks.Checker; +import com.google.copybara.checks.DescriptionChecker; +import com.google.copybara.credentials.CredentialModule.UsernamePasswordIssuer; import com.google.copybara.effect.DestinationEffect; import com.google.copybara.exception.EmptyChangeException; import com.google.copybara.exception.RepoException; @@ -38,6 +43,7 @@ import java.nio.file.PathMatcher; import java.nio.file.StandardCopyOption; import java.util.stream.Stream; +import javax.annotation.Nullable; /** A Perforce (Helix Core) destination that submits each migrated change as a changelist. */ public class PerforceDestination implements Destination { @@ -48,16 +54,26 @@ public class PerforceDestination implements Destination { private final PerforceOptions perforceOptions; private final String stream; private final boolean submitAsAuthor; + @Nullable private final UsernamePasswordIssuer credentials; + @Nullable private final Checker checker; private PerforceDestination( GeneralOptions generalOptions, PerforceOptions perforceOptions, String stream, - boolean submitAsAuthor) { + boolean submitAsAuthor, + @Nullable UsernamePasswordIssuer credentials, + @Nullable Checker checker) { this.generalOptions = checkNotNull(generalOptions); this.perforceOptions = checkNotNull(perforceOptions); this.stream = CharMatcher.is('/').trimTrailingFrom(checkNotNull(stream)); this.submitAsAuthor = submitAsAuthor; + this.credentials = credentials; + this.checker = checker; + } + + private PerforceServer server() throws RepoException, ValidationException { + return perforceOptions.server(credentials); } @Override @@ -99,7 +115,7 @@ class WriterImpl implements Writer { @Override public DestinationStatus getDestinationStatus(Glob destinationFiles, String labelName) throws RepoException, ValidationException { - String baseline = perforceOptions.server().findOriginLabel(stream, labelName); + String baseline = server().findOriginLabel(stream, labelName); return baseline == null ? null : new DestinationStatus(baseline, ImmutableList.of()); } @@ -118,7 +134,7 @@ public ImmutableList write( rejectUnsupportedPaths(transformResult.getPath(), destinationFiles); ensureWorkspace(); - PerforceServer server = perforceOptions.server(); + PerforceServer server = server(); // cleanWorkspace() reverts any files left open by a prior aborted run, so this attempt starts // from a clean, head-aligned state. @@ -128,6 +144,11 @@ public ImmutableList write( console.progress("Perforce Destination: staging transformed files"); mirror(transformResult.getPath(), workspaceRoot, destinationFiles); + // Scan the staged tree and (if supported) the changelist description before submitting. + String description = + applyChecker( + checker, transformResult.getPath(), changeDescription(transformResult), console); + if (dryRun) { // Compute the diff but never create or submit a changelist. int changed = server.previewReconcile(clientName, stream); @@ -152,11 +173,7 @@ public ImmutableList write( console.progress("Perforce Destination: submitting changelist"); int changelist = server.reconcileAndSubmit( - clientName, - stream, - changeDescription(transformResult), - transformResult.getAuthor(), - submitAsAuthor); + clientName, stream, description, transformResult.getAuthor(), submitAsAuthor); console.info(String.format("Perforce Destination: submitted changelist %d", changelist)); return ImmutableList.of( new DestinationEffect( @@ -173,6 +190,20 @@ public void visitChanges(PerforceRevision start, ChangesVisitor visitor) { + " supported"); } + @Override + public DestinationReader getDestinationReader( + Console console, @Nullable Origin.Baseline baseline, Path workdir) + throws ValidationException, RepoException { + return new PerforceDestinationReader(server(), stream, workdir); + } + + @Override + public DestinationReader getDestinationReader( + Console console, @Nullable String baseline, Path workdir) + throws ValidationException, RepoException { + return new PerforceDestinationReader(server(), stream, workdir); + } + private void ensureWorkspace() throws RepoException, ValidationException { if (clientName != null) { return; @@ -187,7 +218,7 @@ private void ensureWorkspace() throws RepoException, ValidationException { } catch (IOException e) { throw new RepoException("Could not create Perforce workspace directory", e); } - perforceOptions.server().ensureClient(clientName, workspaceRoot, stream); + server().ensureClient(clientName, workspaceRoot, stream); } } @@ -237,6 +268,24 @@ private static boolean isManaged(Path path) { return Files.isSymbolicLink(path) || Files.isRegularFile(path); } + /** + * Runs {@code checker} (if any) over the staged tree and, when it is a {@link DescriptionChecker}, + * over the changelist description, returning the possibly-rewritten description. + */ + static String applyChecker( + @Nullable Checker checker, Path tree, String description, Console console) + throws ValidationException, IOException { + if (checker == null) { + return description; + } + console.progress("Perforce Destination: running checker"); + checker.doCheck(tree, console); + if (checker instanceof DescriptionChecker) { + return ((DescriptionChecker) checker).processDescription(description, console); + } + return description; + } + /** Fails loudly if any staged file's name contains a Perforce wildcard character (@ # % *). */ private static void rejectUnsupportedPaths(Path from, Glob destinationFiles) throws ValidationException, RepoException { @@ -285,11 +334,17 @@ private static String sanitize(String value) { } static PerforceDestination newPerforceDestination( - Options options, String stream, boolean submitAsAuthor) { + Options options, + String stream, + boolean submitAsAuthor, + @Nullable UsernamePasswordIssuer credentials, + @Nullable Checker checker) { return new PerforceDestination( options.get(GeneralOptions.class), options.get(PerforceOptions.class), stream, - submitAsAuthor); + submitAsAuthor, + credentials, + checker); } } diff --git a/java/com/google/copybara/perforce/PerforceDestinationReader.java b/java/com/google/copybara/perforce/PerforceDestinationReader.java new file mode 100644 index 000000000..97271cec7 --- /dev/null +++ b/java/com/google/copybara/perforce/PerforceDestinationReader.java @@ -0,0 +1,120 @@ +/* + * Copyright (C) 2026 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.copybara.perforce; + +import static com.google.copybara.config.SkylarkUtil.convertFromNoneable; + +import com.google.copybara.CheckoutPath; +import com.google.copybara.DestinationReader; +import com.google.copybara.exception.RepoException; +import com.google.copybara.exception.ValidationException; +import com.google.copybara.util.FileUtil; +import com.google.copybara.util.Glob; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.PathMatcher; +import java.nio.file.StandardCopyOption; +import java.util.stream.Stream; +import net.starlark.java.eval.EvalException; + +/** + * Reads files from the head of a Perforce stream, for {@code destination_reader().read_file(...)}, + * consistency files and merge-import baselines. + */ +class PerforceDestinationReader extends DestinationReader { + + private final PerforceServer server; + private final String stream; + private final Path workdir; + + PerforceDestinationReader(PerforceServer server, String stream, Path workdir) { + this.server = server; + this.stream = stream; + this.workdir = workdir; + } + + @Override + public String readFile(String path) throws RepoException { + return server.readFileAtHead(stream, path); + } + + @Override + public boolean exists(String path) { + try { + server.readFileAtHead(stream, path); + return true; + } catch (RepoException e) { + return false; + } + } + + @Override + public void copyDestinationFiles(Object globObj, Object path) + throws RepoException, ValidationException, EvalException { + CheckoutPath checkoutPath = convertFromNoneable(path, null); + Glob glob = Glob.wrapGlob(globObj, null); + copyDestinationFilesToDirectory( + glob, + checkoutPath == null + ? workdir + : checkoutPath.getCheckoutDir().resolve(checkoutPath.getPath())); + } + + @Override + public void copyDestinationFilesToDirectory(Glob glob, Path directory) throws RepoException { + Path temp; + try { + temp = Files.createTempDirectory("p4_dest_reader"); + } catch (IOException e) { + throw new RepoException("Could not create temp directory for Perforce destination reader", e); + } + try { + // Materialise the whole stream head, then copy just the glob-matched files into the target. + server.syncStreamHeadTo(stream, temp); + PathMatcher matcher = glob.relativeTo(temp); + try (Stream walk = Files.walk(temp)) { + for (Path file : + (Iterable) + walk.filter(p -> Files.isSymbolicLink(p) || Files.isRegularFile(p)) + .filter(matcher::matches) + ::iterator) { + Path dest = directory.resolve(temp.relativize(file)); + Files.createDirectories(dest.getParent()); + if (Files.isSymbolicLink(file)) { + Files.deleteIfExists(dest); + Files.createSymbolicLink(dest, Files.readSymbolicLink(file)); + } else { + Files.copy( + file, + dest, + StandardCopyOption.REPLACE_EXISTING, + StandardCopyOption.COPY_ATTRIBUTES); + } + } + } + } catch (IOException e) { + throw new RepoException("Error copying Perforce destination files", e); + } finally { + try { + FileUtil.deleteRecursively(temp); + } catch (IOException e) { + // Best-effort cleanup of a temp directory. + } + } + } +} diff --git a/java/com/google/copybara/perforce/PerforceModule.java b/java/com/google/copybara/perforce/PerforceModule.java index 873fc6173..fb0091456 100644 --- a/java/com/google/copybara/perforce/PerforceModule.java +++ b/java/com/google/copybara/perforce/PerforceModule.java @@ -20,12 +20,17 @@ import com.google.common.base.Preconditions; import com.google.copybara.Options; +import com.google.copybara.checks.Checker; import com.google.copybara.config.LabelsAwareModule; +import com.google.copybara.credentials.CredentialModule.UsernamePasswordIssuer; import com.google.copybara.doc.annotations.UsesFlags; +import javax.annotation.Nullable; import net.starlark.java.annot.Param; +import net.starlark.java.annot.ParamType; import net.starlark.java.annot.StarlarkBuiltin; import net.starlark.java.annot.StarlarkMethod; import net.starlark.java.eval.EvalException; +import net.starlark.java.eval.NoneType; import net.starlark.java.eval.StarlarkValue; /** Main module for Perforce (Helix Core) origins and destinations. */ @@ -62,9 +67,22 @@ public PerforceModule(Options options) { "The default reference used to read a revision. Either a submitted changelist" + " number (e.g. \"12345\") or the literal \"head\"" + " for the most recent submitted changelist on the stream."), + @Param( + name = "credentials", + allowedTypes = { + @ParamType(type = UsernamePasswordIssuer.class), + @ParamType(type = NoneType.class) + }, + named = true, + defaultValue = "None", + doc = + "Optional credentials.username_password(...) used to authenticate," + + " preferred over the --perforce-* flags / env vars."), }) - public PerforceOrigin origin(String stream, String ref) throws EvalException { - return PerforceOrigin.newPerforceOrigin(options, checkNotEmpty(stream, "stream"), ref); + public PerforceOrigin origin(String stream, String ref, Object credentials) + throws EvalException { + return PerforceOrigin.newPerforceOrigin( + options, checkNotEmpty(stream, "stream"), ref, asIssuer(credentials)); } @StarlarkMethod( @@ -89,10 +107,40 @@ public PerforceOrigin origin(String stream, String ref) throws EvalException { + " user is created on demand). Requires the connecting user to have permission" + " to create users and submit on their behalf. If false, changelists are" + " submitted as the connecting user and the author is kept in the description."), + @Param( + name = "credentials", + allowedTypes = { + @ParamType(type = UsernamePasswordIssuer.class), + @ParamType(type = NoneType.class) + }, + named = true, + defaultValue = "None", + doc = + "Optional credentials.username_password(...) used to authenticate," + + " preferred over the --perforce-* flags / env vars."), + @Param( + name = "checker", + allowedTypes = {@ParamType(type = Checker.class), @ParamType(type = NoneType.class)}, + named = true, + defaultValue = "None", + doc = + "Optional checker run against the staged files (and the changelist description, if" + + " it is a description checker) before each submit."), }) - public PerforceDestination destination(String stream, boolean submitAsAuthor) + public PerforceDestination destination( + String stream, boolean submitAsAuthor, Object credentials, Object checker) throws EvalException { return PerforceDestination.newPerforceDestination( - options, checkNotEmpty(stream, "stream"), submitAsAuthor); + options, + checkNotEmpty(stream, "stream"), + submitAsAuthor, + asIssuer(credentials), + checker instanceof Checker ? (Checker) checker : null); + } + + @Nullable + private static UsernamePasswordIssuer asIssuer(Object credentials) { + return credentials instanceof UsernamePasswordIssuer ? (UsernamePasswordIssuer) credentials + : null; } } diff --git a/java/com/google/copybara/perforce/PerforceOptions.java b/java/com/google/copybara/perforce/PerforceOptions.java index f7110383a..5d5f687a7 100644 --- a/java/com/google/copybara/perforce/PerforceOptions.java +++ b/java/com/google/copybara/perforce/PerforceOptions.java @@ -23,6 +23,7 @@ import com.google.common.flogger.FluentLogger; import com.google.copybara.GeneralOptions; import com.google.copybara.Option; +import com.google.copybara.credentials.CredentialModule.UsernamePasswordIssuer; import com.google.copybara.exception.RepoException; import com.google.copybara.exception.ValidationException; import com.perforce.p4java.exception.P4JavaException; @@ -92,17 +93,35 @@ public PerforceOptions(GeneralOptions generalOptions) { /** Returns a connected {@link PerforceServer}, building and caching it on first use. */ public PerforceServer server() throws RepoException, ValidationException { + return server(null); + } + + /** + * Returns a connected {@link PerforceServer}. When {@code credentials} is provided (from + * config), its issued username/password are used in preference to the {@code --perforce-*} flags + * and env vars. Cached on first use; later callers reuse the first connection. + */ + public PerforceServer server(@Nullable UsernamePasswordIssuer credentials) + throws RepoException, ValidationException { if (cachedServer == null) { - cachedServer = new PerforceServer(connect()); + cachedServer = new PerforceServer(connect(credentials)); } return cachedServer; } - private IOptionsServer connect() throws RepoException, ValidationException { + private IOptionsServer connect(@Nullable UsernamePasswordIssuer credentials) + throws RepoException, ValidationException { String resolvedPort = firstNonEmpty(port, env("P4PORT")); String resolvedUser = firstNonEmpty(user, env("P4USER")); String resolvedToken = firstNonEmpty(token, env("P4TICKET")); String resolvedPassword = firstNonEmpty(password, env("P4PASSWD")); + + // Config-supplied credentials win over flags/env, and imply password auth (not a ticket). + if (credentials != null) { + resolvedUser = credentials.username().issue().provideSecret(); + resolvedPassword = credentials.password().issue().provideSecret(); + resolvedToken = null; + } String resolvedCharset = firstNonEmpty(charset, env("P4CHARSET")); String resolvedFingerprint = firstNonEmpty(sslFingerprint, env("P4FINGERPRINT")); diff --git a/java/com/google/copybara/perforce/PerforceOrigin.java b/java/com/google/copybara/perforce/PerforceOrigin.java index 08028db75..55a174742 100644 --- a/java/com/google/copybara/perforce/PerforceOrigin.java +++ b/java/com/google/copybara/perforce/PerforceOrigin.java @@ -30,6 +30,7 @@ import com.google.copybara.Origin; import com.google.copybara.Origin.Reader.ChangesResponse.EmptyReason; import com.google.copybara.authoring.Authoring; +import com.google.copybara.credentials.CredentialModule.UsernamePasswordIssuer; import com.google.copybara.exception.CannotResolveRevisionException; import com.google.copybara.exception.RepoException; import com.google.copybara.exception.ValidationException; @@ -50,22 +51,25 @@ public class PerforceOrigin implements Origin { private final PerforceOptions perforceOptions; private final String stream; @Nullable private final String configRef; + @Nullable private final UsernamePasswordIssuer credentials; private PerforceOrigin( GeneralOptions generalOptions, PerforceOptions perforceOptions, String stream, - @Nullable String configRef) { + @Nullable String configRef, + @Nullable UsernamePasswordIssuer credentials) { this.generalOptions = checkNotNull(generalOptions); this.perforceOptions = checkNotNull(perforceOptions); // Streams are referenced without a trailing slash, e.g. "//stream/main". this.stream = CharMatcher.is('/').trimTrailingFrom(checkNotNull(stream)); this.configRef = configRef; + this.credentials = credentials; } @VisibleForTesting public PerforceServer getServer() throws RepoException, ValidationException { - return perforceOptions.server(); + return perforceOptions.server(credentials); } @Override @@ -191,8 +195,14 @@ public String toString() { return String.format("PerforceOrigin{stream = %s, ref = %s}", stream, configRef); } - static PerforceOrigin newPerforceOrigin(Options options, String stream, @Nullable String ref) { + static PerforceOrigin newPerforceOrigin( + Options options, String stream, @Nullable String ref, + @Nullable UsernamePasswordIssuer credentials) { return new PerforceOrigin( - options.get(GeneralOptions.class), options.get(PerforceOptions.class), stream, ref); + options.get(GeneralOptions.class), + options.get(PerforceOptions.class), + stream, + ref, + credentials); } } diff --git a/java/com/google/copybara/perforce/PerforceServer.java b/java/com/google/copybara/perforce/PerforceServer.java index 40b302776..15310880e 100644 --- a/java/com/google/copybara/perforce/PerforceServer.java +++ b/java/com/google/copybara/perforce/PerforceServer.java @@ -45,6 +45,9 @@ import com.perforce.p4java.option.client.SyncOptions; import com.perforce.p4java.option.server.GetChangelistsOptions; import com.perforce.p4java.server.IOptionsServer; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.time.ZoneOffset; import java.time.ZonedDateTime; @@ -140,6 +143,15 @@ Change describe(String stream, int changelist) * removed afterwards so no workspace state leaks between runs. */ void syncStreamTo(String stream, int changelist, Path checkoutDir) throws RepoException { + syncStream(stream, "@" + changelist, checkoutDir); + } + + /** Syncs the head of {@code stream} into {@code checkoutDir} (used by the destination reader). */ + void syncStreamHeadTo(String stream, Path checkoutDir) throws RepoException { + syncStream(stream, "#head", checkoutDir); + } + + private void syncStream(String stream, String revSpec, Path checkoutDir) throws RepoException { String clientName = String.format("copybara_%d_%d", ProcessHandle.current().pid(), System.nanoTime()); boolean created = false; @@ -156,12 +168,12 @@ void syncStreamTo(String stream, int changelist, Path checkoutDir) throws RepoEx server.setCurrentClient(bound); List synced = bound.sync( - FileSpecBuilder.makeFileSpecList(stream + "/...@" + changelist), + FileSpecBuilder.makeFileSpecList(stream + "/..." + revSpec), new SyncOptions().setForceUpdate(true)); logFileSpecErrors(synced); } catch (P4JavaException e) { throw new RepoException( - String.format("Error syncing stream '%s' at changelist %d", stream, changelist), e); + String.format("Error syncing stream '%s' at %s", stream, revSpec), e); } finally { if (created) { try { @@ -173,6 +185,23 @@ void syncStreamTo(String stream, int changelist, Path checkoutDir) throws RepoEx } } + /** Reads the head contents of {@code stream + "/" + relativePath} as a UTF-8 string. */ + String readFileAtHead(String stream, String relativePath) throws RepoException { + String depotPath = stream + "/" + relativePath; + try (InputStream in = + server.getFileContents( + FileSpecBuilder.makeFileSpecList(depotPath), + /* allRevs= */ false, + /* noHeaderLine= */ true)) { + if (in == null) { + throw new RepoException("Could not read " + depotPath + " from Perforce"); + } + return new String(in.readAllBytes(), StandardCharsets.UTF_8); + } catch (P4JavaException | IOException e) { + throw new RepoException("Error reading " + depotPath + " from Perforce", e); + } + } + // =========================================================================================== // Write side (perforce.destination) // =========================================================================================== diff --git a/javatests/com/google/copybara/perforce/BUILD b/javatests/com/google/copybara/perforce/BUILD index 737e02dbf..6a87e1f03 100644 --- a/javatests/com/google/copybara/perforce/BUILD +++ b/javatests/com/google/copybara/perforce/BUILD @@ -27,11 +27,13 @@ all_tests( deps = [ "//java/com/google/copybara:base", "//java/com/google/copybara/authoring", + "//java/com/google/copybara/checks", "//java/com/google/copybara/exception", "//java/com/google/copybara/perforce", "//java/com/google/copybara/revision", "//java/com/google/copybara/testing", "//java/com/google/copybara/util", + "//java/com/google/copybara/util/console/testing", "//third_party:mockito", "//third_party:p4java", "//third_party:truth", diff --git a/javatests/com/google/copybara/perforce/PerforceDestinationTest.java b/javatests/com/google/copybara/perforce/PerforceDestinationTest.java index c57a9a9bc..3d1e0af48 100644 --- a/javatests/com/google/copybara/perforce/PerforceDestinationTest.java +++ b/javatests/com/google/copybara/perforce/PerforceDestinationTest.java @@ -17,6 +17,7 @@ package com.google.copybara.perforce; import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.Mockito.mock; @@ -28,10 +29,16 @@ import com.google.copybara.Destination.Writer; import com.google.copybara.Options; import com.google.copybara.WriterContext; +import com.google.copybara.checks.Checker; +import com.google.copybara.exception.ValidationException; +import com.google.copybara.testing.DummyChecker; import com.google.copybara.testing.OptionsBuilder; import com.google.copybara.util.Glob; +import com.google.copybara.util.console.testing.TestingConsole; import com.perforce.p4java.core.IChangelistSummary; import com.perforce.p4java.server.IOptionsServer; +import java.nio.file.Files; +import java.nio.file.Path; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -56,7 +63,8 @@ public void setUp() { } private PerforceDestination destination(boolean submitAsAuthor) { - return PerforceDestination.newPerforceDestination(options, STREAM, submitAsAuthor); + return PerforceDestination.newPerforceDestination( + options, STREAM, submitAsAuthor, /* credentials= */ null, /* checker= */ null); } private Writer writer() { @@ -130,6 +138,51 @@ public void destinationStatusNullWhenStreamEmpty() throws Exception { assertThat(writer().getDestinationStatus(Glob.ALL_FILES, LABEL)).isNull(); } + @Test + public void checkerRejectsBadFileContent() throws Exception { + Path tree = Files.createTempDirectory("p4checker"); + Files.writeString(tree.resolve("a.txt"), "this has a BADWORD in it"); + Checker checker = new DummyChecker(ImmutableSet.of("badword")); + + ValidationException e = + assertThrows( + ValidationException.class, + () -> PerforceDestination.applyChecker(checker, tree, "desc", new TestingConsole())); + assertThat(e).hasMessageThat().ignoringCase().contains("badword"); + } + + @Test + public void checkerRejectsBadDescription() throws Exception { + Path tree = Files.createTempDirectory("p4checker"); + Files.writeString(tree.resolve("a.txt"), "perfectly fine"); + Checker checker = new DummyChecker(ImmutableSet.of("secret")); + + assertThrows( + ValidationException.class, + () -> + PerforceDestination.applyChecker( + checker, tree, "contains a secret token", new TestingConsole())); + } + + @Test + public void checkerPassesCleanTreeAndDescription() throws Exception { + Path tree = Files.createTempDirectory("p4checker"); + Files.writeString(tree.resolve("a.txt"), "totally fine content"); + Checker checker = new DummyChecker(ImmutableSet.of("badword")); + + assertThat( + PerforceDestination.applyChecker( + checker, tree, "clean description", new TestingConsole())) + .isEqualTo("clean description"); + } + + @Test + public void nullCheckerReturnsDescriptionUnchanged() throws Exception { + Path tree = Files.createTempDirectory("p4checker"); + assertThat(PerforceDestination.applyChecker(null, tree, "desc", new TestingConsole())) + .isEqualTo("desc"); + } + private void stubChangelists(IChangelistSummary... summaries) throws Exception { when(server.getChangelists(anyList(), any())).thenReturn(ImmutableList.copyOf(summaries)); } diff --git a/javatests/com/google/copybara/perforce/PerforceOriginTest.java b/javatests/com/google/copybara/perforce/PerforceOriginTest.java index eded4306c..25b136ec0 100644 --- a/javatests/com/google/copybara/perforce/PerforceOriginTest.java +++ b/javatests/com/google/copybara/perforce/PerforceOriginTest.java @@ -72,7 +72,7 @@ public void setUp() { } private PerforceOrigin origin() { - return PerforceOrigin.newPerforceOrigin(options, STREAM, "head"); + return PerforceOrigin.newPerforceOrigin(options, STREAM, "head", /* credentials= */ null); } private Reader reader() { From e0d42cd0eced3ccc64d27337f390d27574b80b77 Mon Sep 17 00:00:00 2001 From: Jan Harasym Date: Sat, 4 Jul 2026 16:08:07 +0200 Subject: [PATCH 6/6] Fix special characters (encoding support) - seems to work bidirectionally --- .../perforce/PerforceDestination.java | 34 ------------------- .../copybara/perforce/PerforceServer.java | 18 +++++++--- 2 files changed, 14 insertions(+), 38 deletions(-) diff --git a/java/com/google/copybara/perforce/PerforceDestination.java b/java/com/google/copybara/perforce/PerforceDestination.java index 8e5ed5601..0f7f1c779 100644 --- a/java/com/google/copybara/perforce/PerforceDestination.java +++ b/java/com/google/copybara/perforce/PerforceDestination.java @@ -128,11 +128,6 @@ public boolean supportsHistory() { public ImmutableList write( TransformResult transformResult, Glob destinationFiles, Console console) throws ValidationException, RepoException, IOException { - // p4 reconcile silently ignores local files whose names contain Perforce wildcards, which - // would drop them from the changelist without error. Refuse loudly instead. (Full support - // would require per-file encoded add/edit/delete; deferred.) - rejectUnsupportedPaths(transformResult.getPath(), destinationFiles); - ensureWorkspace(); PerforceServer server = server(); @@ -286,35 +281,6 @@ static String applyChecker( return description; } - /** Fails loudly if any staged file's name contains a Perforce wildcard character (@ # % *). */ - private static void rejectUnsupportedPaths(Path from, Glob destinationFiles) - throws ValidationException, RepoException { - PathMatcher matcher = destinationFiles.relativeTo(from); - ImmutableList.Builder offending = ImmutableList.builder(); - int count = 0; - try (Stream walk = Files.walk(from)) { - for (Path file : - (Iterable) walk.filter(PerforceDestination::isManaged).filter(matcher::matches) - ::iterator) { - String relative = from.relativize(file).toString(); - if (CharMatcher.anyOf("@#%*").matchesAnyOf(relative)) { - offending.add(relative); - if (++count >= 10) { - break; - } - } - } - } catch (IOException e) { - throw new RepoException("Error scanning files for Perforce-incompatible names", e); - } - ImmutableList bad = offending.build(); - if (!bad.isEmpty()) { - throw new ValidationException( - "Perforce cannot store filenames containing '@', '#', '%' or '*'; offending file(s): " - + bad); - } - } - /** Builds the changelist description, stamping the origin revision label for incremental syncs. */ private static String changeDescription(TransformResult transformResult) throws ValidationException { diff --git a/java/com/google/copybara/perforce/PerforceServer.java b/java/com/google/copybara/perforce/PerforceServer.java index 15310880e..74ccb0847 100644 --- a/java/com/google/copybara/perforce/PerforceServer.java +++ b/java/com/google/copybara/perforce/PerforceServer.java @@ -270,7 +270,9 @@ int previewReconcile(String clientName, String stream) throws RepoException { new ReconcileFilesOptions() .setOutsideAdd(true) .setOutsideEdit(true) - .setRemoved(true)); + .setRemoved(true) + // -f: encode @ # % * in filenames (e.g. npm @types dirs) instead of skipping them. + .setUseWildcards(true)); int changed = 0; for (IFileSpec spec : opened) { if (spec != null && spec.getOpStatus() == FileSpecOpStatus.VALID) { @@ -333,6 +335,8 @@ int reconcileAndSubmit( .setOutsideAdd(true) .setOutsideEdit(true) .setRemoved(true) + // -f: encode @ # % * in filenames (e.g. npm @types dirs) instead of skipping them. + .setUseWildcards(true) .setChangelistId(pending.getId())); pending.refresh(); @@ -342,9 +346,15 @@ int reconcileAndSubmit( "No changes to submit: the destination already matches the transformed tree"); } - int changelist = pending.getId(); + int pendingId = pending.getId(); String submitErrors = errorMessages(pending.submit(/* reOpen= */ false)); + // p4 renumbers a pending changelist on submit whenever its number isn't the next in + // sequence (a gap opens as soon as one empty change is created and deleted), so the + // submitted number can differ from pendingId. p4java refreshes the changelist id from the + // submit response, so read it back rather than trusting the pending number. + int changelist = pending.getId(); + // A 'p4 submit' can return without throwing yet leave the changelist pending (e.g. the // unlicensed-server file/user cap, or a server-side error). Verify it actually landed and // surface the real Perforce message so the failure is loud rather than a silent drift. @@ -352,11 +362,11 @@ int reconcileAndSubmit( if (submitted == null || submitted.getStatus() != ChangelistStatus.SUBMITTED) { client.revertFiles( FileSpecBuilder.makeFileSpecList(stream + "/..."), new RevertFilesOptions()); - tryDeletePending(changelist); + tryDeletePending(pendingId); throw new RepoException( String.format( "Perforce submit of changelist %d did not complete%s", - changelist, submitErrors.isEmpty() ? "" : ": " + submitErrors)); + pendingId, submitErrors.isEmpty() ? "" : ": " + submitErrors)); } return changelist; } catch (P4JavaException e) {