diff --git a/README_DEBUGGING.md b/README_DEBUGGING.md index 3351e131..97f9bf9d 100644 --- a/README_DEBUGGING.md +++ b/README_DEBUGGING.md @@ -21,7 +21,7 @@ This downloads the latest sdk to the folder `defoldsdk//defoldsdk`; sets t # Environment variables * **DM_DEBUG_COMMANDS** - Prints the command line and result for each command in a build -* **DM_DEBUG_DISABLE_PROGUARD** - Disables building with ProGuard (Android only) +* **DM_DEBUG_DISABLE_R8** - Disables shrinking with R8 and uses D8 directly (Android only) * **DM_DEBUG_JOB_FOLDER** - The uploaded job (and build) will always end up in this folder * **DM_DEBUG_KEEP_JOB_FOLDER** - Always keep the job folders * **DM_DEBUG_JOB_UPLOAD** - Output the file names in the received payload diff --git a/server/build.gradle b/server/build.gradle index 51d5915b..4fc789ab 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -43,6 +43,17 @@ configurations.all { repositories { mavenCentral() + exclusiveContent { + forRepository { + maven { + name = 'GoogleR8' + url = uri('https://dl.google.com/dl/android/maven2/') + } + } + filter { + includeModule('com.android.tools', 'r8') + } + } } dependencies { @@ -102,6 +113,7 @@ dependencies { testImplementation('org.springframework.security:spring-security-test') testImplementation('org.smali:dexlib2:2.5.2') + testImplementation('com.android.tools:r8:8.13.19') testImplementation project(':client') testImplementation('org.wiremock:wiremock-standalone:3.13.2') } diff --git a/server/docker/Dockerfile.android.ndk25_sdk36-env b/server/docker/Dockerfile.android.ndk25_sdk36-env index 4e5b40a9..c21ce5bd 100644 --- a/server/docker/Dockerfile.android.ndk25_sdk36-env +++ b/server/docker/Dockerfile.android.ndk25_sdk36-env @@ -13,6 +13,7 @@ ARG ANDROID_NDK_PATH=${ANDROID_ROOT}/android-ndk-r${ANDROID_NDK_VERSION} ARG ANDROID_SDK_HOME=${ANDROID_ROOT}/.android ARG ANDROID_NDK_BIN_PATH=${ANDROID_NDK_PATH}/toolchains/llvm/prebuilt/linux-x86_64/bin ARG ANDROID_SDK_BUILD_TOOLS_PATH=${ANDROID_HOME}/build-tools/${ANDROID_BUILD_TOOLS_VERSION} +ARG R8_VERSION=8.13.19 FROM europe-west1-docker.pkg.dev/extender-426409/extender-public-registry/extender-build-env:1.1.0 AS build @@ -29,7 +30,7 @@ ARG ANDROID_NDK_VERSION ARG ANDROID_NDK_API_VERSION ARG ANDROID_64_NDK_API_VERSION ARG ANDROID_NDK_FILENAME=android-ndk-r${ANDROID_NDK_VERSION}-linux.tar.gz -ARG R8_VERSION=8.13.19 +ARG R8_VERSION SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN --mount=type=secret,id=DM_PACKAGES_URL,required=true \ @@ -77,6 +78,7 @@ ARG ANDROID_SDK_BUILD_TOOLS_PATH ARG ANDROID_NDK_VERSION ARG ANDROID_NDK_API_VERSION ARG ANDROID_64_NDK_API_VERSION +ARG R8_VERSION ENV ANDROID_ROOT=${ANDROID_ROOT} \ # ANDROID_HOME has been replaced with ANDROID_SDK_ROOT @@ -101,21 +103,15 @@ ENV ANDROID_ROOT=${ANDROID_ROOT} \ ANDROID_NDK_SYSROOT=${ANDROID_NDK_PATH}/toolchains/llvm/prebuilt/linux-x86_64/sysroot \ # We specify it in build_input.yml by setting it the first in PATH PATH=${PATH}:${ANDROID_HOME}/tools:${ANDROID_HOME}/platform-tools:${ANDROID_SDK_BUILD_TOOLS_PATH}:${ANDROID_NDK_BIN_PATH} \ - ANDROID_PROGUARD=/usr/share/java/proguard.jar + ANDROID_R8=${ANDROID_SDK_BUILD_TOOLS_PATH}/lib/d8.jar \ + ANDROID_R8_VERSION=${R8_VERSION} COPY --from=build ${ANDROID_ROOT} ${ANDROID_ROOT} SHELL ["/bin/bash", "-o", "pipefail", "-c"] -# android proguard was version 4.7, this is at least 5.2.1 which seems to work with OpenJDK 11 -RUN \ - apt-get update && \ - apt-get install -y --no-install-recommends proguard && \ - apt-get autoremove -y && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* && \ # Since dotnet cannot really cross compile, we need to create a "ar" shim for "llvm-ar" # As long as it's in the path, it will be picked up - echo '#!/usr/bin/env bash' > /usr/bin/ar && \ +RUN echo '#!/usr/bin/env bash' > /usr/bin/ar && \ LLVM_AR=$(find /opt/platformsdk/android -iname "llvm-ar" | tail -1) && \ echo "${LLVM_AR} \$*" >> /usr/bin/ar && \ chmod +x /usr/bin/ar diff --git a/server/docker/common-services.yml b/server/docker/common-services.yml index 399c54c7..c6a912bd 100644 --- a/server/docker/common-services.yml +++ b/server/docker/common-services.yml @@ -28,7 +28,7 @@ services: environment: - DYNAMO_HOME${DYNAMO_HOME:+=/dynamo_home} - DM_DEBUG_COMMANDS - - DM_DEBUG_DISABLE_PROGUARD + - DM_DEBUG_DISABLE_R8 - DM_DEBUG_JOB_FOLDER - DM_DEBUG_KEEP_JOB_FOLDER - DM_DEBUG_JOB_UPLOAD @@ -37,4 +37,4 @@ services: command: ["--spring.config.additional-location=file:/etc/defold/extender/", "--spring.profiles.active=local-dev,prometheus${STRUCTURED_LOGGING+,logging}"] test_remote_builder: extends: test_builder - command: ["--spring.config.additional-location=file:/etc/defold/extender/", "--spring.profiles.active=local-dev"] \ No newline at end of file + command: ["--spring.config.additional-location=file:/etc/defold/extender/", "--spring.profiles.active=local-dev"] diff --git a/server/src/main/java/com/defold/extender/AsyncBuilder.java b/server/src/main/java/com/defold/extender/AsyncBuilder.java index fcdd459d..d9a87dde 100644 --- a/server/src/main/java/com/defold/extender/AsyncBuilder.java +++ b/server/src/main/java/com/defold/extender/AsyncBuilder.java @@ -39,6 +39,7 @@ public class AsyncBuilder { private GradleService gradleService; private CocoaPodsService cocoaPodsService; private BuildProgressService buildProgressService; + private R8Configuration r8Configuration; private File jobResultLocation; private long resultLifetime; private boolean keepJobDirectory = false; @@ -47,12 +48,14 @@ public AsyncBuilder(DefoldSdkService defoldSdkService, GradleService gradleService, Optional cocoaPodsService, BuildProgressService buildProgressService, + R8Configuration r8Configuration, @Value("${extender.job-result.location}") String jobResultLocation, @Value("${extender.job-result.lifetime:1200000}") long jobResultLifetime) { this.defoldSdkService = defoldSdkService; this.gradleService = gradleService; cocoaPodsService.ifPresent(val -> { this.cocoaPodsService = val; }); this.buildProgressService = buildProgressService; + this.r8Configuration = r8Configuration; this.jobResultLocation = new File(jobResultLocation); this.keepJobDirectory = System.getenv("DM_DEBUG_KEEP_JOB_FOLDER") != null || System.getenv("DM_DEBUG_JOB_FOLDER") != null; this.resultLifetime = jobResultLifetime; @@ -114,6 +117,7 @@ public void asyncBuildEngine(MetricsWriter metricsWriter, String platform, Strin .setBuildDirectory(buildDirectory) .setMetricsWriter(metricsWriter) .setProgressReporter(progressReporter) + .setR8Configuration(r8Configuration) .build(); // Resolve Gradle dependencies and .aar files shipped inside the extensions diff --git a/server/src/main/java/com/defold/extender/Extender.java b/server/src/main/java/com/defold/extender/Extender.java index ba72c139..00abdb5e 100644 --- a/server/src/main/java/com/defold/extender/Extender.java +++ b/server/src/main/java/com/defold/extender/Extender.java @@ -32,6 +32,7 @@ import java.util.Collections; import java.util.Enumeration; import java.util.Set; +import java.util.TreeSet; import java.util.ArrayList; import java.util.HashSet; import java.util.HashMap; @@ -42,6 +43,7 @@ import java.util.zip.ZipEntry; import java.util.zip.ZipFile; +import com.defold.extender.services.GradleArtifact; import com.defold.extender.services.GradleService; import com.defold.extender.services.cocoapods.CocoaPodsService; import com.defold.extender.services.cocoapods.PodBuildSpec; @@ -67,6 +69,7 @@ class Extender { private final TemplateExecutor templateExecutor = new TemplateExecutor(); private final ProcessExecutor processExecutor = new ProcessExecutor(); private final ProgressReporter progressReporter; + private final R8Configuration r8Configuration; private MetricsWriter metricsWriter; // context flags private Boolean needsCSLibraries = false; @@ -77,9 +80,12 @@ class Extender { private List extDirs; private List manifests; // The list of ext.manifests found in the upload - // Unpacked Android dependencies: a .jar file, or a directory named "*.aar" holding an exploded - // .aar. They come from Gradle/Maven, or from .aar files shipped inside an extension. + // Android dependencies: a standalone .jar, or an exploded .aar directory in either the local + // archive layout or AGP's EXPLODED_AAR layout. private List androidPackages; + // Maven AARs keep their legacy externally visible package names even though their content is + // consumed directly from AGP's transform cache. + private Map androidPackageResourceNames; private List outputFiles; private ResolvedPods resolvedPods; private int nameCounter = 0; @@ -111,22 +117,6 @@ class Extender { "ps5", "x86_64-ps5", }; - // This class specifies the set of files that are used when running proguard on - // the project jars that were found during the build process. This class is only - // relevant on Android. - // - // * proGuardFiles - Array of .pro files that contain settings and rules that specify - // what ProGuard should do with the input jars. - // * libraryJars - Array of .jar files should be passed to ProGuard as '-libraryjar' entries. - // Everything from a libraryjar will be kept by ProGuard, i.e no optimization or - // obfuscation will be performed. - private static class ProGuardContext { - public List proGuardFiles = new ArrayList<>(); - public List libraryJars = new ArrayList<>(); - } - - private static final boolean DM_DEBUG_DISABLE_PROGUARD = System.getenv("DM_DEBUG_DISABLE_PROGUARD") != null; - static public class Builder { String platform; File sdk; @@ -136,6 +126,7 @@ static public class Builder { Map env = new HashMap(); MetricsWriter metricsWriter; ProgressReporter progressReporter = ProgressReporter.NOOP; + R8Configuration r8Configuration = new R8Configuration(); public Builder() { } @@ -179,6 +170,11 @@ public Builder setProgressReporter(ProgressReporter progressReporter) { return this; } + public Builder setR8Configuration(R8Configuration r8Configuration) { + this.r8Configuration = r8Configuration; + return this; + } + public Extender build() throws IOException, ExtenderException { return new Extender(this); } @@ -187,7 +183,9 @@ public Extender build() throws IOException, ExtenderException { private Extender(Builder builder) throws IOException, ExtenderException { this.metricsWriter = builder.metricsWriter; this.progressReporter = builder.progressReporter != null ? builder.progressReporter : ProgressReporter.NOOP; + this.r8Configuration = builder.r8Configuration; this.androidPackages = new ArrayList<>(); + this.androidPackageResourceNames = new HashMap<>(); this.outputFiles = new ArrayList<>(); // Read config from SDK @@ -301,6 +299,12 @@ private Extender(Builder builder) throws IOException, ExtenderException { Set keys = this.platformConfig.env.keySet(); for (String k : keys) { + // Older SDKs declare PROGUARD in build.yml. The old command is never + // used, so do not require the removed ANDROID_PROGUARD environment. + // TODO: Remove this compatibility workaround after 2027-02-26. + if (k.equals("PROGUARD")) { + continue; + } String v = this.platformConfig.env.get(k); v = templateExecutor.execute(v, envContext); processExecutor.putEnv(k, v); @@ -355,6 +359,10 @@ private File uniqueTmpFile(String prefix, String suffix) { private String executeCommand(String template, Map context) throws ExtenderException { String command = templateExecutor.execute(template, context); + return executeCommandLine(command); + } + + private String executeCommandLine(String command) throws ExtenderException { try { if (processExecutor.execute(command) != 0) { throw new ExtenderException(processExecutor.getOutput()); @@ -470,6 +478,17 @@ private List getExtensionLibAars(File extDir) { return aars; } + List getExtensionLocalAarJars(File extDir) throws ExtenderException { + Set jars = new TreeSet<>(); + File localAarsDir = new File(buildState.buildDir, "local_aars"); + for (String path : getExtensionLibAars(extDir)) { + File aar = new File(path); + File unpacked = SandboxedPath.resolve(localAarsDir, extDir.getName() + "-" + aar.getName()); + jars.addAll(R8Builder.getAndroidPackageJars(unpacked)); + } + return new ArrayList<>(jars); + } + private List getAllExtensionsLibJars() { List allLibJars = new ArrayList<>(); for (File extDir : this.extDirs) { @@ -479,23 +498,10 @@ private List getAllExtensionsLibJars() { // Where we previously stored the dependencies directly inside the extensions // we now use gradle to resolve the dependencies for (File f : androidPackages) { - if (f.getName().endsWith(".jar")) + if (f.isFile() && f.getName().endsWith(".jar")) allLibJars.add(f.getAbsolutePath()); - else if(f.getName().endsWith(".aar")) { - File classesJar = new File(f, "classes.jar"); - if (classesJar.exists()) { - allLibJars.add(classesJar.getAbsolutePath()); - } - - // There can be an optional libs/ folder with jar files. - // Make sure to copy these! - // https://developer.android.com/studio/projects/android-library.html#aar-contents - File libs = new File(f, "libs"); - if (libs.exists() && libs.isDirectory()) { - for(File lib : libs.listFiles()) { - allLibJars.add(lib.getAbsolutePath()); - } - } + else if (f.isDirectory()) { + allLibJars.addAll(R8Builder.getAndroidPackageJars(f)); } } @@ -1684,6 +1690,48 @@ private static File createDir(File parent, String child) throws IOException { return dir; } + static String getCompiledResourceDirectoryName(int index, File resourceDirectory) { + File packageDirectory = resourceDirectory.getParentFile(); + String packageName = packageDirectory == null ? "resources" : packageDirectory.getName(); + String safePackageName = packageName.replaceAll("[^A-Za-z0-9._-]", "_"); + return String.format("%04d-%s", index, safePackageName); + } + + static List getReturnedResourcePackageNames( + List resourceDirectories, + Map preferredPackageNames) throws IOException { + List baseNames = new ArrayList<>(); + for (String resourceDirectory : resourceDirectories) { + File packageDirectory = new File(resourceDirectory).getParentFile(); + String preferredName = packageDirectory == null + ? null + : preferredPackageNames.get(packageDirectory.getCanonicalFile()); + baseNames.add(preferredName != null + ? preferredName + : packageDirectory == null ? "resources" : packageDirectory.getName()); + } + Set reservedNames = new HashSet<>(baseNames); + Set assignedNames = new HashSet<>(); + List result = new ArrayList<>(); + + for (int index = 0; index < baseNames.size(); index++) { + String baseName = baseNames.get(index); + if (assignedNames.add(baseName)) { + // Preserve the existing externally visible package name whenever possible. + result.add(baseName); + continue; + } + + String candidate = String.format("%s-%04d", baseName, index); + int retry = 1; + while (reservedNames.contains(candidate) || !assignedNames.add(candidate)) { + candidate = String.format("%s-%04d-%d", baseName, index, retry++); + } + result.add(candidate); + } + return result; + } + /** * Compile android resources into "flat" files * https://developer.android.com/studio/build/building-cmdline#compile_and_link_your_apps_resources @@ -1695,13 +1743,15 @@ private File compileAndroidResources(List resourceDirectories, Map context = createContext(mergedAppContext); + int resourceDirectoryIndex = 0; for (String resDir : resourceDirectories) { - // /tmp/.gradle/unpacked/android.arch.lifecycle-livedata-1.1.1.aar/res File resourceDirectory = new File(resDir); - // android.arch.lifecycle-livedata-1.1.1.aar - String packageName = resourceDirectory.getParentFile().getName(); + String packageName = getCompiledResourceDirectoryName( + resourceDirectoryIndex++, + resourceDirectory); - // we compile the package resources to one output directory per package + // Keep one output directory per input. Different Maven groups may publish AARs + // with the same filename, while AGP's exploded directory names omit the group. File packageDirectoryOut = createDir(outputDirectory, packageName); context.put("outputDirectory", packageDirectoryOut.getAbsolutePath()); @@ -1767,9 +1817,15 @@ private Map linkAndroidResources(File compiledResourcesDir, Map buildJavaExtension(File manifest, Map manifestContext, File rJar) throws ExtenderException { + // Returns the compiled extension jar together with its R8 rules/protection context. + private Map.Entry buildJavaExtension(File manifest, Map manifestContext, File rJar) throws ExtenderException { try { // Collect all Java source files File extDir = manifest.getParentFile(); @@ -1880,39 +1934,22 @@ private Map.Entry buildJavaExtension(File manifest, Map proGuardSrcFiles = new ArrayList<>(); - if (manifestDir.isDirectory()) { - proGuardSrcFiles = FileUtils.listFiles(manifestDir, null, true); - proGuardSrcFiles = ExtenderUtil.filterFiles(proGuardSrcFiles, platformConfig.proGuardSourceRe); - } - // We want to collect ProGuards files even if we don't have java or jar files for the build - // because it's possible that this extention depends on some base extension - if (javaSrcFiles.size() == 0 && proGuardSrcFiles.size() == 0) { + List extensionLibJars = getExtensionLibJars(extDir); + extensionLibJars.addAll(getExtensionLocalAarJars(extDir)); + R8Builder.ExtensionContext r8Context = R8Builder.createExtensionContext( + extDir, + extensionLibJars, + platformConfig.r8RuleSourceRe); + + // Rules can apply to a base extension with no Java sources, and a jar-only + // extension still needs a protection context when it has no rules. + if (javaSrcFiles.size() == 0 && r8Context.ruleFiles.isEmpty() && extensionLibJars.size() == 0) { LOGGER.info("No Java sources. Skipping"); return null; } LOGGER.info("Building Java sources with extension source {}", buildState.uploadDir); - ProGuardContext proGuardContext = new ProGuardContext(); - - // * If we found proguard files, we add all of them to the proguard context - // for this extension. It is implied that if there are .pro files present, - // then the extension developer is responsible to make sure the correct classes and symbols are kept. - // * However, if no proguard files were found, we need to add all potential jar files - // from the extension lib folder into the context so that we can set them as -libraryjar when - // running proguard. - if (proGuardSrcFiles.size() > 0) { - for (File pFile : proGuardSrcFiles) { - proGuardContext.proGuardFiles.add(pFile.getAbsolutePath()); - } - } else { - // Get extension supplied Jar libraries - List extJars = getExtensionLibJars(extDir); - proGuardContext.libraryJars = new ArrayList<>(extJars); - } - // Create temp working directory, which will include; // * classes/ - Output directory of javac compilation // * sources.txt - Text file with list of Java sources @@ -1923,11 +1960,9 @@ private Map.Entry buildJavaExtension(File manifest, Map(proguardFakeJar, proGuardContext); + // Preserve the context even when this extension has no compiled jar. + File r8FakeJar = new File(tmpDir, R8Builder.RULES_WITHOUT_JAR); + return new AbstractMap.SimpleEntry(r8FakeJar, r8Context); } File classesDir = new File(tmpDir, "classes"); @@ -1970,20 +2005,15 @@ private Map.Entry buildJavaExtension(File manifest, Map(outputJar, proGuardContext); + return new AbstractMap.SimpleEntry(outputJar, r8Context); } catch (IOException e) { throw new ExtenderException(e, "Building java extension"); } } - // returns: - // a file path to the built jar as well as a (potential) list - // of proguard files that should be applied to the final application jar. - // If the collection contains zero entries, then the jar should be treated as a library jar, - // which means that it should not be obfuscated or optimized. - private Map buildJava(File rJar) throws ExtenderException { - Map builtJars = new HashMap<>(); + private Map buildJava(File rJar) throws ExtenderException { + Map builtJars = new HashMap<>(); if (rJar != null) { builtJars.put(rJar.getAbsolutePath(), null); @@ -1994,7 +2024,7 @@ private Map buildJava(File rJar) throws ExtenderExceptio Map extensionContext = manifestConfigs.get(extensionSymbol); File extensionManifest = manifestFiles.get(extensionSymbol); - Map.Entry javaExtensionsEntry = buildJavaExtension(extensionManifest, extensionContext, rJar); + Map.Entry javaExtensionsEntry = buildJavaExtension(extensionManifest, extensionContext, rJar); if (javaExtensionsEntry != null) { builtJars.put(javaExtensionsEntry.getKey().getAbsolutePath(), javaExtensionsEntry.getValue()); } @@ -2002,19 +2032,15 @@ private Map buildJava(File rJar) throws ExtenderExceptio return builtJars; } - // arguments: - // extensionJarMap - a mapping from a jar file to a list of its corresponding proGuard files // returns: // all jar files from each extension, as well as the engine defined jar files - private List getAllJars(Map extensionJarMap) throws ExtenderException { + private List getAllJars(Map extensionJarMap) throws ExtenderException { List includeJars = ExtenderUtil.getStringList(mergedAppContext, "includeJars"); List excludeJars = ExtenderUtil.getStringList(mergedAppContext, "excludeJars"); List extensionJars = getAllExtensionsLibJars(); - for (Map.Entry extensionJar : extensionJarMap.entrySet()) { - extensionJars.add(extensionJar.getKey()); - } + extensionJars.addAll(R8Builder.getCompiledJars(extensionJarMap)); Map context = createContext(mergedAppContext); List allJars = ExtenderUtil.pruneItems( (List)context.get("engineJars"), includeJars, excludeJars); @@ -2022,122 +2048,6 @@ private List getAllJars(Map extensionJarMap) thr return allJars; } - // arguments: - // jars - the list of all available jar files gathered from the build - // extensionJarMap - a mapping from a jar file to a list of its corresponding proGuard contexts - private Map getProGuardMapping(List jars, Map extensionJarMap) { - Map jarToProGuardContextMap = new HashMap<>(); - - for (String jar : jars) { - jarToProGuardContextMap.put(jar, null); - } - - for (Map.Entry extensionJarEntry : extensionJarMap.entrySet()) { - String jar = extensionJarEntry.getKey(); - ProGuardContext ctx = extensionJarEntry.getValue(); - - // rJars from the buildRJar function will exist in the extensionJarMap, - // but associated with a null context. - if (ctx == null) { - continue; - } - - jarToProGuardContextMap.put(jar,ctx); - - // If we couldn't find any proguard files for this extension, - // we need to make sure that there is a context available - // for all the .jar files we found in the extension - if (ctx.proGuardFiles.size() == 0) { - for (String libraryJar : ctx.libraryJars) { - ProGuardContext libraryCtx = jarToProGuardContextMap.get(libraryJar); - - if (libraryCtx == null) { - libraryCtx = new ProGuardContext(); - libraryCtx.proGuardFiles.addAll(ctx.proGuardFiles); - jarToProGuardContextMap.put(libraryJar,libraryCtx); - } - } - } - } - - return jarToProGuardContextMap; - } - - // arguments: - // allJars - the list of all available jar files gathered from the build - // extensionJarMap - a mapping from a jar file to a list of its corresponding proGuard contexts - // returns: - // a pair of the built & optimized proGuard jar and its corresponding mappings.txt file. - // the mappings file can be uploaded to google play and then used for symbolication - private Map.Entry buildProGuard(List allJars, Map extensionJarMap) throws ExtenderException { - // To support older versions of build.yml where proGuardCmd is not defined: - String proGuardCmd = platformConfig.proGuardCmd; - if (proGuardCmd == null || proGuardCmd.isEmpty() || DM_DEBUG_DISABLE_PROGUARD) { - if (DM_DEBUG_DISABLE_PROGUARD) { - LOGGER.info("ProGuard support disabled by environment flag DM_DEBUG_DISABLE_PROGUARD"); - } else { - LOGGER.info("No SDK support. Skipping ProGuard step."); - } - return null; - } - - File appPro = new File(buildState.uploadDir, "/_app/app.pro"); - if (!appPro.exists()) { - LOGGER.info("No .pro file present. Skipping ProGuard step."); - return null; - } - - LOGGER.info("Building using ProGuard {}", buildState.uploadDir); - - String appProPath = appPro.getAbsolutePath(); - Map allJarsMap = getProGuardMapping(allJars, extensionJarMap); - - List allPro = new ArrayList<>(); - allPro.add(appProPath); - - File targetFile = new File(buildState.buildDir, "dmengine.jar"); - File mappingFile = new File(buildState.buildDir, "mapping.txt"); - - List jarList = new ArrayList<>(); - List jarLibrariesList = new ArrayList<>(); - - for (Map.Entry jarMapEntry : allJarsMap.entrySet()) - { - String jar = jarMapEntry.getKey(); - ProGuardContext jarProGuardContext = jarMapEntry.getValue(); - - // jarProGuardContext is null for all the jars that are affected by the - // 'global' appPro file. We could make a context for them, but it's not necessary - if (jarProGuardContext == null || jarProGuardContext.proGuardFiles.size() > 0) { - jarList.add(jar); - - if (jarProGuardContext != null) { - for (String proGuardFile : jarProGuardContext.proGuardFiles) { - allPro.add(proGuardFile); - } - } - } else { - jarLibrariesList.add(jar); - } - } - //exclude fake `jar` paths for extensions without java code - List excludeJars = new ArrayList<>(); - excludeJars.add("(.*)/proguard_files_without_jar"); - jarLibrariesList = ExtenderUtil.excludeItems(jarLibrariesList, excludeJars); - jarList = ExtenderUtil.excludeItems(jarList, excludeJars); - - Map context = createContext(mergedAppContext); - context.put("jars", jarList); - context.put("libraryjars", jarLibrariesList); - context.put("src", allPro); - context.put("tgt", targetFile.getAbsolutePath()); - context.put("mapping", mappingFile.getAbsolutePath()); - - executeCommand(proGuardCmd, context); - - return new AbstractMap.SimpleEntry(targetFile, mappingFile); - } - public void validateManifestPlatforms(ManifestConfiguration manifestConfig) throws ExtenderException { if (manifestConfig.platforms == null) { return; @@ -2192,7 +2102,7 @@ private File buildMainDexList(List jars) throws ExtenderException { try { mainList.createNewFile(); for (String classFile : mainClassNames) { - // create main dex list in form of Proguards rules. Additional info https://github.com/defold/extender/issues/393 + // Create the main dex list in R8 keep-rule form. Additional info: https://github.com/defold/extender/issues/393 classFile = classFile.replace("/", ".").replace(".class", ""); FileUtils.writeStringToFile(mainList, String.format("-keep class %s { *; }\n", classFile), Charset.defaultCharset(), true); } @@ -2332,11 +2242,6 @@ private void loadManifests(ExtensionManifestValidator validator) throws IOExcept mergedAppContext.put("platform", buildState.fullPlatform); mergedAppContext.put("host_platform", buildState.getHostPlatform()); - //exclude fake `jar` path for extensions without java code - List excludeJars = ExtenderUtil.getStringList(mergedAppContext, "excludeJars"); - excludeJars.add("(.*)/proguard_files_without_jar"); - mergedAppContext.put("excludeJars", excludeJars); - mergedAppContext = ExtenderUtil.mergeContexts(mergedAppContext, debugContext); } @@ -2552,9 +2457,14 @@ private List copyAndroidResourceFolders(List androidResourceFolder List packagesList = new ArrayList<>(); try { - for (String androidResourceFolder : androidResourceFolders) { + List packageNames = getReturnedResourcePackageNames( + androidResourceFolders, + androidPackageResourceNames); + for (int index = 0; index < androidResourceFolders.size(); index++) { + String androidResourceFolder = androidResourceFolders.get(index); File packageResourceDir = new File(androidResourceFolder); - File targetDir = new File(packagesDir, packageResourceDir.getParentFile().getName() + "/res"); + String packageName = packageNames.get(index); + File targetDir = new File(packagesDir, packageName + "/res"); FileUtils.copyDirectory(packageResourceDir, targetDir); String relativePath = ExtenderUtil.getRelativePath(packagesDir, targetDir); @@ -2602,7 +2512,7 @@ private List getExtraPackagesFromAndroidPackages() throws ExtenderExcept Set extraPackages = new HashSet(); try { for (File f : androidPackages) { - if(f.getName().endsWith(".aar")) { + if (f.isDirectory()) { File res = new File(f, "res"); File androidManifest = new File(f, "AndroidManifest.xml"); if (res.exists() && androidManifest.exists()) { @@ -2631,6 +2541,7 @@ private List buildAndroid(String platform) throws ExtenderException { final List androidResourceFolders = getAndroidResourceFolders(platform); File rJavaDir = null; + File aaptKeepRules = null; // 1.2.174 if (platformConfig.aapt2compileCmd != null) { // compile and link all of the resource files @@ -2642,6 +2553,7 @@ private List buildAndroid(String platform) throws ExtenderException { outputFiles.add(files.get("outApkFile")); outputFiles.add(files.get("resourceIdsFile")); rJavaDir = files.get("outJavaDirectory"); + aaptKeepRules = files.get("aaptKeepRules"); } else { rJavaDir = generateRJava(androidResourceFolders, mergedAppContext); @@ -2650,44 +2562,41 @@ private List buildAndroid(String platform) throws ExtenderException { // take the generated R.java files and compile them to jar files File rJar = buildRJar(rJavaDir); - Map extensionJarMap = buildJava(rJar); - List allJars = getAllJars(extensionJarMap); - Map.Entry proGuardFiles = buildProGuard(allJars, extensionJarMap); - - File mainDexList = buildMainDexList(allJars); - - // If we have proGuard support, we need to reset the allJars list so that - // we don't get duplicate symbols. - if (proGuardFiles != null) { - allJars.clear(); - allJars.add(proGuardFiles.getKey().getAbsolutePath()); // built jar - outputFiles.add(proGuardFiles.getValue()); // mappings file - - // Add the jars that were not run through ProGuard - for (Map.Entry extensionJarEntry : extensionJarMap.entrySet()) { - String extensionJar = extensionJarEntry.getKey(); - ProGuardContext proGuardContext = extensionJarEntry.getValue(); - - if (proGuardContext != null && proGuardContext.proGuardFiles.size() == 0) { - allJars.add(extensionJar); - - for (String extensionLibraryJar : proGuardContext.libraryJars) { - allJars.add(extensionLibraryJar); - } - } + Map extensionJarMap = buildJava(rJar); + List allJars = getAllJars(extensionJarMap); + R8Builder r8Builder = new R8Builder( + buildState.uploadDir, + buildState.buildDir, + platformConfig, + androidPackages, + createContext(mergedAppContext), + buildState.getMinAndroidSdkVersion(), + r8Configuration, + templateExecutor, + (command, commandContext) -> executeCommandLine(command)); + R8Builder.BuildOutput r8Output = r8Builder.build( + allJars, + extensionJarMap, + aaptKeepRules); + if (r8Output != null) { + outputFiles.addAll(Arrays.asList(r8Output.dexFiles)); + outputFiles.add(r8Output.mappingFile); + outputFiles.addAll(Arrays.asList(r8Output.metaInformationFiles)); + } else { + File mainDexList = buildMainDexList(allJars); + File[] classesDex = buildClassesDex(allJars, mainDexList); + if (classesDex.length > 0) { + outputFiles.addAll(Arrays.asList(classesDex)); } } - File[] classesDex = buildClassesDex(allJars, mainDexList); - if (classesDex.length > 0) { - outputFiles.addAll(Arrays.asList(classesDex)); - } - outputFiles.addAll(copyAndroidResourceFolders(androidResourceFolders)); outputFiles.addAll(copyAndroidAssetFolders(platform)); outputFiles.addAll(copyAndroidJniFolders(platform)); - outputFiles.addAll(copyMetaInformationFiles(allJars)); + if (r8Output == null) { + outputFiles.addAll(copyMetaInformationFiles(allJars)); + } return outputFiles; } @@ -2835,7 +2744,17 @@ File writeLog() { void resolve(GradleService gradleService) throws ExtenderException { try { - androidPackages.addAll(gradleService.resolveDependencies(this.buildState, this.platformConfig.context, outputFiles)); + for (GradleArtifact artifact : gradleService.resolveDependencies( + this.buildState, + this.platformConfig.context, + outputFiles)) { + androidPackages.add(artifact.getFile()); + if (artifact.getResourcePackageName() != null) { + androidPackageResourceNames.put( + artifact.getFile().getCanonicalFile(), + artifact.getResourcePackageName()); + } + } } catch (IOException e) { throw new ExtenderException(e, "Failed to resolve Gradle dependencies. " + e.getMessage()); @@ -2844,15 +2763,15 @@ void resolve(GradleService gradleService) throws ExtenderException { // Unpack the .aar files shipped inside the extensions into the same exploded layout that the // Gradle service produces, so that their classes.jar, libs/, res/, assets/, jni/ and - // AndroidManifest.xml are consumed just like those of a Maven resolved .aar. + // AndroidManifest.xml are consumed just like AGP's exploded Maven dependencies. void resolveLocalAars() throws ExtenderException { File aarsDir = new File(buildState.buildDir, "local_aars"); for (File extDir : this.extDirs) { for (String path : getExtensionLibAars(extDir)) { File aar = new File(path); - // The name must be unique among all packages and must end with ".aar", since that is - // what the consumers key off, and it also names the resource package of the .aar. + // The name must be unique among all local packages. Keeping the .aar suffix also + // makes retained debug jobs easy to inspect; consumers identify packages by layout. File unpacked = SandboxedPath.resolve(aarsDir, extDir.getName() + "-" + aar.getName()); if (unpacked.exists()) { // the same .aar was found in both lib/android and lib/-android diff --git a/server/src/main/java/com/defold/extender/ExtenderUtil.java b/server/src/main/java/com/defold/extender/ExtenderUtil.java index 9c0ff7a1..88fc1032 100644 --- a/server/src/main/java/com/defold/extender/ExtenderUtil.java +++ b/server/src/main/java/com/defold/extender/ExtenderUtil.java @@ -930,6 +930,7 @@ public static boolean isMetaInfEntryValuable(ZipEntry entry) { && !entryName.endsWith(".kotlin_module") && !entryName.endsWith(".class") && !entryName.endsWith(".pro") + && !R8Builder.isEmbeddedRuleEntryName(entryName) && !entryName.endsWith("pom.xml") && !entryName.endsWith("pom.properties"); } diff --git a/server/src/main/java/com/defold/extender/PlatformConfig.java b/server/src/main/java/com/defold/extender/PlatformConfig.java index 2e668e43..24238397 100644 --- a/server/src/main/java/com/defold/extender/PlatformConfig.java +++ b/server/src/main/java/com/defold/extender/PlatformConfig.java @@ -29,8 +29,12 @@ public class PlatformConfig { public String manifestName; public String manifestMergeCmd; public String bitcodeStripCmd; // deprecated - public String proGuardSourceRe; - public String proGuardCmd; + public String r8RuleSourceRe; + public String r8Cmd; + public String r8Version; + // Legacy SDK deserialization only. Extender deliberately never reads these fields. + @Deprecated public String proGuardCmd; + @Deprecated public String proGuardSourceRe; public String windresCmd; public String symbolCmd; public String symbolsPattern; diff --git a/server/src/main/java/com/defold/extender/R8Builder.java b/server/src/main/java/com/defold/extender/R8Builder.java new file mode 100644 index 00000000..e77f905a --- /dev/null +++ b/server/src/main/java/com/defold/extender/R8Builder.java @@ -0,0 +1,1071 @@ +package com.defold.extender; + +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.DataInputStream; +import java.io.File; +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Collectors; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +/** Builds Android dex files with R8 and owns all R8 rule discovery and assembly. */ +final class R8Builder { + private static final Logger LOGGER = LoggerFactory.getLogger(R8Builder.class); + + static final String RULES_WITHOUT_JAR = "r8_rules_without_jar"; + + private static final boolean DM_DEBUG_DISABLE_R8 = System.getenv("DM_DEBUG_DISABLE_R8") != null; + private static final String APP_RULES_PATH = "_app/app.keep"; + private static final String JAR_LEGACY_RULE_PREFIX = "META-INF/proguard/"; + private static final String JAR_R8_RULE_PREFIX = "META-INF/com.android.tools/r8"; + private static final String GENERATED_EXTENSION_KEEP_ATTRIBUTES = + "-keepattributes *Annotation*,Signature,InnerClasses,EnclosingMethod,MethodParameters,Exceptions"; + static final class ExtensionContext { + final List ruleFiles = new ArrayList<>(); + final List protectedJars = new ArrayList<>(); + } + + static final class BuildOutput { + final File[] dexFiles; + final File mappingFile; + final File[] metaInformationFiles; + + BuildOutput(File[] dexFiles, File mappingFile, File[] metaInformationFiles) { + this.dexFiles = dexFiles; + this.mappingFile = mappingFile; + this.metaInformationFiles = metaInformationFiles; + } + } + + @FunctionalInterface + interface CommandExecutor { + void execute(String commandTemplate, Map context) throws ExtenderException; + } + + private final File uploadDir; + private final File buildDir; + private final PlatformConfig platformConfig; + private final List androidPackages; + private final Map commandContext; + private final int minAndroidSdkVersion; + private final R8Configuration r8Configuration; + private final TemplateExecutor templateExecutor; + private final CommandExecutor commandExecutor; + + R8Builder( + File uploadDir, + File buildDir, + PlatformConfig platformConfig, + List androidPackages, + Map commandContext, + int minAndroidSdkVersion, + R8Configuration r8Configuration, + TemplateExecutor templateExecutor, + CommandExecutor commandExecutor) { + this.uploadDir = uploadDir; + this.buildDir = buildDir; + this.platformConfig = platformConfig; + this.androidPackages = androidPackages; + this.commandContext = commandContext; + this.minAndroidSdkVersion = minAndroidSdkVersion; + this.r8Configuration = r8Configuration; + this.templateExecutor = templateExecutor; + this.commandExecutor = commandExecutor; + } + + static ExtensionContext createExtensionContext( + File extensionDir, + List extensionLibJars, + String ruleSourceRegex) { + ExtensionContext context = new ExtensionContext(); + File manifestDir = new File(extensionDir, "manifests/android"); + if (manifestDir.isDirectory() && ruleSourceRegex != null && !ruleSourceRegex.isBlank()) { + Collection candidates = FileUtils.listFiles(manifestDir, null, true); + List ruleFiles = ExtenderUtil.filterFiles(candidates, ruleSourceRegex); + ruleFiles.sort((left, right) -> left.getAbsolutePath().compareTo(right.getAbsolutePath())); + for (File ruleFile : ruleFiles) { + context.ruleFiles.add(ruleFile.getAbsolutePath()); + } + } + + if (context.ruleFiles.isEmpty()) { + context.protectedJars.addAll(extensionLibJars); + } + return context; + } + + static boolean isRulesOnlyPlaceholder(String path) { + return path.endsWith(RULES_WITHOUT_JAR); + } + + static List getCompiledJars(Map extensionJarMap) { + return extensionJarMap.keySet().stream() + .filter(path -> !isRulesOnlyPlaceholder(path)) + .sorted() + .collect(Collectors.toList()); + } + + static List getAndroidPackageJars(File androidPackage) { + Set jars = new TreeSet<>(); + File classesJar = getAndroidPackageClassesJar(androidPackage); + if (classesJar.isFile()) { + jars.add(classesJar.getAbsolutePath()); + } + for (File libsDir : List.of( + new File(androidPackage, "libs"), + new File(androidPackage, "jars/libs"))) { + File[] libraryJars = libsDir.listFiles(file -> file.isFile() && file.getName().endsWith(".jar")); + if (libraryJars != null) { + for (File libraryJar : libraryJars) { + jars.add(libraryJar.getAbsolutePath()); + } + } + } + return new ArrayList<>(jars); + } + + static File getAndroidPackageClassesJar(File androidPackage) { + File unpackedClassesJar = new File(androidPackage, "classes.jar"); + if (unpackedClassesJar.isFile()) { + return unpackedClassesJar; + } + return new File(androidPackage, "jars/classes.jar"); + } + + private static final class R8SemanticVersion { + private final int major; + private final int minor; + private final int patch; + private final String prerelease; + + private R8SemanticVersion(int major, int minor, int patch, String prerelease) { + this.major = major; + this.minor = minor; + this.patch = patch; + this.prerelease = prerelease; + } + + static R8SemanticVersion parse(String value) { + int firstDot = value.indexOf('.'); + if (firstDot <= 0) { + throw new IllegalArgumentException("Invalid R8 semantic version " + value); + } + int secondDot = value.indexOf('.', firstDot + 1); + if (secondDot <= firstDot + 1) { + throw new IllegalArgumentException("Invalid R8 semantic version " + value); + } + int prereleaseStart = value.indexOf('-', secondDot + 1); + int patchEnd = prereleaseStart < 0 ? value.length() : prereleaseStart; + if (patchEnd <= secondDot + 1) { + throw new IllegalArgumentException("Invalid R8 semantic version " + value); + } + try { + return new R8SemanticVersion( + Integer.parseInt(value.substring(0, firstDot)), + Integer.parseInt(value.substring(firstDot + 1, secondDot)), + Integer.parseInt(value.substring(secondDot + 1, patchEnd)), + prereleaseStart < 0 ? null : value.substring(prereleaseStart + 1)); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Invalid R8 semantic version " + value, e); + } + } + + int compareNumbers(R8SemanticVersion other) { + int comparison = Integer.compare(major, other.major); + if (comparison == 0) { + comparison = Integer.compare(minor, other.minor); + } + if (comparison == 0) { + comparison = Integer.compare(patch, other.patch); + } + return comparison; + } + + boolean isNewer(R8SemanticVersion other) { + return compareNumbers(other) > 0; + } + + boolean isNewerOrEqual(R8SemanticVersion other) { + return isNewer(other) || (compareNumbers(other) == 0 + && (prerelease == null ? other.prerelease == null : prerelease.equals(other.prerelease))); + } + } + + static int compareVersions(String left, String right) { + return R8SemanticVersion.parse(left).compareNumbers(R8SemanticVersion.parse(right)); + } + + static boolean isEmbeddedRuleEntryName(String entryName) { + if (entryName.startsWith(JAR_LEGACY_RULE_PREFIX)) { + return true; + } + if (!entryName.startsWith(JAR_R8_RULE_PREFIX)) { + return false; + } + String suffix = entryName.substring(JAR_R8_RULE_PREFIX.length()); + return suffix.startsWith("/") + || suffix.startsWith("-from-") + || suffix.startsWith("-upto-"); + } + + private static int indexOfEither(String value, char first, char second) { + int firstIndex = value.indexOf(first); + int secondIndex = value.indexOf(second); + if (firstIndex < 0) { + return secondIndex; + } + if (secondIndex < 0) { + return firstIndex; + } + return Math.min(firstIndex, secondIndex); + } + + private static boolean isApplicableR8RuleEntry(String entryName, String r8Version) { + if (!entryName.startsWith(JAR_R8_RULE_PREFIX)) { + return false; + } + String suffix = entryName.substring(JAR_R8_RULE_PREFIX.length()); + if (suffix.startsWith("/")) { + return true; + } + if (!suffix.startsWith("-from-") && !suffix.startsWith("-upto-")) { + return false; + } + + try { + R8SemanticVersion from = new R8SemanticVersion(0, 0, 0, null); + R8SemanticVersion upTo = null; + if (suffix.startsWith("-from-")) { + suffix = suffix.substring("-from-".length()); + int versionEnd = indexOfEither(suffix, '-', '/'); + if (versionEnd < 0) { + return false; + } + from = R8SemanticVersion.parse(suffix.substring(0, versionEnd)); + suffix = suffix.substring(versionEnd); + } + if (suffix.startsWith("-upto-")) { + suffix = suffix.substring("-upto-".length()); + int versionEnd = suffix.indexOf('/'); + if (versionEnd < 0) { + return false; + } + upTo = R8SemanticVersion.parse(suffix.substring(0, versionEnd)); + } + R8SemanticVersion compilerVersion = R8SemanticVersion.parse(r8Version); + return compilerVersion.isNewerOrEqual(from) + && (upTo == null || upTo.isNewer(compilerVersion)); + } catch (IllegalArgumentException e) { + return false; + } + } + + static boolean isApplicableRuleDirectory(String directory, String r8Version) { + if (!directory.startsWith("r8")) { + return false; + } + return isApplicableR8RuleEntry( + JAR_R8_RULE_PREFIX + directory.substring("r8".length()) + "/rules.keep", + r8Version); + } + + static List selectEmbeddedRuleEntries(File jar, String r8Version) + throws IOException, ExtenderException { + return selectEmbeddedRuleEntries(jar, r8Version, new R8Configuration()); + } + + static List selectEmbeddedRuleEntries( + File jar, + String r8Version, + R8Configuration r8Configuration) throws IOException, ExtenderException { + List targetedRules = new ArrayList<>(); + List legacyRules = new ArrayList<>(); + Set candidateNames = new LinkedHashSet<>(); + int candidateCount = 0; + + try (ZipFile zipFile = new ZipFile(jar)) { + Enumeration entries = zipFile.entries(); + while (entries.hasMoreElements()) { + ZipEntry entry = entries.nextElement(); + if (entry.isDirectory()) { + continue; + } + + String entryName = entry.getName(); + if (!isEmbeddedRuleEntryName(entryName)) { + continue; + } + if (!candidateNames.add(entryName)) { + throw new ExtenderException("Duplicate embedded R8 rule entry " + entryName + " in " + jar); + } + if (++candidateCount > r8Configuration.getMaxRuleFiles()) { + throw new ExtenderException(String.format( + "Too many embedded R8 rule files in %s (maximum %d)", + jar, + r8Configuration.getMaxRuleFiles())); + } + if (entryName.startsWith(JAR_LEGACY_RULE_PREFIX)) { + legacyRules.add(entryName); + continue; + } + if (isApplicableR8RuleEntry(entryName, r8Version)) { + targetedRules.add(entryName); + } + } + } + + Collections.sort(targetedRules); + Collections.sort(legacyRules); + return targetedRules.isEmpty() ? legacyRules : targetedRules; + } + + static boolean isRequested(File uploadDir) { + return new File(uploadDir, APP_RULES_PATH).isFile(); + } + + static void validateConfiguration(String r8Cmd, String r8Version) throws ExtenderException { + if (r8Cmd == null || r8Cmd.isBlank() || r8Version == null || r8Version.isBlank()) { + throw new ExtenderException("R8 shrinking was requested, but this Defold SDK does not provide r8Cmd and r8Version"); + } + try { + compareVersions(r8Version, r8Version); + } catch (IllegalArgumentException e) { + throw new ExtenderException("Invalid r8Version in this Defold SDK: " + r8Version); + } + } + + private void validateResolvedR8Environment(Map context) throws ExtenderException { + for (String name : List.of("R8", "R8_VERSION")) { + if (!platformConfig.env.containsKey(name)) { + continue; + } + Object value = context.get("env." + name); + if (!(value instanceof String) || ((String) value).isBlank()) { + validateConfiguration(null, null); + } + } + } + + private static boolean containsTargetedRules(List entries) { + return entries.stream().anyMatch(entry -> entry.startsWith(JAR_R8_RULE_PREFIX)); + } + + static List collectConsumerRules( + List allJars, + List androidPackages, + String r8Version, + File rulesRoot) throws ExtenderException { + return collectConsumerRules( + allJars, + androidPackages, + r8Version, + rulesRoot, + new R8Configuration()); + } + + static List collectConsumerRules( + List allJars, + List androidPackages, + String r8Version, + File rulesRoot, + R8Configuration r8Configuration) throws ExtenderException { + File emptyRuleBase = R8RulePolicy.createEmptyBaseDirectory( + rulesRoot.getAbsoluteFile().getParentFile()); + return collectConsumerRules( + allJars, + androidPackages, + r8Version, + rulesRoot, + r8Configuration, + new R8RulePolicy.Budget(r8Configuration), + emptyRuleBase); + } + + private static List collectConsumerRules( + List allJars, + List androidPackages, + String r8Version, + File rulesRoot, + R8Configuration r8Configuration, + R8RulePolicy.Budget ruleBudget, + File emptyRuleBase) throws ExtenderException { + try { + Files.createDirectories(rulesRoot.toPath()); + } catch (IOException e) { + throw new ExtenderException(e, "Failed to create R8 consumer rules directory " + rulesRoot); + } + + List sortedJars = new ArrayList<>(new LinkedHashSet<>(allJars)); + Collections.sort(sortedJars); + Map jarHasTargetedRules = new HashMap<>(); + List consumerRules = new ArrayList<>(); + int artifactIndex = 0; + + for (String jarPath : sortedJars) { + File jar = new File(jarPath); + if (!jar.isFile() || !jar.getName().endsWith(".jar")) { + continue; + } + + try (ZipFile zipFile = new ZipFile(jar)) { + List selectedEntries = selectEmbeddedRuleEntries( + jar, + r8Version, + r8Configuration); + jarHasTargetedRules.put(jar.getCanonicalPath(), containsTargetedRules(selectedEntries)); + if (selectedEntries.isEmpty()) { + continue; + } + + File artifactRulesDir = new File(rulesRoot, String.format("jar-%04d", artifactIndex++)); + for (int ruleIndex = 0; ruleIndex < selectedEntries.size(); ++ruleIndex) { + String entryName = selectedEntries.get(ruleIndex); + ZipEntry entry = zipFile.getEntry(entryName); + if (entry == null) { + throw new ExtenderException("Missing embedded R8 rule entry " + entryName + " in " + jar); + } + byte[] contents = R8RulePolicy.readAndValidate(zipFile, entry, ruleBudget); + File extractedRule = new File( + artifactRulesDir, + String.format("rule-%04d.keep", ruleIndex)); + R8RulePolicy.writeSanitized(extractedRule, contents, emptyRuleBase); + consumerRules.add(extractedRule.getAbsolutePath()); + } + } catch (IOException e) { + throw new ExtenderException(e, "Failed to read R8 consumer rules from " + jarPath); + } + } + + List sortedPackages = new ArrayList<>(androidPackages); + sortedPackages.sort((left, right) -> left.getAbsolutePath().compareTo(right.getAbsolutePath())); + for (File androidPackage : sortedPackages) { + if (!androidPackage.isDirectory()) { + continue; + } + + File legacyRules = new File(androidPackage, "proguard.txt"); + File classesJar = getAndroidPackageClassesJar(androidPackage); + try { + boolean hasTargetedRules = classesJar.isFile() + && jarHasTargetedRules.getOrDefault(classesJar.getCanonicalPath(), false); + if (legacyRules.isFile() && !hasTargetedRules) { + File extractedRules = new File(rulesRoot, String.format("aar-%04d.keep", artifactIndex++)); + byte[] contents = R8RulePolicy.readAndValidate(legacyRules, ruleBudget); + R8RulePolicy.writeSanitized(extractedRules, contents, emptyRuleBase); + consumerRules.add(extractedRules.getAbsolutePath()); + } + } catch (IOException e) { + throw new ExtenderException(e, "Failed to collect R8 consumer rules from " + androidPackage); + } + } + + return consumerRules; + } + + private static boolean hasEmbeddedRuleEntries(File jar) throws IOException { + try (org.apache.commons.compress.archivers.zip.ZipFile zipFile = + org.apache.commons.compress.archivers.zip.ZipFile.builder().setFile(jar).get()) { + Enumeration entries = zipFile.getEntriesInPhysicalOrder(); + while (entries.hasMoreElements()) { + ZipArchiveEntry entry = entries.nextElement(); + if (!entry.isDirectory() && isEmbeddedRuleEntryName(entry.getName())) { + return true; + } + } + return false; + } + } + + static File stripEmbeddedRuleEntries(File jar, File strippedJar) throws ExtenderException { + try { + if (!hasEmbeddedRuleEntries(jar)) { + return jar; + } + if (strippedJar.toPath().getParent() != null) { + Files.createDirectories(strippedJar.toPath().getParent()); + } + try (org.apache.commons.compress.archivers.zip.ZipFile zipFile = + org.apache.commons.compress.archivers.zip.ZipFile.builder().setFile(jar).get(); + ZipArchiveOutputStream output = new ZipArchiveOutputStream(strippedJar)) { + Enumeration entries = zipFile.getEntriesInPhysicalOrder(); + while (entries.hasMoreElements()) { + ZipArchiveEntry entry = entries.nextElement(); + if (!entry.isDirectory() && isEmbeddedRuleEntryName(entry.getName())) { + continue; + } + try (InputStream rawInput = zipFile.getRawInputStream(entry)) { + if (rawInput == null) { + throw new IOException("Missing raw ZIP data for " + entry.getName()); + } + output.addRawArchiveEntry(new ZipArchiveEntry(entry), rawInput); + } + } + } + return strippedJar; + } catch (IOException e) { + throw new ExtenderException(e, "Failed to remove embedded R8 rules from " + jar); + } + } + + private static List prepareProgramJars(List allJars, File strippedJarsDir) + throws ExtenderException { + List originalProgramJars = allJars.stream() + .filter(jar -> new File(jar).isFile()) + .distinct() + .sorted() + .collect(Collectors.toList()); + List programJars = new ArrayList<>(originalProgramJars.size()); + for (int index = 0; index < originalProgramJars.size(); ++index) { + File originalJar = new File(originalProgramJars.get(index)); + File strippedJar = new File(strippedJarsDir, String.format("program-%04d.jar", index)); + programJars.add(stripEmbeddedRuleEntries(originalJar, strippedJar).getPath()); + } + return programJars; + } + + private List getProtectedJars(Map extensionJarMap) { + Set protectedJars = new TreeSet<>(); + for (Map.Entry extensionEntry : extensionJarMap.entrySet()) { + ExtensionContext context = extensionEntry.getValue(); + if (context == null || !context.ruleFiles.isEmpty()) { + continue; + } + + String extensionJar = extensionEntry.getKey(); + if (!isRulesOnlyPlaceholder(extensionJar)) { + protectedJars.add(extensionJar); + } + protectedJars.addAll(context.protectedJars); + } + return new ArrayList<>(protectedJars); + } + + private static final class ClassFileReadBudget { + private long totalBytes; + } + + private static final class BoundedClassFileInputStream extends FilterInputStream { + private final ClassFileReadBudget budget; + private final int maxClassfileHeaderBytes; + private final long maxTotalClassfileHeaderBytes; + private int classBytes; + + BoundedClassFileInputStream( + InputStream input, + ClassFileReadBudget budget, + R8Configuration r8Configuration) { + super(input); + this.budget = budget; + this.maxClassfileHeaderBytes = r8Configuration.getMaxClassfileHeaderBytes(); + this.maxTotalClassfileHeaderBytes = r8Configuration.getMaxTotalClassfileHeaderBytes(); + } + + private int allowedBytes(int requested) throws IOException { + if (requested == 0) { + return 0; + } + int classRemaining = maxClassfileHeaderBytes - classBytes; + if (classRemaining <= 0) { + throw new IOException(String.format( + "Class file header exceeds the %d-byte read limit", + maxClassfileHeaderBytes)); + } + long totalRemaining = maxTotalClassfileHeaderBytes - budget.totalBytes; + if (totalRemaining <= 0) { + throw new IOException(String.format( + "Class file headers exceed the %d-byte aggregate read limit", + maxTotalClassfileHeaderBytes)); + } + return (int) Math.min(requested, Math.min(classRemaining, totalRemaining)); + } + + private void account(int count) { + if (count > 0) { + classBytes += count; + budget.totalBytes += count; + } + } + + @Override + public int read() throws IOException { + allowedBytes(1); + int value = super.read(); + if (value >= 0) { + account(1); + } + return value; + } + + @Override + public int read(byte[] bytes, int offset, int length) throws IOException { + int allowed = allowedBytes(length); + if (allowed == 0) { + return 0; + } + int count = super.read(bytes, offset, allowed); + account(count); + return count; + } + } + + private static String readClassInternalName( + ZipFile zipFile, + ZipEntry entry, + ClassFileReadBudget budget, + R8Configuration r8Configuration) throws IOException { + try (DataInputStream input = new DataInputStream(new BoundedClassFileInputStream( + zipFile.getInputStream(entry), + budget, + r8Configuration))) { + if (input.readInt() != 0xCAFEBABE) { + throw new IOException("Invalid class file magic"); + } + input.readUnsignedShort(); // minor_version + input.readUnsignedShort(); // major_version + + int constantPoolCount = input.readUnsignedShort(); + if (constantPoolCount <= 1) { + throw new IOException("Invalid class file constant pool count"); + } + byte[] tags = new byte[constantPoolCount]; + String[] utf8Values = new String[constantPoolCount]; + int[] classNameIndexes = new int[constantPoolCount]; + for (int index = 1; index < constantPoolCount; ++index) { + int tag = input.readUnsignedByte(); + tags[index] = (byte) tag; + switch (tag) { + case 1: + utf8Values[index] = input.readUTF(); + break; + case 3: + case 4: + input.readInt(); + break; + case 5: + case 6: + input.readLong(); + if (++index >= constantPoolCount) { + throw new IOException("Long or double overruns the class file constant pool"); + } + break; + case 7: + classNameIndexes[index] = input.readUnsignedShort(); + break; + case 8: + case 16: + case 19: + case 20: + input.readUnsignedShort(); + break; + case 9: + case 10: + case 11: + case 12: + case 17: + case 18: + input.readInt(); + break; + case 15: + input.readUnsignedByte(); + input.readUnsignedShort(); + break; + default: + throw new IOException("Unknown class file constant pool tag " + tag); + } + } + + input.readUnsignedShort(); // access_flags + int thisClass = input.readUnsignedShort(); + input.readUnsignedShort(); // super_class + if (thisClass <= 0 || thisClass >= constantPoolCount || tags[thisClass] != 7) { + throw new IOException("Invalid this_class constant pool index"); + } + int nameIndex = classNameIndexes[thisClass]; + if (nameIndex <= 0 || nameIndex >= constantPoolCount || tags[nameIndex] != 1) { + throw new IOException("Invalid class name constant pool index"); + } + return utf8Values[nameIndex]; + } + } + + private static boolean isR8LiteralSimpleNameCodePoint(int codePoint) { + boolean isDexSimpleNameCodePoint = (codePoint >= 'A' && codePoint <= 'Z') + || (codePoint >= 'a' && codePoint <= 'z') + || (codePoint >= '0' && codePoint <= '9') + || codePoint == '$' + || codePoint == '-' + || codePoint == '_' + || (codePoint >= 0x00A1 && codePoint <= 0x1FFF) + || (codePoint >= 0x2010 && codePoint <= 0x2027) + || (codePoint >= 0x2030 && codePoint <= 0xD7FF) + || (codePoint >= 0xE000 && codePoint <= 0xFEFE) + || (codePoint >= 0xFF00 && codePoint <= 0xFFEF) + || (codePoint >= 0x10000 && codePoint <= 0x10FFFF); + boolean isUnicodeSpace = codePoint == ' ' + || codePoint == 0x00A0 + || codePoint == 0x1680 + || (codePoint >= 0x2000 && codePoint <= 0x200A) + || codePoint == 0x202F + || codePoint == 0x205F + || codePoint == 0x3000; + return isDexSimpleNameCodePoint && !isUnicodeSpace; + } + + private static String toR8ClassPattern(String internalName) throws IOException { + if (internalName == null || internalName.isEmpty()) { + throw new IOException("Class file has an empty internal name"); + } + + StringBuilder pattern = new StringBuilder(internalName.length()); + boolean segmentIsEmpty = true; + for (int offset = 0; offset < internalName.length();) { + int codePoint = internalName.codePointAt(offset); + offset += Character.charCount(codePoint); + if (codePoint == '/') { + if (segmentIsEmpty || offset == internalName.length()) { + throw new IOException("Class file has an empty internal-name segment"); + } + pattern.append('.'); + segmentIsEmpty = true; + } else { + if (codePoint == '.' || codePoint == ';' || codePoint == '[') { + throw new IOException(String.format( + "Class file internal name contains forbidden character U+%04X", + codePoint)); + } + pattern.appendCodePoint(isR8LiteralSimpleNameCodePoint(codePoint) ? codePoint : '?'); + segmentIsEmpty = false; + } + } + return pattern.toString(); + } + + static File writeProtectedJarKeepRules(List jarPaths, File outputFile) + throws IOException, ExtenderException { + return writeProtectedJarKeepRules(jarPaths, outputFile, new R8Configuration()); + } + + static File writeProtectedJarKeepRules( + List jarPaths, + File outputFile, + R8Configuration r8Configuration) throws IOException, ExtenderException { + Set classNames = new TreeSet<>(); + int classEntryCount = 0; + ClassFileReadBudget classFileReadBudget = new ClassFileReadBudget(); + long generatedBytes = jarPaths.isEmpty() + ? 0 + : GENERATED_EXTENSION_KEEP_ATTRIBUTES.getBytes(StandardCharsets.UTF_8).length + 1L; + for (String jarPath : jarPaths) { + File jar = new File(jarPath); + if (!jar.isFile()) { + continue; + } + + try (ZipFile zipFile = new ZipFile(jar)) { + Enumeration entries = zipFile.entries(); + while (entries.hasMoreElements()) { + ZipEntry entry = entries.nextElement(); + String entryName = entry.getName(); + if (entry.isDirectory() + || !entryName.endsWith(".class") + || entryName.startsWith("META-INF/")) { + continue; + } + if (++classEntryCount > r8Configuration.getMaxGeneratedExtensionClasses()) { + throw new ExtenderException(String.format( + "Too many classes need generated R8 keep rules (maximum %d)", + r8Configuration.getMaxGeneratedExtensionClasses())); + } + + String internalName; + try { + internalName = readClassInternalName( + zipFile, + entry, + classFileReadBudget, + r8Configuration); + } catch (IOException e) { + throw new ExtenderException(e, String.format( + "Failed to read class file %s from %s: %s", + entryName, + jar, + e.getMessage())); + } + if (internalName.equals("module-info")) { + continue; + } + + String className; + try { + className = toR8ClassPattern(internalName); + } catch (IOException e) { + throw new ExtenderException(e, String.format( + "Invalid class name in %s from %s: %s", + entryName, + jar, + e.getMessage())); + } + if (classNames.add(className)) { + String rule = String.format("-keep class %s { *; }\n", className); + generatedBytes += rule.getBytes(StandardCharsets.UTF_8).length; + if (generatedBytes > r8Configuration.getMaxRuleFileBytes()) { + throw new ExtenderException(String.format( + "Generated R8 keep rules are too large (maximum %d bytes)", + r8Configuration.getMaxRuleFileBytes())); + } + } + } + } + } + + StringBuilder rules = new StringBuilder((int) generatedBytes); + if (!jarPaths.isEmpty()) { + rules.append(GENERATED_EXTENSION_KEEP_ATTRIBUTES).append('\n'); + } + for (String className : classNames) { + rules.append("-keep class ").append(className).append(" { *; }\n"); + } + Files.writeString(outputFile.toPath(), rules, StandardCharsets.UTF_8); + return outputFile; + } + + static String escapeDoubleQuotedCommandValue(String value) { + return value.replace("\\", "\\\\").replace("\"", "\\\""); + } + + static Map createR8CommandContext(Map context) { + Map escaped = new HashMap<>(context); + for (String key : List.of( + "env.R8", + "env.LIBRARYJAR", + "classes_dex_dir", + "mapping")) { + Object value = escaped.get(key); + if (value instanceof String) { + escaped.put(key, escapeDoubleQuotedCommandValue((String) value)); + } + } + for (String key : List.of("jars", "rules")) { + Object value = escaped.get(key); + if (value instanceof List) { + List escapedValues = ((List) value).stream() + .map(Object::toString) + .map(R8Builder::escapeDoubleQuotedCommandValue) + .collect(Collectors.toList()); + escaped.put(key, escapedValues); + } + } + return escaped; + } + + private File createR8OutputDirectory() throws ExtenderException { + File r8OutputDir = new File(buildDir, "r8-output"); + try { + FileUtils.deleteDirectory(r8OutputDir); + Files.createDirectories(r8OutputDir.toPath()); + } catch (IOException e) { + throw new ExtenderException(e, "Failed to create R8 output directory " + r8OutputDir); + } + return r8OutputDir; + } + + private File moveR8OutputFile(File r8OutputDir, File source) throws ExtenderException, IOException { + String relativePath = r8OutputDir.toPath() + .relativize(source.toPath()) + .toString() + .replace(File.separatorChar, '/'); + File target = SandboxedPath.resolve(buildDir, relativePath); + Files.createDirectories(target.getParentFile().toPath()); + Files.move(source.toPath(), target.toPath(), StandardCopyOption.REPLACE_EXISTING); + return target; + } + + private File[] collectR8DexFiles(File r8OutputDir) throws ExtenderException { + File[] generatedDexFiles = ExtenderUtil.listFilesMatching( + r8OutputDir, + "^classes(|[0-9]+)\\.dex$"); + Arrays.sort(generatedDexFiles, (left, right) -> left.getName().compareTo(right.getName())); + + File[] dexFiles = new File[generatedDexFiles.length]; + try { + for (int index = 0; index < generatedDexFiles.length; ++index) { + dexFiles[index] = moveR8OutputFile(r8OutputDir, generatedDexFiles[index]); + } + } catch (IOException e) { + throw new ExtenderException(e, "Failed to collect R8 dex output"); + } + return dexFiles; + } + + private File[] collectR8MetaInformationFiles(File r8OutputDir) throws ExtenderException { + File metaInfDir = new File(r8OutputDir, "META-INF"); + if (!metaInfDir.isDirectory()) { + return new File[0]; + } + + List generatedFiles = new ArrayList<>(FileUtils.listFiles(metaInfDir, null, true)); + generatedFiles.sort((left, right) -> left.getAbsolutePath().compareTo(right.getAbsolutePath())); + List metaInformationFiles = new ArrayList<>(); + try { + for (File generatedFile : generatedFiles) { + String entryName = r8OutputDir.toPath() + .relativize(generatedFile.toPath()) + .toString() + .replace(File.separatorChar, '/'); + if (ExtenderUtil.isMetaInfEntryValuable(new ZipEntry(entryName))) { + metaInformationFiles.add(moveR8OutputFile(r8OutputDir, generatedFile)); + } + } + } catch (IOException e) { + throw new ExtenderException(e, "Failed to collect R8 META-INF output"); + } + return metaInformationFiles.toArray(File[]::new); + } + + BuildOutput build( + List allJars, + Map extensionJarMap, + File aaptGeneratedRules) throws ExtenderException { + File appRules = new File(uploadDir, APP_RULES_PATH); + if (!isRequested(uploadDir)) { + LOGGER.info("No app.keep file present. Skipping R8 step."); + return null; + } + if (DM_DEBUG_DISABLE_R8) { + LOGGER.info("R8 support disabled by environment flag DM_DEBUG_DISABLE_R8"); + return null; + } + + Map context = new HashMap<>(commandContext); + if (platformConfig.r8Cmd == null || platformConfig.r8Cmd.isBlank() + || platformConfig.r8Version == null || platformConfig.r8Version.isBlank()) { + validateConfiguration(platformConfig.r8Cmd, platformConfig.r8Version); + } + validateResolvedR8Environment(context); + String r8Version; + try { + r8Version = templateExecutor.executeOnceWithoutLogging(platformConfig.r8Version, context); + } catch (RuntimeException e) { + throw new ExtenderException(e, "R8 shrinking was requested, but its configuration could not be resolved"); + } + validateConfiguration(platformConfig.r8Cmd, r8Version); + if (aaptGeneratedRules == null || !aaptGeneratedRules.isFile()) { + throw new ExtenderException( + "R8 shrinking was requested, but aapt2 did not generate aapt-generated.keep"); + } + LOGGER.info("Building classes.dex using R8 {}", r8Version); + + Set ruleFiles = new LinkedHashSet<>(); + R8RulePolicy.Budget ruleBudget = new R8RulePolicy.Budget(r8Configuration); + File emptyRuleBase = R8RulePolicy.createEmptyBaseDirectory(buildDir); + File sanitizedRulesDir = new File(buildDir, "r8-sanitized-rules"); + File sanitizedAppRules = new File(sanitizedRulesDir, "app.keep"); + R8RulePolicy.writeSanitized( + sanitizedAppRules, + R8RulePolicy.readAndValidate(appRules, ruleBudget), + emptyRuleBase); + ruleFiles.add(sanitizedAppRules.getAbsolutePath()); + File sanitizedAaptRules = new File(sanitizedRulesDir, "aapt-generated.keep"); + R8RulePolicy.writeSanitized( + sanitizedAaptRules, + R8RulePolicy.readAndValidate(aaptGeneratedRules, ruleBudget), + emptyRuleBase); + ruleFiles.add(sanitizedAaptRules.getAbsolutePath()); + + Set seenUntrustedRules = new LinkedHashSet<>(); + int sanitizedExtensionRuleIndex = 0; + List extensionJars = new ArrayList<>(extensionJarMap.keySet()); + Collections.sort(extensionJars); + for (String extensionJar : extensionJars) { + ExtensionContext extensionContext = extensionJarMap.get(extensionJar); + if (extensionContext != null) { + List extensionRules = new ArrayList<>(extensionContext.ruleFiles); + Collections.sort(extensionRules); + for (String extensionRule : extensionRules) { + if (seenUntrustedRules.add(extensionRule)) { + File sanitizedExtensionRule = new File( + sanitizedRulesDir, + String.format("extension-%04d.keep", sanitizedExtensionRuleIndex++)); + R8RulePolicy.writeSanitized( + sanitizedExtensionRule, + R8RulePolicy.readAndValidate(new File(extensionRule), ruleBudget), + emptyRuleBase); + ruleFiles.add(sanitizedExtensionRule.getAbsolutePath()); + } + } + } + } + + File consumerRulesDir = new File(buildDir, "r8-consumer-rules"); + ruleFiles.addAll(collectConsumerRules( + allJars, + androidPackages, + r8Version, + consumerRulesDir, + r8Configuration, + ruleBudget, + emptyRuleBase)); + + List protectedJars = getProtectedJars(extensionJarMap); + if (!protectedJars.isEmpty()) { + File generatedRules = new File(buildDir, "generated-extension-rules.keep"); + try { + writeProtectedJarKeepRules(protectedJars, generatedRules, r8Configuration); + } catch (IOException e) { + throw new ExtenderException(e, "Failed to generate R8 rules for unconfigured extension jars"); + } + R8RulePolicy.account(generatedRules, ruleBudget); + ruleFiles.add(generatedRules.getAbsolutePath()); + } + + List programJars = prepareProgramJars( + allJars, + new File(buildDir, "r8-program-jars")); + File r8OutputDir = createR8OutputDirectory(); + File mappingFile = new File(buildDir, "mapping.txt"); + + context.put("classes_dex_dir", r8OutputDir.getAbsolutePath()); + context.put("jars", programJars); + context.put("rules", new ArrayList<>(ruleFiles)); + context.put("mapping", mappingFile.getAbsolutePath()); + context.put("minAndroidSdkVersion", minAndroidSdkVersion); + R8RulePolicy.requireEmptyBaseDirectory(emptyRuleBase); + String r8Command; + try { + r8Command = templateExecutor.executeOnceWithoutLogging( + platformConfig.r8Cmd, + createR8CommandContext(context)); + } catch (RuntimeException e) { + throw new ExtenderException(e, "R8 shrinking was requested, but its configuration could not be resolved"); + } + commandExecutor.execute(r8Command, context); + + File[] dexFiles = collectR8DexFiles(r8OutputDir); + if (dexFiles.length == 0 || !mappingFile.isFile()) { + throw new ExtenderException("R8 completed without producing classes.dex and mapping.txt"); + } + File[] metaInformationFiles = collectR8MetaInformationFiles(r8OutputDir); + return new BuildOutput(dexFiles, mappingFile, metaInformationFiles); + } +} diff --git a/server/src/main/java/com/defold/extender/R8Configuration.java b/server/src/main/java/com/defold/extender/R8Configuration.java new file mode 100644 index 00000000..3ae09e9a --- /dev/null +++ b/server/src/main/java/com/defold/extender/R8Configuration.java @@ -0,0 +1,84 @@ +package com.defold.extender; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +/** Server-side resource limits for R8 rule processing and generated keep rules. */ +@Component +@ConfigurationProperties(prefix = "extender.r8") +public class R8Configuration { + private int maxGeneratedExtensionClasses = 32 * 1024; + private int maxClassfileHeaderBytes = 4 * 1024 * 1024; + private long maxTotalClassfileHeaderBytes = 64L * 1024L * 1024L; + private int maxRuleFiles = 1024; + private long maxRuleFileBytes = 1024L * 1024L; + private long maxTotalRuleBytes = 16L * 1024L * 1024L; + + public int getMaxGeneratedExtensionClasses() { + return maxGeneratedExtensionClasses; + } + + public void setMaxGeneratedExtensionClasses(int maxGeneratedExtensionClasses) { + this.maxGeneratedExtensionClasses = requirePositive( + maxGeneratedExtensionClasses, + "max-generated-extension-classes"); + } + + public int getMaxClassfileHeaderBytes() { + return maxClassfileHeaderBytes; + } + + public void setMaxClassfileHeaderBytes(int maxClassfileHeaderBytes) { + this.maxClassfileHeaderBytes = requirePositive( + maxClassfileHeaderBytes, + "max-classfile-header-bytes"); + } + + public long getMaxTotalClassfileHeaderBytes() { + return maxTotalClassfileHeaderBytes; + } + + public void setMaxTotalClassfileHeaderBytes(long maxTotalClassfileHeaderBytes) { + this.maxTotalClassfileHeaderBytes = requirePositive( + maxTotalClassfileHeaderBytes, + "max-total-classfile-header-bytes"); + } + + public int getMaxRuleFiles() { + return maxRuleFiles; + } + + public void setMaxRuleFiles(int maxRuleFiles) { + this.maxRuleFiles = requirePositive(maxRuleFiles, "max-rule-files"); + } + + public long getMaxRuleFileBytes() { + return maxRuleFileBytes; + } + + public void setMaxRuleFileBytes(long maxRuleFileBytes) { + this.maxRuleFileBytes = requirePositive(maxRuleFileBytes, "max-rule-file-bytes"); + } + + public long getMaxTotalRuleBytes() { + return maxTotalRuleBytes; + } + + public void setMaxTotalRuleBytes(long maxTotalRuleBytes) { + this.maxTotalRuleBytes = requirePositive(maxTotalRuleBytes, "max-total-rule-bytes"); + } + + private static int requirePositive(int value, String property) { + if (value <= 0) { + throw new IllegalArgumentException("extender.r8." + property + " must be positive"); + } + return value; + } + + private static long requirePositive(long value, String property) { + if (value <= 0) { + throw new IllegalArgumentException("extender.r8." + property + " must be positive"); + } + return value; + } +} diff --git a/server/src/main/java/com/defold/extender/R8RulePolicy.java b/server/src/main/java/com/defold/extender/R8RulePolicy.java new file mode 100644 index 00000000..09377db9 --- /dev/null +++ b/server/src/main/java/com/defold/extender/R8RulePolicy.java @@ -0,0 +1,283 @@ +package com.defold.extender; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Objects; +import java.util.stream.Stream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +/** Applies the server-side policy for untrusted R8/ProGuard-compatible rule files. */ +final class R8RulePolicy { + // Match exact lowercase prefixes anywhere in the raw text. R8 only accepts these lowercase + // spellings. Deliberate false positives in comments, strings, or malformed tokens keep this + // check independent of R8's evolving grammar. + private static final List BLOCKED_OPTION_PREFIXES = List.of( + "include", + "basedirectory", + "injars", + "outjars", + "libraryjars", + "applymapping", + "obfuscationdictionary", + "classobfuscationdictionary", + "packageobfuscationdictionary", + "print", + "dump", + "protomapping", + "laststageoutput"); + + static final class Budget { + private final R8Configuration configuration; + private int fileCount; + private long totalBytes; + + Budget() { + this(new R8Configuration()); + } + + Budget(R8Configuration configuration) { + this.configuration = Objects.requireNonNull(configuration); + } + + private void beginFile(String source, long declaredSize) throws ExtenderException { + ++fileCount; + if (fileCount > configuration.getMaxRuleFiles()) { + throw new ExtenderException(String.format( + "Too many R8 rule files (maximum %d), while reading %s", + configuration.getMaxRuleFiles(), + source)); + } + if (declaredSize > configuration.getMaxRuleFileBytes()) { + throw new ExtenderException(String.format( + "R8 rule file is too large (maximum %d bytes): %s", + configuration.getMaxRuleFileBytes(), + source)); + } + if (declaredSize >= 0 + && totalBytes + declaredSize > configuration.getMaxTotalRuleBytes()) { + throw new ExtenderException(String.format( + "R8 rule files are too large in total (maximum %d bytes), while reading %s", + configuration.getMaxTotalRuleBytes(), + source)); + } + } + + private void addBytes(String source, long fileBytes, int bytesRead) throws ExtenderException { + if (fileBytes > configuration.getMaxRuleFileBytes()) { + throw new ExtenderException(String.format( + "R8 rule file is too large (maximum %d bytes): %s", + configuration.getMaxRuleFileBytes(), + source)); + } + totalBytes += bytesRead; + if (totalBytes > configuration.getMaxTotalRuleBytes()) { + throw new ExtenderException(String.format( + "R8 rule files are too large in total (maximum %d bytes), while reading %s", + configuration.getMaxTotalRuleBytes(), + source)); + } + } + + private long getMaxRuleFileBytes() { + return configuration.getMaxRuleFileBytes(); + } + } + + private R8RulePolicy() {} + + static File createEmptyBaseDirectory(File parent) throws ExtenderException { + try { + Files.createDirectories(parent.toPath()); + return Files.createTempDirectory(parent.toPath(), "r8-rule-base-").toFile(); + } catch (IOException e) { + throw new ExtenderException(e, "Failed to create the R8 rule sandbox directory"); + } + } + + static void requireEmptyBaseDirectory(File emptyBase) throws ExtenderException { + Path basePath = emptyBase.toPath(); + if (!Files.isDirectory(basePath)) { + throw new ExtenderException("R8 rule sandbox is not a directory: " + emptyBase); + } + try (Stream entries = Files.list(basePath)) { + if (entries.findAny().isPresent()) { + throw new ExtenderException("R8 rule sandbox is not empty: " + emptyBase); + } + } catch (IOException e) { + throw new ExtenderException(e, "Failed to inspect the R8 rule sandbox " + emptyBase); + } + } + + static byte[] readAndValidate(ZipFile zipFile, ZipEntry entry, Budget budget) + throws ExtenderException { + String source = zipFile.getName() + "!/" + entry.getName(); + try (InputStream input = zipFile.getInputStream(entry)) { + return normalizeAndValidate( + readBounded(input, entry.getSize(), source, budget), + source); + } catch (IOException e) { + throw new ExtenderException(e, "Failed to read R8 consumer rules from " + source); + } + } + + static byte[] readAndValidate(File ruleFile, Budget budget) throws ExtenderException { + String source = ruleFile.getAbsolutePath(); + try (InputStream input = new FileInputStream(ruleFile)) { + return normalizeAndValidate( + readBounded(input, ruleFile.length(), source, budget), + source); + } catch (IOException e) { + throw new ExtenderException(e, "Failed to read R8 rule file " + source); + } + } + + static void account(File trustedRuleFile, Budget budget) throws ExtenderException { + String source = trustedRuleFile.getAbsolutePath(); + try (InputStream input = new FileInputStream(trustedRuleFile)) { + readBounded(input, trustedRuleFile.length(), source, budget); + } catch (IOException e) { + throw new ExtenderException(e, "Failed to read R8 rule file " + source); + } + } + + static void writeSanitized(File target, byte[] contents, File emptyBase) + throws ExtenderException { + requireEmptyBaseDirectory(emptyBase); + try { + String base = emptyBase.getCanonicalPath(); + char quote; + if (!base.contains("\"") && base.indexOf('\n') < 0 && base.indexOf('\r') < 0) { + quote = '"'; + } else if (!base.contains("'") && base.indexOf('\n') < 0 && base.indexOf('\r') < 0) { + quote = '\''; + } else { + throw new ExtenderException("R8 rule sandbox path cannot be quoted safely: " + emptyBase); + } + + byte[] prefix = String.format("-basedirectory %c%s%c\n", quote, base, quote) + .getBytes(StandardCharsets.UTF_8); + ByteArrayOutputStream output = new ByteArrayOutputStream(prefix.length + contents.length); + output.write(prefix); + output.write(contents); + Files.createDirectories(target.getParentFile().toPath()); + Files.write(target.toPath(), output.toByteArray()); + } catch (IOException e) { + throw new ExtenderException(e, "Failed to write sanitized R8 rules to " + target); + } + } + + private static byte[] readBounded( + InputStream input, + long declaredSize, + String source, + Budget budget) throws IOException, ExtenderException { + budget.beginFile(source, declaredSize); + ByteArrayOutputStream output = new ByteArrayOutputStream( + (int) Math.min( + Math.min(Math.max(declaredSize, 0), budget.getMaxRuleFileBytes()), + Integer.MAX_VALUE)); + byte[] buffer = new byte[8192]; + long fileBytes = 0; + int read; + while ((read = input.read(buffer)) != -1) { + fileBytes += read; + budget.addBytes(source, fileBytes, read); + output.write(buffer, 0, read); + } + return output.toByteArray(); + } + + private static byte[] normalizeAndValidate(byte[] contents, String source) + throws ExtenderException { + final String decoded; + try { + decoded = StandardCharsets.UTF_8.newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(contents)) + .toString(); + } catch (CharacterCodingException e) { + throw new ExtenderException(e, "R8 rule file is not valid UTF-8: " + source); + } + String rules = decoded.startsWith("\uFEFF") ? decoded.substring(1) : decoded; + validateRules(rules, source); + return rules.getBytes(StandardCharsets.UTF_8); + } + + static void validateRules(String rules, String source) throws ExtenderException { + int line = 1; + for (int index = 0; index < rules.length(); ++index) { + char current = rules.charAt(index); + if (current == '\n') { + ++line; + continue; + } + if (current == '-') { + for (String blockedPrefix : BLOCKED_OPTION_PREFIXES) { + if (rules.startsWith(blockedPrefix, index + 1)) { + throw blockedDirective(source, line, "-" + blockedPrefix); + } + } + } else if (current == '@' && hasPathLikeOperand(rules, index + 1)) { + throw blockedDirective(source, line, "path-like @file include"); + } + } + } + + private static boolean hasPathLikeOperand(String rules, int start) { + int index = start; + while (index < rules.length()) { + while (index < rules.length() && Character.isWhitespace(rules.charAt(index))) { + ++index; + } + if (index >= rules.length() || rules.charAt(index) != '#') { + break; + } + while (index < rules.length() + && rules.charAt(index) != '\n' + && rules.charAt(index) != '\r') { + ++index; + } + } + if (index >= rules.length()) { + return false; + } + + char first = rules.charAt(index); + if (first == '/' || first == '\\' || first == '.' || first == '~' + || first == '\'' || first == '"' || first == '<') { + return true; + } + + for (; index < rules.length() && !Character.isWhitespace(rules.charAt(index)); ++index) { + char current = rules.charAt(index); + if (current == '/' || current == '\\' || current == ':' + || current == '<' || current == '>') { + return true; + } + if (current == '.' && index + 1 < rules.length() && rules.charAt(index + 1) == '.') { + return true; + } + } + return false; + } + + private static ExtenderException blockedDirective(String source, int line, String directive) { + return new ExtenderException(String.format( + "R8 rule file %s:%d uses forbidden filesystem directive %s", + source, + line, + directive)); + } +} diff --git a/server/src/main/java/com/defold/extender/TemplateExecutor.java b/server/src/main/java/com/defold/extender/TemplateExecutor.java index 35246911..010452e6 100644 --- a/server/src/main/java/com/defold/extender/TemplateExecutor.java +++ b/server/src/main/java/com/defold/extender/TemplateExecutor.java @@ -13,14 +13,22 @@ public class TemplateExecutor { private static final Logger LOGGER = LoggerFactory.getLogger(TemplateExecutor.class); + String executeOnceWithoutLogging(String template, Map context) { + return Mustache.compiler().compile(template).execute(context); + } + + String executeWithoutLogging(String template, Map context) { + String result = executeOnceWithoutLogging(template, context); + while (!result.equals(template)) { + template = result; + result = executeOnceWithoutLogging(template, context); + } + return result; + } + public String execute(String template, Map context) { try { - String result = Mustache.compiler().compile(template).execute(context); - while (!result.equals(template)) { - template = result; - result = Mustache.compiler().compile(template).execute(context); - } - return result; + return executeWithoutLogging(template, context); } catch (Exception e) { LOGGER.error(Markers.COMPILATION_ERROR, String.format("Failed to substitute string '%s'", (String)template)); ExtenderUtil.debugPrint(context, 0); diff --git a/server/src/main/java/com/defold/extender/services/GradleArtifact.java b/server/src/main/java/com/defold/extender/services/GradleArtifact.java new file mode 100644 index 00000000..349a1ef6 --- /dev/null +++ b/server/src/main/java/com/defold/extender/services/GradleArtifact.java @@ -0,0 +1,49 @@ +package com.defold.extender.services; + +import java.io.File; + +public final class GradleArtifact { + public enum Kind { + EXPLODED_AAR, + JAR + } + + private final File file; + private final String component; + private final String originalFileName; + private final Kind kind; + private final String resourcePackageName; + + GradleArtifact( + File file, + String component, + String originalFileName, + Kind kind, + String resourcePackageName) { + this.file = file; + this.component = component; + this.originalFileName = originalFileName; + this.kind = kind; + this.resourcePackageName = resourcePackageName; + } + + public File getFile() { + return file; + } + + public String getComponent() { + return component; + } + + public String getOriginalFileName() { + return originalFileName; + } + + public Kind getKind() { + return kind; + } + + public String getResourcePackageName() { + return resourcePackageName; + } +} diff --git a/server/src/main/java/com/defold/extender/services/GradleService.java b/server/src/main/java/com/defold/extender/services/GradleService.java index d3566270..075d8b9f 100644 --- a/server/src/main/java/com/defold/extender/services/GradleService.java +++ b/server/src/main/java/com/defold/extender/services/GradleService.java @@ -24,7 +24,7 @@ public GradleService(GradleServiceInterface service, Gauge.builder("extender.job.gradle.cacheSize", this, GradleService::getCacheSize).baseUnit(BaseUnits.BYTES).register(registry); } - public List resolveDependencies(ExtenderBuildState buildState, Map env, List outputFiles) + public List resolveDependencies(ExtenderBuildState buildState, Map env, List outputFiles) throws IOException, ExtenderException { return gradleService.resolveDependencies(buildState, env, outputFiles); } diff --git a/server/src/main/java/com/defold/extender/services/GradleServiceInterface.java b/server/src/main/java/com/defold/extender/services/GradleServiceInterface.java index caac966e..cf23cdf5 100644 --- a/server/src/main/java/com/defold/extender/services/GradleServiceInterface.java +++ b/server/src/main/java/com/defold/extender/services/GradleServiceInterface.java @@ -10,7 +10,7 @@ public interface GradleServiceInterface { // Resolve dependencies, download them, extract to - public List resolveDependencies(ExtenderBuildState buildState, Map env, List outputFiles) throws IOException, ExtenderException; + public List resolveDependencies(ExtenderBuildState buildState, Map env, List outputFiles) throws IOException, ExtenderException; public long getCacheSize() throws IOException; } diff --git a/server/src/main/java/com/defold/extender/services/MockGradleService.java b/server/src/main/java/com/defold/extender/services/MockGradleService.java index 60b7e0c7..e0bfd10f 100644 --- a/server/src/main/java/com/defold/extender/services/MockGradleService.java +++ b/server/src/main/java/com/defold/extender/services/MockGradleService.java @@ -15,7 +15,7 @@ @ConditionalOnProperty(name = "extender.gradle.enabled", havingValue = "false", matchIfMissing = true) public class MockGradleService implements GradleServiceInterface { @Override - public List resolveDependencies(ExtenderBuildState buildState, Map env, List outputFiles) + public List resolveDependencies(ExtenderBuildState buildState, Map env, List outputFiles) throws IOException, ExtenderException { return List.of(); } diff --git a/server/src/main/java/com/defold/extender/services/RealGradleService.java b/server/src/main/java/com/defold/extender/services/RealGradleService.java index b412f4b9..8ac409a5 100644 --- a/server/src/main/java/com/defold/extender/services/RealGradleService.java +++ b/server/src/main/java/com/defold/extender/services/RealGradleService.java @@ -4,13 +4,13 @@ import com.defold.extender.ExtenderException; import com.defold.extender.ExtenderUtil; import com.defold.extender.TemplateExecutor; -import com.defold.extender.Timer; -import com.defold.extender.ZipUtils; -import com.defold.extender.log.Markers; import com.defold.extender.metrics.MetricsWriter; import com.defold.extender.process.ProcessUtils; -import org.apache.commons.io.FileUtils; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; @@ -23,19 +23,20 @@ import java.io.File; import java.io.IOException; -import java.io.InputStream; -import java.io.FileInputStream; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; import java.util.ArrayList; -import java.util.List; import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; import java.util.Map; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Stream; @Service @ConditionalOnProperty(name = "extender.gradle.enabled", havingValue = "true") @@ -49,12 +50,13 @@ public class RealGradleService implements GradleServiceInterface { ); private static final String GRADLE_USER_HOME = System.getenv("GRADLE_USER_HOME"); private static final String GRADLE_PLUGIN_VERSION = System.getenv("GRADLE_PLUGIN_VERSION"); + private static final String ARTIFACT_KIND_EXPLODED_AAR = "exploded-aar"; + private static final String ARTIFACT_KIND_JAR = "jar"; private final TemplateExecutor templateExecutor = new TemplateExecutor(); private final String gradleHome; - private final File baseDirectory; private final MeterRegistry meterRegistry; private final String buildGradleTemplateContents; private final String gradlePropertiesTemplateContents; @@ -68,19 +70,12 @@ public class RealGradleService implements GradleServiceInterface { this.gradleHome = GRADLE_USER_HOME; } else { File f = new File(".gradle"); - if (!f.exists()) { - f.mkdirs(); - } this.gradleHome = f.getAbsolutePath(); } + Files.createDirectories(Paths.get(this.gradleHome)); this.meterRegistry = meterRegistry; - this.baseDirectory = new File(this.gradleHome, "unpacked"); - if (!this.baseDirectory.exists()) { - Files.createDirectories(this.baseDirectory.toPath()); - } - this.buildGradleTemplateContents = ExtenderUtil.readContentFromResource(buildGradleTemplate); this.gradlePropertiesTemplateContents = ExtenderUtil.readContentFromResource(gradlePropertiesTemplate); this.localPropertiesTemplateContents = ExtenderUtil.readContentFromResource(localPropertiesTemplate); @@ -96,7 +91,7 @@ private Map createJobEnvContext(Map env) { } @Override - public List resolveDependencies(ExtenderBuildState buildState, Map env, List outputFiles) throws IOException, ExtenderException { + public List resolveDependencies(ExtenderBuildState buildState, Map env, List outputFiles) throws IOException, ExtenderException { // cwd -> jobDir File workDir = buildState.getJobDir(); File buildDir = buildState.getBuildDir(); @@ -106,7 +101,24 @@ public List resolveDependencies(ExtenderBuildState buildState, Map gradleFiles = ExtenderUtil.listFilesMatchingRecursive(workDir, "build\\.gradle"); // This file might exist when testing and debugging the extender using a debug job folder gradleFiles.remove(mainGradleFile); - createBuildGradleFile(mainGradleFile, gradleFiles, jobEnvContext); + boolean hasDependencies = createBuildGradleFile(mainGradleFile, gradleFiles, jobEnvContext); + + Files.createDirectories(buildDir.toPath()); + File lockFile = new File(buildDir, "gradle.lockfile"); + File dependencyTreeFile = new File(buildDir, "gradle.dependencytree"); + File artifactManifestFile = new File(buildDir, "gradle-artifacts.json"); + outputFiles.add(lockFile); + outputFiles.add(dependencyTreeFile); + + if (!hasDependencies) { + Files.deleteIfExists(artifactManifestFile.toPath()); + Files.writeString(lockFile.toPath(), "", StandardCharsets.UTF_8); + Files.writeString( + dependencyTreeFile.toPath(), + "No Gradle dependencies were declared.\n", + StandardCharsets.UTF_8); + return List.of(); + } // create gradle.properties File gradlePropertiesFile = new File(workDir, "gradle.properties"); @@ -116,27 +128,19 @@ public List resolveDependencies(ExtenderBuildState buildState, Map unpackedDependencies = downloadDependencies(workDir); - // add gradle lockfile to outputs - // configured in template.build.gradle - outputFiles.add(new File(buildDir, "gradle.lockfile")); - - // write dependency tree and add to outputs - File dependencyTreeFile = new File(buildDir, "gradle.dependencytree"); - writeDependencyTree(dependencyTreeFile, workDir); - outputFiles.add(dependencyTreeFile); - - return unpackedDependencies; + // Resolve AGP-processed dependencies and reuse their cache paths directly. + return resolveGradleArtifacts(workDir, artifactManifestFile, dependencyTreeFile); } @Override public long getCacheSize() throws IOException { - Path folder = Paths.get(GRADLE_USER_HOME); - return Files.walk(folder) - .filter(p -> p.toFile().isFile()) - .mapToLong(p -> p.toFile().length()) - .sum(); + Path folder = Paths.get(this.gradleHome); + try (Stream paths = Files.walk(folder)) { + return paths + .filter(Files::isRegularFile) + .mapToLong(path -> path.toFile().length()) + .sum(); + } } /////////////////////////////////////////////////////////////////////////////////////////////// @@ -192,7 +196,7 @@ private List extractBlockContent(String content, String blockName) { return lines; } - private void createBuildGradleFile(File mainGradleFile, List gradleFiles, Map jobEnvContext) throws IOException, ExtenderException { + private boolean createBuildGradleFile(File mainGradleFile, List gradleFiles, Map jobEnvContext) throws IOException, ExtenderException { List userDependencies = new ArrayList<>(); List userRepositories = new ArrayList<>(); for (File file : gradleFiles) { @@ -208,142 +212,133 @@ private void createBuildGradleFile(File mainGradleFile, List gradleFiles, envContext.put("gradle-plugin-version", GRADLE_PLUGIN_VERSION); String contents = templateExecutor.execute(buildGradleTemplateContents, envContext); Files.write(mainGradleFile.toPath(), contents.getBytes()); + return !userDependencies.isEmpty(); } - // Helper function to move files/directories - private static void Move(Path source, Path target) { - try { - Files.move(source, target, StandardCopyOption.ATOMIC_MOVE); - } catch (IOException e) { - // If the target path suddenly exists, and the source path still exists, - // then we failed with the atomic move, and we assume another job succeeded with the download - if (Files.exists(source) && Files.exists(target)) { - LOGGER.info("Gradle package {} was downloaded by another job in the meantime", source.toString()); - try { - FileUtils.deleteDirectory(source.toFile()); - } catch (IOException e2) { - LOGGER.error(Markers.SERVER_ERROR, "Failed to delete temp directory {}: {}", source.toString(), e2.getMessage()); - } - } + private static String getResourcePackageName(String component, String originalFileName) { + String[] coordinate = component.split(":", 3); + if (coordinate.length == 3 + && !coordinate[0].isBlank() + && !coordinate[1].isBlank() + && !coordinate[2].isBlank()) { + // Match the package directory names returned by the pre-AGP-cache implementation. + return (coordinate[0] + "-" + coordinate[1] + "-" + coordinate[2] + ".aar") + .replaceAll("[^A-Za-z0-9._-]", "_"); } - } - private Map parseDependencies(String log) { - // The output comes from template.build.gradle - Pattern p = Pattern.compile("PATH:\\s*([\\w-.\\/]*)\\sEXTENSION:\\s*([\\w-.\\/]*)\\sTYPE:\\s*([\\w-.\\/]*)\\sMODULE_GROUP:\\s*([\\w-.\\/]*)\\sMODULE_NAME:\\s*([\\w-.\\/]*)\\sMODULE_VERSION:\\s*([\\w-.\\/]*)"); - - Map dependencies = new HashMap<>(); - String[] lines = log.split(System.getProperty("line.separator")); - for (String line : lines) { - Matcher m = p.matcher(line); - if (m.matches()) { - String path = m.group(1); - String extension = m.group(2); - String group = m.group(4); - String name = m.group(5); - String version = m.group(6); - - // Map the new name to the original file path - dependencies.put(String.format("%s-%s-%s.%s", group, name, version, extension), path); - } + String fileName = new File(originalFileName).getName(); + if (fileName.isBlank()) { + fileName = "dependency.aar"; } - - return dependencies; - } - - private File resolveDependencyAAR(File dependency, String name, File jobDir) throws IOException { - File unpackedTarget = new File(baseDirectory, name); - if (unpackedTarget.exists()) { - return unpackedTarget; + if (!fileName.toLowerCase(Locale.ROOT).endsWith(".aar")) { + fileName += ".aar"; } - // use job folder as tmp location - File unpackedTmp = new File(jobDir, dependency.getName() + ".tmp"); - try (InputStream fis = new FileInputStream(dependency)) { - ZipUtils.unzip(fis, unpackedTmp.toPath()); - } - Move(unpackedTmp.toPath(), unpackedTarget.toPath()); - return unpackedTarget; + // Gradle uses incomplete coordinates such as :LocalAar: for flatDir dependencies. The + // externally visible name is cosmetic, so fall back instead of rejecting a valid build. + return fileName.replaceAll("[^A-Za-z0-9._-]", "_"); } - private File resolveDependencyJAR(File dependency, String name, File jobDir) throws IOException { - File targetFile = new File(baseDirectory, name); - if (targetFile.exists()) { - return targetFile; + static List parseGradleArtifacts(File artifactManifest) throws IOException, ExtenderException { + final Object parsed; + try { + parsed = new JSONParser().parse(Files.readString( + artifactManifest.toPath(), + StandardCharsets.UTF_8)); + } catch (ParseException e) { + throw new ExtenderException(e, "Invalid Gradle artifact manifest: " + artifactManifest); + } + if (!(parsed instanceof JSONArray)) { + throw new ExtenderException("Gradle artifact manifest must contain a JSON array: " + artifactManifest); } - // use job folder as tmp location - File tmpFile = new File(jobDir, dependency.getName() + ".tmp"); - FileUtils.copyFile(dependency, tmpFile); - Move(tmpFile.toPath(), targetFile.toPath()); - return targetFile; - } - - private List unpackDependencies(Map dependencies, File jobDir) throws IOException { - List resolvedDependencies = new ArrayList<>(); - Timer timer = new Timer(); - timer.start(); - for (String newName : dependencies.keySet()) { - String dependency = dependencies.get(newName); - - File file = new File(dependency); - if (!file.exists()) { - throw new IOException("File does not exist: %s" + dependency); + List artifacts = new ArrayList<>(); + Set seenPaths = new LinkedHashSet<>(); + for (Object value : (JSONArray) parsed) { + if (!(value instanceof JSONObject)) { + throw new ExtenderException("Invalid entry in Gradle artifact manifest: " + value); + } + JSONObject entry = (JSONObject) value; + Object componentValue = entry.get("component"); + Object originalFileNameValue = entry.get("originalFileName"); + Object kindValue = entry.get("kind"); + Object pathValue = entry.get("path"); + if (!(componentValue instanceof String) + || !(originalFileNameValue instanceof String) + || !(kindValue instanceof String) + || !(pathValue instanceof String)) { + throw new ExtenderException( + "Gradle artifact entry must contain string component, originalFileName, kind and path fields: " + + entry); } - if (dependency.endsWith(".aar")) { - resolvedDependencies.add(resolveDependencyAAR(file, newName, jobDir)); - } else if (dependency.endsWith(".jar")) { - resolvedDependencies.add(resolveDependencyJAR(file, newName, jobDir)); + + String component = (String) componentValue; + String originalFileName = (String) originalFileNameValue; + String kind = (String) kindValue; + File artifact = new File((String) pathValue).getCanonicalFile(); + GradleArtifact.Kind artifactKind; + String resourcePackageName = null; + if (ARTIFACT_KIND_EXPLODED_AAR.equals(kind)) { + if (!artifact.isDirectory()) { + throw new ExtenderException("Gradle exploded AAR does not exist: " + artifact); + } + artifactKind = GradleArtifact.Kind.EXPLODED_AAR; + resourcePackageName = getResourcePackageName(component, originalFileName); + } else if (ARTIFACT_KIND_JAR.equals(kind)) { + if (!artifact.isFile() || !artifact.getName().endsWith(".jar")) { + throw new ExtenderException("Gradle JAR does not exist: " + artifact); + } + artifactKind = GradleArtifact.Kind.JAR; } else { - resolvedDependencies.add(file); + throw new ExtenderException("Unsupported Gradle artifact kind '" + kind + "': " + artifact); + } + + if (seenPaths.add(artifact.getAbsolutePath())) { + artifacts.add(new GradleArtifact( + artifact, + component, + originalFileName, + artifactKind, + resourcePackageName)); } } - long duration = timer.start(); - MetricsWriter.metricsTimer(meterRegistry, "extender.service.gradle.unpack", duration); - return resolvedDependencies; + return artifacts; } - private List downloadDependencies(File cwd) throws IOException, ExtenderException { - long methodStart = System.currentTimeMillis(); - LOGGER.info("Resolving dependencies"); - - // add --info for additional logging - String log = ProcessUtils.execCommand(List.of( + static List getGradleResolveCommand() { + return List.of( "gradle", "downloadDependencies", + "dependencies", + "--configuration", + "releaseCompileClasspath", "--write-locks", "--stacktrace", "--warning-mode", "all", - "--no-daemon" - ), cwd, - Map.of("GRADLE_USER_HOME", this.gradleHome)); - LOGGER.debug("\n" + log); - - Map dependencies = parseDependencies(log); - - List unpackedDependencies = unpackDependencies(dependencies, cwd); - - MetricsWriter.metricsTimer(meterRegistry, "extender.service.gradle.get", System.currentTimeMillis() - methodStart); - return unpackedDependencies; + "--no-daemon"); } - private void writeDependencyTree(File out, File cwd) throws IOException, ExtenderException { + private List resolveGradleArtifacts( + File cwd, + File artifactManifest, + File dependencyTree) throws IOException, ExtenderException { long methodStart = System.currentTimeMillis(); - LOGGER.info("Writing dependency tree"); + LOGGER.info("Resolving dependencies"); + Files.deleteIfExists(artifactManifest.toPath()); - String treelog = ProcessUtils.execCommand(List.of( - "gradle", - "dependencies", - "--configuration", - "releaseCompileClasspath", - "--no-daemon" - ), cwd, Map.of("GRADLE_USER_HOME", this.gradleHome)); - LOGGER.debug("\n" + treelog); + String log = ProcessUtils.execCommand(getGradleResolveCommand(), cwd, + Map.of("GRADLE_USER_HOME", this.gradleHome)); + LOGGER.debug("\n" + log); + Files.writeString(dependencyTree.toPath(), log, StandardCharsets.UTF_8); - Files.write(out.toPath(), treelog.getBytes()); + if (!artifactManifest.isFile()) { + throw new ExtenderException("Gradle did not produce its artifact manifest: " + artifactManifest); + } + List artifacts = parseGradleArtifacts(artifactManifest); - MetricsWriter.metricsTimer(meterRegistry, "extender.service.gradle.dependencytree", System.currentTimeMillis() - methodStart); + MetricsWriter.metricsTimer(meterRegistry, "extender.service.gradle.get", System.currentTimeMillis() - methodStart); + return artifacts; } } diff --git a/server/src/main/resources/application.yml b/server/src/main/resources/application.yml index 871666d4..a7f9f94c 100644 --- a/server/src/main/resources/application.yml +++ b/server/src/main/resources/application.yml @@ -38,6 +38,14 @@ extender: gradle: enabled: false location: /tmp/.gradle + # Resource limits for untrusted R8 input; byte limits use expanded sizes. + r8: + max-generated-extension-classes: 32768 + max-classfile-header-bytes: 4194304 + max-total-classfile-header-bytes: 67108864 + max-rule-files: 1024 + max-rule-file-bytes: 1048576 + max-total-rule-bytes: 16777216 cocoapods: enabled: false cdn-concurrency: 10 # value for COCOAPODS_CDN_MAX_CONCURRENCY diff --git a/server/src/main/resources/template.build.gradle b/server/src/main/resources/template.build.gradle index 8517ced4..2d895818 100644 --- a/server/src/main/resources/template.build.gradle +++ b/server/src/main/resources/template.build.gradle @@ -1,3 +1,7 @@ +import com.android.build.gradle.internal.publishing.AndroidArtifacts +import groovy.json.JsonOutput +import org.gradle.api.artifacts.component.ModuleComponentIdentifier + buildscript { repositories { google() @@ -53,13 +57,84 @@ dependencyLocking { task downloadDependencies { doLast { - project.configurations.releaseRuntimeClasspath.getResolvedConfiguration().getResolvedArtifacts().each { - println "PATH: " + it.file + \ - " EXTENSION: " + it.extension + \ - " TYPE: " + it.type + \ - " MODULE_GROUP: " + it.moduleVersion.id.group + \ - " MODULE_NAME: " + it.moduleVersion.id.name + \ - " MODULE_VERSION: " + it.moduleVersion.id.version + def runtimeClasspath = project.configurations.releaseRuntimeClasspath + def useJetifier = (project.findProperty("android.enableJetifier") ?: "false").toBoolean() + def rawModuleArtifacts = runtimeClasspath.incoming.artifacts.artifacts.findAll { + it.id.componentIdentifier instanceof ModuleComponentIdentifier + } + def standaloneJarComponents = rawModuleArtifacts.findAll { + it.file.name.toLowerCase(Locale.ROOT).endsWith(".jar") + }.collect { + it.id.componentIdentifier + } as Set + + def explodedAars = runtimeClasspath.incoming.artifactView { + componentFilter { + it instanceof ModuleComponentIdentifier + } + attributes { + attribute( + AndroidArtifacts.ARTIFACT_TYPE, + AndroidArtifacts.ArtifactType.EXPLODED_AAR.type) + } + }.artifacts.artifacts + def getOriginalFileName = { artifact -> + def identifier = artifact.id + identifier.metaClass.hasProperty(identifier, "originalFileName") != null + ? identifier.originalFileName + : artifact.file.name + } + def aarArtifactKeys = explodedAars.collect { + [it.id.componentIdentifier, getOriginalFileName(it)] + } as Set + + def standaloneJars + if (useJetifier && !standaloneJarComponents.isEmpty()) { + standaloneJars = runtimeClasspath.incoming.artifactView { + componentFilter { + it instanceof ModuleComponentIdentifier && standaloneJarComponents.contains(it) + } + attributes { + attribute( + AndroidArtifacts.ARTIFACT_TYPE, + AndroidArtifacts.ArtifactType.PROCESSED_JAR.type) + } + }.artifacts.artifacts.findAll { + def artifactKey = [it.id.componentIdentifier, getOriginalFileName(it)] + !aarArtifactKeys.contains(artifactKey) + } + } else { + standaloneJars = rawModuleArtifacts.findAll { + it.file.name.toLowerCase(Locale.ROOT).endsWith(".jar") + } } + + def artifacts = [] + explodedAars.each { + artifacts << [ + component: it.id.componentIdentifier.toString(), + originalFileName: getOriginalFileName(it), + kind: "exploded-aar", + path: it.file.absolutePath + ] + } + standaloneJars.each { + artifacts << [ + component: it.id.componentIdentifier.toString(), + originalFileName: getOriginalFileName(it), + kind: "jar", + path: it.file.absolutePath + ] + } + artifacts.sort { left, right -> + def componentComparison = left.component <=> right.component + if (componentComparison != 0) return componentComparison + def fileComparison = left.originalFileName <=> right.originalFileName + fileComparison != 0 ? fileComparison : left.kind <=> right.kind + } + + def artifactManifest = file("$buildDir/gradle-artifacts.json") + artifactManifest.parentFile.mkdirs() + artifactManifest.text = JsonOutput.prettyPrint(JsonOutput.toJson(artifacts)) } } diff --git a/server/src/main/resources/template.gradle.properties b/server/src/main/resources/template.gradle.properties index f3c63750..fad9a594 100644 --- a/server/src/main/resources/template.gradle.properties +++ b/server/src/main/resources/template.gradle.properties @@ -1,7 +1,7 @@ # Gradle will use the Jetifier tool to migrate dependencies to Android X if android.enableJetifier is true android.enableJetifier={{android-enable-jetifier}} -# Gradle will stop resolving dependencies if android.useAndroidX is false and a dependency is using Android X -android.useAndroidX={{android-enable-jetifier}} +# Dependency graphs may use AndroidX even when none of their artifacts need Jetifier. +android.useAndroidX=true org.gradle.java.home=/usr/local/jdk-25+36 diff --git a/server/src/test/java/com/defold/extender/ExtenderTest.java b/server/src/test/java/com/defold/extender/ExtenderTest.java index e7f4b120..8b0753bd 100644 --- a/server/src/test/java/com/defold/extender/ExtenderTest.java +++ b/server/src/test/java/com/defold/extender/ExtenderTest.java @@ -16,6 +16,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -30,8 +31,61 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.zip.ZipEntry; public class ExtenderTest { + // Verifies that compiled Android resource directories include their input index so equal AAR names cannot collide. + @Test + public void testCompiledResourceDirectoryNamesAreCollisionSafe(@TempDir File temporaryDirectory) { + File first = new File(temporaryDirectory, "first/common-1.0/res"); + File second = new File(temporaryDirectory, "second/common-1.0/res"); + + assertEquals("0000-common-1.0", Extender.getCompiledResourceDirectoryName(0, first)); + assertEquals("0001-common-1.0", Extender.getCompiledResourceDirectoryName(1, second)); + assertNotEquals( + Extender.getCompiledResourceDirectoryName(0, first), + Extender.getCompiledResourceDirectoryName(1, second)); + } + + // Verifies that returned resource packages keep legacy names when unique and add deterministic suffixes on collisions. + @Test + public void testReturnedResourcePackageNamesPreserveLegacyNamesAndResolveCollisions( + @TempDir File temporaryDirectory) throws IOException { + List resourceDirectories = List.of( + new File(temporaryDirectory, "first/common-1.0/res").getAbsolutePath(), + new File(temporaryDirectory, "second/common-1.0/res").getAbsolutePath(), + new File(temporaryDirectory, "third/common-1.0-0001/res").getAbsolutePath(), + new File(temporaryDirectory, "fourth/unique-1.0/res").getAbsolutePath()); + + assertEquals( + List.of( + "common-1.0", + "common-1.0-0001-1", + "common-1.0-0001", + "unique-1.0"), + Extender.getReturnedResourcePackageNames(resourceDirectories, Map.of())); + } + + // Verifies that Gradle artifact identities, rather than transient exploded-directory names, determine returned package names. + @Test + public void testReturnedResourcePackageNamesUseGradleArtifactIdentity( + @TempDir File temporaryDirectory) throws IOException { + File firstPackage = new File(temporaryDirectory, "transforms/first/jetified-library"); + File secondPackage = new File(temporaryDirectory, "transforms/second/jetified-library"); + List resourceDirectories = List.of( + new File(firstPackage, "res").getAbsolutePath(), + new File(secondPackage, "res").getAbsolutePath()); + String legacyName = "com.example-library-1.0.aar"; + + assertEquals( + List.of(legacyName, legacyName + "-0001"), + Extender.getReturnedResourcePackageNames( + resourceDirectories, + Map.of( + firstPackage.getCanonicalFile(), legacyName, + secondPackage.getCanonicalFile(), legacyName))); + } + static Map createEnv() { Map env = new HashMap<>(); @@ -247,7 +301,8 @@ public void testCollectLibraries() { static Map createAndroidEnv() { Map env = createEnv(); - env.put("ANDROID_PROGUARD", "/opt/android/proguard.jar"); + env.put("ANDROID_R8", "/opt/android/r8.jar"); + env.put("ANDROID_R8_VERSION", "8.13.19"); env.put("ANDROID_LIBRARYJAR", "/opt/android/android.jar"); env.put("ANDROID_NDK_PATH", "/opt/android/ndk"); env.put("ANDROID_NDK_SYSROOT", "/opt/android/ndk/sysroot"); @@ -259,6 +314,122 @@ static Map createAndroidEnv() return env; } + // Verifies that the Android SDK fixture uses only R8 configuration and discovers only the new .keep rule format. + @Test + @SuppressWarnings("deprecation") + public void testAndroidSdkUsesOnlyR8Configuration() throws Exception { + File root = new File("test-data"); + File sdk = new File(root, "sdk/a/defoldsdk"); + Configuration config = Extender.loadYaml(root, new File(sdk, "extender/build.yml"), Configuration.class); + PlatformConfig android = mergePlatformConfig(config, "armv7-android"); + + assertTrue(android.r8Cmd.contains("com.android.tools.r8.R8")); + assertFalse(android.r8Cmd.contains("--main-dex-rules")); + assertTrue(android.r8Cmd.contains("--pg-conf \"{{{.}}}\"")); + assertTrue(android.r8Cmd.contains("{{#jars}}\"{{{.}}}\"")); + assertTrue(android.dxCmd.contains("--min-api {{minAndroidSdkVersion}}")); + assertEquals("{{env.R8_VERSION}}", android.r8Version); + assertEquals("(?i).+(\\.keep)$", android.r8RuleSourceRe); + assertTrue(android.aapt2linkCmd.contains("{{#useR8}}--proguard \"{{{aaptKeepRules}}}\"")); + assertFalse(android.aapt2linkCmd.contains("--proguard-main-dex")); + assertNull(android.proGuardCmd); + assertNull(android.proGuardSourceRe); + + Collection candidates = List.of( + new File("manifests/android/extension.keep"), + new File("manifests/android/extension.pro")); + List rules = ExtenderUtil.filterFiles(candidates, android.r8RuleSourceRe); + assertEquals(List.of(new File("manifests/android/extension.keep")), rules); + } + + // Verifies that legacy ProGuard-era SDK YAML still loads while app.pro remains ignored and does not request R8. + @Test + @SuppressWarnings("deprecation") + public void testLegacyAndroidSdkProguardConfigurationIsAcceptedAndIgnored(@TempDir File tempDir) throws Exception { + String buildYaml = Files.readString( + new File("test-data/sdk/a/defoldsdk/extender/build.yml").toPath()); + buildYaml = buildYaml + .replace(" R8: \"{{env.ANDROID_R8}}\"\n", "") + .replace(" R8_VERSION: \"{{env.ANDROID_R8_VERSION}}\"\n", "") + .replace( + " LIBRARYJAR: \"{{env.ANDROID_LIBRARYJAR}}\"\n", + " PROGUARD: \"{{env.ANDROID_PROGUARD}}\"\n" + + " LIBRARYJAR: \"{{env.ANDROID_LIBRARYJAR}}\"\n") + .replace( + " r8Cmd: 'java -cp \"{{{env.R8}}}\" com.android.tools.r8.R8 --release --min-api {{minAndroidSdkVersion}} --lib \"{{{env.LIBRARYJAR}}}\" --pg-map-output \"{{{mapping}}}\" --output \"{{{classes_dex_dir}}}\" {{#rules}}--pg-conf \"{{{.}}}\" {{/rules}} {{#jars}}\"{{{.}}}\" {{/jars}}'\n" + + " r8Version: '{{env.R8_VERSION}}'\n" + + " r8RuleSourceRe: '(?i).+(\\.keep)$'\n", + " proGuardCmd: 'legacy-proguard-command'\n" + + " proGuardSourceRe: '(?i).+(\\.pro)$'\n") + .replace( + " {{#useR8}}--proguard \"{{{aaptKeepRules}}}\" {{/useR8}}", + " "); + + File sdk = new File(tempDir, "legacy-sdk"); + File sdkExtenderDir = new File(sdk, "extender"); + assertTrue(sdkExtenderDir.mkdirs()); + File buildFile = new File(sdkExtenderDir, "build.yml"); + Files.writeString(buildFile.toPath(), buildYaml); + + Configuration config = Extender.loadYaml(tempDir, buildFile, Configuration.class); + PlatformConfig android = mergePlatformConfig(config, "armv7-android"); + assertEquals("legacy-proguard-command", android.proGuardCmd); + assertEquals("(?i).+(\\.pro)$", android.proGuardSourceRe); + assertEquals("{{env.ANDROID_PROGUARD}}", android.env.get("PROGUARD")); + assertNull(android.r8Cmd); + assertFalse(android.aapt2linkCmd.contains("useR8")); + assertFalse(android.aapt2linkCmd.contains("aaptKeepRules")); + assertFalse(android.aapt2linkCmd.contains("--proguard")); + + File uploadWithoutProguard = new File(tempDir, "upload-without-proguard"); + File buildWithoutProguard = new File(tempDir, "build-without-proguard"); + assertTrue(uploadWithoutProguard.mkdirs()); + assertTrue(buildWithoutProguard.mkdirs()); + assertFalse(R8Builder.isRequested(uploadWithoutProguard)); + + assertDoesNotThrow(() -> new Extender.Builder() + .setPlatform("armv7-android") + .setSdk(sdk) + .setJobDirectory(tempDir) + .setUploadDirectory(uploadWithoutProguard) + .setBuildDirectory(buildWithoutProguard) + .setEnv(createAndroidEnv()) + .build()); + + File uploadWithProguard = new File(tempDir, "upload-with-proguard"); + File appDir = new File(uploadWithProguard, "_app"); + File buildWithProguard = new File(tempDir, "build-with-proguard"); + assertTrue(appDir.mkdirs()); + assertTrue(buildWithProguard.mkdirs()); + Files.writeString(new File(appDir, "app.pro").toPath(), "-keep class Legacy"); + assertFalse(R8Builder.isRequested(uploadWithProguard)); + + assertDoesNotThrow(() -> new Extender.Builder() + .setPlatform("armv7-android") + .setSdk(sdk) + .setJobDirectory(tempDir) + .setUploadDirectory(uploadWithProguard) + .setBuildDirectory(buildWithProguard) + .setEnv(createAndroidEnv()) + .build()); + } + + // Verifies that consumer rules and legacy .pro metadata are excluded from runtime META-INF copying. + @Test + public void testConsumerRulesAreNotCopiedAsRuntimeMetaInfResources() { + assertFalse(ExtenderUtil.isMetaInfEntryValuable(new ZipEntry("META-INF/proguard/rules.pro"))); + assertFalse(ExtenderUtil.isMetaInfEntryValuable(new ZipEntry("META-INF/proguard/rules.keep"))); + assertFalse(ExtenderUtil.isMetaInfEntryValuable(new ZipEntry("META-INF/com.android.tools/r8/rules.keep"))); + assertFalse(ExtenderUtil.isMetaInfEntryValuable(new ZipEntry( + "META-INF/com.android.tools/r8-from-0.0.0-arbitrary/rules.pro"))); + assertTrue(ExtenderUtil.isMetaInfEntryValuable(new ZipEntry( + "META-INF/com.android.tools/r8foo/not-a-rule.txt"))); + assertTrue(ExtenderUtil.isMetaInfEntryValuable(new ZipEntry( + "META-INF/com.android.tools/lint/model.xml"))); + assertTrue(ExtenderUtil.isMetaInfEntryValuable(new ZipEntry("META-INF/services/com.example.Service"))); + assertFalse(ExtenderUtil.isMetaInfEntryValuable(new ZipEntry("META-INF/example/info.pro"))); + } + // An .aar in an extension is unpacked into the same exploded layout as a Maven resolved .aar: // a directory named "*.aar" holding classes.jar, libs/, res/, assets/ and the AndroidManifest. @Test @@ -293,6 +464,18 @@ public void testResolveLocalAars() throws IOException, ExtenderException { assertTrue(new File(unpacked, "assets/local_aar.txt").exists()); assertTrue(new File(unpacked, "AndroidManifest.xml").exists()); + List extensionOwnedJars = extender.getExtensionLocalAarJars(extDir); + assertEquals( + List.of( + new File(unpacked, "classes.jar").getAbsolutePath(), + new File(unpacked, "libs/InnerJar.jar").getAbsolutePath()), + extensionOwnedJars); + R8Builder.ExtensionContext r8Context = R8Builder.createExtensionContext( + extDir, + extensionOwnedJars, + "(?i).+(\\.keep)$"); + assertEquals(extensionOwnedJars, r8Context.protectedJars); + FileUtils.deleteQuietly(jobDir); } diff --git a/server/src/test/java/com/defold/extender/IntegrationTest.java b/server/src/test/java/com/defold/extender/IntegrationTest.java index 66280e93..cc614f5e 100644 --- a/server/src/test/java/com/defold/extender/IntegrationTest.java +++ b/server/src/test/java/com/defold/extender/IntegrationTest.java @@ -20,6 +20,7 @@ import org.springframework.boot.logging.LoggingSystem; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -30,6 +31,8 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; @@ -38,6 +41,10 @@ import java.util.concurrent.TimeUnit; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; +import java.util.jar.JarOutputStream; + +import javax.tools.JavaCompiler; +import javax.tools.ToolProvider; @Tag("integration") @Execution(ExecutionMode.SAME_THREAD) @@ -479,7 +486,7 @@ public void buildEngineWithBaseExtension(TestConfiguration configuration) throws doBuild(sourceFiles, configuration); } - private boolean checkClassesDexClasses(File buildZip, List classes) throws IOException { + private Set getClassesDexClasses(File buildZip) throws IOException { Set dexClasses = new HashSet<>(); try (ZipFile zipFile = new ZipFile(buildZip)) { @@ -503,6 +510,12 @@ private boolean checkClassesDexClasses(File buildZip, List classes) thro } } + return dexClasses; + } + + private boolean checkClassesDexClasses(File buildZip, List classes) throws IOException { + Set dexClasses = getClassesDexClasses(buildZip); + for (String cls : classes) { if (!dexClasses.contains(cls)) { System.err.println(String.format("Missing class %s", cls)); @@ -512,6 +525,170 @@ private boolean checkClassesDexClasses(File buildZip, List classes) thro return true; } + private Path createGradleHandoffClassifierJar(Path fixtureDirectory) throws IOException { + Path source = fixtureDirectory.resolve("android/support/annotation/Nullable.java"); + Path classes = fixtureDirectory.resolve("classifier-classes"); + Files.createDirectories(source.getParent()); + Files.createDirectories(classes); + Files.writeString( + source, + "package android.support.annotation;\n" + + "public @interface Nullable {}\n", + StandardCharsets.UTF_8); + + JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); + assertNotNull(compiler, "The integration test requires a JDK"); + assertEquals( + 0, + compiler.run( + null, + null, + null, + "--release", + "8", + "-d", + classes.toString(), + source.toString())); + + Path outputJar = fixtureDirectory.resolve("handoff-1.0-extras.jar"); + try (ZipFile sourceJar = new ZipFile("test-data/ext/lib/android/JarDep.jar"); + OutputStream fileOutput = Files.newOutputStream(outputJar); + JarOutputStream jarOutput = new JarOutputStream(fileOutput)) { + ZipEntry jarDependency = sourceJar.getEntry("com/defold/JarDep.class"); + assertNotNull(jarDependency); + jarOutput.putNextEntry(new ZipEntry(jarDependency.getName())); + try (InputStream input = sourceJar.getInputStream(jarDependency)) { + input.transferTo(jarOutput); + } + jarOutput.closeEntry(); + + Path nullableClass = classes.resolve("android/support/annotation/Nullable.class"); + jarOutput.putNextEntry(new ZipEntry("android/support/annotation/Nullable.class")); + Files.copy(nullableClass, jarOutput); + jarOutput.closeEntry(); + } + return outputJar; + } + + private TestConfiguration latestAndroidConfiguration() { + return data().stream() + .filter(configuration -> configuration.platform.endsWith("-android")) + .max(Comparator + .comparingInt((TestConfiguration configuration) -> configuration.version.version.major) + .thenComparingInt(configuration -> configuration.version.version.middle) + .thenComparingInt(configuration -> configuration.version.version.minor) + .thenComparing(configuration -> configuration.platform)) + .orElseThrow(() -> new IllegalArgumentException("No Android integration configuration selected")); + } + + private List gradleArtifactHandoffResources( + Path fixtureDirectory, + TestConfiguration configuration, + boolean useJetifier) throws IOException { + Path repositoryVersion = fixtureDirectory.resolve("repo/com/defold/test/handoff/1.0"); + Files.createDirectories(repositoryVersion); + Path aar = repositoryVersion.resolve("handoff-1.0.aar"); + Path classifierJar = repositoryVersion.resolve("handoff-1.0-extras.jar"); + Path pom = repositoryVersion.resolve("handoff-1.0.pom"); + Files.copy( + Path.of("test-data/ext/lib/android/LocalAar.aar"), + aar, + StandardCopyOption.REPLACE_EXISTING); + Files.copy( + createGradleHandoffClassifierJar(fixtureDirectory), + classifierJar, + StandardCopyOption.REPLACE_EXISTING); + Files.writeString( + pom, + "\n" + + " 4.0.0\n" + + " com.defold.test\n" + + " handoff\n" + + " 1.0\n" + + " aar\n" + + "\n", + StandardCharsets.UTF_8); + + Path buildGradle = fixtureDirectory.resolve("build.gradle"); + Files.writeString( + buildGradle, + "repositories {\n" + + " maven { url uri(\"$rootDir/upload/ext/manifests/android/repo\") }\n" + + "}\n" + + "dependencies {\n" + + " implementation 'com.defold.test:handoff:1.0@aar'\n" + + " implementation 'com.defold.test:handoff:1.0:extras@jar'\n" + + "}\n", + StandardCharsets.UTF_8); + Path appManifest = fixtureDirectory.resolve("app.manifest"); + Files.writeString( + appManifest, + "platforms:\n" + + " android:\n" + + " context:\n" + + " jetifier: " + useJetifier + "\n", + StandardCharsets.UTF_8); + + String repositoryZipRoot = "ext/manifests/android/repo/com/defold/test/handoff/1.0/"; + return Lists.newArrayList( + new FileExtenderResource("test-data/AndroidManifest.xml", "AndroidManifest.xml"), + new FileExtenderResource("test-data/ext/ext.manifest"), + new FileExtenderResource("test-data/ext/src/test_ext.cpp"), + new FileExtenderResource("test-data/ext/src/TestGradleHandoff.java"), + new FileExtenderResource( + String.format("test-data/ext/lib/%s/libalib.a", configuration.platform)), + new FileExtenderResource(buildGradle.toString(), "ext/manifests/android/build.gradle"), + new FileExtenderResource(appManifest.toString(), "_app/app.manifest"), + new FileExtenderResource(aar.toString(), repositoryZipRoot + aar.getFileName()), + new FileExtenderResource( + classifierJar.toString(), + repositoryZipRoot + classifierJar.getFileName()), + new FileExtenderResource(pom.toString(), repositoryZipRoot + pom.getFileName())); + } + + // Verifies that AAR and classifier-JAR artifacts survive Gradle handoff, Jetifier selection, dexing, and packaging end to end. + @Test + public void buildAndroidGradleArtifactHandoff(@org.junit.jupiter.api.io.TempDir Path fixtureDirectory) + throws IOException, ExtenderClientException { + Set selectedPlatforms = TestUtils.selectedPlatforms(); + assumeTrue( + selectedPlatforms.isEmpty() + || selectedPlatforms.stream().anyMatch(platform -> platform.endsWith("-android")), + "This test is only run when an Android target is selected"); + TestConfiguration configuration = latestAndroidConfiguration(); + + for (boolean useJetifier : List.of(true, false)) { + Path buildFixture = Files.createDirectory( + fixtureDirectory.resolve(useJetifier ? "jetifier-on" : "jetifier-off")); + File destination = doBuild( + gradleArtifactHandoffResources(buildFixture, configuration, useJetifier), + configuration); + + Set dexClasses = getClassesDexClasses(destination); + assertTrue(dexClasses.containsAll(List.of( + "Lcom/defold/GradleHandoffTest;", + "Lcom/defold/JarDep;", + "Lcom/defold/localaar/InnerJar;", + "Lcom/defold/localaar/LocalAar;", + "Lcom/defold/localaar/R;"))); + String expectedAnnotation = useJetifier + ? "Landroidx/annotation/Nullable;" + : "Landroid/support/annotation/Nullable;"; + String unexpectedAnnotation = useJetifier + ? "Landroid/support/annotation/Nullable;" + : "Landroidx/annotation/Nullable;"; + assertTrue(dexClasses.contains(expectedAnnotation)); + assertFalse(dexClasses.contains(unexpectedAnnotation)); + try (ZipFile zipFile = new ZipFile(destination)) { + assertNotNull(zipFile.getEntry("assets/local_aar.txt")); + assertNotNull(zipFile.getEntry( + "packages/com.defold.test-handoff-1.0.aar/res/values/strings.xml")); + assertNotNull(zipFile.getEntry("gradle.lockfile")); + assertNotNull(zipFile.getEntry("gradle.dependencytree")); + } + } + } + @ParameterizedTest(name = "[{index}] {displayName} {arguments}") @MethodSource("data") public void buildAndroidCheckClassesDex(TestConfiguration configuration) throws IOException, ExtenderClientException { diff --git a/server/src/test/java/com/defold/extender/R8BuilderTest.java b/server/src/test/java/com/defold/extender/R8BuilderTest.java new file mode 100644 index 00000000..74eb99bd --- /dev/null +++ b/server/src/test/java/com/defold/extender/R8BuilderTest.java @@ -0,0 +1,1287 @@ +package com.defold.extender; + +import com.defold.extender.process.CommandLineTokenizer; + +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.zip.CRC32; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; +import java.util.zip.ZipOutputStream; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class R8BuilderTest { + private static final String KEEP_RULE_REGEX = "(?i).+(\\.keep)$"; + + private static File createJar(File jar, Map entries) throws IOException { + try (ZipOutputStream output = new ZipOutputStream(new FileOutputStream(jar))) { + for (Map.Entry entry : entries.entrySet()) { + output.putNextEntry(new ZipEntry(entry.getKey())); + output.write(entry.getValue().getBytes(StandardCharsets.UTF_8)); + output.closeEntry(); + } + } + return jar; + } + + private static File createJarWithBytes(File jar, Map entries) throws IOException { + try (ZipOutputStream output = new ZipOutputStream(new FileOutputStream(jar))) { + for (Map.Entry entry : entries.entrySet()) { + output.putNextEntry(new ZipEntry(entry.getKey())); + output.write(entry.getValue()); + output.closeEntry(); + } + } + return jar; + } + + private static File createJarWithDuplicateEntries( + File jar, + String entryName, + String firstContents, + String secondContents) throws IOException { + try (ZipArchiveOutputStream output = new ZipArchiveOutputStream(jar)) { + for (String contents : List.of(firstContents, secondContents)) { + byte[] bytes = contents.getBytes(StandardCharsets.UTF_8); + ZipArchiveEntry entry = new ZipArchiveEntry(entryName); + output.putArchiveEntry(entry); + output.write(bytes); + output.closeArchiveEntry(); + } + } + return jar; + } + + private static void writeZipEntry( + ZipOutputStream output, + String name, + byte[] contents, + int method) throws IOException { + ZipEntry entry = new ZipEntry(name); + entry.setMethod(method); + if (method == ZipEntry.STORED) { + CRC32 crc = new CRC32(); + crc.update(contents); + entry.setSize(contents.length); + entry.setCompressedSize(contents.length); + entry.setCrc(crc.getValue()); + } + output.putNextEntry(entry); + output.write(contents); + output.closeEntry(); + } + + private static List zipEntryNames(File jar) throws IOException { + List names = new ArrayList<>(); + try (ZipFile zipFile = new ZipFile(jar)) { + zipFile.entries().asIterator().forEachRemaining(entry -> names.add(entry.getName())); + } + return names; + } + + private static byte[] createMinimalClassFile(String internalName) throws IOException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (DataOutputStream output = new DataOutputStream(bytes)) { + output.writeInt(0xCAFEBABE); + output.writeShort(0); // minor_version + output.writeShort(52); // major_version (Java 8) + output.writeShort(5); // constant_pool_count + output.writeByte(1); // CONSTANT_Utf8 + output.writeUTF(internalName); + output.writeByte(7); // CONSTANT_Class + output.writeShort(1); + output.writeByte(1); // CONSTANT_Utf8 + output.writeUTF("java/lang/Object"); + output.writeByte(7); // CONSTANT_Class + output.writeShort(3); + output.writeShort(0x0021); // public, super + output.writeShort(2); // this_class + output.writeShort(4); // super_class + output.writeShort(0); // interfaces_count + output.writeShort(0); // fields_count + output.writeShort(0); // methods_count + output.writeShort(0); // attributes_count + } + return bytes.toByteArray(); + } + + private static byte[] createOversizedClassFileHeader(int maxClassfileHeaderBytes) throws IOException { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(maxClassfileHeaderBytes + 65536); + try (DataOutputStream output = new DataOutputStream(bytes)) { + output.writeInt(0xCAFEBABE); + output.writeShort(0); + output.writeShort(52); + int utf8Count = maxClassfileHeaderBytes / 65538 + 2; + output.writeShort(utf8Count + 1); + String padding = "a".repeat(65535); + for (int index = 0; index < utf8Count; ++index) { + output.writeByte(1); + output.writeUTF(padding); + } + } + return bytes.toByteArray(); + } + + private static String sanitizedRuleBody(File rule) throws IOException { + String contents = Files.readString(rule.toPath()); + assertTrue(contents.startsWith("-basedirectory ")); + int firstLineEnd = contents.indexOf('\n'); + assertTrue(firstLineEnd >= 0); + return contents.substring(firstLineEnd + 1); + } + + private static File sanitizedRuleBase(File rule) throws IOException { + String firstLine = Files.readString(rule.toPath()).lines().findFirst().orElseThrow(); + int quoteStart = firstLine.indexOf('"'); + char quote = '"'; + if (quoteStart < 0) { + quoteStart = firstLine.indexOf('\''); + quote = '\''; + } + assertTrue(quoteStart >= 0); + int quoteEnd = firstLine.indexOf(quote, quoteStart + 1); + assertTrue(quoteEnd > quoteStart); + return new File(firstLine.substring(quoteStart + 1, quoteEnd)); + } + + // Verifies that only _app/app.keep opts into R8; app.pro and extension consumer rules alone do not. + @Test + public void testOnlyAppKeepRequestsR8(@TempDir File uploadDir) throws Exception { + File appDir = new File(uploadDir, "_app"); + assertTrue(appDir.mkdirs()); + Files.writeString(new File(appDir, "app.pro").toPath(), "-keep class Legacy"); + assertFalse(R8Builder.isRequested(uploadDir)); + + File extensionRulesDir = new File(uploadDir, "extension/manifests/android"); + assertTrue(extensionRulesDir.mkdirs()); + Files.writeString( + new File(extensionRulesDir, "consumer.keep").toPath(), + "-keep class ExtensionConsumer"); + assertFalse(R8Builder.isRequested(uploadDir)); + + Files.writeString(new File(appDir, "app.keep").toPath(), "-keep class Current"); + assertTrue(R8Builder.isRequested(uploadDir)); + } + + // Verifies that a build without _app/app.keep skips R8 before requiring aapt rules or executing its command. + @Test + public void testNoAppKeepSkipsR8WithoutAaptRules(@TempDir File tempDir) throws Exception { + File uploadDir = new File(tempDir, "upload"); + File buildDir = new File(tempDir, "build"); + assertTrue(uploadDir.mkdirs()); + assertTrue(buildDir.mkdirs()); + PlatformConfig config = new PlatformConfig(); + + R8Builder builder = new R8Builder( + uploadDir, + buildDir, + config, + List.of(), + new HashMap<>(), + 21, + new R8Configuration(), + new TemplateExecutor(), + (command, context) -> { + throw new AssertionError("R8 command must not execute without app.keep"); + }); + + assertNull(builder.build(List.of(), Map.of(), null)); + } + + // Verifies that R8/aapt commands preserve exact quoting and literals, conditional rule flags, and data resources. + @Test + public void testConfiguredCommandPreservesQuotedPathSpecialCharacters() throws Exception { + File root = new File("test-data"); + File sdk = new File(root, "sdk/a/defoldsdk"); + Configuration config = Extender.loadYaml( + root, + new File(sdk, "extender/build.yml"), + Configuration.class); + PlatformConfig android = ExtenderTest.mergePlatformConfig(config, "armv7-android"); + + String r8Jar = "/tmp/R8 tools/{{literal}}/r8\\\"tool.jar"; + String androidJar = "/tmp/Android SDK/android\\lib.jar"; + String mapping = "/tmp/Output dir/mapping\\symbols.txt"; + String dexDir = "/tmp/Output dir/dex {{literal}} & files"; + String appRules = "/tmp/Rules dir/app\"rules.keep"; + String aaptRules = "/tmp/Rules dir/aapt&generated.keep"; + String programJar = "/tmp/Program jars/game\\code.jar"; + + Map context = new HashMap<>(); + context.put("env.R8", r8Jar); + context.put("env.LIBRARYJAR", androidJar); + context.put("minAndroidSdkVersion", 21); + context.put("mapping", mapping); + context.put("classes_dex_dir", dexDir); + context.put("rules", List.of(appRules, aaptRules)); + context.put("jars", List.of(programJar)); + + String rendered = new TemplateExecutor().executeOnceWithoutLogging( + android.r8Cmd, + R8Builder.createR8CommandContext(context)); + List arguments = CommandLineTokenizer.parse(rendered); + + assertFalse(rendered.contains("&")); + assertFalse(arguments.contains("--no-data-resources")); + assertTrue(rendered.contains("{{literal}}")); + assertTrue(arguments.contains(r8Jar)); + assertTrue(arguments.contains(androidJar)); + assertTrue(arguments.contains(mapping)); + assertTrue(arguments.contains(dexDir)); + assertTrue(arguments.contains(appRules)); + assertTrue(arguments.contains(aaptRules)); + assertTrue(arguments.contains(programJar)); + + context.put("env.ANDROID_BUILD_TOOLS_PATH", "/tmp/Android SDK/build tools"); + context.put("manifestFile", "/tmp/Manifest dir/AndroidManifest.xml"); + context.put("outJavaDirectory", "/tmp/Generated Java"); + context.put("outApkFile", "/tmp/Compiled resources/resources.apk"); + context.put("resourceIdsFile", "/tmp/Compiled resources/resource ids.txt"); + context.put("aaptKeepRules", aaptRules); + context.put("resourceListFile", "/tmp/Compiled resources/list.txt"); + context.put("extraPackages", ""); + context.put("useR8", false); + String d8AaptCommand = new TemplateExecutor().execute(android.aapt2linkCmd, context); + assertFalse(d8AaptCommand.contains("--proguard")); + + context.put("useR8", true); + String r8AaptCommand = new TemplateExecutor().execute(android.aapt2linkCmd, context); + assertTrue(r8AaptCommand.contains("--proguard")); + assertTrue(CommandLineTokenizer.parse(r8AaptCommand).contains(aaptRules)); + } + + // Verifies that requested R8 builds fail before command execution when aapt-generated keep rules are missing. + @Test + public void testMissingAaptGeneratedRulesIsAnError(@TempDir File tempDir) throws Exception { + File uploadDir = new File(tempDir, "upload"); + File appDir = new File(uploadDir, "_app"); + File buildDir = new File(tempDir, "build"); + assertTrue(appDir.mkdirs()); + assertTrue(buildDir.mkdirs()); + Files.writeString(new File(appDir, "app.keep").toPath(), "-keep class App"); + PlatformConfig config = new PlatformConfig(); + config.r8Cmd = "r8-command"; + config.r8Version = "8.13.19"; + boolean[] commandExecuted = {false}; + R8Builder builder = new R8Builder( + uploadDir, + buildDir, + config, + List.of(), + new HashMap<>(), + 21, + new R8Configuration(), + new TemplateExecutor(), + (command, context) -> commandExecuted[0] = true); + + ExtenderException exception = assertThrows( + ExtenderException.class, + () -> builder.build( + List.of(), + Map.of(), + new File(buildDir, "missing-aapt-generated.keep"))); + assertTrue(exception.getMessage().contains("aapt-generated.keep")); + assertFalse(commandExecuted[0]); + } + + // Verifies that aapt-generated keep rules pass the filesystem-directive policy before R8 can execute. + @Test + public void testAaptGeneratedRulesUseFilesystemPolicy(@TempDir File tempDir) throws Exception { + File uploadDir = new File(tempDir, "upload"); + File appDir = new File(uploadDir, "_app"); + File buildDir = new File(tempDir, "build"); + assertTrue(appDir.mkdirs()); + assertTrue(buildDir.mkdirs()); + Files.writeString(new File(appDir, "app.keep").toPath(), "-keep class App"); + File aaptRules = new File(buildDir, "aapt-generated.keep"); + Files.writeString(aaptRules.toPath(), "}-include /etc/hosts"); + PlatformConfig config = new PlatformConfig(); + config.r8Cmd = "r8-command"; + config.r8Version = "8.13.19"; + boolean[] commandExecuted = {false}; + R8Builder builder = new R8Builder( + uploadDir, + buildDir, + config, + List.of(), + new HashMap<>(), + 24, + new R8Configuration(), + new TemplateExecutor(), + (command, context) -> commandExecuted[0] = true); + + ExtenderException exception = assertThrows( + ExtenderException.class, + () -> builder.build(List.of(), Map.of(), aaptRules)); + + assertTrue(exception.getMessage().contains("forbidden filesystem directive")); + assertFalse(commandExecuted[0]); + } + + // Verifies that missing SDK R8 configuration is reported before secondary missing-rule validation. + @Test + public void testMissingR8ConfigurationIsReportedBeforeMissingAaptRules(@TempDir File tempDir) throws Exception { + File uploadDir = new File(tempDir, "upload"); + File appDir = new File(uploadDir, "_app"); + File buildDir = new File(tempDir, "build"); + assertTrue(appDir.mkdirs()); + assertTrue(buildDir.mkdirs()); + Files.writeString(new File(appDir, "app.keep").toPath(), "-keep class App"); + R8Builder builder = new R8Builder( + uploadDir, + buildDir, + new PlatformConfig(), + List.of(), + new HashMap<>(), + 21, + new R8Configuration(), + new TemplateExecutor(), + (command, context) -> { + throw new AssertionError("R8 command must not execute with missing configuration"); + }); + + ExtenderException exception = assertThrows( + ExtenderException.class, + () -> builder.build(List.of(), Map.of(), null)); + assertTrue(exception.getMessage().contains("does not provide r8Cmd and r8Version")); + assertFalse(exception.getMessage().contains("aapt-generated.keep")); + } + + // Verifies that a blank resolved R8 path produces a clear configuration error without invoking R8. + @Test + public void testBlankResolvedR8EnvironmentProducesClearConfigurationError(@TempDir File tempDir) throws Exception { + File uploadDir = new File(tempDir, "upload"); + File appDir = new File(uploadDir, "_app"); + File buildDir = new File(tempDir, "build"); + assertTrue(appDir.mkdirs()); + assertTrue(buildDir.mkdirs()); + Files.writeString(new File(appDir, "app.keep").toPath(), "-keep class App"); + File aaptRules = new File(buildDir, "aapt-generated.keep"); + Files.writeString(aaptRules.toPath(), "-keep class AaptGenerated"); + PlatformConfig config = new PlatformConfig(); + config.env.put("R8", "{{env.ANDROID_R8}}"); + config.r8Cmd = "java -cp \"{{{env.R8}}}\" com.android.tools.r8.R8"; + config.r8Version = "8.13.19"; + Map commandContext = new HashMap<>(); + commandContext.put("env.R8", " "); + + R8Builder builder = new R8Builder( + uploadDir, + buildDir, + config, + List.of(), + commandContext, + 21, + new R8Configuration(), + new TemplateExecutor(), + (command, context) -> { + throw new AssertionError("R8 command must not execute with unresolved configuration"); + }); + + ExtenderException exception = assertThrows( + ExtenderException.class, + () -> builder.build(List.of(), Map.of(), aaptRules)); + assertTrue(exception.getOutput().contains("does not provide r8Cmd and r8Version")); + } + + // Verifies that absent R8 command fields and malformed pinned versions are rejected with distinct errors. + @Test + public void testMissingConfigurationIsAnError() { + ExtenderException missing = assertThrows( + ExtenderException.class, + () -> R8Builder.validateConfiguration(null, "8.13.19")); + assertTrue(missing.getMessage().contains("does not provide r8Cmd and r8Version")); + + ExtenderException invalid = assertThrows( + ExtenderException.class, + () -> R8Builder.validateConfiguration("r8", "8.13-dev")); + assertTrue(invalid.getMessage().contains("Invalid r8Version")); + } + + // Verifies that supported R8 rule-directory ranges and suffixes use inclusive from and exclusive upto matching. + @Test + public void testRuleVersionRanges() { + assertTrue(R8Builder.isApplicableRuleDirectory("r8", "8.13.19")); + assertTrue(R8Builder.isApplicableRuleDirectory("r8-from-8.13.19", "8.13.19")); + assertTrue(R8Builder.isApplicableRuleDirectory("r8-from-8.13.18-dev", "8.13.19")); + assertTrue(R8Builder.isApplicableRuleDirectory("r8-from-0.0.0-arbitrary", "8.13.19")); + assertTrue(R8Builder.isApplicableRuleDirectory("r8-upto-8.13.20", "8.13.19")); + assertTrue(R8Builder.isApplicableRuleDirectory("r8-from-8.0.0-upto-9.0.0", "8.13.19")); + assertFalse(R8Builder.isApplicableRuleDirectory("r8-upto-8.13.19", "8.13.19")); + assertFalse(R8Builder.isApplicableRuleDirectory("r8-from-8.13.20", "8.13.19")); + assertFalse(R8Builder.isApplicableRuleDirectory("r8-from-invalid", "8.13.19")); + assertFalse(R8Builder.isApplicableRuleDirectory("r8foo", "8.13.19")); + } + + // Verifies that targeted-rule selection is deterministic and falls back to legacy META-INF/proguard rules when needed. + @Test + public void testSelectTargetedRulesWithLegacyFallback(@TempDir File tempDir) throws Exception { + Map entries = new HashMap<>(); + entries.put("META-INF/proguard/legacy.pro", "-keep class Legacy"); + entries.put("META-INF/com.android.tools/r8-upto-8.0.0/old.keep", "-keep class Old"); + entries.put("META-INF/com.android.tools/r8-from-8.0.0/z.keep", "-keep class Z"); + entries.put("META-INF/com.android.tools/r8-from-8.0.0/a.keep", "-keep class A"); + File targetedJar = createJar(new File(tempDir, "targeted.jar"), entries); + + assertEquals( + List.of( + "META-INF/com.android.tools/r8-from-8.0.0/a.keep", + "META-INF/com.android.tools/r8-from-8.0.0/z.keep"), + R8Builder.selectEmbeddedRuleEntries(targetedJar, "8.13.19")); + + entries.remove("META-INF/com.android.tools/r8-from-8.0.0/a.keep"); + entries.remove("META-INF/com.android.tools/r8-from-8.0.0/z.keep"); + File legacyJar = createJar(new File(tempDir, "legacy.jar"), entries); + assertEquals( + List.of("META-INF/proguard/legacy.pro"), + R8Builder.selectEmbeddedRuleEntries(legacyJar, "8.13.19")); + } + + // Verifies that pinned R8 handles prerelease, arbitrary, future, and lookalike rule-directory suffixes correctly. + @Test + public void testSelectTargetedRulesMatchesPinnedR8SuffixHandling(@TempDir File tempDir) + throws Exception { + File jar = createJar( + new File(tempDir, "suffixes.jar"), + Map.of( + "META-INF/com.android.tools/r8-from-8.13.18-dev/prerelease.keep", + "-keep class Prerelease", + "META-INF/com.android.tools/r8-from-0.0.0-arbitrary/malformed.keep", + "-keep class MalformedSuffix", + "META-INF/com.android.tools/r8-from-9.0.0-future/future.keep", + "-keep class Future", + "META-INF/com.android.tools/r8foo/not-a-rule.txt", + "metadata")); + + assertEquals( + List.of( + "META-INF/com.android.tools/r8-from-0.0.0-arbitrary/malformed.keep", + "META-INF/com.android.tools/r8-from-8.13.18-dev/prerelease.keep"), + R8Builder.selectEmbeddedRuleEntries(jar, "8.13.19")); + } + + // Verifies that duplicate embedded consumer-rule entry names are rejected instead of being applied ambiguously. + @Test + public void testDuplicateEmbeddedRuleNamesAreRejected(@TempDir File tempDir) throws Exception { + File jar = createJarWithDuplicateEntries( + new File(tempDir, "duplicate-rules.jar"), + "META-INF/proguard/rules.pro", + "-keep class First", + "-keep class Second"); + + ExtenderException exception = assertThrows( + ExtenderException.class, + () -> R8Builder.selectEmbeddedRuleEntries(jar, "8.13.19")); + + assertTrue(exception.getMessage().contains("Duplicate embedded R8 rule entry")); + } + + // Verifies that stripping consumer rules removes every supported rule namespace while preserving all other JAR entries. + @Test + public void testStripEmbeddedRulesRetainsAllOtherRawJarEntries(@TempDir File tempDir) + throws Exception { + File original = createJar( + new File(tempDir, "original.jar"), + new LinkedHashMap<>(Map.of( + "classes/example/Retained.class", "class bytes", + "assets/data.bin", "retained data", + "META-INF/proguard/legacy.pro", "-keep class Legacy", + "META-INF/com.android.tools/r8/rules.keep", "-keep class R8", + "META-INF/com.android.tools/r8-from-0.0.0-arbitrary/hidden.pro", "-keep class Hidden", + "META-INF/com.android.tools/r8-upto-99.0.0/future.keep", "-keep class Future", + "META-INF/com.android.tools/r8foo/not-a-rule.txt", "retained r8 metadata", + "META-INF/com.android.tools/lint/model.xml", "retained lint metadata", + "META-INF/proguarded/not-a-rule.txt", "retained proguard metadata"))); + File requestedOutput = new File(tempDir, "stripped/program-0000.jar"); + + File stripped = R8Builder.stripEmbeddedRuleEntries(original, requestedOutput); + + assertEquals(requestedOutput.getCanonicalFile(), stripped.getCanonicalFile()); + assertEquals( + List.of( + "META-INF/com.android.tools/lint/model.xml", + "META-INF/com.android.tools/r8foo/not-a-rule.txt", + "META-INF/proguarded/not-a-rule.txt", + "assets/data.bin", + "classes/example/Retained.class"), + zipEntryNames(stripped).stream().sorted().toList()); + try (ZipFile originalZip = new ZipFile(original); + ZipFile strippedZip = new ZipFile(stripped)) { + ZipEntry originalClass = originalZip.getEntry("classes/example/Retained.class"); + ZipEntry strippedClass = strippedZip.getEntry("classes/example/Retained.class"); + assertEquals(originalClass.getMethod(), strippedClass.getMethod()); + assertEquals(originalClass.getCrc(), strippedClass.getCrc()); + assertEquals(originalClass.getCompressedSize(), strippedClass.getCompressedSize()); + assertEquals( + "class bytes", + new String(strippedZip.getInputStream(strippedClass).readAllBytes(), StandardCharsets.UTF_8)); + } + } + + // Verifies that a JAR without embedded consumer rules is returned unchanged and no replacement JAR is written. + @Test + public void testStripEmbeddedRulesReturnsOriginalWhenNoRulesExist(@TempDir File tempDir) + throws Exception { + File original = createJar( + new File(tempDir, "original.jar"), + Map.of( + "classes/example/Retained.class", "class bytes", + "META-INF/com.android.tools/r8foo/not-a-rule.txt", "metadata")); + File requestedOutput = new File(tempDir, "stripped/program-0000.jar"); + + File result = R8Builder.stripEmbeddedRuleEntries(original, requestedOutput); + + assertEquals(original.getCanonicalFile(), result.getCanonicalFile()); + assertFalse(requestedOutput.exists()); + } + + // Verifies that rule stripping preserves directory entries, compression methods, and retained entry contents. + @Test + public void testStripEmbeddedRulesPreservesStoredDeflatedAndDirectoryEntries(@TempDir File tempDir) + throws Exception { + File original = new File(tempDir, "mixed.jar"); + try (ZipOutputStream output = new ZipOutputStream(new FileOutputStream(original))) { + writeZipEntry(output, "assets/", new byte[0], ZipEntry.STORED); + writeZipEntry( + output, + "assets/stored.bin", + "stored bytes".getBytes(StandardCharsets.UTF_8), + ZipEntry.STORED); + writeZipEntry( + output, + "assets/deflated.bin", + "deflated bytes".getBytes(StandardCharsets.UTF_8), + ZipEntry.DEFLATED); + writeZipEntry(output, "META-INF/proguard/", new byte[0], ZipEntry.STORED); + writeZipEntry( + output, + "META-INF/proguard/rules.pro", + "-keep class Removed".getBytes(StandardCharsets.UTF_8), + ZipEntry.DEFLATED); + } + + File stripped = R8Builder.stripEmbeddedRuleEntries( + original, + new File(tempDir, "stripped/mixed.jar")); + + assertEquals( + List.of( + "META-INF/proguard/", + "assets/", + "assets/deflated.bin", + "assets/stored.bin"), + zipEntryNames(stripped).stream().sorted().toList()); + try (ZipFile zipFile = new ZipFile(stripped)) { + ZipEntry stored = zipFile.getEntry("assets/stored.bin"); + ZipEntry deflated = zipFile.getEntry("assets/deflated.bin"); + assertEquals(ZipEntry.STORED, stored.getMethod()); + assertEquals(ZipEntry.DEFLATED, deflated.getMethod()); + assertEquals( + "stored bytes", + new String(zipFile.getInputStream(stored).readAllBytes(), StandardCharsets.UTF_8)); + assertEquals( + "deflated bytes", + new String(zipFile.getInputStream(deflated).readAllBytes(), StandardCharsets.UTF_8)); + } + } + + // Verifies that JAR discovery supports both locally unpacked AARs and AGP exploded-AAR directory layouts. + @Test + public void testAndroidPackageJarsSupportLocalAndAgpExplodedLayouts(@TempDir File tempDir) + throws Exception { + File localAar = new File(tempDir, "local.aar"); + assertTrue(new File(localAar, "libs").mkdirs()); + File localClasses = new File(localAar, "classes.jar"); + File localLibrary = new File(localAar, "libs/local-library.jar"); + Files.writeString(localClasses.toPath(), "classes"); + Files.writeString(localLibrary.toPath(), "library"); + + File explodedAar = new File(tempDir, "jetified-library"); + assertTrue(new File(explodedAar, "jars/libs").mkdirs()); + File explodedClasses = new File(explodedAar, "jars/classes.jar"); + File explodedLibrary = new File(explodedAar, "jars/libs/embedded-library.jar"); + Files.writeString(explodedClasses.toPath(), "classes"); + Files.writeString(explodedLibrary.toPath(), "library"); + + assertEquals(localClasses, R8Builder.getAndroidPackageClassesJar(localAar)); + assertEquals(explodedClasses, R8Builder.getAndroidPackageClassesJar(explodedAar)); + assertEquals( + List.of(localClasses.getAbsolutePath(), localLibrary.getAbsolutePath()), + R8Builder.getAndroidPackageJars(localAar)); + assertEquals( + List.of(explodedClasses.getAbsolutePath(), explodedLibrary.getAbsolutePath()), + R8Builder.getAndroidPackageJars(explodedAar)); + } + + // Verifies that JAR/AAR consumer rules have deterministic ordering, targeted precedence, and proguard.txt fallback. + @Test + public void testCollectJarAndAarConsumerRulesDeterministically(@TempDir File tempDir) throws Exception { + File standaloneJar = createJar( + new File(tempDir, "standalone.jar"), + Map.of( + "META-INF/proguard/legacy.pro", "-keep class JarLegacy", + "META-INF/com.android.tools/r8-from-8.0.0/rules.keep", "-keep class JarTarget")); + + File targetedAar = new File(tempDir, "jetified-targeted"); + assertTrue(new File(targetedAar, "jars").mkdirs()); + File targetedClasses = createJar( + new File(targetedAar, "jars/classes.jar"), + Map.of("META-INF/com.android.tools/r8/rules.keep", "-keep class AarTarget")); + Files.writeString(new File(targetedAar, "proguard.txt").toPath(), "-keep class AarRootIgnored"); + + File legacyAar = new File(tempDir, "legacy.aar"); + assertTrue(legacyAar.mkdirs()); + File legacyClasses = createJar(new File(legacyAar, "classes.jar"), Map.of("example/Legacy.class", "class")); + Files.writeString(new File(legacyAar, "proguard.txt").toPath(), "-keep class AarLegacy"); + + List rules = R8Builder.collectConsumerRules( + List.of(targetedClasses.getAbsolutePath(), standaloneJar.getAbsolutePath(), legacyClasses.getAbsolutePath()), + List.of(targetedAar, legacyAar), + "8.13.19", + new File(tempDir, "rules")); + List contents = new ArrayList<>(); + for (String rule : rules) { + contents.add(sanitizedRuleBody(new File(rule))); + } + + assertEquals( + List.of("-keep class AarTarget", "-keep class JarTarget", "-keep class AarLegacy"), + contents); + } + + // Verifies that valid bytecode names generate protection rules across Unicode, package-info, and module edge cases. + @Test + public void testGeneratedRulesProtectValidJarClasses(@TempDir File tempDir) throws Exception { + Map entries = new LinkedHashMap<>(); + entries.put("com/example/Outer.class", createMinimalClassFile("com/example/Outer")); + entries.put("com/example/Outer$Inner.class", createMinimalClassFile("com/example/Outer$Inner")); + entries.put("com/example/Foo-Bar.class", createMinimalClassFile("com/example/Foo-Bar")); + entries.put("com/example/9Patch.class", createMinimalClassFile("com/example/9Patch")); + entries.put("com/example/Ɯber.class", createMinimalClassFile("com/example/Ɯber")); + entries.put("com/example/Supplementary.class", createMinimalClassFile("com/example/š€Name")); + entries.put("com/example/NonJavaIdentifier.class", createMinimalClassFile("com/example/Ā”Name")); + entries.put("com/example/Combining.class", createMinimalClassFile("com/example/eĢName")); + entries.put("com/example/UnicodeSpace.class", createMinimalClassFile("com/example/įš€Name")); + entries.put("com/example/UnsupportedSpace.class", createMinimalClassFile("com/example/ā€€Name")); + entries.put("META-INF/versions/9/com/example/Versioned.class", "not parsed".getBytes(StandardCharsets.UTF_8)); + entries.put("decoy/NotModule.class", createMinimalClassFile("module-info")); + entries.put("module-info.class", createMinimalClassFile("com/example/ActuallyClass")); + entries.put("com/example/package-info.class", createMinimalClassFile("com/example/package-info")); + File jar = createJarWithBytes(new File(tempDir, "extension.jar"), entries); + File rules = R8Builder.writeProtectedJarKeepRules( + List.of(jar.getAbsolutePath()), + new File(tempDir, "generated.keep")); + String contents = Files.readString(rules.toPath()); + + assertTrue(contents.startsWith( + "-keepattributes *Annotation*,Signature,InnerClasses,EnclosingMethod,MethodParameters,Exceptions")); + assertTrue(contents.contains("-keep class com.example.Outer { *; }")); + assertTrue(contents.contains("-keep class com.example.Outer$Inner { *; }")); + assertTrue(contents.contains("-keep class com.example.Foo-Bar { *; }")); + assertTrue(contents.contains("-keep class com.example.9Patch { *; }")); + assertTrue(contents.contains("-keep class com.example.Ɯber { *; }")); + assertTrue(contents.contains("-keep class com.example.š€Name { *; }")); + assertTrue(contents.contains("-keep class com.example.Ā”Name { *; }")); + assertTrue(contents.contains("-keep class com.example.eĢName { *; }")); + assertTrue(contents.contains("-keep class com.example.?Name { *; }")); + assertFalse(contents.contains("com.example.įš€Name")); + assertFalse(contents.contains("com.example.ā€€Name")); + assertTrue(contents.contains("-keep class com.example.ActuallyClass { *; }")); + assertTrue(contents.contains("-keep class com.example.package-info { *; }")); + assertFalse(contents.contains("Versioned")); + assertFalse(contents.contains("module-info")); + } + + // Verifies that generated keep rules trust the class file's internal name rather than its potentially misleading ZIP path. + @Test + public void testGeneratedRulesUseClassFileNameInsteadOfZipEntryName(@TempDir File tempDir) + throws Exception { + File jar = createJarWithBytes( + new File(tempDir, "mismatched.jar"), + Map.of("x/Entry.class", createMinimalClassFile("test/LambdaProbe"))); + + String contents = Files.readString(R8Builder.writeProtectedJarKeepRules( + List.of(jar.getAbsolutePath()), + new File(tempDir, "generated.keep")).toPath()); + + assertTrue(contents.contains("-keep class test.LambdaProbe { *; }")); + assertFalse(contents.contains("x.Entry")); + } + + // Verifies that class-name metacharacters and directive-like text are neutralized in generated keep patterns. + @Test + public void testGeneratedRulesNeutralizeClassNameMetacharacters(@TempDir File tempDir) + throws Exception { + String internalName = "safe/Bad#\"'{}()\\\n\r\t @*!?%,:=~<>!&|-include /etc/hosts"; + File jar = createJarWithBytes( + new File(tempDir, "metacharacters.jar"), + Map.of("safe/Decoy.class", createMinimalClassFile(internalName))); + + String contents = Files.readString(R8Builder.writeProtectedJarKeepRules( + List.of(jar.getAbsolutePath()), + new File(tempDir, "generated.keep")).toPath()); + List lines = contents.lines().toList(); + assertEquals(2, lines.size()); + String prefix = "-keep class "; + String suffix = " { *; }"; + assertTrue(lines.get(1).startsWith(prefix)); + assertTrue(lines.get(1).endsWith(suffix)); + String pattern = lines.get(1).substring(prefix.length(), lines.get(1).length() - suffix.length()); + + assertTrue(pattern.startsWith("safe.Bad")); + assertTrue(pattern.endsWith(".etc.hosts")); + assertTrue(pattern.contains("?")); + assertFalse(pattern.contains("#")); + assertFalse(pattern.contains("\"")); + assertFalse(pattern.contains("'")); + assertFalse(pattern.contains("{")); + assertFalse(pattern.contains("}")); + assertFalse(pattern.contains("\\")); + assertFalse(pattern.contains("/")); + assertFalse(pattern.codePoints().anyMatch(Character::isWhitespace)); + } + + // Verifies that a malicious ZIP entry name cannot inject directives when the bytecode contains a safe class name. + @Test + public void testGeneratedRulesIgnoreMaliciousZipEntryName(@TempDir File tempDir) + throws Exception { + File jar = createJarWithBytes( + new File(tempDir, "malicious-entry.jar"), + Map.of( + "decoy/Entry.class\n-include /etc/hosts.class", + createMinimalClassFile("safe/Actual"))); + + String contents = Files.readString(R8Builder.writeProtectedJarKeepRules( + List.of(jar.getAbsolutePath()), + new File(tempDir, "generated.keep")).toPath()); + + assertTrue(contents.contains("-keep class safe.Actual { *; }")); + assertFalse(contents.contains("-include")); + assertFalse(contents.contains("/etc/hosts")); + assertEquals(2, contents.lines().count()); + } + + // Verifies that malformed JVM internal class names are rejected without leaving a partial generated rules file. + @ParameterizedTest + @ValueSource(strings = {"p//Foo", "p/Foo.Bar", "p/Foo;Bar", "p/Foo[Bar", "/Foo", "Foo/"}) + public void testGeneratedRulesRejectMalformedInternalClassNames( + String internalName, + @TempDir File tempDir) throws Exception { + File jar = createJarWithBytes( + new File(tempDir, "malformed-name.jar"), + Map.of("p/Entry.class", createMinimalClassFile(internalName))); + File output = new File(tempDir, "generated.keep"); + + ExtenderException exception = assertThrows( + ExtenderException.class, + () -> R8Builder.writeProtectedJarKeepRules(List.of(jar.getAbsolutePath()), output)); + + assertTrue(exception.getOutput().contains("Invalid class name")); + assertFalse(output.exists()); + } + + // Verifies that malformed class-file bytes produce a precise error and no generated rules output. + @Test + public void testGeneratedRulesRejectMalformedClassFile(@TempDir File tempDir) throws Exception { + File jar = createJar( + new File(tempDir, "malformed-class.jar"), + Map.of("p/Broken.class", "not a class file")); + File output = new File(tempDir, "generated.keep"); + + ExtenderException exception = assertThrows( + ExtenderException.class, + () -> R8Builder.writeProtectedJarKeepRules(List.of(jar.getAbsolutePath()), output)); + + assertTrue(exception.getOutput().contains("Failed to read class file")); + assertTrue(exception.getMessage().contains("Invalid class file magic")); + assertFalse(output.exists()); + } + + // Verifies that class-file header inflation is bounded before a compressed input can exhaust memory. + @Test + public void testGeneratedRulesBoundClassFileHeaderDecompression(@TempDir File tempDir) + throws Exception { + R8Configuration r8Configuration = new R8Configuration(); + r8Configuration.setMaxClassfileHeaderBytes(1024); + File jar = createJarWithBytes( + new File(tempDir, "oversized-class-header.jar"), + Map.of( + "p/Oversized.class", + createOversizedClassFileHeader( + r8Configuration.getMaxClassfileHeaderBytes()))); + File output = new File(tempDir, "generated.keep"); + + ExtenderException exception = assertThrows( + ExtenderException.class, + () -> R8Builder.writeProtectedJarKeepRules( + List.of(jar.getAbsolutePath()), + output, + r8Configuration)); + + assertTrue(exception.getMessage().contains("Class file header exceeds")); + assertFalse(output.exists()); + } + + // Verifies that generated extension rules enforce the maximum number of enumerated classes. + @Test + public void testGeneratedRuleClassCountIsBoundedDuringJarEnumeration(@TempDir File tempDir) + throws Exception { + R8Configuration r8Configuration = new R8Configuration(); + r8Configuration.setMaxGeneratedExtensionClasses(3); + Map entries = new LinkedHashMap<>(); + byte[] classFile = createMinimalClassFile("p/Repeated"); + for (int index = 0; + index <= r8Configuration.getMaxGeneratedExtensionClasses(); + ++index) { + entries.put(String.format("p/C%05d.class", index), classFile); + } + File jar = createJarWithBytes(new File(tempDir, "too-many-classes.jar"), entries); + File output = new File(tempDir, "generated.keep"); + + ExtenderException exception = assertThrows( + ExtenderException.class, + () -> R8Builder.writeProtectedJarKeepRules( + List.of(jar.getAbsolutePath()), + output, + r8Configuration)); + + assertTrue(exception.getMessage().contains("Too many classes")); + assertFalse(output.exists()); + } + + // Verifies that generated extension rules enforce a total output-size budget during JAR enumeration. + @Test + public void testGeneratedRuleBytesAreBoundedDuringJarEnumeration(@TempDir File tempDir) + throws Exception { + R8Configuration r8Configuration = new R8Configuration(); + r8Configuration.setMaxRuleFileBytes(512); + Map entries = new LinkedHashMap<>(); + String longName = "a".repeat(200); + for (int index = 0; index < 3; ++index) { + String internalName = String.format("p/C%03d%s", index, longName); + entries.put(internalName + ".class", createMinimalClassFile(internalName)); + } + File jar = createJarWithBytes(new File(tempDir, "oversized-generated-rules.jar"), entries); + File output = new File(tempDir, "generated.keep"); + + ExtenderException exception = assertThrows( + ExtenderException.class, + () -> R8Builder.writeProtectedJarKeepRules( + List.of(jar.getAbsolutePath()), + output, + r8Configuration)); + + assertTrue(exception.getMessage().contains("Generated R8 keep rules are too large")); + assertFalse(output.exists()); + } + + // Verifies that .keep discovery is recursive only under manifests/android and explicit rules disable automatic JAR protection. + @Test + public void testExtensionRuleDiscoveryIsRecursiveAndFolderScoped(@TempDir File tempDir) throws Exception { + File extensionDir = new File(tempDir, "extension"); + File manifestsDir = new File(extensionDir, "manifests/android"); + File nestedDir = new File(manifestsDir, "nested"); + assertTrue(nestedDir.mkdirs()); + + File rootKeep = new File(manifestsDir, "root.keep"); + File nestedKeep = new File(nestedDir, "nested.keep"); + Files.writeString(rootKeep.toPath(), "-keep class Root"); + Files.writeString(nestedKeep.toPath(), "-keep class Nested"); + Files.writeString(new File(manifestsDir, "legacy.pro").toPath(), "-keep class Legacy"); + Files.writeString(new File(extensionDir, "outside.keep").toPath(), "-keep class Outside"); + + R8Builder.ExtensionContext context = R8Builder.createExtensionContext( + extensionDir, + List.of("extension.jar"), + KEEP_RULE_REGEX); + + assertEquals( + List.of(nestedKeep.getAbsolutePath(), rootKeep.getAbsolutePath()), + context.ruleFiles); + assertTrue(context.protectedJars.isEmpty()); + } + + // Verifies that a rules-only extension placeholder is never forwarded to R8 as a program JAR. + @Test + public void testRulesOnlyPlaceholderNeverBecomesAProgramJar() { + R8Builder.ExtensionContext context = new R8Builder.ExtensionContext(); + Map extensionJars = new HashMap<>(); + extensionJars.put("/tmp/real.jar", context); + extensionJars.put("/tmp/" + R8Builder.RULES_WITHOUT_JAR, context); + + assertEquals(List.of("/tmp/real.jar"), R8Builder.getCompiledJars(extensionJars)); + } + + // Verifies that a full build assembles and sanitizes rules, strips consumer entries, and returns dex and mapping outputs. + @Test + public void testBuildAssemblesExtensionAndAarRulesAndReturnsMapping(@TempDir File tempDir) throws Exception { + File uploadDir = new File(tempDir, "upload"); + File appDir = new File(uploadDir, "_app"); + File buildDir = new File(tempDir, "build"); + assertTrue(appDir.mkdirs()); + assertTrue(buildDir.mkdirs()); + File appRules = new File(appDir, "app.keep"); + Files.writeString(appRules.toPath(), "-keep class App"); + File unselectedEngineRules = new File(appDir, "dmengine.keep"); + Files.writeString(unselectedEngineRules.toPath(), "-keep class UnselectedEngine"); + File aaptRules = new File(buildDir, "aapt-generated.keep"); + Files.writeString(aaptRules.toPath(), "-keep class AaptGenerated"); + + File extensionDir = new File(uploadDir, "extension"); + File extensionRulesDir = new File(extensionDir, "manifests/android"); + assertTrue(extensionRulesDir.mkdirs()); + File extensionRules = new File(extensionRulesDir, "extension.keep"); + Files.writeString(extensionRules.toPath(), "-keep class Extension"); + Files.writeString(new File(extensionRulesDir, "ignored.pro").toPath(), "-keep class Ignored"); + File extensionJar = createJar( + new File(tempDir, "extension.jar"), + Map.of("com/example/Extension.class", "class")); + R8Builder.ExtensionContext extensionContext = R8Builder.createExtensionContext( + extensionDir, + List.of(extensionJar.getAbsolutePath()), + KEEP_RULE_REGEX); + + File dependencyAar = new File(tempDir, "dependency.aar"); + assertTrue(dependencyAar.mkdirs()); + File dependencyJar = createJar( + new File(dependencyAar, "classes.jar"), + Map.of( + "com/example/Dependency.class", "class", + "META-INF/com.android.tools/r8-from-0.0.0-arbitrary/dependency.keep", + "-keep class Dependency")); + Files.writeString(new File(dependencyAar, "proguard.txt").toPath(), "-keep class Dependency"); + + PlatformConfig config = new PlatformConfig(); + config.r8Cmd = "r8-command {{#jars}}\"{{{.}}}\" {{/jars}}"; + config.r8Version = "8.13.19"; + + Map executedContext = new HashMap<>(); + Map executedCommand = new HashMap<>(); + R8Builder builder = new R8Builder( + uploadDir, + buildDir, + config, + List.of(dependencyAar), + new HashMap<>(), + 24, + new R8Configuration(), + new TemplateExecutor(), + (command, context) -> { + assertTrue(command.startsWith("r8-command ")); + executedCommand.put("command", command); + executedContext.putAll(context); + try { + Files.writeString( + new File((String) context.get("classes_dex_dir"), "classes.dex").toPath(), + "dex"); + Files.writeString(new File((String) context.get("mapping")).toPath(), "mapping"); + } catch (IOException e) { + throw new ExtenderException(e, "Failed to create fake R8 outputs"); + } + }); + + R8Builder.BuildOutput output = builder.build( + List.of(extensionJar.getAbsolutePath(), dependencyJar.getAbsolutePath()), + Map.of(extensionJar.getAbsolutePath(), extensionContext), + aaptRules); + + assertNotNull(output); + assertEquals(List.of("classes.dex"), Arrays.stream(output.dexFiles).map(File::getName).toList()); + assertEquals("mapping.txt", output.mappingFile.getName()); + assertEquals(0, output.metaInformationFiles.length); + assertEquals(output.mappingFile.getAbsolutePath(), executedContext.get("mapping")); + + @SuppressWarnings("unchecked") + List programJars = (List) executedContext.get("jars"); + assertTrue(programJars.contains(extensionJar.getAbsolutePath())); + assertFalse(programJars.contains(dependencyJar.getAbsolutePath())); + String strippedDependencyJar = programJars.stream() + .filter(path -> path.contains("r8-program-jars")) + .findFirst() + .orElseThrow(); + assertTrue(new File(strippedDependencyJar).isFile()); + assertTrue(zipEntryNames(new File(strippedDependencyJar)).contains("com/example/Dependency.class")); + assertFalse(zipEntryNames(new File(strippedDependencyJar)).stream() + .anyMatch(R8Builder::isEmbeddedRuleEntryName)); + assertTrue(executedCommand.get("command").contains(strippedDependencyJar)); + assertFalse(executedCommand.get("command").contains(dependencyJar.getAbsolutePath())); + + @SuppressWarnings("unchecked") + List assembledRules = (List) executedContext.get("rules"); + assertFalse(assembledRules.contains(appRules.getAbsolutePath())); + assertFalse(assembledRules.contains(aaptRules.getAbsolutePath())); + assertFalse(assembledRules.contains(unselectedEngineRules.getAbsolutePath())); + assertFalse(assembledRules.contains(extensionRules.getAbsolutePath())); + assertFalse(assembledRules.stream().anyMatch(path -> path.endsWith("ignored.pro"))); + assertFalse(executedContext.containsKey("mainDexRules")); + List sanitizedRules = assembledRules.stream() + .map(File::new) + .filter(rule -> { + try { + return Files.readString(rule.toPath()).startsWith("-basedirectory "); + } catch (IOException e) { + return false; + } + }) + .toList(); + assertEquals(4, sanitizedRules.size()); + assertTrue(sanitizedRules.stream().anyMatch(rule -> { + try { + return sanitizedRuleBody(rule).contains("-keep class App"); + } catch (IOException e) { + return false; + } + })); + assertTrue(sanitizedRules.stream().anyMatch(rule -> { + try { + return sanitizedRuleBody(rule).contains("-keep class Extension"); + } catch (IOException e) { + return false; + } + })); + assertTrue(sanitizedRules.stream().anyMatch(rule -> { + try { + return sanitizedRuleBody(rule).contains("-keep class AaptGenerated"); + } catch (IOException e) { + return false; + } + })); + assertTrue(sanitizedRules.stream().anyMatch(rule -> { + try { + return sanitizedRuleBody(rule).contains("-keep class Dependency"); + } catch (IOException e) { + return false; + } + })); + File ruleBase = sanitizedRuleBase(sanitizedRules.get(0)); + assertTrue(ruleBase.isDirectory()); + assertEquals(0, ruleBase.listFiles().length); + for (File sanitizedRule : sanitizedRules) { + assertEquals(ruleBase.getCanonicalFile(), sanitizedRuleBase(sanitizedRule).getCanonicalFile()); + assertFalse(sanitizedRule.toPath().startsWith(ruleBase.toPath())); + } + } + + // Verifies that supported annotation-based R8 class and member specifications pass rule validation. + @Test + public void testRulePolicyAllowsAnnotationRules() { + String rules = String.join("\n", + "# Annotation-based Gson and Android rules are supported", + "-keep,allowoptimization @com.example.JsonAdapter class *", + "-keep @com.example.Outer$Marker class *", + "-keep,allowoptimization", + "@com.example.MultilineAdapter class *", + "-keep @interface androidx.annotation.Keep", + "-keep public class * extends @com.example.BaseType ** {", + " @androidx.annotation.Keep ;", + " java.lang.String source();", + "}", + "-maximumremovedandroidloglevel 2 @com.example.Marker class * { *; }"); + + assertDoesNotThrow(() -> R8RulePolicy.validateRules(rules, "safe.keep")); + } + + // Verifies that rule sanitization normalizes a BOM, uses an empty isolated base, and rejects a contaminated sandbox. + @Test + public void testSanitizedRuleUsesSeparateEmptyBaseAndNormalizesBom(@TempDir File tempDir) + throws Exception { + File source = new File(tempDir, "source.keep"); + Files.writeString( + source.toPath(), + "\uFEFF@local.keep\n-keep @com.example.Marker class *"); + File emptyBase = R8RulePolicy.createEmptyBaseDirectory(tempDir); + File sanitized = new File(tempDir, "sanitized/rule.keep"); + + R8RulePolicy.writeSanitized( + sanitized, + R8RulePolicy.readAndValidate(source, new R8RulePolicy.Budget()), + emptyBase); + + assertEquals( + "@local.keep\n-keep @com.example.Marker class *", + sanitizedRuleBody(sanitized)); + assertEquals(emptyBase.getCanonicalFile(), sanitizedRuleBase(sanitized).getCanonicalFile()); + assertEquals(0, emptyBase.listFiles().length); + assertFalse(sanitized.toPath().startsWith(emptyBase.toPath())); + + Files.writeString(new File(emptyBase, "unexpected.keep").toPath(), "-keep class Unexpected"); + ExtenderException exception = assertThrows( + ExtenderException.class, + () -> R8RulePolicy.writeSanitized( + new File(tempDir, "second.keep"), + "-keep class Second".getBytes(StandardCharsets.UTF_8), + emptyBase)); + assertTrue(exception.getMessage().contains("sandbox is not empty")); + } + + // Verifies that the pinned R8-only class-spec options accepted by dependencies pass policy validation. + @ParameterizedTest + @ValueSource(strings = { + "-alwaysinline", + "-checkenumstringsdiscarded", + "-assumenoexternalsideeffects", + "-checkenumunboxed", + "-alwaysclassinline" + }) + public void testRulePolicyAllowsPinnedR8ClassSpecOptions(String option) { + assertDoesNotThrow(() -> R8RulePolicy.validateRules( + option + "\n@com.example.Marker class *", + "class-spec.keep")); + } + + // Verifies that direct and obfuscated filesystem directives are rejected across includes, paths, and output options. + @ParameterizedTest + @ValueSource(strings = { + "@/etc/passwd", + "@ /etc/hosts", + "@# hidden operand\n/etc/hosts", + "@\"/etc/hosts\"", + "@file:///etc/hosts", + "@../other-job/rules.keep", + "@~/rules.keep", + "@C:\\server\\rules.keep", + "@${user.home}/rules.keep", + "@/rules.keep", + "-keepkotlinmetadata\n@/etc/hosts", + "-maximumremovedandroidloglevel 4\n@/etc/hosts", + "-keepkotlinmetadata\r@/etc/hosts", + "# -include /etc/passwd", + "# harmless comment\r-include /etc/hosts", + "\uFEFF-include /etc/hosts", + "-renamesourcefileattribute \"-include /etc/hosts\"", + "-renamesourcefileattribute \"foo\\\" -include /etc/hosts \"", + "-renamesourcefileattribute {\n@/etc/hosts", + "}-include /etc/hosts", + "-keep class Safe @../other-job/rules.keep", + "-include /etc/passwd", + "-basedirectory /", + "-injars /tmp/input.jar", + "-outjars /tmp/output.jar", + "-libraryjars /tmp/library.jar", + "-applymapping /tmp/mapping.txt", + "-obfuscationdictionary /tmp/words.txt", + "-classobfuscationdictionary /tmp/classes.txt", + "-packageobfuscationdictionary /tmp/packages.txt", + "-printusage /tmp/usage.txt", + "-printconfiguration /tmp/configuration.txt", + "-printmapping /tmp/mapping.txt", + "-printseeds /tmp/seeds.txt", + "-dump /tmp/dump.txt" + }) + public void testRulePolicyRejectsFilesystemDirectives(String rules) { + ExtenderException exception = assertThrows( + ExtenderException.class, + () -> R8RulePolicy.validateRules(rules, "unsafe.keep")); + + assertTrue(exception.getMessage().contains("forbidden filesystem directive")); + } + + // Verifies that the number of embedded consumer-rule files is bounded before extraction. + @Test + public void testEmbeddedRuleEntryCountIsBounded(@TempDir File tempDir) throws Exception { + R8Configuration r8Configuration = new R8Configuration(); + r8Configuration.setMaxRuleFiles(3); + Map entries = new LinkedHashMap<>(); + for (int index = 0; index <= r8Configuration.getMaxRuleFiles(); ++index) { + entries.put(String.format("META-INF/proguard/rule-%04d.pro", index), "-keep class Example"); + } + File jar = createJar(new File(tempDir, "too-many-rules.jar"), entries); + + ExtenderException exception = assertThrows( + ExtenderException.class, + () -> R8Builder.selectEmbeddedRuleEntries( + jar, + "8.13.19", + r8Configuration)); + + assertTrue(exception.getMessage().contains("Too many embedded R8 rule files")); + } + + // Verifies that expanded embedded rule data is size-bounded and leaves no partial extracted files on failure. + @Test + public void testEmbeddedRuleExpandedSizeIsBounded(@TempDir File tempDir) throws Exception { + R8Configuration r8Configuration = new R8Configuration(); + r8Configuration.setMaxRuleFileBytes(32); + String oversizedRule = " ".repeat((int) r8Configuration.getMaxRuleFileBytes() + 1); + File jar = createJar( + new File(tempDir, "oversized-rule.jar"), + Map.of("META-INF/proguard/oversized.pro", oversizedRule)); + File outputDir = new File(tempDir, "collected"); + + ExtenderException exception = assertThrows( + ExtenderException.class, + () -> R8Builder.collectConsumerRules( + List.of(jar.getAbsolutePath()), + List.of(), + "8.13.19", + outputDir, + r8Configuration)); + + assertTrue(exception.getMessage().contains("R8 rule file is too large")); + assertEquals(0, outputDir.listFiles().length); + } + + // Verifies that legacy embedded consumer rules are subjected to the same filesystem-directive policy as app rules. + @Test + public void testEmbeddedConsumerRuleUsesFilesystemPolicy(@TempDir File tempDir) throws Exception { + File jar = createJar( + new File(tempDir, "unsafe-consumer-rule.jar"), + Map.of("META-INF/proguard/unsafe.pro", "-include /etc/passwd")); + + ExtenderException exception = assertThrows( + ExtenderException.class, + () -> R8Builder.collectConsumerRules( + List.of(jar.getAbsolutePath()), + List.of(), + "8.13.19", + new File(tempDir, "collected"))); + + assertTrue(exception.getMessage().contains("forbidden filesystem directive")); + } + + // Verifies that rules under malformed-but-applicable R8 suffix directories cannot bypass filesystem policy checks. + @Test + public void testMalformedR8SuffixConsumerRuleUsesFilesystemPolicy(@TempDir File tempDir) + throws Exception { + File jar = createJar( + new File(tempDir, "unsafe-malformed-suffix.jar"), + Map.of( + "META-INF/com.android.tools/r8-from-0.0.0-arbitrary/unsafe.pro", + "-include /etc/passwd")); + + ExtenderException exception = assertThrows( + ExtenderException.class, + () -> R8Builder.collectConsumerRules( + List.of(jar.getAbsolutePath()), + List.of(), + "8.13.19", + new File(tempDir, "collected"))); + + assertTrue(exception.getMessage().contains("forbidden filesystem directive")); + } +} diff --git a/server/src/test/java/com/defold/extender/R8ConfigurationTest.java b/server/src/test/java/com/defold/extender/R8ConfigurationTest.java new file mode 100644 index 00000000..212bb0eb --- /dev/null +++ b/server/src/test/java/com/defold/extender/R8ConfigurationTest.java @@ -0,0 +1,31 @@ +package com.defold.extender; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.Map; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.context.properties.bind.Binder; +import org.springframework.boot.context.properties.source.MapConfigurationPropertySource; + +public class R8ConfigurationTest { + @Test + public void testR8LimitsBindFromServerConfiguration() { + R8Configuration configuration = new Binder(new MapConfigurationPropertySource(Map.of( + "extender.r8.max-generated-extension-classes", 11, + "extender.r8.max-classfile-header-bytes", 12, + "extender.r8.max-total-classfile-header-bytes", 13, + "extender.r8.max-rule-files", 14, + "extender.r8.max-rule-file-bytes", 15, + "extender.r8.max-total-rule-bytes", 16))) + .bind("extender.r8", R8Configuration.class) + .get(); + + assertEquals(11, configuration.getMaxGeneratedExtensionClasses()); + assertEquals(12, configuration.getMaxClassfileHeaderBytes()); + assertEquals(13, configuration.getMaxTotalClassfileHeaderBytes()); + assertEquals(14, configuration.getMaxRuleFiles()); + assertEquals(15, configuration.getMaxRuleFileBytes()); + assertEquals(16, configuration.getMaxTotalRuleBytes()); + } +} diff --git a/server/src/test/java/com/defold/extender/R8LambdaRegressionTest.java b/server/src/test/java/com/defold/extender/R8LambdaRegressionTest.java new file mode 100644 index 00000000..56fbfd40 --- /dev/null +++ b/server/src/test/java/com/defold/extender/R8LambdaRegressionTest.java @@ -0,0 +1,174 @@ +package com.defold.extender; + +import com.android.tools.r8.CompilationMode; +import com.android.tools.r8.OutputMode; +import com.android.tools.r8.R8; +import com.android.tools.r8.R8Command; +import com.android.tools.r8.Version; +import com.android.tools.r8.origin.Origin; + +import org.jf.dexlib2.DexFileFactory; +import org.jf.dexlib2.Opcodes; +import org.jf.dexlib2.iface.ClassDef; +import org.jf.dexlib2.iface.DexFile; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import javax.tools.JavaCompiler; +import javax.tools.ToolProvider; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.jar.JarEntry; +import java.util.jar.JarOutputStream; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class R8LambdaRegressionTest { + private static Path writeSource(Path sourceRoot, String relativePath, String source) throws IOException { + Path sourceFile = sourceRoot.resolve(relativePath); + Files.createDirectories(sourceFile.getParent()); + Files.writeString(sourceFile, source); + return sourceFile; + } + + private static void compileJava8(Path outputDir, List sources) throws IOException { + JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); + assertNotNull(compiler, "Tests must run on a JDK, not a JRE"); + Files.createDirectories(outputDir); + + List arguments = new ArrayList<>(List.of( + "-Xlint:-options", + "--release", "8", + "-d", outputDir.toString())); + for (Path source : sources) { + arguments.add(source.toString()); + } + + ByteArrayOutputStream diagnostics = new ByteArrayOutputStream(); + int result = compiler.run(null, diagnostics, diagnostics, arguments.toArray(String[]::new)); + assertEquals(0, result, diagnostics.toString(StandardCharsets.UTF_8)); + } + + private static void createJar(Path classesDir, Path jar) throws IOException { + List classFiles; + try (Stream paths = Files.walk(classesDir)) { + classFiles = paths.filter(Files::isRegularFile).sorted().toList(); + } + + try (JarOutputStream output = new JarOutputStream(Files.newOutputStream(jar))) { + for (Path classFile : classFiles) { + String entryName = classesDir.relativize(classFile).toString().replace('\\', '/'); + JarEntry entry = new JarEntry(entryName); + entry.setTime(0L); + output.putNextEntry(entry); + Files.copy(classFile, output); + output.closeEntry(); + } + } + } + + // Verifies that pinned R8 8.13.19 desugars a real Java 8 lambda without requiring LambdaMetafactory or suppression rules. + @Test + void pinnedR8CompilesJava8LambdaWithoutLambdaMetafactorySuppression(@TempDir Path tempDir) + throws Exception { + assertEquals(8, Version.getMajorVersion()); + assertEquals(13, Version.getMinorVersion()); + assertEquals(19, Version.getPatchVersion()); + + Path programSources = tempDir.resolve("program-sources"); + Path programClasses = tempDir.resolve("program-classes"); + Path programSource = writeSource( + programSources, + "com/defold/r8test/LambdaProgram.java", + """ + package com.defold.r8test; + + import java.util.function.Function; + + public final class LambdaProgram { + public static String run(String value) { + Function prefix = input -> "lambda:" + input; + return prefix.apply(value); + } + } + """); + compileJava8(programClasses, List.of(programSource)); + + Path lambdaClass = programClasses.resolve("com/defold/r8test/LambdaProgram.class"); + String classFileConstants = new String(Files.readAllBytes(lambdaClass), StandardCharsets.ISO_8859_1); + assertTrue(classFileConstants.contains("java/lang/invoke/LambdaMetafactory"), + "The regression input must contain a real Java 8 invokedynamic lambda"); + + Path librarySources = tempDir.resolve("library-sources"); + Path libraryClasses = tempDir.resolve("library-classes"); + List librarySourceFiles = List.of( + writeSource(librarySources, "java/lang/Object.java", """ + package java.lang; + public class Object { + public Object() {} + } + """), + writeSource(librarySources, "java/lang/String.java", """ + package java.lang; + public final class String extends Object {} + """), + writeSource(librarySources, "java/lang/StringBuilder.java", """ + package java.lang; + public final class StringBuilder extends Object { + public StringBuilder() {} + public StringBuilder append(String value) { return this; } + public String toString() { return null; } + } + """), + writeSource(librarySources, "java/util/function/Function.java", """ + package java.util.function; + public interface Function { + R apply(T value); + } + """)); + compileJava8(libraryClasses, librarySourceFiles); + + Path programJar = tempDir.resolve("program.jar"); + Path libraryJar = tempDir.resolve("android-minimal.jar"); + createJar(programClasses, programJar); + createJar(libraryClasses, libraryJar); + assertFalse(Files.exists(libraryClasses.resolve("java/lang/invoke/LambdaMetafactory.class")), + "The Android-like library must not provide LambdaMetafactory"); + + List keepRules = List.of("-keep class com.defold.r8test.LambdaProgram { *; }"); + assertFalse(keepRules.stream().anyMatch(rule -> rule.contains("dontwarn"))); + assertFalse(keepRules.stream().anyMatch(rule -> rule.contains("LambdaMetafactory"))); + + Path outputDir = tempDir.resolve("r8-output"); + Path mapping = tempDir.resolve("mapping.txt"); + Files.createDirectories(outputDir); + R8Command command = R8Command.builder() + .addProgramFiles(programJar) + .addLibraryFiles(libraryJar) + .setMode(CompilationMode.RELEASE) + .setMinApiLevel(21) + .setOutput(outputDir, OutputMode.DexIndexed) + .addProguardConfiguration(keepRules, Origin.unknown()) + .setProguardMapOutputPath(mapping) + .build(); + R8.run(command); + + Path classesDex = outputDir.resolve("classes.dex"); + assertTrue(Files.size(classesDex) > 0); + assertTrue(Files.isRegularFile(mapping)); + + DexFile dexFile = DexFileFactory.loadDexFile(classesDex.toFile().getAbsolutePath(), Opcodes.forApi(21)); + assertTrue(dexFile.getClasses().stream() + .map(ClassDef::getType) + .anyMatch("Lcom/defold/r8test/LambdaProgram;"::equals)); + } +} diff --git a/server/src/test/java/com/defold/extender/R8ServiceLoaderRegressionTest.java b/server/src/test/java/com/defold/extender/R8ServiceLoaderRegressionTest.java new file mode 100644 index 00000000..62e2e034 --- /dev/null +++ b/server/src/test/java/com/defold/extender/R8ServiceLoaderRegressionTest.java @@ -0,0 +1,237 @@ +package com.defold.extender; + +import com.android.tools.r8.CompilationFailedException; +import com.android.tools.r8.CompilationMode; +import com.android.tools.r8.OutputMode; +import com.android.tools.r8.R8; +import com.android.tools.r8.R8Command; +import com.android.tools.r8.Version; +import com.android.tools.r8.origin.Origin; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import javax.tools.JavaCompiler; +import javax.tools.ToolProvider; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.jar.JarEntry; +import java.util.jar.JarOutputStream; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class R8ServiceLoaderRegressionTest { + private static final String SERVICE_TYPE = "com.defold.r8test.Greeting"; + private static final String PROVIDER_TYPE = "com.defold.r8test.GreetingProvider"; + + private static Path writeSource(Path sourceRoot, String relativePath, String source) throws IOException { + Path sourceFile = sourceRoot.resolve(relativePath); + Files.createDirectories(sourceFile.getParent()); + Files.writeString(sourceFile, source); + return sourceFile; + } + + private static void compileJava8(Path outputDir, List sources) throws IOException { + JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); + assertNotNull(compiler, "Tests must run on a JDK, not a JRE"); + Files.createDirectories(outputDir); + + List arguments = new ArrayList<>(List.of( + "-Xlint:-options", + "--release", "8", + "-d", outputDir.toString())); + sources.stream().map(Path::toString).forEach(arguments::add); + + ByteArrayOutputStream diagnostics = new ByteArrayOutputStream(); + int result = compiler.run(null, diagnostics, diagnostics, arguments.toArray(String[]::new)); + assertEquals(0, result, diagnostics.toString(StandardCharsets.UTF_8)); + } + + private static void createJar(Path classesDir, Path jar, Map resources) + throws IOException { + List classFiles; + try (Stream paths = Files.walk(classesDir)) { + classFiles = paths.filter(Files::isRegularFile).sorted().toList(); + } + + try (JarOutputStream output = new JarOutputStream(Files.newOutputStream(jar))) { + for (Path classFile : classFiles) { + String entryName = classesDir.relativize(classFile).toString().replace('\\', '/'); + JarEntry entry = new JarEntry(entryName); + entry.setTime(0L); + output.putNextEntry(entry); + Files.copy(classFile, output); + output.closeEntry(); + } + for (Map.Entry resource : resources.entrySet()) { + JarEntry entry = new JarEntry(resource.getKey()); + entry.setTime(0L); + output.putNextEntry(entry); + output.write(resource.getValue().getBytes(StandardCharsets.UTF_8)); + output.closeEntry(); + } + } + } + + @SuppressWarnings("unchecked") + private static void runR8(Map context, Path libraryJar) throws ExtenderException { + try { + List programJars = ((List) context.get("jars")).stream() + .map(Path::of) + .toList(); + List rules = new ArrayList<>(); + for (String ruleFile : (List) context.get("rules")) { + rules.addAll(Files.readAllLines(Path.of(ruleFile), StandardCharsets.UTF_8)); + } + + R8Command command = R8Command.builder() + .addProgramFiles(programJars) + .addLibraryFiles(libraryJar) + .setMode(CompilationMode.RELEASE) + .setMinApiLevel((Integer) context.get("minAndroidSdkVersion")) + .setOutput(Path.of((String) context.get("classes_dex_dir")), OutputMode.DexIndexed) + .addProguardConfiguration(rules, Origin.unknown()) + .setProguardMapOutputPath(Path.of((String) context.get("mapping"))) + .build(); + R8.run(command); + } catch (CompilationFailedException | IOException e) { + throw new ExtenderException(e, "Failed to run the ServiceLoader R8 regression fixture"); + } + } + + // Verifies that R8 output returns the renamed ServiceLoader descriptor and provider instead of stale original names. + @Test + void returnsR8RewrittenServiceDescriptorInsteadOfOriginalNames(@TempDir Path tempDir) + throws Exception { + assertEquals(8, Version.getMajorVersion()); + assertEquals(13, Version.getMinorVersion()); + assertEquals(19, Version.getPatchVersion()); + + Path programSources = tempDir.resolve("program-sources"); + Path programClasses = tempDir.resolve("program-classes"); + List programSourceFiles = List.of( + writeSource(programSources, "com/defold/r8test/Greeting.java", """ + package com.defold.r8test; + public interface Greeting { + String greet(); + } + """), + writeSource(programSources, "com/defold/r8test/GreetingProvider.java", """ + package com.defold.r8test; + public final class GreetingProvider implements Greeting { + public GreetingProvider() {} + public String greet() { return "hello"; } + } + """), + writeSource(programSources, "com/defold/r8test/ServiceMain.java", """ + package com.defold.r8test; + import java.util.ServiceLoader; + public final class ServiceMain { + public static String load() { + return ServiceLoader.load(Greeting.class).iterator().next().greet(); + } + } + """)); + compileJava8(programClasses, programSourceFiles); + + Path programJar = tempDir.resolve("program.jar"); + createJar( + programClasses, + programJar, + Map.of("META-INF/services/" + SERVICE_TYPE, PROVIDER_TYPE + "\n")); + + Path librarySources = tempDir.resolve("library-sources"); + Path libraryClasses = tempDir.resolve("library-classes"); + List librarySourceFiles = List.of( + writeSource(librarySources, "java/lang/Object.java", """ + package java.lang; + public class Object { public Object() {} } + """), + writeSource(librarySources, "java/lang/String.java", """ + package java.lang; + public final class String extends Object {} + """), + writeSource(librarySources, "java/lang/Class.java", """ + package java.lang; + public final class Class extends Object {} + """), + writeSource(librarySources, "java/util/Iterator.java", """ + package java.util; + public interface Iterator { E next(); } + """), + writeSource(librarySources, "java/util/ServiceLoader.java", """ + package java.util; + public final class ServiceLoader { + public static ServiceLoader load(Class service) { return null; } + public Iterator iterator() { return null; } + } + """)); + compileJava8(libraryClasses, librarySourceFiles); + Path libraryJar = tempDir.resolve("android-minimal.jar"); + createJar(libraryClasses, libraryJar, Map.of()); + + Path uploadDir = tempDir.resolve("upload"); + Path appDir = uploadDir.resolve("_app"); + Path buildDir = tempDir.resolve("build"); + Files.createDirectories(appDir); + Files.createDirectories(buildDir); + Files.writeString( + appDir.resolve("app.keep"), + "-keep class com.defold.r8test.ServiceMain { public static java.lang.String load(); }\n"); + Path aaptRules = buildDir.resolve("aapt-generated.keep"); + Files.writeString(aaptRules, "-keep class com.defold.r8test.ServiceMain\n"); + + PlatformConfig config = new PlatformConfig(); + config.r8Cmd = "in-process-r8 --output \"{{{classes_dex_dir}}}\""; + config.r8Version = "8.13.19"; + R8Builder builder = new R8Builder( + uploadDir.toFile(), + buildDir.toFile(), + config, + List.of(), + new HashMap<>(), + 21, + new R8Configuration(), + new TemplateExecutor(), + (command, context) -> runR8(context, libraryJar)); + + R8Builder.BuildOutput output = builder.build( + List.of(programJar.toString()), + Map.of(), + aaptRules.toFile()); + + assertEquals(1, output.dexFiles.length); + assertEquals(buildDir.resolve("classes.dex"), output.dexFiles[0].toPath()); + assertTrue(Files.size(output.dexFiles[0].toPath()) > 0); + assertEquals(1, output.metaInformationFiles.length); + + String mapping = Files.readString(output.mappingFile.toPath()); + Matcher providerMapping = Pattern.compile( + "^" + Pattern.quote(PROVIDER_TYPE) + " -> ([^:]+):$", + Pattern.MULTILINE).matcher(mapping); + assertTrue(providerMapping.find(), mapping); + String renamedProvider = providerMapping.group(1); + assertNotEquals(PROVIDER_TYPE, renamedProvider); + + Path serviceDescriptor = output.metaInformationFiles[0].toPath(); + String relativeDescriptor = buildDir.relativize(serviceDescriptor).toString().replace('\\', '/'); + assertTrue(relativeDescriptor.startsWith("META-INF/services/"), relativeDescriptor); + assertNotEquals("META-INF/services/" + SERVICE_TYPE, relativeDescriptor); + assertEquals(renamedProvider, Files.readString(serviceDescriptor).trim()); + assertFalse(Files.exists(buildDir.resolve("META-INF/services/" + SERVICE_TYPE))); + } +} diff --git a/server/src/test/java/com/defold/extender/services/RealGradleServiceTest.java b/server/src/test/java/com/defold/extender/services/RealGradleServiceTest.java new file mode 100644 index 00000000..beda471e --- /dev/null +++ b/server/src/test/java/com/defold/extender/services/RealGradleServiceTest.java @@ -0,0 +1,214 @@ +package com.defold.extender.services; + +import com.defold.extender.ExtenderBuildState; +import com.defold.extender.ExtenderException; + +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.core.io.ByteArrayResource; +import org.springframework.core.io.ClassPathResource; + +import java.io.File; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class RealGradleServiceTest { + private static String readResource(String path) throws Exception { + try (InputStream input = RealGradleServiceTest.class.getResourceAsStream(path)) { + assertNotNull(input); + return new String(input.readAllBytes(), StandardCharsets.UTF_8); + } + } + + // Verifies that the Gradle template reuses AGP's exploded AARs and processed JARs and writes a structured artifact manifest. + @Test + public void testBuildTemplateReusesAgpArtifacts() throws Exception { + String template = readResource("/template.build.gradle"); + + assertTrue(template.contains("AndroidArtifacts.ArtifactType.EXPLODED_AAR.type")); + assertTrue(template.contains("AndroidArtifacts.ArtifactType.PROCESSED_JAR.type")); + assertFalse(template.contains("AndroidArtifacts.ArtifactType.PROCESSED_AAR.type")); + assertTrue(template.contains("standaloneJarComponents")); + assertTrue(template.contains("standaloneJarComponents.contains(it)")); + assertTrue(template.contains("if (useJetifier && !standaloneJarComponents.isEmpty())")); + assertTrue(template.contains("kind: \"exploded-aar\"")); + assertTrue(template.contains("kind: \"jar\"")); + assertTrue(template.contains("gradle-artifacts.json")); + assertTrue(template.contains("JsonOutput.toJson(artifacts)")); + assertFalse(template.contains("println \"PATH:")); + } + + // Verifies that AndroidX stays enabled even when Jetifier itself is disabled. + @Test + public void testAndroidXIsIndependentFromJetifier() throws Exception { + String template = readResource("/template.gradle.properties"); + + assertTrue(template.contains("android.enableJetifier={{android-enable-jetifier}}")); + assertTrue(template.contains("android.useAndroidX=true")); + assertFalse(template.contains("android.useAndroidX={{android-enable-jetifier}}")); + } + + // Verifies that dependency resolution, reporting, and lock generation use one deterministic Gradle invocation. + @Test + public void testDependencyResolutionAndReportShareOneGradleInvocation() { + assertEquals( + List.of( + "gradle", + "downloadDependencies", + "dependencies", + "--configuration", + "releaseCompileClasspath", + "--write-locks", + "--stacktrace", + "--warning-mode", + "all", + "--no-daemon"), + RealGradleService.getGradleResolveCommand()); + } + + // Verifies that artifact-manifest parsing preserves canonical Gradle cache paths, metadata, ordering, and de-duplicates entries. + @Test + @SuppressWarnings("unchecked") + public void testArtifactManifestReturnsGradleCachePathsDirectly(@TempDir Path temporaryDirectory) + throws Exception { + Path explodedAar = Files.createDirectory(temporaryDirectory.resolve("jetified-library")); + Files.createDirectories(explodedAar.resolve("jars")); + Files.writeString(explodedAar.resolve("jars/classes.jar"), "classes"); + Path flatDirAar = Files.createDirectory(temporaryDirectory.resolve("jetified-local-aar")); + Files.createDirectories(flatDirAar.resolve("jars")); + Files.writeString(flatDirAar.resolve("jars/classes.jar"), "classes"); + Path classifierAar = Files.createDirectory(temporaryDirectory.resolve("jetified-classifier-aar")); + Files.createDirectories(classifierAar.resolve("jars")); + Files.writeString(classifierAar.resolve("jars/classes.jar"), "classes"); + Path jar = temporaryDirectory.resolve("jetified-library.jar"); + Files.writeString(jar, "jar"); + + JSONArray manifest = new JSONArray(); + JSONObject aarEntry = new JSONObject(); + aarEntry.put("component", "com.example:library:1.0"); + aarEntry.put("originalFileName", "library-1.0.aar"); + aarEntry.put("kind", "exploded-aar"); + aarEntry.put("path", explodedAar.toString()); + manifest.add(aarEntry); + JSONObject flatDirEntry = new JSONObject(); + flatDirEntry.put("component", ":LocalAar:"); + flatDirEntry.put("originalFileName", "LocalAar.aar"); + flatDirEntry.put("kind", "exploded-aar"); + flatDirEntry.put("path", flatDirAar.toString()); + manifest.add(flatDirEntry); + JSONObject classifierEntry = new JSONObject(); + classifierEntry.put("component", "com.example:library:1.0"); + classifierEntry.put("originalFileName", "library-1.0-debug.aar"); + classifierEntry.put("kind", "exploded-aar"); + classifierEntry.put("path", classifierAar.toString()); + manifest.add(classifierEntry); + JSONObject jarEntry = new JSONObject(); + jarEntry.put("component", "com.example:library:1.0"); + jarEntry.put("originalFileName", "library-1.0.jar"); + jarEntry.put("kind", "jar"); + jarEntry.put("path", jar.toString()); + manifest.add(jarEntry); + manifest.add(jarEntry); + + Path manifestFile = temporaryDirectory.resolve("gradle-artifacts.json"); + Files.writeString(manifestFile, manifest.toJSONString()); + + List artifacts = RealGradleService.parseGradleArtifacts(manifestFile.toFile()); + assertEquals(4, artifacts.size()); + assertEquals(explodedAar.toFile().getCanonicalFile(), artifacts.get(0).getFile()); + assertEquals("com.example:library:1.0", artifacts.get(0).getComponent()); + assertEquals("library-1.0.aar", artifacts.get(0).getOriginalFileName()); + assertEquals(GradleArtifact.Kind.EXPLODED_AAR, artifacts.get(0).getKind()); + assertEquals("com.example-library-1.0.aar", artifacts.get(0).getResourcePackageName()); + assertEquals(flatDirAar.toFile().getCanonicalFile(), artifacts.get(1).getFile()); + assertEquals("LocalAar.aar", artifacts.get(1).getResourcePackageName()); + assertEquals(classifierAar.toFile().getCanonicalFile(), artifacts.get(2).getFile()); + assertEquals("com.example-library-1.0.aar", artifacts.get(2).getResourcePackageName()); + assertEquals(jar.toFile().getCanonicalFile(), artifacts.get(3).getFile()); + assertEquals(GradleArtifact.Kind.JAR, artifacts.get(3).getKind()); + assertNull(artifacts.get(3).getResourcePackageName()); + } + + // Verifies that missing artifact paths and malformed artifact-manifest JSON are rejected with Extender errors. + @Test + @SuppressWarnings("unchecked") + public void testArtifactManifestRejectsInvalidEntries(@TempDir Path temporaryDirectory) + throws Exception { + Path missing = temporaryDirectory.resolve("missing.jar"); + JSONObject entry = new JSONObject(); + entry.put("kind", "jar"); + entry.put("path", missing.toString()); + JSONArray manifest = new JSONArray(); + manifest.add(entry); + + Path manifestFile = temporaryDirectory.resolve("gradle-artifacts.json"); + Files.writeString(manifestFile, manifest.toJSONString()); + assertThrows( + ExtenderException.class, + () -> RealGradleService.parseGradleArtifacts(manifestFile.toFile())); + + Files.writeString(manifestFile, "not-json"); + assertThrows( + ExtenderException.class, + () -> RealGradleService.parseGradleArtifacts(manifestFile.toFile())); + } + + // Verifies that an empty dependency set skips Gradle while still emitting empty lock and explanatory dependency reports. + @Test + public void testNoDependenciesSkipsGradle(@TempDir Path temporaryDirectory) throws Exception { + Path jobDirectory = Files.createDirectory(temporaryDirectory.resolve("job")); + Path buildDirectory = Files.createDirectory(jobDirectory.resolve("build")); + ExtenderBuildState buildState = mock(ExtenderBuildState.class); + when(buildState.getJobDir()).thenReturn(jobDirectory.toFile()); + when(buildState.getBuildDir()).thenReturn(buildDirectory.toFile()); + when(buildState.isUsedJetifier()).thenReturn(true); + + RealGradleService service = new RealGradleService( + new ByteArrayResource(( + "dependencies {\n" + + "{{#user-dependencies}}{{{.}}}{{/user-dependencies}}\n" + + "}\n").getBytes(StandardCharsets.UTF_8)), + new ClassPathResource("template.gradle.properties"), + new ClassPathResource("template.local.properties"), + new SimpleMeterRegistry()); + List outputFiles = new ArrayList<>(); + + List artifacts = service.resolveDependencies( + buildState, + Map.of( + "env.ANDROID_SDK_ROOT", "/unused/android-sdk", + "env.ANDROID_SDK_VERSION", "36"), + outputFiles); + + assertTrue(artifacts.isEmpty()); + assertEquals( + List.of( + buildDirectory.resolve("gradle.lockfile").toFile(), + buildDirectory.resolve("gradle.dependencytree").toFile()), + outputFiles); + assertEquals("", Files.readString(buildDirectory.resolve("gradle.lockfile"))); + assertEquals( + "No Gradle dependencies were declared.\n", + Files.readString(buildDirectory.resolve("gradle.dependencytree"))); + assertFalse(Files.exists(jobDirectory.resolve("gradle.properties"))); + assertFalse(Files.exists(jobDirectory.resolve("local.properties"))); + assertFalse(Files.exists(buildDirectory.resolve("gradle-artifacts.json"))); + } +} diff --git a/server/test-data/ext/src/TestGradleHandoff.java b/server/test-data/ext/src/TestGradleHandoff.java new file mode 100644 index 00000000..2149f75e --- /dev/null +++ b/server/test-data/ext/src/TestGradleHandoff.java @@ -0,0 +1,10 @@ +package com.defold; + +import com.defold.localaar.InnerJar; +import com.defold.localaar.LocalAar; + +class GradleHandoffTest { + static String doStuff() { + return LocalAar.DoStuff() + InnerJar.DoStuff() + JarDep.DoStuff(); + } +} diff --git a/server/test-data/sdk/a/defoldsdk/extender/build.yml b/server/test-data/sdk/a/defoldsdk/extender/build.yml index 8785d2f0..d2c96d3c 100644 --- a/server/test-data/sdk/a/defoldsdk/extender/build.yml +++ b/server/test-data/sdk/a/defoldsdk/extender/build.yml @@ -268,7 +268,8 @@ platforms: android: env: - PROGUARD: "{{env.ANDROID_PROGUARD}}" + R8: "{{env.ANDROID_R8}}" + R8_VERSION: "{{env.ANDROID_R8_VERSION}}" LIBRARYJAR: "{{env.ANDROID_LIBRARYJAR}}" NDK_PATH: "{{env.ANDROID_NDK_PATH}}" SYSROOT: "{{env.ANDROID_NDK_SYSROOT}}" @@ -303,11 +304,12 @@ platforms: javacCmd: 'javac -proc:none -source 1.8 -target 1.8 -J-Xms2048m -J-Xmx2048m -classpath {{env.LIBRARYJAR}}:{{classPath}} -d {{classesDir}} @{{sourcesListFile}}' jarCmd: 'jar cf {{outputJar}} -C {{classesDir}} .' # mainDexList is automatically created by listing all classes inside the engine jars - dxCmd: '{{env.ANDROID_BUILD_TOOLS_PATH}}/d8 --main-dex-rules {{mainDexList}} --output {{classes_dex_dir}} --release --lib {{env.LIBRARYJAR}} {{#jars}}{{.}} {{/jars}}' - proGuardCmd: 'java -jar {{env.PROGUARD}} {{#src}}-include {{.}} {{/src}} -libraryjars {{env.LIBRARYJAR}} {{#jars}}-injars {{.}} {{/jars}} {{#libraryjars}}-libraryjars {{.}} {{/libraryjars}} -outjar {{tgt}} -printmapping {{mapping}}' - proGuardSourceRe: '(?i).+(\.pro)$' + dxCmd: '{{env.ANDROID_BUILD_TOOLS_PATH}}/d8 --min-api {{minAndroidSdkVersion}} --main-dex-rules {{mainDexList}} --output {{classes_dex_dir}} --release --lib {{env.LIBRARYJAR}} {{#jars}}{{.}} {{/jars}}' + r8Cmd: 'java -cp "{{{env.R8}}}" com.android.tools.r8.R8 --release --min-api {{minAndroidSdkVersion}} --lib "{{{env.LIBRARYJAR}}}" --pg-map-output "{{{mapping}}}" --output "{{{classes_dex_dir}}}" {{#rules}}--pg-conf "{{{.}}}" {{/rules}} {{#jars}}"{{{.}}}" {{/jars}}' + r8Version: '{{env.R8_VERSION}}' + r8RuleSourceRe: '(?i).+(\.keep)$' aapt2compileCmd: '{{env.ANDROID_BUILD_TOOLS_PATH}}/aapt2 compile {{resourceFile}} -o {{outputDirectory}}' - aapt2linkCmd: '{{env.ANDROID_BUILD_TOOLS_PATH}}/aapt2 link {{#extraPackages.length}}--extra-packages {{#extraPackages}}{{.}}{{/extraPackages}}{{/extraPackages.length}} --proto-format --non-final-ids --auto-add-overlay --manifest {{manifestFile}} -I {{env.LIBRARYJAR}} --java {{outJavaDirectory}} -o {{outApkFile}} --emit-ids {{resourceIdsFile}} -R @{{resourceListFile}}' + aapt2linkCmd: '{{env.ANDROID_BUILD_TOOLS_PATH}}/aapt2 link {{#extraPackages.length}}--extra-packages {{#extraPackages}}{{.}}{{/extraPackages}}{{/extraPackages.length}} --proto-format --non-final-ids --auto-add-overlay --manifest {{manifestFile}} -I {{env.LIBRARYJAR}} --java {{outJavaDirectory}} -o {{outApkFile}} --emit-ids {{resourceIdsFile}} {{#useR8}}--proguard "{{{aaptKeepRules}}}" {{/useR8}}-R @{{resourceListFile}}' manifestName: 'AndroidManifest.xml' manifestMergeCmd: 'java -jar {{env.MANIFEST_MERGE_TOOL}} --platform {{platform}} --main {{mainManifest}} {{#libraries}} --lib {{.}} {{/libraries}} --out {{target}}'