diff --git a/abseil-cpp.spec b/abseil-cpp.spec index 89e8aeea3c8..b5665643d6d 100644 --- a/abseil-cpp.spec +++ b/abseil-cpp.spec @@ -1,4 +1,4 @@ -### RPM external abseil-cpp 20230802.2 +### RPM external abseil-cpp 20250814.1 ## INCLUDE cpp-standard Source: https://github.com/abseil/abseil-cpp/archive/%{realversion}.tar.gz @@ -7,7 +7,7 @@ BuildRequires: cmake gmake %prep %setup -n %{n}-%{realversion} -patch -p1 <%{_sourcedir}/1732.diff +#patch -p1 <%{_sourcedir}/1732.diff %build rm -rf ../build diff --git a/acts.spec b/acts.spec index 8fd4f9a39d9..c5526e2c0a0 100644 --- a/acts.spec +++ b/acts.spec @@ -1,13 +1,13 @@ -### RPM external acts v44.0.1 +### RPM external acts 46.8.1 ## INITENV +PATH PYTHON3PATH %{i}/python ## INCLUDE microarch_flags ## INCLUDE cuda-flags ## INCLUDE rocm-flags ## INCLUDE geant4-deps -%define tag 30fb4ea -%define branch cms/%{realversion} -%define github_user cms-externals +%define tag v%{realversion} +%define branch main +%define github_user acts-project Source: git+https://github.com/%{github_user}/%{n}.git?obj=%{branch}/%{tag}&export=%{n}-%{realversion}&output=/%{n}-%{realversion}-%{tag}.tgz Source99: scram-tools.file/tools/eigen/env @@ -71,6 +71,7 @@ cmake ../%{n}-%{realversion} \ -DCMAKE_BUILD_TYPE="Release" \ -DCMAKE_INSTALL_PREFIX="%{i}" \ -DCMAKE_SKIP_INSTALL_RPATH="ON" \ + -DEigen3_DIR="$EIGEN_ROOT/share/eigen3/cmake" \ %if 0%{!?without_cuda:1} -DCMAKE_CUDA_ARCHITECTURES="$(echo %{cuda_arch} | sed -e 's/ \+/;/g')" \ -DCMAKE_CUDA_FLAGS="-Wno-deprecated-gpu-targets" \ @@ -114,6 +115,8 @@ cmake ../%{n}-%{realversion} \ -DPython_EXECUTABLE=$(which python3) \ -DACTS_BUILD_EXAMPLES_PYTHON_BINDINGS="ON" \ -DTRACCC_BUILD_TESTING="ON" \ + -DTRACCC_BUILD_SIMULATION="ON" \ + -DTRACCC_BUILD_PERFORMANCE="ON" \ -DCMAKE_GTEST_DISCOVER_TESTS_DISCOVERY_MODE=PRE_TEST \ %endif -L diff --git a/bazel.spec b/bazel.spec index 4260b94f9a5..2fb8ea918d9 100644 --- a/bazel.spec +++ b/bazel.spec @@ -1,41 +1,12 @@ -### RPM external bazel 6.5.0 +### RPM external bazel 7.7.0 ## INCLUDE cpp-standard - -Source: https://github.com/bazelbuild/bazel/releases/download/%{realversion}/bazel-%{realversion}-dist.zip - -BuildRequires: java-env python3 python-python3 - -# For some build steps, bazel uses a process-wrapper that is executed in an empty environment. -# Therefore, the wrapper is linked to the system library /lib64/libstdc++.so.6, and complains about -# a missing GLIBCXX_3.4.21 version when (e.g.) used during the compilation of tensorflow python -# modules invoked via swig. This mechanism of bazel is actually only useful in combination with its -# remote compilation features. When disabling the process-wrapper, the local environment is taken -# into account which is the desired behavior for us. For example, see: -# - https://github.com/bazelbuild/bazel/issues/4137 -# - https://github.com/bazelbuild/bazel/issues/4510 -# - https://github.com/tensorflow/tensorboard/issues/1611 - -# configuration issue -# https://github.com/bazelbuild/bazel/issues/9392 -Patch0: bazel-3.7.0-patches -Patch1: bazel-absl - +Source: https://github.com/bazelbuild/bazel/releases/download/%{realversion}/bazel-%{realversion}-linux-x86_64 +BuildRequires: java-env %prep -%setup -q -c -n bazel-%{realversion} - -%patch0 -p1 -%patch1 -p1 - +%setup -T -c -n bazel-%{realversion} +cp %{_sourcedir}/bazel-%{realversion}-linux-x86_64 bazel %build - -export EXTRA_BAZEL_ARGS="--define=ABSOLUTE_JAVABASE=${JAVA_HOME} --jobs %{compiling_processes}" -export BAZEL_CXXOPTS="-Wno-error=deprecated:-std=c++%{cms_cxx_standard}" -${JAVA_HOME}/bin/java -version 2>&1 | grep -E -i 'openjdk version "[1-9]' -if [ $(${JAVA_HOME}/bin/java -version 2>&1 | grep -E -i 'openjdk version "[1-9]' | sed -E 's|.* "([0-9]+)[.].*|\1|') -ge 17 ] ; then - export JNI_FLAGS="--add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED" -fi -bash ./compile.sh - +# nothing to build — prebuilt binary %install -mkdir %{i}/bin -cp output/bazel %{i}/bin/. +mkdir -p %{i}/bin +install -m 0755 bazel %{i}/bin/bazel diff --git a/eigen.spec b/eigen.spec index 5066543fde2..660b88914ef 100644 --- a/eigen.spec +++ b/eigen.spec @@ -1,11 +1,8 @@ -### RPM external eigen c1d637433e3b3f9012b226c2c9125c494b470ae6 +### RPM external eigen 5.0.1 ## INITENV +PATH PKG_CONFIG_PATH %{i}/share/pkgconfig ## NOCOMPILER ## INCLUDE cpp-standard -%define tag b25e86af3379e35cd267d337693684dcdbdfd5d1 -%define branch cms/master/%{realversion} -%define github_user cms-externals -Source: git+https://github.com/%{github_user}/eigen-git-mirror.git?obj=%{branch}/%{tag}&export=%{n}-%{realversion}&output=/%{n}-%{realversion}-%{tag}.tgz +Source: git+https://gitlab.com/libeigen/eigen.git?tag=%{realversion}&export=%{n}-%{realversion}&output=/%{n}-%{realversion}.tgz BuildRequires: cmake %prep diff --git a/fasthadd.spec b/fasthadd.spec index 1ce06e4d2f2..ca98f8235af 100644 --- a/fasthadd.spec +++ b/fasthadd.spec @@ -5,7 +5,7 @@ %define user cms-sw Source0: https://raw.githubusercontent.com/%user/cmssw/%commit/DQMServices/Components/bin/fastHadd.cc Source1: https://raw.githubusercontent.com/%user/cmssw/%commit/DQMServices/Core/src/ROOTFilePB.proto -Requires: protobuf root +Requires: protobuf root abseil-cpp %prep @@ -17,7 +17,9 @@ cp %{_sourcedir}/ROOTFilePB.proto . protoc -I ./ --cpp_out=./ ROOTFilePB.proto perl -p -i -e 's|DQMServices/Core/interface/||' ROOTFilePB.pb.cc fastHadd.cc g++ -O2 -o %i/bin/fastHadd ROOTFilePB.pb.cc ./fastHadd.cc \ - -I$PROTOBUF_ROOT/include -L$PROTOBUF_ROOT/lib -lprotobuf \ + -I$PROTOBUF_ROOT/include -I$ABSEIL_CPP_ROOT/include \ + -L$PROTOBUF_ROOT/lib -lprotobuf \ + -L$ABSEIL_CPP_ROOT/lib $ABSEIL_CPP_ROOT/lib/libabsl_*.so \ `root-config --cflags --libs` %install diff --git a/flatbuffers.spec b/flatbuffers.spec index 00404d73be7..b1787fcdf09 100644 --- a/flatbuffers.spec +++ b/flatbuffers.spec @@ -1,4 +1,4 @@ -### RPM external flatbuffers 24.3.25 +### RPM external flatbuffers 25.9.23 ## INITENV +PATH LD_LIBRARY_PATH %{i}/lib64 ## INCLUDE cpp-standard %define tag v%{realversion} diff --git a/grpc.spec b/grpc.spec index 553cf2ded32..b00d055178e 100644 --- a/grpc.spec +++ b/grpc.spec @@ -1,4 +1,4 @@ -### RPM external grpc 1.35.0 +### RPM external grpc 1.82.0 ## INCLUDE cpp-standard Source: git+https://github.com/grpc/grpc.git?obj=master/v%{realversion}&export=%{n}-%{realversion}&submodules=1&output=/%{n}-%{realversion}.tgz @@ -11,8 +11,8 @@ Requires: protobuf zlib pcre c-ares abseil-cpp re2 %prep %setup -n %{n}-%{realversion} -patch -p1 <%{_sourcedir}/28212.patch -%patch1 -p1 +#patch -p1 <%{_sourcedir}/28212.patch +#%patch1 -p1 %build rm -rf ../build @@ -42,3 +42,5 @@ ninja -v %{makeprocesses} cd ../build ninja -v %{makeprocesses} install ln -sf ../../../abseil-cpp/${ABSEIL_CPP_VERSION}/include/absl %{i}/include/absl + + diff --git a/java-env.spec b/java-env.spec index b3b22ae0817..fc47edb7309 100644 --- a/java-env.spec +++ b/java-env.spec @@ -6,15 +6,7 @@ #For RHEL: Use java 17 for el8/el9 and 21 for above #Nor non RHEL e.g FC: use default system java -%define java_home /usr/lib/jvm/java -%if 0%{?rhel:1} -%if 0%{rhel} < 10 -%define java_home /usr/lib/jvm/java-17 -%endif -%if 0%{?rhel} >= 10 %define java_home /usr/lib/jvm/java-21 -%endif -%endif Source: none %prep diff --git a/lwtnn-assert-fix.patch b/lwtnn-assert-fix.patch index e4728e5bc41..013ed0c5636 100644 --- a/lwtnn-assert-fix.patch +++ b/lwtnn-assert-fix.patch @@ -34,3 +34,19 @@ index 7678ac1..7de3018 100644 namespace { int run_on_files(const lwt::JSONConfig& config, +diff --git a/cmake/FindEigen3.cmake b/cmake/FindEigen3.cmake +--- a/cmake/FindEigen3.cmake ++++ b/cmake/FindEigen3.cmake +@@ -30,7 +30,11 @@ if(NOT Eigen3_FIND_VERSION) + endif(NOT Eigen3_FIND_VERSION) + + macro(_eigen3_check_version) +- file(READ "${EIGEN3_INCLUDE_DIR}/Eigen/src/Core/util/Macros.h" _eigen3_version_header) ++ if(EXISTS "${EIGEN3_INCLUDE_DIR}/Eigen/Version") ++ file(READ "${EIGEN3_INCLUDE_DIR}/Eigen/Version" _eigen3_version_header) ++ else() ++ file(READ "${EIGEN3_INCLUDE_DIR}/Eigen/src/Core/util/Macros.h" _eigen3_version_header) ++ endif() + + string(REGEX MATCH "define[ \t]+EIGEN_WORLD_VERSION[ \t]+([0-9]+)" _eigen3_world_version_match "${_eigen3_version_header}") + set(EIGEN3_WORLD_VERSION "${CMAKE_MATCH_1}") diff --git a/patches/pip/pyg-lib.patch b/patches/pip/pyg-lib.patch index 2876d30a941..2905313558e 100644 --- a/patches/pip/pyg-lib.patch +++ b/patches/pip/pyg-lib.patch @@ -23,3 +23,14 @@ index 52399d2..542ca41 100644 packages=find_packages(), ext_modules=ext_modules, cmdclass=cmdclass, +--- a/setup.py ++++ b/setup.py +@@ -65,7 +65,7 @@ + f'-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={extdir}', + f'-DCMAKE_RUNTIME_OUTPUT_DIRECTORY={extdir}', + f'-DCMAKE_BUILD_TYPE={self.build_type}', +- f'-DCMAKE_PREFIX_PATH={torch.utils.cmake_prefix_path}', ++ f'-DCMAKE_PREFIX_PATH={torch.utils.cmake_prefix_path};{os.environ.get("CMAKE_PREFIX_PATH", "")}', + ] + + if WITH_CUDA and not os.getenv('TORCH_CUDA_ARCH_LIST'): diff --git a/pip/files/pyg-lib-common.file b/pip/files/pyg-lib-common.file index 4fd6d8d97c3..01dd8585ec4 100644 --- a/pip/files/pyg-lib-common.file +++ b/pip/files/pyg-lib-common.file @@ -1,7 +1,7 @@ ## INCLUDE cpp-standard ## INCLUDE compilation_flags %define default_pkgname py3-pyg-lib -Requires: zlib ninja +Requires: zlib ninja abseil-cpp Patch0: patches/pip/pyg-lib %define source0 git+https://github.com/pyg-team/pyg-lib?obj=master/%{realversion}&export=pyg-lib-%{realversion}&submodules=1&output=/source.tar.gz %define patchsrc0 \ diff --git a/pip/files/torch-extension-build.file b/pip/files/torch-extension-build.file index 3675241614c..22caba1a2ac 100644 --- a/pip/files/torch-extension-build.file +++ b/pip/files/torch-extension-build.file @@ -2,7 +2,7 @@ ## INCLUDE compilation_flags ## INCLUDE microarch_flags BuildRequires: cmake ninja -Requires: zlib +Requires: zlib abseil-cpp %define build_flags -O2 %{?arch_build_flags} %{default_microarch} %define PipPreBuild \ diff --git a/pip/onnx.file b/pip/onnx.file index 8d7a6bc303c..067d677c566 100644 --- a/pip/onnx.file +++ b/pip/onnx.file @@ -1,6 +1,6 @@ ## INCLUDE cpp-standard Patch0: patches/pip/py3-onnx Requires: cmake protobuf py3-protobuf py3-six py3-typing-extensions py3-numpy py3-pytest-runner -Requires: py3-ml_dtypes +Requires: py3-ml_dtypes abseil-cpp %define runpath_opts -n pb -n onnx %define PipPreBuild export ONNX_ML=1;export CMAKE_ARGS="-DCMAKE_CXX_STANDARD=%{cms_cxx_standard} -DONNX_USE_PROTOBUF_SHARED_LIBS=ON -DProtobuf_USE_STATIC_LIBS=OFF -DCMAKE_PREFIX_PATH=%{cmake_prefix_path}" diff --git a/pip/requirements.txt b/pip/requirements.txt index 77d2ea128e3..4f86f652ec8 100644 --- a/pip/requirements.txt +++ b/pip/requirements.txt @@ -309,7 +309,7 @@ prettytable==3.17.0 prometheus-client==0.24.1 prompt_toolkit==3.0.52 propcache==0.4.1 -protobuf==5.29.6 +protobuf==6.31.1 prwlock==0.4.1 psutil==7.2.2 ptyprocess==0.7.0 @@ -413,7 +413,7 @@ tabulate==0.9.0 tblib==3.2.2 tenacity==9.1.4 #NO_AUTO_UPDATE:1: Force to use tensorflow 2.17.0; this should match the version in tensorflow-sources.spec -tensorflow==2.17.0 +tensorflow==2.21.0 tensorboard==2.20.0 tensorflow-io-gcs-filesystem==0.37.1 tensorflow-estimator==2.15.0 diff --git a/pip/torch.file b/pip/torch.file index a3b5f1168db..ce554e5b099 100644 --- a/pip/torch.file +++ b/pip/torch.file @@ -10,7 +10,7 @@ Patch1: patches/pytorch-typename-decltype Patch2: patches/pytorch-hipblaslt-include BuildRequires: cmake ninja python-python3 py3-packaging -Requires: fftw3 eigen fxdiv numactl openmpi protobuf psimd python3 py3-PyYAML +Requires: fftw3 eigen fxdiv numactl openmpi protobuf psimd python3 py3-PyYAML abseil-cpp Requires: OpenBLAS zlib protobuf fmt py3-pybind11 py3-typing-extensions Requires: py3-filelock py3-fsspec py3-Jinja2 py3-networkx py3-sympy diff --git a/protobuf.spec b/protobuf.spec index b5d07546a9e..3e982cfeb96 100644 --- a/protobuf.spec +++ b/protobuf.spec @@ -1,4 +1,4 @@ -### RPM external protobuf 3.21.9 +### RPM external protobuf 6.31.1 ## INCLUDE cpp-standard #============= IMPORTANT NOTE ========================# # When changing the version of protobuf, remember to regenerate protobuf objects in CMSSW @@ -12,7 +12,7 @@ %define keep_archives true Source: https://github.com/protocolbuffers/protobuf/archive/v%{realversion}.zip -Requires: zlib +Requires: zlib abseil-cpp BuildRequires: cmake ninja # improves text_format printing Patch0: protobuf_text_format @@ -20,12 +20,12 @@ Patch1: protobuf-non-virtual-dtor %prep %setup -n %{n}-%{realversion} -%patch0 -p1 -%patch1 -p1 +#%patch0 -p1 +#%patch1 -p1 # Make sure the default c++sdt stand is c++11 -grep -q 'CMAKE_CXX_STANDARD *11' CMakeLists.txt -sed -i -e 's|CMAKE_CXX_STANDARD *11|CMAKE_CXX_STANDARD %{cms_cxx_standard}|' CMakeLists.txt +#grep -q 'CMAKE_CXX_STANDARD *11' CMakeLists.txt +#sed -i -e 's|CMAKE_CXX_STANDARD *11|CMAKE_CXX_STANDARD %{cms_cxx_standard}|' CMakeLists.txt %build rm -rf ../build mkdir ../build @@ -43,7 +43,8 @@ cmake ../%{n}-%{realversion} \ -DCMAKE_CXX_FLAGS="-I${ZLIB_ROOT}/include" \ -DCMAKE_C_FLAGS="-I${ZLIB_ROOT}/include" \ -DCMAKE_SHARED_LINKER_FLAGS="-L${ZLIB_ROOT}/lib" \ - -DCMAKE_PREFIX_PATH="${ZLIB_ROOT}" + -DCMAKE_PREFIX_PATH="${ABSEIL_CPP_ROOT};${ZLIB_ROOT}" \ + -Dprotobuf_ABSL_PROVIDER=package ninja -v %{makeprocesses} diff --git a/tensorflow-requires.file b/tensorflow-requires.file index a3058e283ba..1e0e83af4e7 100644 --- a/tensorflow-requires.file +++ b/tensorflow-requires.file @@ -1,7 +1,7 @@ Requires: python3 py3-numpy py3-mock py3-typing-extensions Requires: py3-keras-applications py3-keras-preprocessing py3-future py3-wrapt py3-gast py3-setuptools Requires: py3-cython py3-protobuf py3-astor py3-six py3-termcolor py3-absl-py -Requires: py3-flatbuffers +Requires: py3-flatbuffers llvm hwloc Requires: py3-psutil py3-lit py3-tblib py3-portpicker Requires: eigen protobuf zlib libpng libjpeg-turbo curl giflib sqlite grpc flatbuffers py3-pybind11 %define enable_gpu 0 diff --git a/tensorflow-sources.file b/tensorflow-sources.file index 3eaaac7c799..ab0417955ea 100644 --- a/tensorflow-sources.file +++ b/tensorflow-sources.file @@ -29,12 +29,19 @@ Source: git+https://github.com/%{github_user}/tensorflow.git?obj=%{branch}/%{tag %setup -q -n tensorflow-%{realversion} sed -i -e 's|lib/python[^/]*/site-packages/|lib/python%{cms_python3_major_minor_version}/site-packages/|' third_party/systemlibs/pybind11.BUILD +grep -c '^[a-zA-Z].*==' requirements_lock_3_12.txt %build export PYTHON_BIN_PATH="$(which python3)" export USE_DEFAULT_PYTHON_LIB_PATH=1 export GCC_HOST_COMPILER_PATH="$(which gcc)" +# TF 2.21's XLA CPU codegen intrinsics (xla/codegen/intrinsic/cpp, cc_to_llvm_ir) +# compile C++ to LLVM IR with clang-only flags (-emit-llvm, -fno-sanitize=all, ...), +# so gcc cannot build them. Use clang; on Linux it defaults to libstdc++ (the same +# std lib as gcc13), so the result stays ABI-compatible with the rest of CMSSW. +export TF_NEED_CLANG=1 +export CLANG_COMPILER_PATH="${LLVM_ROOT}/bin/clang" export CC_OPT_FLAGS="-Wno-sign-compare" BAZEL_OPTS="--batch --output_user_root ../build" @@ -54,12 +61,36 @@ DISTINCT_HOST_CONFIG="true" %endif BAZEL_OPTS="$BAZEL_OPTS --copt=-DEIGEN_USE_AVX512_GEMM_KERNELS=0" %endif -BAZEL_OPTS="$BAZEL_OPTS --distinct_host_configuration=${DISTINCT_HOST_CONFIG}" +BAZEL_OPTS="$BAZEL_OPTS" %if "%{?arch_build_flags}" BAZEL_OPTS="$BAZEL_OPTS $(echo %{arch_build_flags} | tr ' ' '\n' | grep -v '^$' | sed -e 's|^|--copt=|' | tr '\n' ' ')" %endif -BAZEL_OPTS="$BAZEL_OPTS --config=%{build_type} --cxxopt=-std=c++%{cms_cxx_standard} --host_cxxopt=-std=c++%{cms_cxx_standard} %{makeprocesses}" -BAZEL_OPTS="$BAZEL_OPTS --config=noaws --config=nogcp --config=nohdfs --config=nonccl" +BAZEL_OPTS="$BAZEL_OPTS --config=%{build_type} --cxxopt=-std=c++%{cms_cxx_standard} --host_cxxopt=-std=c++%{cms_cxx_standard} --jobs=auto --local_resources=memory=HOST_RAM*0.5" +BAZEL_OPTS="$BAZEL_OPTS --config=nogcp --config=nonccl" +# On Linux, XLA always compiles numa_hwloc.cc (//xla/tsl:with_numa_support = match_any +# linux_any), but the CMS hwloc system BUILD only links -lhwloc under +# --define=with_numa_support=true. Without it the hwloc symbols are undefined at link. +BAZEL_OPTS="$BAZEL_OPTS --define=with_numa_support=true" +# TF 2.21 defaults to the hermetic clang toolchain (rules_ml_toolchain / llvm18). +# The 'clang_local' config disables hermetic cc and toolchain resolution, falling +# back to @local_config_cc, which ./configure populates from the CMS clang +# (TF_NEED_CLANG=1 + CLANG_COMPILER_PATH set above). Clang is required because +# XLA's cc_to_llvm_ir intrinsics emit LLVM IR; it uses libstdc++ so the result +# is ABI-compatible with the gcc13-built rest of CMSSW. +BAZEL_OPTS="$BAZEL_OPTS --config=clang_local" +# oneDNN 3.7.3 bundles fmt 11.0.2 whose consteval format-string checks clang 21 +# rejects under C++20. oneDNN is a C++17 codebase, so build just its files with +# -std=c++17 (per_file_copt is applied after the global --cxxopt=-std=c++20, so it +# wins). Under C++17 fmt sets FMT_USE_CONSTEVAL 0 (FMT_CPLUSPLUS < 201709L) and the +# error disappears -- reliable, unlike patching the bundled fmt via ctx.patch. +BAZEL_OPTS="$BAZEL_OPTS --per_file_copt=external/onednn/.*[.]cpp@-std=c++17" +BAZEL_OPTS="$BAZEL_OPTS --per_file_copt=external/jpegxl/.*enc_fast_lossless.*@-O1" +# Build-time tools (protoc_minimal, ...) run under bazel with `env -` (cleared env), +# so they lose LD_LIBRARY_PATH and can't find the CMS gcc13 libstdc++ (GLIBCXX_3.4.31) -- +# nor any other external lib -- falling back to the older system /lib64. Forward the full +# CMS ${LD_LIBRARY_PATH} into the action environment so tool executions resolve all their +# runtime deps at build time. Supplied per-build, baked into NO artifact -> relocatable. +BAZEL_OPTS="$BAZEL_OPTS --host_action_env=LD_LIBRARY_PATH=${LD_LIBRARY_PATH} --action_env=LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" %if "%{enable_tf_mkldnn}" == "0" BAZEL_OPTS="$BAZEL_OPTS --define=zendnn=true --define=build_with_mkl_dnn_only=false --define=build_with_mkl=false --define=enable_mkl=false --define=tensorflow_mkldnn_contraction_kernel=0 --define=build_with_mkl_opensource=false" %endif @@ -77,7 +108,9 @@ export TF_CUDA_CLANG=0 export cuda=Y %endif export TF_NEED_CUDA=%{enable_gpu} -export TF_NEED_CLANG=0 +# TF_NEED_CLANG / CLANG_COMPILER_PATH are set earlier (clang is required for XLA's +# LLVM-IR intrinsics); keep the value consistent here. +export TF_NEED_CLANG=1 export TF_NEED_JEMALLOC=0 export TF_NEED_HDFS=0 export TF_NEED_GCP=0 @@ -108,6 +141,7 @@ echo "curl:${CURL_ROOT}" >> ${TF_CMS_EXTERNALS} #echo "com_google_protobuf:${PROTOBUF_ROOT}" >> ${TF_CMS_EXTERNALS} echo "com_github_grpc_grpc:${GRPC_ROOT}" >> ${TF_CMS_EXTERNALS} echo "gif:${GIFLIB_ROOT}" >> ${TF_CMS_EXTERNALS} +echo "hwloc:${HWLOC_ROOT}" >> ${TF_CMS_EXTERNALS} echo "org_sqlite:${SQLITE_ROOT}" >> ${TF_CMS_EXTERNALS} echo "cython:" >> ${TF_CMS_EXTERNALS} echo "flatbuffers:${FLATBUFFERS_ROOT}" >> ${TF_CMS_EXTERNALS} @@ -129,6 +163,27 @@ for name in $( grep '^[a-zA-Z].*==' ${tf_requirement} | sed 's| *==.*||;s|-|_|g' done rm -f ${tf_requirement}; touch ${tf_requirement} +# TF 2.21 switched rules_python to the "hub" layout: pip packages are referenced +# as @pypi// (and numpy headers as @pypi//numpy:numpy_headers) instead of +# the older per-package @pypi_ repos. The CMS build provides each package as +# its own local repo (@pypi_, from third_party/cms/.BUILD), so remap the +# hub labels back to the per-package repos in all BUILD/.bzl files. The +# '@pypi//:requirements.bzl' load in WORKSPACE is intentionally left untouched +# (the empty hermetic @pypi hub still generates it). +find . \( -name BUILD -o -name BUILD.bazel -o -name '*.bzl' -o -name '*.BUILD' \) -type f -print0 \ + | xargs -0 --no-run-if-empty grep -lZ '@pypi//' \ + | xargs -0 --no-run-if-empty sed -i \ + -e 's|@pypi//numpy:numpy_headers|@pypi_numpy//:numpy_headers|g' \ + -e 's|@pypi//\([a-zA-Z0-9_][a-zA-Z0-9_]*\)|@pypi_\1//:pkg|g' + +# gif is a CMS system lib (TF_SYSTEM_LIBS=gif), but @gif is a linkopt-only cc_library. +# Under cc_shared_library its standalone '-lgif' linker_input is dropped while +# libgif_internal.pic.a is still linked into libtensorflow_cc.so.2, so the link fails +# with undefined DGifOpen/DGifSlurp/GifErrorString/... . Put the link flags on +# gif_internal itself so they travel in the same linker_input as its objects. +grep -q '"//conditions:default": \["-ldl"\],' tensorflow/core/lib/gif/BUILD +sed -i -e 's|"//conditions:default": \["-ldl"\],|"//conditions:default": ["-ldl", "-Lexternal/gif/lib", "-lgif"],|' tensorflow/core/lib/gif/BUILD + # clear the build dir and configure if [ -d ../build ] ; then chmod -R u+w ../build @@ -156,8 +211,9 @@ bazel $BAZEL_OPTS //tensorflow/tools/pip_package:wheel mkdir %{i}/lib-xla-runtime find bazel-out/%{bazel_dir}/bin -path '*/pip_package/wheel_house/tensorflow-%{realversion}*.whl' | xargs --no-run-if-empty -i cp '{}' %{i}/ find bazel-out/%{bazel_dir}/bin -path '*/external/ducc/libfft*.pic.a' | xargs --no-run-if-empty -i cp '{}' %{i}/lib-xla-runtime -find bazel-out/%{bazel_dir}/bin -path '*/external/local_tsl/tsl/*/libmutex.pic.a' | xargs --no-run-if-empty -i cp '{}' %{i}/lib-xla-runtime -find bazel-out/%{bazel_dir}/bin -path '*/external/nsync/libnsync_cpp.pic.a' | xargs --no-run-if-empty -i cp '{}' %{i}/lib-xla-runtime -for lib in libfft.pic.a libfft_wrapper.pic.a libmutex.pic.a libnsync_cpp.pic.a ; do +# TF 2.21 dropped nsync and the standalone TSL mutex lib in favour of abseil +# synchronisation, so libmutex.pic.a / libnsync_cpp.pic.a no longer exist (and +# local_tsl was renamed to tsl). Only the ducc FFT runtime libs remain here. +for lib in libfft.pic.a libfft_wrapper.pic.a ; do test -e %{i}/lib-xla-runtime/${lib} done diff --git a/tensorflow-sources.spec b/tensorflow-sources.spec index c5c12d8c2ec..2afd608d4c1 100644 --- a/tensorflow-sources.spec +++ b/tensorflow-sources.spec @@ -1,7 +1,7 @@ -### RPM external tensorflow-sources 2.17.0 -%define tag f48e52b5a63d0c54dc209bc8354c5e46afbe8889 +### RPM external tensorflow-sources 2.21.0 +%define tag a64812892be30f3f2d01cdb6fd25a1aa2983a894 %define branch cms/v%{realversion} -%define github_user cms-externals +%define github_user akritkbehera %define build_type opt %define pythonOnly no ## INCLUDE tensorflow-sources diff --git a/tensorflow-xla-runtime-absl.patch b/tensorflow-xla-runtime-absl.patch index f4ba31c5293..70b80a57506 100644 --- a/tensorflow-xla-runtime-absl.patch +++ b/tensorflow-xla-runtime-absl.patch @@ -9,4 +9,4 @@ +add_library(tf_xla_runtime SHARED $ ) -+target_link_libraries(tf_xla_runtime absl::strings absl::str_format_internal) ++target_link_libraries(tf_xla_runtime absl::strings absl::str_format_internal absl::log absl::check absl::synchronization absl::status) diff --git a/tensorflow-xla-runtime.spec b/tensorflow-xla-runtime.spec index 3500ae7545e..0637d179e6e 100644 --- a/tensorflow-xla-runtime.spec +++ b/tensorflow-xla-runtime.spec @@ -1,4 +1,4 @@ -### RPM external tensorflow-xla-runtime 2.17.0 +### RPM external tensorflow-xla-runtime 2.21.0 ## INCLUDE cpp-standard ## INCLUDE compilation_flags ## INCLUDE microarch_flags @@ -35,7 +35,7 @@ pushd xla-aot-runtime/src -DCMAKE_CXX_FLAGS="${CXXFLAGS} -I${TENSORFLOW_ROOT}/include" \ -DCMAKE_CXX_STANDARD=%{cms_cxx_standard} \ -DCMAKE_PREFIX_PATH=${ABSEIL_CPP_ROOT} \ - -DCMAKE_SHARED_LINKER_FLAGS="-L../lib -Wl,--whole-archive -l:libfft_wrapper.pic.a -Wl,--no-whole-archive -l:libfft.pic.a -l:libmutex.pic.a -l:libnsync_cpp.pic.a" \ + -DCMAKE_SHARED_LINKER_FLAGS="-L../lib -Wl,--whole-archive -l:libfft_wrapper.pic.a -Wl,--no-whole-archive -l:libfft.pic.a" \ -DBUILD_SHARED_LIBS=ON make %{makeprocesses} popd diff --git a/tensorflow.spec b/tensorflow.spec index eae39db134f..cc4619cb6ed 100644 --- a/tensorflow.spec +++ b/tensorflow.spec @@ -1,4 +1,4 @@ -### RPM external tensorflow 2.17.0 +### RPM external tensorflow 2.21.0 %if "%{?vectorized_package:set}" != "set" %define source_package tensorflow-sources %else diff --git a/triton-inference-client.spec b/triton-inference-client.spec index e5a35339541..55951b6b648 100644 --- a/triton-inference-client.spec +++ b/triton-inference-client.spec @@ -53,7 +53,7 @@ cmake ${PROJ_DIR} \ -DTRITON_KEEP_TYPEINFO=ON \ -DTRITON_COMMON_REPO_TAG=${common_tag} \ -DTRITON_ENABLE_GPU=${TRITON_ENABLE_GPU_VALUE} \ - -DCMAKE_CXX_FLAGS="-Wno-error -fPIC" \ + -DCMAKE_CXX_FLAGS="-Wno-error -Wno-error=deprecated-declarations -fPIC" \ -DFETCHCONTENT_SOURCE_DIR_REPO-COMMON=${COMMON_DIR} \ -DCMAKE_PREFIX_PATH="${GRPC_ROOT};${ABSEIL_CPP_ROOT};${RE2_ROOT};${RAPIDJSON_ROOT}"