diff --git a/.github/workflows/bazel_test_centipede.yml b/.github/workflows/bazel_test_centipede.yml index 1bbcba0d4..f28d5654a 100644 --- a/.github/workflows/bazel_test_centipede.yml +++ b/.github/workflows/bazel_test_centipede.yml @@ -30,6 +30,8 @@ jobs: # TODO(xinhaoyuan): Bump to 24.04 after https://github.com/llvm/llvm-project/issues/102443 # is fixed. runs-on: ubuntu-22.04 + permissions: + contents: read timeout-minutes: 60 strategy: matrix: @@ -39,13 +41,13 @@ jobs: run: | sudo sysctl -w kernel.core_pattern="" - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install dependencies run: | sudo apt-get update && sudo apt-get install -yq \ clang llvm libssl-dev - name: Restore latest cache - uses: actions/cache/restore@v4 + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: "~/.cache/bazel" key: bazel-centipede-cache-${{ matrix.config }} @@ -84,19 +86,21 @@ jobs: bazel test --no//fuzztest:use_riegeli --test_output=errors --linkopt=-fsanitize=address --copt=-fsanitize=address --test_env=ASAN_OPTIONS=detect_leaks=0 --platform_suffix=asan --test_timeout=600 centipede/puzzles:all - name: Save new cache based on main if: github.ref == 'refs/heads/main' - uses: actions/cache/save@v4 + uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: "~/.cache/bazel" key: bazel-centipede-cache-${{ matrix.config }}-${{ github.run_id }} run_tests_mac: name: Run Centipede tests (MacOS) runs-on: macos-15 + permissions: + contents: read timeout-minutes: 60 steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Restore latest cache - uses: actions/cache/restore@v4 + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: "~/.cache/bazel" key: bazel-centipede-cache-mac @@ -127,7 +131,62 @@ jobs: bazel --output_user_root="${HOME}/.cache/bazel" test --test_output=errors --no//fuzztest:use_riegeli --linkopt=-fsanitize=address --copt=-fsanitize=address --test_env=ASAN_OPTIONS=detect_leaks=0 --platform_suffix=asan --test_timeout=600 centipede/puzzles:all - name: Save new cache based on main if: github.ref == 'refs/heads/main' - uses: actions/cache/save@v4 + uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: "~/.cache/bazel" key: bazel-centipede-cache-mac-${{ github.run_id }} + run_tests_win: + name: Run Centipede tests (Windows) + runs-on: windows-latest + permissions: + contents: read + timeout-minutes: 60 + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Restore latest cache + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 + with: + path: "~/.cache/bazel" + key: bazel-centipede-cache-win + restore-keys: bazel-centipede-cache-win- + - name: Set environment variable + run: echo "USE_BAZEL_VERSION=8.7.0" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + - name: Run unit tests + if: ${{ !cancelled() }} + run: | + <# Only supported libraries are tested here. #> ` + bazelisk test --disk_cache=~/.cache/bazel --local_test_jobs=1 --test_output=errors --no//fuzztest:use_riegeli ` + --extra_toolchains=@local_config_cc//:cc-toolchain-x64_windows-clang-cl ` + --extra_execution_platforms=//:x64_windows-clang-cl --enable_runfiles -- ` + centipede:call_graph_test ` + centipede:callstack_test ` + centipede:command_test ` + centipede:concurrent_bitset_test ` + centipede:concurrent_byteset_test ` + centipede:config_file_test ` + centipede:crash_summary_test ` + centipede:execution_metadata_test ` + centipede:feature_set_test ` + centipede:feature_test ` + centipede:foreach_nonzero_test ` + centipede:hashed_ring_buffer_test ` + centipede:int_utils_test ` + centipede:knobs_test ` + centipede:mutation_data_test ` + centipede:pc_info_test ` + centipede:periodic_action_test ` + centipede:reverse_pc_table_test ` + centipede:rolling_hash_test ` + centipede:runner_flags_test ` + centipede:runner_result_test ` + centipede:rusage_stats_test ` + centipede:shared_memory_blob_sequence_test ` + centipede:stop_test ` + centipede:util_test + - name: Save new cache based on main + if: github.ref == 'refs/heads/main' + uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 + with: + path: "~/.cache/bazel" + key: bazel-centipede-cache-win-${{ github.run_id }} diff --git a/BUILD b/BUILD index 17cbe3f7f..e8b953e49 100644 --- a/BUILD +++ b/BUILD @@ -13,3 +13,12 @@ # limitations under the License. exports_files(["MODULE.bazel"]) + +platform( + name = "x64_windows-clang-cl", + constraint_values = [ + "@platforms//cpu:x86_64", + "@platforms//os:windows", + "@bazel_tools//tools/cpp:clang-cl", + ], +) diff --git a/MODULE.bazel b/MODULE.bazel index 0a0dc3eec..f99e39cf6 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -23,6 +23,10 @@ bazel_dep( name = "rules_cc", version = "0.2.17", ) + +cc_configure = use_extension("@rules_cc//cc:extensions.bzl", "cc_configure_extension") +use_repo(cc_configure, "local_config_cc") + bazel_dep( name = "rules_shell", version = "0.6.1", diff --git a/centipede/BUILD b/centipede/BUILD index 3c045cac4..9d7986f74 100644 --- a/centipede/BUILD +++ b/centipede/BUILD @@ -228,6 +228,7 @@ cc_library( "@com_google_fuzztest//common:hash", "@com_google_fuzztest//common:logging", "@com_google_fuzztest//common:remote_file", + "@com_google_fuzztest//common:windows_includes", ], ) @@ -296,6 +297,7 @@ cc_library( "@abseil-cpp//absl/strings:str_format", "@abseil-cpp//absl/time", "@com_google_fuzztest//common:logging", + "@com_google_fuzztest//common:windows_includes", ], ) @@ -420,8 +422,10 @@ cc_library( "-lrt", # for shm_open ], }), - deps = ["@abseil-cpp//absl/base:nullability"], - # don't add any dependencies. + deps = [ + "@abseil-cpp//absl/base:nullability", + "@com_google_fuzztest//common:windows_includes", + ], ) cc_library( @@ -597,6 +601,8 @@ cc_library( ":stop", ":util", "@abseil-cpp//absl/base:core_headers", + "@abseil-cpp//absl/cleanup", + "@abseil-cpp//absl/container:btree", "@abseil-cpp//absl/status", "@abseil-cpp//absl/status:statusor", "@abseil-cpp//absl/strings", @@ -604,6 +610,8 @@ cc_library( "@abseil-cpp//absl/synchronization", "@abseil-cpp//absl/time", "@com_google_fuzztest//common:logging", + "@com_google_fuzztest//common:windows_includes", + "@com_google_fuzztest//fuzztest/internal:escaping", ], ) @@ -1393,7 +1401,12 @@ cc_test( cc_test( name = "util_test", srcs = ["util_test.cc"], - copts = ["-fno-signed-char"], + copts = select({ + "@platforms//os:windows": [ + "/J", # Make unsigned char the default + ], + "//conditions:default": ["-fno-signed-char"], + }), deps = [ ":feature", ":thread_pool", @@ -1558,6 +1571,7 @@ cc_test( srcs = ["shared_memory_blob_sequence_test.cc"], deps = [ ":shared_memory_blob_sequence", + "@com_google_fuzztest//common:windows_includes", "@googletest//:gtest_main", ], ) @@ -1740,10 +1754,14 @@ cc_binary( name = "command_test_helper", srcs = ["command_test_helper.cc"], deps = [ - ":runner_fork_server", "@abseil-cpp//absl/base:nullability", + "@abseil-cpp//absl/strings", "@abseil-cpp//absl/time", - ], + "@com_google_fuzztest//common:windows_includes", + ] + select({ + "@platforms//os:windows": [], + "//conditions:default": [":runner_fork_server"], + }), ) cc_test( diff --git a/centipede/analyze_corpora_test.cc b/centipede/analyze_corpora_test.cc index c349ec308..f0baad1bc 100644 --- a/centipede/analyze_corpora_test.cc +++ b/centipede/analyze_corpora_test.cc @@ -42,7 +42,8 @@ using ::testing::Not; // Returns path to test_fuzz_target. static std::string GetTargetPath() { - return GetDataDependencyFilepath("centipede/testing/test_fuzz_target"); + return GetDataDependencyFilepath("centipede/testing/test_fuzz_target") + .string(); } // TODO(ussuri): Implement. @@ -91,7 +92,7 @@ TEST(DumpCoverageReport, SimpleCoverageResults) { auto corpus_records = RunInputsAndCollectCorpusRecords(env, {"func1"}); ASSERT_EQ(corpus_records.size(), 1); - const std::string test_tmpdir = GetTestTempDir(test_info_->name()); + const std::string test_tmpdir = GetTestTempDir(test_info_->name()).string(); BinaryInfo binary_info; binary_info.InitializeFromSanCovBinary(GetTargetPath(), /*env_diff=*/{}, GetObjDumpPath(), @@ -100,7 +101,7 @@ TEST(DumpCoverageReport, SimpleCoverageResults) { GetCoverage(corpus_records, std::move(binary_info)); const std::string coverage_report_path = - std::filesystem::path{test_tmpdir} / "covered_symbol_table"; + (std::filesystem::path{test_tmpdir} / "covered_symbol_table").string(); DumpCoverageReport(coverage_results, coverage_report_path); std::string symbol_table_contents; ASSERT_OK(RemoteFileGetContents(coverage_report_path, symbol_table_contents)); diff --git a/centipede/binary_info.cc b/centipede/binary_info.cc index 1ef77b6cc..a8222fb01 100644 --- a/centipede/binary_info.cc +++ b/centipede/binary_info.cc @@ -66,7 +66,7 @@ void BinaryInfo::InitializeFromSanCovBinary( ":arg3=", EscapeEngineFlag(dso_table_path.path()), ":")); Command::Options cmd_options; cmd_options.env_diff = std::move(env_diff); - cmd_options.stdout_file_prefix = log_prefix; + cmd_options.stdout_file_prefix = log_prefix.string(); Command cmd{binary_path_with_args, std::move(cmd_options)}; int exit_code = cmd.Execute(); if (exit_code != EXIT_SUCCESS) { @@ -131,20 +131,20 @@ void BinaryInfo::Read(std::string_view dir) { std::string symbol_table_contents; // TODO(b/295978603): move calculation of paths into WorkDir class. FUZZTEST_CHECK_OK(RemoteFileGetContents( - (std::filesystem::path(dir) / kSymbolTableFileName).c_str(), + (std::filesystem::path(dir) / kSymbolTableFileName).string(), symbol_table_contents)); std::istringstream symbol_table_stream(symbol_table_contents); symbols.ReadFromLLVMSymbolizer(symbol_table_stream); std::string pc_table_contents; FUZZTEST_CHECK_OK(RemoteFileGetContents( - (std::filesystem::path(dir) / kPCTableFileName).c_str(), + (std::filesystem::path(dir) / kPCTableFileName).string(), pc_table_contents)); std::istringstream pc_table_stream(pc_table_contents); pc_table = ReadPcTable(pc_table_stream); cf_table = - ReadCfTable((std::filesystem::path(dir) / kCfTableFileName).c_str()); + ReadCfTable((std::filesystem::path(dir) / kCfTableFileName).string()); } void BinaryInfo::Write(std::string_view dir) { @@ -152,19 +152,19 @@ void BinaryInfo::Write(std::string_view dir) { symbols.WriteToLLVMSymbolizer(symbol_table_stream); // TODO(b/295978603): move calculation of paths into WorkDir class. FUZZTEST_CHECK_OK(RemoteFileSetContents( - (std::filesystem::path(dir) / kSymbolTableFileName).c_str(), + (std::filesystem::path(dir) / kSymbolTableFileName).string(), symbol_table_stream.str())); std::ostringstream pc_table_stream; WritePcTable(pc_table, pc_table_stream); FUZZTEST_CHECK_OK(RemoteFileSetContents( - (std::filesystem::path(dir) / kPCTableFileName).c_str(), + (std::filesystem::path(dir) / kPCTableFileName).string(), pc_table_stream.str())); std::ostringstream cf_table_stream; WriteCfTable(cf_table, cf_table_stream); FUZZTEST_CHECK_OK(RemoteFileSetContents( - (std::filesystem::path(dir) / kCfTableFileName).c_str(), + (std::filesystem::path(dir) / kCfTableFileName).string(), cf_table_stream.str())); } diff --git a/centipede/binary_info_test.cc b/centipede/binary_info_test.cc index 2b730f945..02fe1103e 100644 --- a/centipede/binary_info_test.cc +++ b/centipede/binary_info_test.cc @@ -28,7 +28,7 @@ namespace fuzztest::internal { namespace { TEST(BinaryInfoTest, SerializesAndDeserializesBinaryInfoSuccessfully) { - const std::string temp_dir = GetTestTempDir(test_info_->name()); + const std::string temp_dir = GetTestTempDir(test_info_->name()).string(); const PCTable input_pcs = {{/*pc=*/0, /*flags=*/1}, {/*pc=*/2, /*flags=*/3}}; std::string input_symbols = @@ -57,7 +57,7 @@ TEST(BinaryInfoTest, SerializesAndDeserializesBinaryInfoSuccessfully) { } TEST(BinaryInfoTest, SerializesAndDeserializesEmptyBinaryInfoSuccessfully) { - const std::string temp_dir = GetTestTempDir(test_info_->name()); + const std::string temp_dir = GetTestTempDir(test_info_->name()).string(); const PCTable input_pcs = {}; std::string input_symbols = ""; diff --git a/centipede/blob_file_converter.cc b/centipede/blob_file_converter.cc index 7b8888959..5d0bd6a8a 100644 --- a/centipede/blob_file_converter.cc +++ b/centipede/blob_file_converter.cc @@ -100,7 +100,7 @@ void Convert( // FUZZTEST_CHECK(RemotePathExists(in)) << VV(in); FUZZTEST_CHECK_OK( - RemoteMkdir(std::filesystem::path{out}.parent_path().c_str())); + RemoteMkdir(std::filesystem::path{out}.parent_path().string())); // Open blob file reader and writer. diff --git a/centipede/centipede.cc b/centipede/centipede.cc index 7566883b2..02baa7d44 100644 --- a/centipede/centipede.cc +++ b/centipede/centipede.cc @@ -114,14 +114,15 @@ std::vector InputsToMutantRefs(const std::vector& inputs) { void SaveReproducer(const WorkDir& wd, std::string_view log_prefix, size_t input_idx, ByteSpan reproducer, const BatchResult& result) { - auto hash = Hash(reproducer); - auto crash_dir = wd.CrashReproducerDirPaths().MyShard(); + const auto hash = Hash(reproducer); + const auto crash_dir = wd.CrashReproducerDirPaths().MyShard(); FUZZTEST_CHECK_OK(RemoteMkdir(crash_dir)); - std::string input_file_path = std::filesystem::path(crash_dir) / hash; - auto crash_metadata_dir = wd.CrashMetadataDirPaths().MyShard(); + const auto input_file_path = + (std::filesystem::path(crash_dir) / hash).string(); + const auto crash_metadata_dir = wd.CrashMetadataDirPaths().MyShard(); FUZZTEST_CHECK_OK(RemoteMkdir(crash_metadata_dir)); - std::string crash_metadata_path_prefix = - std::filesystem::path(crash_metadata_dir) / hash; + const auto crash_metadata_path_prefix = + (std::filesystem::path(crash_metadata_dir) / hash).string(); FUZZTEST_LOG(INFO) << log_prefix << "Detected crash-reproducing input:" << "\nInput index : " << input_idx @@ -169,13 +170,15 @@ Centipede::Centipede(const Environment& env, CentipedeCallbacks& user_callbacks, coverage_logger_(coverage_logger), stats_(stats), input_filter_path_(std::filesystem::path(TemporaryLocalDirPath()) - .append("filter-input")), + .append("filter-input") + .string()), input_filter_cmd_{[&] { Command::Options cmd_options; cmd_options.args = {input_filter_path_}; cmd_options.stdout_file_prefix = std::filesystem::path(TemporaryLocalDirPath()) - .append("filter-input.log"); + .append("filter-input.log") + .string(); cmd_options.stderr_file_prefix = cmd_options.stdout_file_prefix; return Command{env_.input_filter, std::move(cmd_options)}; }()}, @@ -274,7 +277,8 @@ absl::Status Centipede::CrashesToFiles(const Environment &env, std::vector reproducer_paths, RemoteListFiles(reproducer_dir, /*recursively=*/false)); for (const auto &reproducer_path : reproducer_paths) { - std::string id = std::filesystem::path{reproducer_path}.filename(); + std::string id = + std::filesystem::path{reproducer_path}.filename().string(); if (auto [_it, inserted] = crash_ids.insert(id); !inserted) { continue; } @@ -1172,21 +1176,25 @@ void Centipede::ReportCrash(std::string_view binary, FUZZTEST_CHECK_OK(RemoteMkdir(crash_dir)); std::string crashing_batch_name = absl::StrCat("crashing_batch-", suspect_hash); - std::string save_dir = std::filesystem::path(crash_dir) / crashing_batch_name; + std::string save_dir = + (std::filesystem::path(crash_dir) / crashing_batch_name).string(); FUZZTEST_CHECK_OK(RemoteMkdir(save_dir)); FUZZTEST_LOG(INFO) << log_prefix << "Saving used inputs from batch to: " << save_dir; for (int i = 0; i <= suspect_input_idx; ++i) { const auto &one_input = input_vec[i]; auto hash = Hash(one_input); - std::string file_path = std::filesystem::path(save_dir).append( - absl::StrFormat("input-%010d-%s", i, hash)); + std::string file_path = + std::filesystem::path(save_dir) + .append(absl::StrFormat("input-%010d-%s", i, hash)) + .string(); FUZZTEST_CHECK_OK(RemoteFileSetContents(file_path, one_input)); } auto crash_metadata_dir = wd_.CrashMetadataDirPaths().MyShard(); FUZZTEST_CHECK_OK(RemoteMkdir(crash_metadata_dir)); std::string crash_metadata_file_path = - std::filesystem::path(crash_metadata_dir) / crashing_batch_name; + (std::filesystem::path(crash_metadata_dir) / crashing_batch_name) + .string(); FUZZTEST_LOG(INFO) << log_prefix << "Saving crash metadata to: " << crash_metadata_file_path; FUZZTEST_CHECK_OK(RemoteFileSetContents(crash_metadata_file_path, diff --git a/centipede/centipede_callbacks.cc b/centipede/centipede_callbacks.cc index c41e38a73..b32c6aaf7 100644 --- a/centipede/centipede_callbacks.cc +++ b/centipede/centipede_callbacks.cc @@ -403,8 +403,8 @@ CentipedeCallbacks::GetOrCreateCommandContextForBinary( absl::StrCat( ":shmem_size_mb=", env_.shmem_size_mb, ":test=", EscapeEngineFlag(env_.test_name), - ":arg1=", EscapeEngineFlag(inputs_blobseq_.path()), - ":arg2=", EscapeEngineFlag(outputs_blobseq_.path()), + ":arg1=", EscapeEngineFlag(inputs_blobseq_->path()), + ":arg2=", EscapeEngineFlag(outputs_blobseq_->path()), ":failure_description_path=", EscapeEngineFlag(failure_description_path_), ":failure_signature_path=", EscapeEngineFlag(failure_signature_path_), @@ -547,8 +547,8 @@ int CentipedeCallbacks::ExecuteCentipedeSancovBinaryWithShmem( batch_result.ClearAndResize(inputs.size()); // Reset the blobseqs. - inputs_blobseq_.Reset(); - outputs_blobseq_.Reset(); + inputs_blobseq_->Reset(); + outputs_blobseq_->Reset(); size_t num_inputs_written = 0; @@ -558,7 +558,7 @@ int CentipedeCallbacks::ExecuteCentipedeSancovBinaryWithShmem( num_inputs_written = 1; } else { // Feed the inputs to inputs_blobseq_. - num_inputs_written = RequestExecution(inputs, inputs_blobseq_); + num_inputs_written = RequestExecution(inputs, *inputs_blobseq_); } if (num_inputs_written != inputs.size()) { @@ -570,16 +570,16 @@ int CentipedeCallbacks::ExecuteCentipedeSancovBinaryWithShmem( // Run. const auto batch_start_time = absl::Now(); const int exit_code = RunBatchForBinary(binary); - inputs_blobseq_.ReleaseSharedMemory(); // Inputs are already consumed. + inputs_blobseq_->ReleaseSharedMemory(); // Inputs are already consumed. const bool batch_timed_out = env_.timeout_per_batch > 0 && absl::Now() - batch_start_time > absl::Seconds(env_.timeout_per_batch); // Get results. batch_result.exit_code() = exit_code; - const bool read_success = batch_result.Read(outputs_blobseq_); + const bool read_success = batch_result.Read(*outputs_blobseq_); FUZZTEST_LOG_IF(ERROR, !read_success) << "Failed to read batch result!"; - outputs_blobseq_.ReleaseSharedMemory(); // Outputs are already consumed. + outputs_blobseq_->ReleaseSharedMemory(); // Outputs are already consumed. // We may have fewer feature blobs than inputs if // * some inputs were not written (i.e. num_inputs_written < inputs.size). @@ -696,7 +696,7 @@ bool CentipedeCallbacks::GetSeedsViaExternalBinary( std::vector seed_input_filenames; for (const auto& dir_ent : std::filesystem::directory_iterator(output_dir)) { - seed_input_filenames.push_back(dir_ent.path().filename()); + seed_input_filenames.push_back(dir_ent.path().filename().string()); } std::sort(seed_input_filenames.begin(), seed_input_filenames.end()); num_avail_seeds = seed_input_filenames.size(); @@ -720,11 +720,10 @@ bool CentipedeCallbacks::GetSeedsViaExternalBinary( // See also: `DumpSerializedTargetConfigToFile()`. bool CentipedeCallbacks::GetSerializedTargetConfigViaExternalBinary( std::string_view binary, std::string& serialized_config) { - const auto config_file_path = - std::filesystem::path{temp_dir_} / "configuration"; + const std::string config_file_path = temp_dir_ + "/configuration"; std::string centipede_runner_flags = absl::StrCat("CENTIPEDE_RUNNER_FLAGS=:dump_configuration:arg1=", - EscapeEngineFlag(config_file_path.string()), ":"); + EscapeEngineFlag(config_file_path), ":"); if (!env_.runner_dl_path_suffix.empty()) { absl::StrAppend(¢ipede_runner_flags, "dl_path_suffix=", EscapeEngineFlag(env_.runner_dl_path_suffix), ":"); @@ -741,7 +740,7 @@ bool CentipedeCallbacks::GetSerializedTargetConfigViaExternalBinary( if (is_success) { if (std::filesystem::exists(config_file_path)) { - ReadFromLocalFile(config_file_path.string(), serialized_config); + ReadFromLocalFile(config_file_path, serialized_config); } else { serialized_config = ""; } @@ -764,17 +763,17 @@ MutationResult CentipedeCallbacks::MutateViaExternalBinary( << "Standalone binary does not support custom mutator"; auto start_time = absl::Now(); - inputs_blobseq_.Reset(); - outputs_blobseq_.Reset(); + inputs_blobseq_->Reset(); + outputs_blobseq_->Reset(); size_t num_inputs_written = - RequestMutation(num_mutants, inputs, inputs_blobseq_); + RequestMutation(num_mutants, inputs, *inputs_blobseq_); FUZZTEST_LOG_IF(INFO, num_inputs_written != inputs.size()) << VV(num_inputs_written) << VV(inputs.size()); // Execute. const int exit_code = RunBatchForBinary(binary); - inputs_blobseq_.ReleaseSharedMemory(); // Inputs are already consumed. + inputs_blobseq_->ReleaseSharedMemory(); // Inputs are already consumed. if (exit_code != EXIT_SUCCESS) { FUZZTEST_LOG(WARNING) << "Custom mutator failed with exit code: " @@ -786,8 +785,8 @@ MutationResult CentipedeCallbacks::MutateViaExternalBinary( MutationResult result; result.exit_code() = exit_code; - result.Read(num_mutants, outputs_blobseq_); - outputs_blobseq_.ReleaseSharedMemory(); // Outputs are already consumed. + result.Read(num_mutants, *outputs_blobseq_); + outputs_blobseq_->ReleaseSharedMemory(); // Outputs are already consumed. FUZZTEST_VLOG(1) << __FUNCTION__ << " took " << (absl::Now() - start_time); return result; diff --git a/centipede/centipede_callbacks.h b/centipede/centipede_callbacks.h index 38d6bc35c..de2652ab2 100644 --- a/centipede/centipede_callbacks.h +++ b/centipede/centipede_callbacks.h @@ -53,10 +53,12 @@ class CentipedeCallbacks { stop_condition_(stop_condition), byte_array_mutator_(env.knobs, GetRandomSeed(env.seed)), fuzztest_mutator_(env.knobs, GetRandomSeed(env.seed)), - inputs_blobseq_(shmem_name1_.c_str(), env.shmem_size_mb << 20, - env.use_posix_shmem), - outputs_blobseq_(shmem_name2_.c_str(), env.shmem_size_mb << 20, - env.use_posix_shmem) { + inputs_blobseq_(CreateSharedMemoryBlobSequence(shmem_name1_.c_str(), + env.shmem_size_mb << 20, + env.use_posix_shmem)), + outputs_blobseq_(CreateSharedMemoryBlobSequence(shmem_name2_.c_str(), + env.shmem_size_mb << 20, + env.use_posix_shmem)) { if (env.use_legacy_default_mutator) FUZZTEST_CHECK(byte_array_mutator_.set_max_len(env.max_len)); else @@ -188,23 +190,23 @@ class CentipedeCallbacks { // They are computed in CTOR, to avoid extra computation in the hot loop. std::string temp_dir_ = TemporaryLocalDirPath(); std::string temp_input_file_path_ = - std::filesystem::path(temp_dir_).append("temp_input_file"); + std::filesystem::path(temp_dir_).append("temp_input_file").string(); const std::string execute_log_prefix_ = - std::filesystem::path(temp_dir_).append("log"); + std::filesystem::path(temp_dir_).append("log").string(); // An owned file path to save the last execution log before it goes out of // scope. const std::string saved_execute_log_path_ = - std::filesystem::path(temp_dir_).append("saved_log"); + std::filesystem::path(temp_dir_).append("saved_log").string(); std::string last_execute_log_path_; std::string failure_description_path_ = - std::filesystem::path(temp_dir_).append("failure_description"); + std::filesystem::path(temp_dir_).append("failure_description").string(); std::string failure_signature_path_ = - std::filesystem::path(temp_dir_).append("failure_signature"); + std::filesystem::path(temp_dir_).append("failure_signature").string(); const std::string shmem_name1_ = ProcessAndThreadUniqueID("/ctpd-shm1-"); const std::string shmem_name2_ = ProcessAndThreadUniqueID("/ctpd-shm2-"); - SharedMemoryBlobSequence inputs_blobseq_; - SharedMemoryBlobSequence outputs_blobseq_; + std::unique_ptr absl_nonnull inputs_blobseq_; + std::unique_ptr absl_nonnull outputs_blobseq_; // Need unique_ptr indirection because CommandContext is not movable/copyable // due to Command. diff --git a/centipede/centipede_interface.cc b/centipede/centipede_interface.cc index 4cd9f7530..4cb6dc964 100644 --- a/centipede/centipede_interface.cc +++ b/centipede/centipede_interface.cc @@ -83,7 +83,7 @@ constexpr absl::Duration kDefaultRegressionTtl = absl::Hours(24 * 7); void ForEachBlob(const Environment& env, StopCondition& stop_condition) { auto tmpdir = TemporaryLocalDirPath(); CreateLocalDirRemovedAtExit(tmpdir); - std::string tmpfile = std::filesystem::path(tmpdir).append("t"); + std::string tmpfile = (std::filesystem::path(tmpdir) / "t").string(); for (const auto& arg : env.args) { FUZZTEST_LOG(INFO) << "Running '" << env.for_each_blob << "' on " << arg; @@ -212,7 +212,8 @@ BinaryInfo PopulateBinaryInfoAndSavePCsIfNecessary( binary_info.Write(binary_info_dir); } if (binary_info.uses_legacy_trace_pc_instrumentation) { - pcs_file_path = std::filesystem::path(TemporaryLocalDirPath()) / "pcs"; + pcs_file_path = + (std::filesystem::path(TemporaryLocalDirPath()) / "pcs").string(); SavePCTableToFile(binary_info.pc_table, pcs_file_path); } if (env.use_pcpair_features) { @@ -336,30 +337,31 @@ SeedCorpusConfig GetSeedCorpusConfig(const Environment& env, std::string_view coverage_dir) { const WorkDir workdir{env}; SeedCorpusSource regression; - regression.dir_glob = std::string(regression_dir); + regression.src_dirs = {std::string(regression_dir)}; regression.num_recent_dirs = 1; - regression.individual_input_rel_glob = "*"; + regression.individual_input_rel_prefix = ""; regression.sampled_fraction_or_count = 1.0f; std::vector sources = {std::move(regression)}; if (!coverage_dir.empty()) { SeedCorpusSource coverage; - coverage.dir_glob = std::string(coverage_dir); + coverage.src_dirs = {std::string(coverage_dir)}; coverage.num_recent_dirs = 1; // We're using the previously distilled corpus files as seeds. - coverage.shard_rel_glob = - std::filesystem::path{ - workdir.DistilledCorpusFilePaths().AllShardsGlob()} - .filename(); - coverage.individual_input_rel_glob = "*"; + coverage.shard_rel_prefix = + std::filesystem::path{workdir.DistilledCorpusFilePaths().prefix()} + .filename() + .string(); + coverage.individual_input_rel_prefix = ""; coverage.sampled_fraction_or_count = 1.0f; sources.push_back(std::move(coverage)); } SeedCorpusDestination destination; destination.dir_path = env.workdir; // We're seeding the current corpus files. - destination.shard_rel_glob = - std::filesystem::path{workdir.CorpusFilePaths().AllShardsGlob()} - .filename(); + destination.shard_rel_prefix = + std::filesystem::path{workdir.CorpusFilePaths().prefix()} + .filename() + .string(); destination.shard_index_digits = WorkDir::kDigitsInShardIndex; destination.num_shards = static_cast(env.num_threads); return { @@ -453,22 +455,22 @@ void RecordFuzzingResults(const Environment& env, const DatabasePaths& db_paths, if (update_coverage_db) { // Distill and store the coverage corpus. Distill(env); - if (RemotePathExists(db_paths.coverage_dir.c_str())) { + if (RemotePathExists(db_paths.coverage_dir.string())) { // In the future, we will store k latest coverage corpora for some k, but // for now we only keep the latest one. - FUZZTEST_CHECK_OK(RemotePathDelete(db_paths.coverage_dir.c_str(), + FUZZTEST_CHECK_OK(RemotePathDelete(db_paths.coverage_dir.string(), /*recursively=*/true)); } - FUZZTEST_CHECK_OK(RemoteMkdir(db_paths.coverage_dir.c_str())); + FUZZTEST_CHECK_OK(RemoteMkdir(db_paths.coverage_dir.string())); std::vector distilled_corpus_files; FUZZTEST_CHECK_OK( RemoteGlobMatch(workdir.DistilledCorpusFilePaths().AllShardsGlob(), distilled_corpus_files)); for (const std::string& corpus_file : distilled_corpus_files) { const std::string file_name = - std::filesystem::path(corpus_file).filename(); + std::filesystem::path(corpus_file).filename().string(); FUZZTEST_CHECK_OK(RemoteFileRename( - corpus_file, (db_paths.coverage_dir / file_name).c_str())); + corpus_file, (db_paths.coverage_dir / file_name).string())); } } } @@ -532,7 +534,7 @@ void UpdateCorpusDatabase(Environment env, stop_condition.SetStopTime(absl::InfiniteFuture()); if (!is_workdir_specified) { - env.workdir = base_workdir_path / env.test_name; + env.workdir = (base_workdir_path / env.test_name).string(); } const auto execution_id_path = (base_workdir_path / absl::StrCat(env.test_name, ".execution_id")) @@ -602,9 +604,9 @@ void UpdateCorpusDatabase(Environment env, // inputs from the previous fuzzing session. if (!is_resuming) { FUZZTEST_CHECK_OK(GenerateSeedCorpusFromConfig( - GetSeedCorpusConfig(env, db_paths.regression_dir.c_str(), + GetSeedCorpusConfig(env, db_paths.regression_dir.string(), env.fuzztest_replay_coverage_inputs - ? db_paths.coverage_dir.c_str() + ? db_paths.coverage_dir.string() : ""), env.binary_name, env.binary_hash)) << "while generating the seed corpus"; @@ -613,7 +615,7 @@ void UpdateCorpusDatabase(Environment env, absl::Duration time_limit = env.fuzztest_time_limit_per_test; absl::Duration time_spent = absl::ZeroDuration(); const std::string fuzzing_time_file = - std::filesystem::path(env.workdir) / "fuzzing_time"; + (std::filesystem::path(env.workdir) / "fuzzing_time").string(); if (is_resuming && RemotePathExists(fuzzing_time_file)) { time_spent = ReadFuzzingTime(fuzzing_time_file); time_limit = std::max(time_limit - time_spent, absl::ZeroDuration()); @@ -646,10 +648,11 @@ void UpdateCorpusDatabase(Environment env, if (!stats_root_path.empty()) { const auto stats_dir = stats_root_path / env.test_name; - FUZZTEST_CHECK_OK(RemoteMkdir(stats_dir.c_str())); + FUZZTEST_CHECK_OK(RemoteMkdir(stats_dir.string())); FUZZTEST_CHECK_OK(RemoteFileRename( workdir.FuzzingStatsPath(), - (stats_dir / absl::StrCat("fuzzing_stats_", execution_stamp)).c_str())); + (stats_dir / absl::StrCat("fuzzing_stats_", execution_stamp)) + .string())); } if (stop_condition.StopRequested(&stop_request)) { @@ -686,7 +689,8 @@ int ListCrashIds(const Environment& env) { std::vector results; results.reserve(crash_paths.size()); for (const auto& crash_path : crash_paths) { - std::string crash_id = std::filesystem::path{crash_path}.filename(); + std::string crash_id = + std::filesystem::path{crash_path}.filename().string(); results.push_back(std::move(crash_id)); } FUZZTEST_CHECK_OK(RemoteFileSetContents(env.list_crash_ids_file, @@ -707,21 +711,24 @@ void ReplayCrash(const Environment& env, "crashing"; const WorkDir workdir{env}; SeedCorpusSource crash_corpus_source; - crash_corpus_source.dir_glob = crash_dir; + crash_corpus_source.src_dirs = {crash_dir.string()}; crash_corpus_source.num_recent_dirs = 1; - crash_corpus_source.individual_input_rel_glob = env.crash_id; + crash_corpus_source.individual_input_rel_prefix = env.crash_id; crash_corpus_source.sampled_fraction_or_count = 1.0f; - const SeedCorpusConfig crash_corpus_config = { - /*sources=*/{crash_corpus_source}, - /*destination=*/{ - /*dir_path=*/env.workdir, - /*shard_rel_glob=*/ - std::filesystem::path{workdir.CorpusFilePaths().AllShardsGlob()} - .filename(), - /*shard_index_digits=*/WorkDir::kDigitsInShardIndex, - /*num_shards=*/1}}; - FUZZTEST_CHECK_OK(GenerateSeedCorpusFromConfig( - crash_corpus_config, env.binary_name, env.binary_hash)); + { + SeedCorpusDestination crash_corpus_destination; + crash_corpus_destination.dir_path = env.workdir; + crash_corpus_destination.shard_rel_prefix = + std::filesystem::path{workdir.CorpusFilePaths().prefix()} + .filename() + .string(); + crash_corpus_destination.shard_index_digits = WorkDir::kDigitsInShardIndex; + crash_corpus_destination.num_shards = 1; + FUZZTEST_CHECK_OK( + GenerateSeedCorpusFromConfig({/*sources=*/{crash_corpus_source}, + /*destination=*/crash_corpus_destination}, + env.binary_name, env.binary_hash)); + } Environment run_crash_env = env; run_crash_env.load_shards_only = true; run_crash_env.persistent_mode = false; @@ -752,11 +759,12 @@ int ExportCrash(const Environment& env) { env.fuzztest_binary_identifier / env.test_name / "crashing"; std::string crash_contents; - const auto read_status = - RemoteFileGetContents((crash_dir / env.crash_id).c_str(), crash_contents); + const auto read_status = RemoteFileGetContents( + (crash_dir / env.crash_id).string(), crash_contents); if (!read_status.ok()) { FUZZTEST_LOG(ERROR) << "Failed reading the crash " << env.crash_id - << " from " << crash_dir.c_str() << ": " << read_status; + << " from " << crash_dir.string() << ": " + << read_status; return EXIT_FAILURE; } const auto write_status = diff --git a/centipede/centipede_test.cc b/centipede/centipede_test.cc index 3c45b83b7..9a60ea98e 100644 --- a/centipede/centipede_test.cc +++ b/centipede/centipede_test.cc @@ -152,7 +152,7 @@ TEST(Centipede, MockTest) { TempCorpusDir tmp_dir{test_info_->name()}; Environment env; env.log_level = 0; // Disable most of the logging in the test. - env.workdir = tmp_dir.path(); + env.workdir = tmp_dir.path().string(); env.num_runs = 100000; // Enough to run through all 1- and 2-byte inputs. env.batch_size = 7; // Just some small number. env.require_pc_table = false; // No PC table here. @@ -180,11 +180,11 @@ TEST(Centipede, ReadFirstCorpusDir) { TempDir corpus_dir{test_info_->name(), "corpus"}; Environment env; env.log_level = 0; // Disable most of the logging in the test. - env.workdir = workdir_1.path(); + env.workdir = workdir_1.path().string(); env.num_runs = 100000; // Enough to run through all 1- and 2-byte inputs. env.batch_size = 7; // Just some small number. env.require_pc_table = false; // No PC table here. - env.corpus_dir.push_back(corpus_dir.path()); + env.corpus_dir.push_back(corpus_dir.path().string()); // Need to wrap each CentipedeMain in a scope to make sure the shmem is // released before the next call. Otherwise it may fail in MacOS. @@ -201,7 +201,7 @@ TEST(Centipede, ReadFirstCorpusDir) { { // Second, run without fuzzing using the same corpus_dir. - env.workdir = workdir_2.path(); + env.workdir = workdir_2.path().string(); env.num_runs = 0; CentipedeMock mock_2(env); NonOwningCallbacksFactory factory_2(mock_2); @@ -217,11 +217,11 @@ TEST(Centipede, DoesNotReadFirstCorpusDirIfOutputOnly) { TempDir corpus_dir{test_info_->name(), "corpus"}; Environment env; env.log_level = 0; // Disable most of the logging in the test. - env.workdir = workdir_1.path(); + env.workdir = workdir_1.path().string(); env.num_runs = 100000; // Enough to run through all 1- and 2-byte inputs. env.batch_size = 7; // Just some small number. env.require_pc_table = false; // No PC table here. - env.corpus_dir.push_back(corpus_dir.path()); + env.corpus_dir.push_back(corpus_dir.path().string()); { // First, generate corpus files in corpus_dir. @@ -237,7 +237,7 @@ TEST(Centipede, DoesNotReadFirstCorpusDirIfOutputOnly) { { // Second, run without fuzzing using the same corpus_dir, but as // output-only. - env.workdir = workdir_2.path(); + env.workdir = workdir_2.path().string(); env.num_runs = 0; env.first_corpus_dir_output_only = true; CentipedeMock mock_2(env); @@ -252,7 +252,7 @@ TEST(Centipede, SkipsOutputIfFirstCorpusDirIsEmptyPath) { TempCorpusDir tmp_dir{test_info_->name()}; Environment env; env.log_level = 0; // Disable most of the logging in the test. - env.workdir = tmp_dir.path(); + env.workdir = tmp_dir.path().string(); env.num_runs = 100000; // Enough to run through all 1- and 2-byte inputs. env.batch_size = 7; // Just some small number. env.require_pc_table = false; // No PC table here. @@ -273,7 +273,7 @@ TEST(Centipede, SkipsOutputIfFirstCorpusDirIsEmptyPath) { TEST(Centipede, ShardsAndDistillTest) { TempCorpusDir tmp_dir{test_info_->name()}; Environment env; - env.workdir = tmp_dir.path(); + env.workdir = tmp_dir.path().string(); env.log_level = 0; // Disable most of the logging in the test. size_t combined_num_runs = 100000; // Enough to run through all inputs. env.total_shards = 20; @@ -336,13 +336,14 @@ TEST(Centipede, ShardsAndDistillTest) { TEST(Centipede, InputFilter) { TempCorpusDir tmp_dir{test_info_->name()}; Environment env; - env.workdir = tmp_dir.path(); + env.workdir = tmp_dir.path().string(); env.num_runs = 256; // Enough to run through all 1- byte inputs. env.log_level = 0; // Disable most of the logging in the test. env.require_pc_table = false; // No PC table here. // Add %f so that test_input_filter doesn't need to be linked with forkserver. - env.input_filter = "%f" + std::string{GetDataDependencyFilepath( - "centipede/testing/test_input_filter")}; + env.input_filter = + "%f" + + GetDataDependencyFilepath("centipede/testing/test_input_filter").string(); CentipedeMock mock(env); NonOwningCallbacksFactory factory(mock); CentipedeMain(env, factory); // Run fuzzing. @@ -404,10 +405,10 @@ class CentipedeWithTemporaryLocalDir : public testing::Test { TEST_F(CentipedeWithTemporaryLocalDir, MutateViaExternalBinary) { // This binary contains a test-friendly custom mutator. const std::string binary_with_custom_mutator = - GetDataDependencyFilepath("centipede/testing/test_fuzz_target"); + GetDataDependencyFilepath("centipede/testing/test_fuzz_target").string(); // This binary does not contain a custom mutator. const std::string binary_without_custom_mutator = - GetDataDependencyFilepath("centipede/testing/abort_fuzz_target"); + GetDataDependencyFilepath("centipede/testing/abort_fuzz_target").string(); // Mutate a couple of different inputs. std::vector inputs = {{0, 1, 2}, {3, 4}}; // The custom mutator in the test binary will revert the order of bytes @@ -547,7 +548,7 @@ TEST(Centipede, MergeFromOtherCorpus) { // each. TempCorpusDir work_tmp_dir{test_info_->name(), "workdir"}; Environment env; - env.workdir = work_tmp_dir.path(); + env.workdir = work_tmp_dir.path().string(); env.num_runs = 3; // Just a few runs. env.require_pc_table = false; // No PC table here. MergeMock mock(env); @@ -563,7 +564,7 @@ TEST(Centipede, MergeFromOtherCorpus) { // the seed {0} each. TempCorpusDir merge_tmp_dir(test_info_->name(), "merge_from"); Environment merge_env; - merge_env.workdir = merge_tmp_dir.path(); + merge_env.workdir = merge_tmp_dir.path().string(); merge_env.num_runs = 4; merge_env.require_pc_table = false; // No PC table here. mock.Reset(); @@ -578,7 +579,7 @@ TEST(Centipede, MergeFromOtherCorpus) { // Merge shards of `merge_env` into shards of `env`. // Shard 0 will receive one extra input: {4} // Shard 1 will receive two extra inputs: {7}, {8} - env.merge_from = merge_tmp_dir.path(); + env.merge_from = merge_tmp_dir.path().string(); env.num_runs = 0; for (env.my_shard_index = 0; env.my_shard_index < 2; ++env.my_shard_index) { CentipedeMain(env, factory); @@ -648,12 +649,13 @@ class FunctionFilterMock : public CentipedeCallbacks { static std::vector RunWithFunctionFilter( std::string_view function_filter, const TempDir& tmp_dir) { Environment env; - env.workdir = tmp_dir.path(); + env.workdir = tmp_dir.path().string(); env.seed = 1; // make the runs predictable. env.exec_time_weight_scaling = false; env.num_runs = 100; env.batch_size = 10; - env.binary = GetDataDependencyFilepath("centipede/testing/test_fuzz_target"); + env.binary = + GetDataDependencyFilepath("centipede/testing/test_fuzz_target").string(); env.coverage_binary = env.binary; // Must symbolize in order for the filter to work. env.symbolizer_path = GetLLVMSymbolizerPath(); @@ -768,8 +770,9 @@ MATCHER_P(HasFilesWithContents, expected_files_and_contents, "") { const std::string& dir_path = arg; std::vector files_and_contents; for (const auto& dir_ent : std::filesystem::directory_iterator(dir_path)) { - auto file_and_contents = FileAndContents{dir_ent.path().filename()}; - ReadFromLocalFile(dir_ent.path().c_str(), file_and_contents.contents); + auto file_and_contents = + FileAndContents{dir_ent.path().filename().string()}; + ReadFromLocalFile(dir_ent.path().string(), file_and_contents.contents); files_and_contents.push_back(std::move(file_and_contents)); } return ExplainMatchResult(expected_files_and_contents, files_and_contents, @@ -782,7 +785,7 @@ MATCHER_P(HasFilesWithContents, expected_files_and_contents, "") { TEST(Centipede, ExtraBinaries) { TempDir tmp_dir{test_info_->name()}; Environment env; - env.workdir = tmp_dir.path(); + env.workdir = tmp_dir.path().string(); env.num_runs = 100; env.batch_size = 10; env.log_level = 1; @@ -910,7 +913,7 @@ TEST(Centipede, UndetectedCrashingInput) { TempDir temp_dir{test_info_->name()}; Environment env; - env.workdir = temp_dir.path(); + env.workdir = temp_dir.path().string(); env.num_runs = kBatchSize * kNumBatches; env.batch_size = kBatchSize; // No real binary: prevent attempts by Centipede to read a PCtable from it. @@ -935,7 +938,7 @@ TEST(Centipede, UndetectedCrashingInput) { std::vector found_crash_file_names; for (auto const& dir_ent : std::filesystem::directory_iterator(crashes_dir_path)) { - found_crash_file_names.push_back(dir_ent.path().filename()); + found_crash_file_names.push_back(dir_ent.path().filename().string()); } // TODO(ussuri): Verify exact names/contents of the files, not just count. EXPECT_EQ(found_crash_file_names.size(), kCrashingInputIdxInBatch + 1); @@ -947,7 +950,7 @@ TEST(Centipede, UndetectedCrashingInput) { // Verify that when `env.batch_triage_suspect_only` is set, only triage the // suspect. TempDir suspect_only_temp_dir{test_info_->name()}; - env.workdir = suspect_only_temp_dir.path(); + env.workdir = suspect_only_temp_dir.path().string(); env.batch_triage_suspect_only = true; UndetectedCrashingInputMock suspect_only_mock(env, kCrashingInputIdx); NonOwningCallbacksFactory suspect_only_factory(suspect_only_mock); @@ -1081,8 +1084,8 @@ TEST(Centipede, SingleInputNonPersistentSkipsTriageReplay) { TEST_F(CentipedeWithTemporaryLocalDir, GetsSeedInputs) { Environment env; - env.binary = - GetDataDependencyFilepath("centipede/testing/seeded_fuzz_target"); + env.binary = GetDataDependencyFilepath("centipede/testing/seeded_fuzz_target") + .string(); CentipedeDefaultCallbacks callbacks(env, stop_condition); std::vector seeds; @@ -1094,7 +1097,8 @@ TEST_F(CentipedeWithTemporaryLocalDir, GetsSeedInputs) { TEST_F(CentipedeWithTemporaryLocalDir, GetsSerializedTargetConfig) { Environment env; env.binary = - GetDataDependencyFilepath("centipede/testing/fuzz_target_with_config"); + GetDataDependencyFilepath("centipede/testing/fuzz_target_with_config") + .string(); CentipedeDefaultCallbacks callbacks(env, stop_condition); const auto serialized_config = callbacks.GetSerializedTargetConfig(); @@ -1107,7 +1111,7 @@ TEST_F(CentipedeWithTemporaryLocalDir, Environment env; env.binary = absl::StrCat( GetDataDependencyFilepath("centipede/testing/fuzz_target_with_config") - .c_str(), + .string(), " --simulate_failure"); CentipedeDefaultCallbacks callbacks(env, stop_condition); @@ -1118,7 +1122,8 @@ TEST_F(CentipedeWithTemporaryLocalDir, TEST_F(CentipedeWithTemporaryLocalDir, CleansUpMetadataAfterStartup) { Environment env; env.binary = GetDataDependencyFilepath( - "centipede/testing/expensive_startup_fuzz_target"); + "centipede/testing/expensive_startup_fuzz_target") + .string(); CentipedeDefaultCallbacks callbacks(env, stop_condition); BatchResult batch_result; @@ -1165,7 +1170,7 @@ class FakeCentipedeCallbacksForThreadChecking : public CentipedeCallbacks { TEST(Centipede, RunsExecuteCallbackInTheCurrentThreadWhenFuzzingWithOneThread) { TempDir temp_dir{test_info_->name()}; Environment env; - env.workdir = temp_dir.path(); + env.workdir = temp_dir.path().string(); env.require_pc_table = false; ASSERT_EQ(env.num_threads, 1); FakeCentipedeCallbacksForThreadChecking callbacks(env, @@ -1180,7 +1185,8 @@ TEST(Centipede, RunsExecuteCallbackInTheCurrentThreadWhenFuzzingWithOneThread) { TEST_F(CentipedeWithTemporaryLocalDir, DetectsStackOverflow) { Environment env; - env.binary = GetDataDependencyFilepath("centipede/testing/test_fuzz_target"); + env.binary = + GetDataDependencyFilepath("centipede/testing/test_fuzz_target").string(); env.stack_limit_kb = 64; CentipedeDefaultCallbacks callbacks(env, stop_condition); @@ -1223,7 +1229,7 @@ TEST(Centipede, ReturnsFailureOnSetupFailure) { TempDir temp_dir{test_info_->name()}; Environment env; env.log_level = 0; // Disable most of the logging in the test. - env.workdir = temp_dir.path(); + env.workdir = temp_dir.path().string(); env.batch_size = 7; // Just some small number. env.require_pc_table = false; // No PC table here. SetupFailureCallbacks mock(env); @@ -1263,7 +1269,7 @@ TEST(Centipede, ReturnsSuccessOnSkippedTest) { TempDir temp_dir{test_info_->name()}; Environment env; env.log_level = 0; // Disable most of the logging in the test. - env.workdir = temp_dir.path(); + env.workdir = temp_dir.path().string(); env.batch_size = 7; // Just some small number. env.require_pc_table = false; // No PC table here. SkippedTestCallbacks mock(env); @@ -1303,7 +1309,7 @@ TEST(Centipede, KeepsRunningAndReturnsSuccessWithIgnoredFailures) { TempDir temp_dir{test_info_->name()}; Environment env; env.log_level = 0; // Disable most of the logging in the test. - env.workdir = temp_dir.path(); + env.workdir = temp_dir.path().string(); env.batch_size = 7; // Just some small number. env.num_runs = 100; env.require_pc_table = false; // No PC table here. @@ -1349,7 +1355,7 @@ class CentipedeMockForInputReduction : public CentipedeCallbacks { TEST(Centipede, DoesNotReduceInputWhenTheOptionIsUnset) { TempCorpusDir tmp_dir{test_info_->name()}; Environment env; - env.workdir = tmp_dir.path(); + env.workdir = tmp_dir.path().string(); env.num_runs = 1000000; // Should be enough env.batch_size = 7; // Just some small number. env.require_pc_table = false; @@ -1370,7 +1376,7 @@ TEST(Centipede, DoesNotReduceInputWhenTheOptionIsUnset) { TEST(Centipede, ReducesInputWhenTheOptionIsSet) { TempCorpusDir tmp_dir{test_info_->name()}; Environment env; - env.workdir = tmp_dir.path(); + env.workdir = tmp_dir.path().string(); env.num_runs = 1000000; // Should be enough env.batch_size = 7; // Just some small number. env.require_pc_table = false; @@ -1391,7 +1397,8 @@ TEST(Centipede, ReducesInputWhenTheOptionIsSet) { TEST_F(CentipedeWithTemporaryLocalDir, UsesProvidedCustomMutator) { Environment env; env.binary = GetDataDependencyFilepath( - "centipede/testing/fuzz_target_with_custom_mutator"); + "centipede/testing/fuzz_target_with_custom_mutator") + .string(); CentipedeDefaultCallbacks callbacks(env, stop_condition); const std::vector inputs = {{99}}; @@ -1408,7 +1415,7 @@ TEST_F(CentipedeWithTemporaryLocalDir, FailsOnMisbehavingCustomMutator) { env.binary = absl::StrCat(GetDataDependencyFilepath( "centipede/testing/fuzz_target_with_custom_mutator") - .c_str(), + .string(), " --simulate_failure"); StopCondition stop_condition; CentipedeDefaultCallbacks callbacks(env, stop_condition); @@ -1425,7 +1432,8 @@ TEST_F(CentipedeWithTemporaryLocalDir, FailsOnMisbehavingCustomMutator) { TEST_F(CentipedeWithTemporaryLocalDir, FallsBackToBuiltInMutatorWhenCustomMutatorNotProvided) { Environment env; - env.binary = GetDataDependencyFilepath("centipede/testing/abort_fuzz_target"); + env.binary = + GetDataDependencyFilepath("centipede/testing/abort_fuzz_target").string(); CentipedeDefaultCallbacks callbacks(env, stop_condition); const std::vector inputs = {{1}, {2}, {3}, {4}, {5}, {6}}; @@ -1454,7 +1462,8 @@ TEST_F(CentipedeWithTemporaryLocalDir, TEST_F(CentipedeWithTemporaryLocalDir, HangingFuzzTargetExitsAfterTimeout) { Environment env; env.binary = - GetDataDependencyFilepath("centipede/testing/hanging_fuzz_target"); + GetDataDependencyFilepath("centipede/testing/hanging_fuzz_target") + .string(); BatchResult batch_result; const std::vector inputs = {{0}}; CentipedeDefaultCallbacks callbacks(env, stop_condition); @@ -1469,7 +1478,8 @@ TEST_F(CentipedeWithTemporaryLocalDir, HangingFuzzTargetExitsAfterTimeout) { TEST_F(CentipedeWithTemporaryLocalDir, ExecuteEndsAfterCustomFailure) { Environment env; - env.binary = GetDataDependencyFilepath("centipede/testing/test_fuzz_target"); + env.binary = + GetDataDependencyFilepath("centipede/testing/test_fuzz_target").string(); CentipedeDefaultCallbacks callbacks(env, stop_condition); BatchResult result; std::vector inputs = { @@ -1489,7 +1499,8 @@ TEST_F(CentipedeWithTemporaryLocalDir, ExecuteEndsAfterCustomFailure) { TEST_F(CentipedeWithTemporaryLocalDir, ToleratesAsyncFailureInMutation) { Environment env; env.binary = - GetDataDependencyFilepath("centipede/testing/async_failing_target"); + GetDataDependencyFilepath("centipede/testing/async_failing_target") + .string(); StopCondition stop_condition; CentipedeDefaultCallbacks callbacks(env, stop_condition); BatchResult result; @@ -1534,9 +1545,10 @@ TEST_F(CentipedeWithTemporaryLocalDir, CentipedeMainWritesStopReason) { TEST_F(CentipedeWithTemporaryLocalDir, EngineWorksInWorkerMode) { TempCorpusDir tmp_dir{test_info_->name()}; Environment env; - env.workdir = tmp_dir.path(); + env.workdir = tmp_dir.path().string(); env.binary = GetDataDependencyFilepath( - "centipede/testing/test_binary_for_engine_testing"); + "centipede/testing/test_binary_for_engine_testing") + .string(); env.test_name = "some_test"; env.populate_binary_info = false; env.fork_server = true; diff --git a/centipede/command.cc b/centipede/command.cc index 43100e481..ed66e717a 100644 --- a/centipede/command.cc +++ b/centipede/command.cc @@ -14,6 +14,7 @@ #include "./centipede/command.h" +#ifndef _WIN32 #include #include #include @@ -22,6 +23,7 @@ #include #include #include +#endif // _WIN32 #ifdef __APPLE__ #include @@ -35,6 +37,7 @@ #include #include // NOLINT #include +#include #include #include #include @@ -58,8 +61,15 @@ #include "./centipede/stop.h" #include "./centipede/util.h" #include "./common/logging.h" +#include "./fuzztest/internal/escaping.h" +#ifdef _WIN32 +#include "absl/cleanup/cleanup.h" +#include "absl/container/btree_map.h" +#include "absl/strings/ascii.h" // NOLINT +#include "./common/windows_includes.h" +#endif -#if !defined(_MSC_VER) +#ifndef _WIN32 // Needed to pass the current environment to posix_spawn, which needs an // explicit envp without an option to inherit implicitly. extern char** environ; @@ -72,8 +82,11 @@ namespace { constexpr std::string_view kCommandLineSeparator(" \\\n"); constexpr std::string_view kNoForkServerRequestPrefix("%f"); +#ifdef _WIN32 +// Do not define `GetProcessCreationStamp`, which is for fork servers. +#else absl::StatusOr GetProcessCreationStamp(pid_t pid) { -#ifdef __APPLE__ +#if defined(__APPLE__) struct proc_bsdinfo info = {}; if (proc_pidinfo(pid, PROC_PIDTBSDINFO, 0, &info, PROC_PIDTBSDINFO_SIZE) != PROC_PIDTBSDINFO_SIZE) { @@ -110,8 +123,9 @@ absl::StatusOr GetProcessCreationStamp(pid_t pid) { ": ", proc_stat_line)); } return std::string(fields[kFieldIndexOfStartTimeAfterComm]); -#endif +#endif // __APPLE__ } +#endif // _WIN32 std::string GetUniqueSuffix() { static std::atomic suffix_counter = {0}; @@ -122,7 +136,17 @@ std::string GetUniqueSuffix() { // TODO(ussuri): Encapsulate as much of the fork server functionality from // this source as possible in this struct, and make it a class. -struct Command::ForkServerProps { +#ifdef _WIN32 +struct Command::PlatformContext { + HANDLE win_process_handle = INVALID_HANDLE_VALUE; + ~PlatformContext() { + if (win_process_handle != INVALID_HANDLE_VALUE) { + CloseHandle(win_process_handle); + } + } +}; +#else +struct ForkServerProps { // The file paths of the comms pipes. std::string fifo_path_[2]; // The file descriptors of the comms pipes. @@ -177,17 +201,30 @@ struct Command::ForkServerProps { } }; +struct Command::PlatformContext { + pid_t pid = -1; + std::unique_ptr fork_server; +}; +#endif // _WIN32 + // NOTE: Because std::unique_ptr requires T to be a complete type wherever // the deleter is instantiated, the special member functions must be defined -// out-of-line here, now that ForkServerProps is complete (that's by-the-book +// out-of-line here, now that PlatformContext is complete (that's by-the-book // PIMPL). Command::~Command() { if (is_executing()) { FUZZTEST_LOG(WARNING) << "Destructing Command object for " << path() << " with " - << (fork_server_ ? absl::StrCat("fork server PID ", - fork_server_->pid_) - : absl::StrCat("PID ", pid_)) +#ifdef _WIN32 + << GetProcessId(platform_context_->win_process_handle) +#else + << (platform_context_->fork_server + ? absl::StrCat( + "fork server PID ", + platform_context_->fork_server->pid_) + : absl::StrCat("PID ", + platform_context_->pid)) +#endif << " still running. Requesting it to force-stop " "without waiting for it..."; RequestStop(/*force=*/true); @@ -196,14 +233,51 @@ Command::~Command() { } Command::Command(std::string_view path, Options options) - : path_(path), options_(std::move(options)) {} + : path_(path), + options_(std::move(options)), + platform_context_(std::make_unique()) {} Command::Command(std::string_view path) : Command{path, {}} {} std::string Command::ToString() const { +#ifdef _WIN32 + std::string path = path_; + if (absl::StartsWith(path, kNoForkServerRequestPrefix)) { + path = path.substr(kNoForkServerRequestPrefix.size()); + } + constexpr std::string_view kTempFileWildCard = "@@"; + if (absl::StrContains(path, kTempFileWildCard)) { + FUZZTEST_CHECK(!options_.temp_file_path.empty()); + std::string temp_file = options_.temp_file_path; + path = absl::StrReplaceAll(path, {{kTempFileWildCard, temp_file}}); + } + std::string binary_cmd = path; + auto Escape = [](std::string_view s) { + std::string r = "\""; + size_t num_bs = 0; + for (size_t i = 0; i < s.size(); ++i) { + if (s[i] == '"') { + r.append(num_bs + 1, '\\'); + num_bs = 0; + } else if (s[i] == '\\') { + ++num_bs; + } else { + num_bs = 0; + } + r += s[i]; + } + r.append(num_bs, '\\'); + r += '"'; + return r; + }; + for (const auto& arg : options_.args) { + absl::StrAppend(&binary_cmd, " ", Escape(arg)); + } + return binary_cmd; +#else // _WIN32 std::vector ss; ss.reserve(/*env*/ 1 + options_.env_diff.size() + /*path*/ 1 + - /*args*/ options_.args.size() + /*out/err*/ 2); + /*args*/ options_.args.size() + /*in/out/err*/ 3); // env. ss.push_back("exec env"); std::vector env_to_set; @@ -217,7 +291,7 @@ std::string Command::ToString() const { } } for (auto& var : env_to_set) { - ss.push_back(std::move(var)); + ss.push_back(ShellEscape(var)); } // path. std::string path = path_; @@ -235,46 +309,57 @@ std::string Command::ToString() const { ss.push_back(std::move(path)); // args. for (const auto& arg : options_.args) { - ss.push_back(arg); + ss.push_back(ShellEscape(arg)); + } + // in/out/err. + if (!options_.stdin_file_path.empty()) { + ss.push_back(absl::StrCat("< ", ShellEscape(options_.stdin_file_path))); } - // out/err. if (!stdout_file_.empty()) { - ss.push_back(absl::StrCat("> ", stdout_file_)); + ss.push_back(absl::StrCat("> ", ShellEscape(stdout_file_))); } if (!stderr_file_.empty()) { if (stdout_file_ != stderr_file_) { - ss.push_back(absl::StrCat("2> ", stderr_file_)); + ss.push_back(absl::StrCat("2> ", ShellEscape(stderr_file_))); } else { ss.push_back("2>&1"); } } // Trim trailing space and return. return absl::StrJoin(ss, kCommandLineSeparator); +#endif // _WIN32 } bool Command::StartForkServer(std::string_view temp_dir_path, std::string_view prefix) { +#ifdef _WIN32 + return false; +#else if (absl::StartsWith(path_, kNoForkServerRequestPrefix)) { FUZZTEST_VLOG(2) << "Fork server disabled for " << path(); return false; } - FUZZTEST_CHECK(!is_executing_ && !fork_server_); + FUZZTEST_CHECK(!is_executing_ && !platform_context_->fork_server); FUZZTEST_VLOG(2) << "Starting fork server for " << path(); ResetRedirectionFiles(GetUniqueSuffix()); command_line_ = ToString(); - fork_server_.reset(new ForkServerProps); - fork_server_->fifo_path_[0] = std::filesystem::path(temp_dir_path) - .append(absl::StrCat(prefix, "_FIFO0")); - fork_server_->fifo_path_[1] = std::filesystem::path(temp_dir_path) - .append(absl::StrCat(prefix, "_FIFO1")); + platform_context_->fork_server = std::make_unique(); + platform_context_->fork_server->fifo_path_[0] = + std::filesystem::path(temp_dir_path) + .append(absl::StrCat(prefix, "_FIFO0")); + platform_context_->fork_server->fifo_path_[1] = + std::filesystem::path(temp_dir_path) + .append(absl::StrCat(prefix, "_FIFO1")); const std::string pid_file_path = std::filesystem::path(temp_dir_path).append("pid"); (void)std::filesystem::create_directory(temp_dir_path); // it may not exist. for (int i = 0; i < 2; ++i) { - FUZZTEST_PCHECK(mkfifo(fork_server_->fifo_path_[i].c_str(), 0600) == 0) - << VV(i) << VV(fork_server_->fifo_path_[i]); + FUZZTEST_PCHECK( + mkfifo(platform_context_->fork_server->fifo_path_[i].c_str(), 0600) == + 0) + << VV(i) << VV(platform_context_->fork_server->fifo_path_[i]); } // NOTE: A background process does not return its exit status to the subshell, @@ -290,8 +375,9 @@ bool Command::StartForkServer(std::string_view temp_dir_path, printf "%%s" $! > "%s" )sh"; const std::string fork_server_command = absl::StrFormat( - kForkServerCommandStub, fork_server_->fifo_path_[0], - fork_server_->fifo_path_[1], command_line_, pid_file_path); + kForkServerCommandStub, platform_context_->fork_server->fifo_path_[0], + platform_context_->fork_server->fifo_path_[1], command_line_, + pid_file_path); FUZZTEST_VLOG(1) << "Fork server command:" << fork_server_command; const int exit_code = system(fork_server_command.c_str()); @@ -316,10 +402,12 @@ bool Command::StartForkServer(std::string_view temp_dir_path, // it. // See more at // https://www.gnu.org/software/libc/manual/html_node/Operating-Modes.html. - if ((fork_server_->pipe_[0] = open(fork_server_->fifo_path_[0].c_str(), - O_RDWR | O_NONBLOCK)) < 0 || - (fork_server_->pipe_[1] = open(fork_server_->fifo_path_[1].c_str(), - O_RDONLY | O_NONBLOCK)) < 0) { + if ((platform_context_->fork_server->pipe_[0] = + open(platform_context_->fork_server->fifo_path_[0].c_str(), + O_RDWR | O_NONBLOCK)) < 0 || + (platform_context_->fork_server->pipe_[1] = + open(platform_context_->fork_server->fifo_path_[1].c_str(), + O_RDONLY | O_NONBLOCK)) < 0) { LogProblemInfo( "Failed to establish communication with fork server; will proceed " "without it"); @@ -328,8 +416,11 @@ bool Command::StartForkServer(std::string_view temp_dir_path, std::string pid_str; ReadFromLocalFile(pid_file_path, pid_str); - FUZZTEST_CHECK(absl::SimpleAtoi(pid_str, &fork_server_->pid_)) << VV(pid_str); - auto creation_stamp = GetProcessCreationStamp(fork_server_->pid_); + FUZZTEST_CHECK( + absl::SimpleAtoi(pid_str, &platform_context_->fork_server->pid_)) + << VV(pid_str); + auto creation_stamp = + GetProcessCreationStamp(platform_context_->fork_server->pid_); if (!creation_stamp.ok()) { LogProblemInfo( absl::StrCat("Failed to get the fork server's creation stamp; will " @@ -338,8 +429,9 @@ bool Command::StartForkServer(std::string_view temp_dir_path, creation_stamp.status(), ")")); return false; } - fork_server_->creation_stamp = *std::move(creation_stamp); + platform_context_->fork_server->creation_stamp = *std::move(creation_stamp); return true; +#endif // _WIN32 } void Command::ResetRedirectionFiles(std::string_view new_suffix) { @@ -364,37 +456,218 @@ void Command::ResetRedirectionFiles(std::string_view new_suffix) { } absl::Status Command::VerifyForkServerIsHealthy() { +#ifdef _WIN32 + return absl::UnimplementedError("Fork server not supported on Windows"); +#else // Preconditions: the callers (`Execute()`) should call us only when the fork // server is presumed to be running (`fork_server_pid_` >= 0). If it is, the // comms pipes are guaranteed to be opened by `StartForkServer()`. - FUZZTEST_CHECK(fork_server_ != nullptr) << "Fork server wasn't started"; - FUZZTEST_CHECK(fork_server_->pid_ >= 0) + FUZZTEST_CHECK(platform_context_->fork_server != nullptr) + << "Fork server wasn't started"; + FUZZTEST_CHECK(platform_context_->fork_server->pid_ >= 0) << "Fork server process failed to start"; - FUZZTEST_CHECK(fork_server_->pipe_[0] >= 0 && fork_server_->pipe_[1] >= 0) + FUZZTEST_CHECK(platform_context_->fork_server->pipe_[0] >= 0 && + platform_context_->fork_server->pipe_[1] >= 0) << "Failed to connect to fork server"; // A process with the fork server PID exists (_some_ process, possibly with a // recycled PID)... - if (kill(fork_server_->pid_, 0) != EXIT_SUCCESS) { - return absl::UnknownError(absl::StrCat( - "Can't communicate with fork server, PID=", fork_server_->pid_)); + if (kill(platform_context_->fork_server->pid_, 0) != EXIT_SUCCESS) { + return absl::UnknownError( + absl::StrCat("Can't communicate with fork server, PID=", + platform_context_->fork_server->pid_)); } // ...and it is a process has the same creation stamp, so it's practically // guaranteed to be our original fork server process. - const auto creation_stamp = GetProcessCreationStamp(fork_server_->pid_); + const auto creation_stamp = + GetProcessCreationStamp(platform_context_->fork_server->pid_); if (!creation_stamp.ok()) return creation_stamp.status(); - if (*creation_stamp != fork_server_->creation_stamp) { + if (*creation_stamp != platform_context_->fork_server->creation_stamp) { return absl::UnknownError(absl::StrCat( "Fork server's creation stamp changed (new process?) - expected ", - fork_server_->creation_stamp, ", but got ", *creation_stamp)); + platform_context_->fork_server->creation_stamp, ", but got ", + *creation_stamp)); } return absl::OkStatus(); +#endif // _WIN32 } bool Command::ExecuteAsync() { FUZZTEST_CHECK(!is_executing()); - if (fork_server_ != nullptr) { +#ifdef _WIN32 + FUZZTEST_CHECK_EQ(platform_context_->win_process_handle, + INVALID_HANDLE_VALUE); + ResetRedirectionFiles(GetUniqueSuffix()); + command_line_ = ToString(); + + struct CaseInsensitiveCompare { + using is_transparent = void; + bool operator()(std::string_view a, std::string_view b) const { + return std::lexicographical_compare( + a.begin(), a.end(), b.begin(), b.end(), + [](unsigned char ca, unsigned char cb) { + return static_cast(absl::ascii_tolower(ca)) < + static_cast(absl::ascii_tolower(cb)); + }); + } + }; + + absl::btree_map env_map; + + LPCH env_strings = GetEnvironmentStringsA(); + if (env_strings != nullptr) { + const char* ptr = env_strings; + while (*ptr != '\0') { + std::string_view entry(ptr); + ptr += entry.size() + 1; + size_t eq_pos = entry.find('=', 1); + if (eq_pos != entry.npos) { + env_map[entry.substr(0, eq_pos)] = + std::string(entry.substr(eq_pos + 1)); + } else { + env_map[entry] = ""; + } + } + FreeEnvironmentStringsA(env_strings); + } + + for (std::string_view env_var : options_.env_diff) { + if (absl::StartsWith(env_var, "-")) { + std::string_view key = env_var.substr(1); + if (absl::EndsWith(key, "=")) { + key = key.substr(0, key.size() - 1); + } + env_map.erase(key); + } else { + auto pos = env_var.find('='); + if (pos != env_var.npos) { + env_map[env_var.substr(0, pos)] = std::string(env_var.substr(pos + 1)); + } + } + } + + std::string env_block = absl::StrJoin(env_map, absl::string_view("\0", 1), + absl::PairFormatter("=")); + env_block.append(2, '\0'); + + STARTUPINFOEXA si = {}; + si.StartupInfo.cb = sizeof(si); + si.StartupInfo.dwFlags = STARTF_USESTDHANDLES; + PROCESS_INFORMATION pi = {}; + + SECURITY_ATTRIBUTES sa = {sizeof(sa), NULL, TRUE}; + HANDLE handle_in = INVALID_HANDLE_VALUE; + HANDLE handle_out = INVALID_HANDLE_VALUE; + HANDLE handle_err = INVALID_HANDLE_VALUE; + + absl::Cleanup close_handles = [&] { + if (handle_in != INVALID_HANDLE_VALUE) CloseHandle(handle_in); + if (handle_out != INVALID_HANDLE_VALUE) CloseHandle(handle_out); + if (handle_err != INVALID_HANDLE_VALUE) CloseHandle(handle_err); + }; + + if (!options_.stdin_file_path.empty()) { + handle_in = CreateFileA(options_.stdin_file_path.c_str(), FILE_READ_DATA, + /*dwShareMode=*/0, &sa, OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, NULL); + if (handle_in == INVALID_HANDLE_VALUE) { + FUZZTEST_LOG(ERROR) << "Failed to open stdin file: " + << options_.stdin_file_path; + return false; + } + si.StartupInfo.hStdInput = handle_in; + } else { + si.StartupInfo.hStdInput = GetStdHandle(STD_INPUT_HANDLE); + } + + if (!stdout_file_.empty()) { + handle_out = + CreateFileA(stdout_file_.c_str(), GENERIC_WRITE, FILE_SHARE_READ, &sa, + CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + if (handle_out == INVALID_HANDLE_VALUE) { + FUZZTEST_LOG(ERROR) << "Failed to open stdout file: " << stdout_file_; + return false; + } + si.StartupInfo.hStdOutput = handle_out; + } else { + si.StartupInfo.hStdOutput = GetStdHandle(STD_OUTPUT_HANDLE); + } + + if (!stderr_file_.empty()) { + if (stderr_file_ == stdout_file_) { + si.StartupInfo.hStdError = si.StartupInfo.hStdOutput; + } else { + handle_err = + CreateFileA(stderr_file_.c_str(), GENERIC_WRITE, FILE_SHARE_READ, &sa, + CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + if (handle_err == INVALID_HANDLE_VALUE) { + FUZZTEST_LOG(ERROR) << "Failed to open stderr file: " << stderr_file_; + return false; + } + si.StartupInfo.hStdError = handle_err; + } + } else { + si.StartupInfo.hStdError = GetStdHandle(STD_ERROR_HANDLE); + } + + std::vector handles_to_inherit; + for (HANDLE h : {si.StartupInfo.hStdInput, si.StartupInfo.hStdOutput, + si.StartupInfo.hStdError}) { + if (h != nullptr && h != INVALID_HANDLE_VALUE && + std::find(handles_to_inherit.begin(), handles_to_inherit.end(), h) == + handles_to_inherit.end()) { + SetHandleInformation(h, HANDLE_FLAG_INHERIT, HANDLE_FLAG_INHERIT); + handles_to_inherit.push_back(h); + } + } + + SIZE_T attr_list_size = 0; + InitializeProcThreadAttributeList(nullptr, 1, 0, &attr_list_size); + auto attr_list_buffer = std::make_unique(attr_list_size); + si.lpAttributeList = + reinterpret_cast(attr_list_buffer.get()); + if (!InitializeProcThreadAttributeList(si.lpAttributeList, 1, 0, + &attr_list_size)) { + DWORD err = GetLastError(); + FUZZTEST_LOG(ERROR) << "InitializeProcThreadAttributeList failed: " << err; + return false; + } + absl::Cleanup delete_attr_list = [&] { + DeleteProcThreadAttributeList(si.lpAttributeList); + }; + + if (!handles_to_inherit.empty()) { + if (!UpdateProcThreadAttribute( + si.lpAttributeList, 0, PROC_THREAD_ATTRIBUTE_HANDLE_LIST, + handles_to_inherit.data(), + handles_to_inherit.size() * sizeof(HANDLE), nullptr, nullptr)) { + DWORD err = GetLastError(); + FUZZTEST_LOG(ERROR) << "UpdateProcThreadAttribute failed: " << err; + return false; + } + } + + std::string cmd = command_line_; + const BOOL cp_res = CreateProcessA( + NULL, cmd.data(), NULL, NULL, + /*bInheritHandles=*/!handles_to_inherit.empty(), + EXTENDED_STARTUPINFO_PRESENT, env_block.empty() ? NULL : env_block.data(), + NULL, &si.StartupInfo, &pi); + + if (!cp_res) { + DWORD err = GetLastError(); + FUZZTEST_LOG(ERROR) << "CreateProcessA failed for '" << cmd + << "': error=" << err; + return false; + } + + platform_context_->win_process_handle = pi.hProcess; + CloseHandle(pi.hThread); + is_executing_ = true; + return true; +#else // _WIN32 + if (platform_context_->fork_server != nullptr) { FUZZTEST_VLOG(1) << "Sending execution request to fork server"; if (const auto status = VerifyForkServerIsHealthy(); !status.ok()) { @@ -405,7 +678,7 @@ bool Command::ExecuteAsync() { // Wake up the fork server. char x = ' '; - if (write(fork_server_->pipe_[0], &x, 1) != 1) { + if (write(platform_context_->fork_server->pipe_[0], &x, 1) != 1) { LogProblemInfo( absl::StrCat("Failed to write to fork server pipe. Errno: ", errno)); return false; @@ -413,12 +686,13 @@ bool Command::ExecuteAsync() { // Read the one-byte ack. // Use 60s as an arbitrary duration to wait for the process to load and // enter the fork server. - if (!fork_server_->ReadPipe(absl::Now() + absl::Seconds(60), x)) { + if (!platform_context_->fork_server->ReadPipe( + absl::Now() + absl::Seconds(60), x)) { LogProblemInfo("Failed to read from fork server pipe."); return false; } } else { - FUZZTEST_CHECK_EQ(pid_, -1); + FUZZTEST_CHECK_EQ(platform_context_->pid, -1); ResetRedirectionFiles(GetUniqueSuffix()); command_line_ = ToString(); @@ -431,24 +705,58 @@ bool Command::ExecuteAsync() { argv.push_back(argv_str.data()); } argv.push_back(nullptr); - FUZZTEST_PCHECK(posix_spawn(&pid_, argv[0], /*file_actions=*/nullptr, + FUZZTEST_PCHECK(posix_spawn(&platform_context_->pid, argv[0], + /*file_actions=*/nullptr, /*attrp=*/nullptr, argv.data(), environ) == 0); } is_executing_ = true; return true; +#endif // _WIN32 } std::optional Command::Wait(absl::Time deadline, StopCondition* stop_condition) { FUZZTEST_CHECK(is_executing()); +#ifdef _WIN32 + FUZZTEST_CHECK_NE(platform_context_->win_process_handle, + INVALID_HANDLE_VALUE); + DWORD timeout_ms = INFINITE; + if (deadline != absl::InfiniteFuture()) { + auto dur = deadline - absl::Now(); + if (dur <= absl::ZeroDuration()) { + timeout_ms = 0; + } else { + timeout_ms = static_cast(absl::ToInt64Milliseconds(dur)); + } + } + DWORD res = + WaitForSingleObject(platform_context_->win_process_handle, timeout_ms); + if (res == WAIT_TIMEOUT) { + VlogProblemInfo( + absl::StrCat("Timeout while waiting for command process: deadline is ", + deadline), + /*vlog_level=*/1); + return std::nullopt; + } + DWORD exit_code = 0; + GetExitCodeProcess(platform_context_->win_process_handle, &exit_code); + CloseHandle(platform_context_->win_process_handle); + platform_context_->win_process_handle = INVALID_HANDLE_VALUE; + is_executing_ = false; + if (exit_code == STATUS_CONTROL_C_EXIT && stop_condition != nullptr) { + stop_condition->RequestStop( + EXIT_FAILURE, "Command killed: signal=SIGINT (likely Ctrl-C)"); + } + return static_cast(exit_code); +#else // _WIN32 int exit_code = EXIT_SUCCESS; - if (fork_server_ != nullptr) { + if (platform_context_->fork_server != nullptr) { // The fork server forks, the child is running. Block until some readable // data appears in the pipe (that is, after the fork server writes the // execution result to it). - if (!fork_server_->ReadPipe(deadline, exit_code)) { + if (!platform_context_->fork_server->ReadPipe(deadline, exit_code)) { VlogProblemInfo( absl::StrCat("Waiting for fork server failed, deadline is ", deadline), @@ -456,11 +764,13 @@ std::optional Command::Wait(absl::Time deadline, return std::nullopt; } } else { - FUZZTEST_CHECK_NE(pid_, -1); + FUZZTEST_CHECK_NE(platform_context_->pid, -1); while (true) { - const pid_t r = waitpid(pid_, &exit_code, WNOHANG); + const pid_t r = waitpid(platform_context_->pid, &exit_code, WNOHANG); FUZZTEST_CHECK_NE(r, -1); - if (r == pid_ && (WIFEXITED(exit_code) || WIFSIGNALED(exit_code))) break; + if (r == platform_context_->pid && + (WIFEXITED(exit_code) || WIFSIGNALED(exit_code))) + break; FUZZTEST_CHECK_EQ(r, 0); const auto timeout = deadline - absl::Now(); if (timeout > absl::ZeroDuration()) { @@ -477,7 +787,7 @@ std::optional Command::Wait(absl::Time deadline, return std::nullopt; } } - pid_ = -1; + platform_context_->pid = -1; } is_executing_ = false; @@ -535,20 +845,27 @@ std::optional Command::Wait(absl::Time deadline, } return exit_code; +#endif // _WIN32 } void Command::RequestStop(bool force) { FUZZTEST_CHECK(is_executing()); - if (fork_server_) { - FUZZTEST_CHECK_NE(fork_server_->pid_, -1); +#ifdef _WIN32 + FUZZTEST_CHECK_NE(platform_context_->win_process_handle, + INVALID_HANDLE_VALUE); + TerminateProcess(platform_context_->win_process_handle, 1); +#else + if (platform_context_->fork_server) { + FUZZTEST_CHECK_NE(platform_context_->fork_server->pid_, -1); // Cannot send SIGKILL to the fork server as it kills only the parent // process, but not the child. The fork server would send SIGKILL to the // child on SIGUSR1. - kill(fork_server_->pid_, force ? SIGUSR1 : SIGTERM); + kill(platform_context_->fork_server->pid_, force ? SIGUSR1 : SIGTERM); return; } - FUZZTEST_CHECK_NE(pid_, -1); - kill(pid_, force ? SIGKILL : SIGTERM); + FUZZTEST_CHECK_NE(platform_context_->pid, -1); + kill(platform_context_->pid, force ? SIGKILL : SIGTERM); +#endif // _WIN32 } std::string Command::ReadRedirectedStdout() const { diff --git a/centipede/command.h b/centipede/command.h index 33c1d1482..15d4d70aa 100644 --- a/centipede/command.h +++ b/centipede/command.h @@ -52,6 +52,8 @@ class Command final { // `Command` automatically unlinks any previous redirected files on // execution and destruction. std::string stderr_file_prefix; + // Redirect stdin from this file path if non-empty. + std::string stdin_file_path; // "@@" in the command will be replaced with `temp_file_path`. std::string temp_file_path; }; @@ -125,9 +127,8 @@ class Command final { const std::string& stderr_file() const { return stderr_file_; } private: - struct ForkServerProps; + struct PlatformContext; - int pid_ = -1; bool is_executing_ = false; // Derived from Options::{stdout,stderr}_file_prefix, with the realized suffix @@ -163,7 +164,7 @@ class Command final { const Options options_; std::string command_line_; - std::unique_ptr fork_server_; + std::unique_ptr platform_context_; }; // Get the shared mutex for execution logging for preventing confusing diff --git a/centipede/command_test.cc b/centipede/command_test.cc index 48f4d0730..ec2380f4e 100644 --- a/centipede/command_test.cc +++ b/centipede/command_test.cc @@ -15,7 +15,9 @@ #include "./centipede/command.h" #include +#ifndef _WIN32 #include // NOLINT(for WTERMSIG) +#endif #include #include // NOLINT @@ -26,6 +28,7 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" +#include "absl/strings/str_cat.h" #include "absl/strings/substitute.h" #include "absl/time/clock.h" #include "absl/time/time.h" @@ -33,152 +36,214 @@ #include "./centipede/util.h" #include "./common/test_util.h" +#ifdef _WIN32 +#define setenv(n, v, _r) _putenv_s(n, v) +#endif + namespace fuzztest::internal { namespace { +using ::testing::AllOf; +using ::testing::HasSubstr; using ::testing::Optional; -TEST(CommandTest, ToString) { - EXPECT_EQ(Command{"x"}.ToString(), "exec env \\\nx"); - { - Command::Options cmd_options; - cmd_options.args = {"arg1", "arg2"}; - EXPECT_EQ((Command{"path", std::move(cmd_options)}.ToString()), - "exec env \\\npath \\\narg1 \\\narg2"); - } - { - Command::Options cmd_options; - cmd_options.env_diff = {"K1=V1", "K2=V2", "-K3"}; - EXPECT_EQ((Command{"x", std::move(cmd_options)}.ToString()), - "exec env \\\n-u K3 \\\nK1=V1 \\\nK2=V2 \\\nx"); - } -} - TEST(CommandTest, Execute) { + const std::string helper = + GetDataDependencyFilepath("centipede/command_test_helper").string(); StopCondition stop_condition; // Check for default exit code. - Command echo{"echo"}; - EXPECT_EQ(echo.Execute(&stop_condition), 0); + Command::Options options_success; + options_success.args = {"success"}; + Command success_cmd{helper, std::move(options_success)}; + EXPECT_EQ(success_cmd.Execute(&stop_condition), 0); EXPECT_FALSE(stop_condition.ShouldStop()); // Check for exit code 7. - Command exit7{"bash -c 'exit 7'"}; + Command::Options options_ret7; + options_ret7.args = {"ret7"}; + Command exit7{helper, std::move(options_ret7)}; EXPECT_EQ(exit7.Execute(&stop_condition), 7); EXPECT_FALSE(stop_condition.ShouldStop()); } TEST(CommandTest, HandlesInterruptedCommand) { + const std::string helper = + GetDataDependencyFilepath("centipede/command_test_helper").string(); StopCondition stop_condition; - Command self_sigint{"bash -c 'kill -SIGINT $$'"}; - self_sigint.ExecuteAsync(); - self_sigint.Wait(absl::InfiniteFuture(), &stop_condition); + Command::Options options_ctrlc; + options_ctrlc.args = {"ctrlc"}; + Command self_ctrlc{helper, std::move(options_ctrlc)}; + // Cannot set to SIG_IGN as the command would inherit that. + signal(SIGINT, [](int) {}); + self_ctrlc.ExecuteAsync(); + signal(SIGINT, SIG_DFL); + self_ctrlc.Wait(absl::InfiniteFuture(), &stop_condition); EXPECT_TRUE(stop_condition.ShouldStop()); } +TEST(CommandTest, ExecuteWithOptions) { + const std::filesystem::path test_tmpdir = GetTestTempDir(test_info_->name()); + const std::string helper = + GetDataDependencyFilepath("centipede/command_test_helper").string(); + + { + const std::string log_prefix = (test_tmpdir / "args").string(); + Command::Options cmd_options; + cmd_options.args = {"echo_args", "arg1", "arg2"}; + cmd_options.stdout_file_prefix = log_prefix; + Command cmd{helper, std::move(cmd_options)}; + EXPECT_EQ(cmd.Execute(), 0); + std::string log_contents; + ReadFromLocalFile(cmd.stdout_file(), log_contents); + EXPECT_THAT(log_contents, + AllOf(HasSubstr("arg[0]=arg1\n"), HasSubstr("arg[1]=arg2\n"))); + } + + { + setenv("K3", "V3", 1); + const std::string log_prefix = (test_tmpdir / "env").string(); + Command::Options cmd_options; + cmd_options.args = {"echo_env", "K1", "K2", "K3"}; + cmd_options.env_diff = {"K1=V1", "K2=V2", "-K3"}; + cmd_options.stdout_file_prefix = log_prefix; + Command cmd{helper, std::move(cmd_options)}; + EXPECT_EQ(cmd.Execute(), 0); + std::string log_contents; + ReadFromLocalFile(cmd.stdout_file(), log_contents); + EXPECT_THAT(log_contents, AllOf(HasSubstr("K1=V1\n"), HasSubstr("K2=V2\n"), + HasSubstr("K3=\n"))); + } + + { + const std::string stdin_file = (test_tmpdir / "input.txt").string(); + WriteToLocalFile(stdin_file, "hello stdin"); + const std::string log_prefix = (test_tmpdir / "stdin").string(); + Command::Options cmd_options; + cmd_options.args = {"echo_stdin"}; + cmd_options.stdin_file_path = stdin_file; + cmd_options.stdout_file_prefix = log_prefix; + Command cmd{helper, std::move(cmd_options)}; + EXPECT_EQ(cmd.Execute(), 0); + std::string log_contents; + ReadFromLocalFile(cmd.stdout_file(), log_contents); + EXPECT_THAT(log_contents, HasSubstr("hello stdin")); + } +} + TEST(CommandTest, InputFileWildCard) { + const std::filesystem::path test_tmpdir = GetTestTempDir(test_info_->name()); + const std::string helper = + GetDataDependencyFilepath("centipede/command_test_helper").string(); + const std::string log_prefix = (test_tmpdir / "wildcard").string(); + Command::Options cmd_options; cmd_options.temp_file_path = "TEMP_FILE"; - Command cmd{"foo bar @@ baz", std::move(cmd_options)}; - EXPECT_EQ(cmd.ToString(), "exec env \\\nfoo bar TEMP_FILE baz"); + cmd_options.stdout_file_prefix = log_prefix; + Command cmd{absl::StrCat(helper, " @@"), std::move(cmd_options)}; + EXPECT_EQ(cmd.Execute(), 17); + std::string log_contents; + ReadFromLocalFile(cmd.stdout_file(), log_contents); + EXPECT_EQ(log_contents, "Got input: TEMP_FILE\n"); } +#ifndef _WIN32 TEST(CommandTest, ForkServer) { - const std::string test_tmpdir = GetTestTempDir(test_info_->name()); + const std::filesystem::path test_tmpdir = GetTestTempDir(test_info_->name()); const std::string helper = - GetDataDependencyFilepath("centipede/command_test_helper"); + GetDataDependencyFilepath("centipede/command_test_helper").string(); // TODO(ussuri): Dedupe these testcases. { const std::string input = "success"; - const std::string log_prefix = std::filesystem::path{test_tmpdir} / input; + const std::string log_prefix = (test_tmpdir / input).string(); Command::Options cmd_options; cmd_options.args = {input}; cmd_options.stdout_file_prefix = log_prefix; cmd_options.stderr_file_prefix = log_prefix; Command cmd{helper, std::move(cmd_options)}; - EXPECT_TRUE(cmd.StartForkServer(test_tmpdir, "ForkServer")); + EXPECT_TRUE(cmd.StartForkServer(test_tmpdir.string(), "ForkServer")); EXPECT_EQ(cmd.Execute(), EXIT_SUCCESS); std::string log_contents; ReadFromLocalFile(cmd.stdout_file(), log_contents); - EXPECT_EQ(log_contents, absl::Substitute("Got input: $0", input)); + EXPECT_EQ(log_contents, absl::Substitute("Got input: $0\n", input)); } { const std::string input = "fail"; - const std::string log_prefix = std::filesystem::path{test_tmpdir} / input; + const std::string log_prefix = test_tmpdir / input; Command::Options cmd_options; cmd_options.args = {input}; cmd_options.stdout_file_prefix = log_prefix; cmd_options.stderr_file_prefix = log_prefix; Command cmd{helper, std::move(cmd_options)}; - EXPECT_TRUE(cmd.StartForkServer(test_tmpdir, "ForkServer")); + EXPECT_TRUE(cmd.StartForkServer(test_tmpdir.string(), "ForkServer")); EXPECT_EQ(cmd.Execute(), EXIT_FAILURE); std::string log_contents; ReadFromLocalFile(cmd.stdout_file(), log_contents); - EXPECT_EQ(log_contents, absl::Substitute("Got input: $0", input)); + EXPECT_EQ(log_contents, absl::Substitute("Got input: $0\n", input)); } { const std::string input = "ret42"; - const std::string log_prefix = std::filesystem::path{test_tmpdir} / input; + const std::string log_prefix = test_tmpdir / input; Command::Options cmd_options; cmd_options.args = {input}; cmd_options.stdout_file_prefix = log_prefix; cmd_options.stderr_file_prefix = log_prefix; Command cmd{helper, std::move(cmd_options)}; - EXPECT_TRUE(cmd.StartForkServer(test_tmpdir, "ForkServer")); + EXPECT_TRUE(cmd.StartForkServer(test_tmpdir.string(), "ForkServer")); EXPECT_EQ(cmd.Execute(), 42); std::string log_contents; ReadFromLocalFile(cmd.stdout_file(), log_contents); - EXPECT_EQ(log_contents, absl::Substitute("Got input: $0", input)); + EXPECT_EQ(log_contents, absl::Substitute("Got input: $0\n", input)); } { const std::string input = "abort"; - const std::string log_prefix = std::filesystem::path{test_tmpdir} / input; + const std::string log_prefix = test_tmpdir / input; Command::Options cmd_options; cmd_options.args = {input}; cmd_options.stdout_file_prefix = log_prefix; cmd_options.stderr_file_prefix = log_prefix; Command cmd{helper, std::move(cmd_options)}; - EXPECT_TRUE(cmd.StartForkServer(test_tmpdir, "ForkServer")); + EXPECT_TRUE(cmd.StartForkServer(test_tmpdir.string(), "ForkServer")); // WTERMSIG() needs an lvalue on some platforms. const int ret = cmd.Execute(); EXPECT_EQ(WTERMSIG(ret), SIGABRT); std::string log_contents; ReadFromLocalFile(cmd.stdout_file(), log_contents); - EXPECT_EQ(log_contents, absl::Substitute("Got input: $0", input)); + EXPECT_EQ(log_contents, absl::Substitute("Got input: $0\n", input)); } { const std::string input = "sleep"; - const std::string log_prefix = std::filesystem::path{test_tmpdir} / input; + const std::string log_prefix = test_tmpdir / input; Command::Options cmd_options; cmd_options.args = {input}; cmd_options.stdout_file_prefix = log_prefix; cmd_options.stderr_file_prefix = log_prefix; Command cmd{helper, std::move(cmd_options)}; - ASSERT_TRUE(cmd.StartForkServer(test_tmpdir, "ForkServer")); + ASSERT_TRUE(cmd.StartForkServer(test_tmpdir.string(), "ForkServer")); ASSERT_TRUE(cmd.ExecuteAsync()); EXPECT_EQ(cmd.Wait(absl::Now() + absl::Seconds(2)), std::nullopt); cmd.RequestStop(/*force=*/false); EXPECT_THAT(cmd.Wait(absl::Now() + absl::Seconds(2)), Optional(SIGTERM)); std::string log_contents; ReadFromLocalFile(cmd.stdout_file(), log_contents); - EXPECT_EQ(log_contents, absl::Substitute("Got input: $0", input)); + EXPECT_EQ(log_contents, absl::Substitute("Got input: $0\n", input)); } { const std::string input = "hang"; - const std::string log_prefix = std::filesystem::path{test_tmpdir} / input; + const std::string log_prefix = test_tmpdir / input; Command::Options cmd_options; cmd_options.args = {input}; cmd_options.stdout_file_prefix = log_prefix; cmd_options.stderr_file_prefix = log_prefix; Command cmd{helper, std::move(cmd_options)}; - ASSERT_TRUE(cmd.StartForkServer(test_tmpdir, "ForkServer")); + ASSERT_TRUE(cmd.StartForkServer(test_tmpdir.string(), "ForkServer")); ASSERT_TRUE(cmd.ExecuteAsync()); EXPECT_EQ(cmd.Wait(absl::Now() + absl::Seconds(2)), std::nullopt); cmd.RequestStop(/*force=*/false); @@ -187,11 +252,12 @@ TEST(CommandTest, ForkServer) { EXPECT_THAT(cmd.Wait(absl::Now() + absl::Seconds(2)), Optional(SIGKILL)); std::string log_contents; ReadFromLocalFile(cmd.stdout_file(), log_contents); - EXPECT_EQ(log_contents, absl::Substitute("Got input: $0", input)); + EXPECT_EQ(log_contents, absl::Substitute("Got input: $0\n", input)); } // TODO(kcc): [impl] test what happens if the child is interrupted. } +#endif } // namespace } // namespace fuzztest::internal diff --git a/centipede/command_test_helper.cc b/centipede/command_test_helper.cc index d3153a853..75fe519d1 100644 --- a/centipede/command_test_helper.cc +++ b/centipede/command_test_helper.cc @@ -12,33 +12,92 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include - #include #include #include #include #include +#ifdef _WIN32 +#include +#include + +#include "./common/windows_includes.h" +#endif + #include "absl/base/nullability.h" +#include "absl/strings/match.h" +#include "absl/strings/numbers.h" +#include "absl/strings/string_view.h" #include "absl/time/clock.h" #include "absl/time/time.h" // A binary linked with the fork server that exits/crashes in different ways. int main(int argc, char** absl_nonnull argv) { - assert(argc == 2); - printf("Got input: %s", argv[1]); +#ifdef _WIN32 + // Disable the automatic \n -> \r\n conversion. + _setmode(_fileno(stdout), _O_BINARY); +#endif + + assert(argc >= 2); + printf("Got input: %s\n", argv[1]); + + if (!strcmp(argv[1], "echo_args")) { + for (int i = 2; i < argc; ++i) { + printf("arg[%d]=%s\n", i - 2, argv[i]); + } + fflush(stdout); + return EXIT_SUCCESS; + } + + if (!strcmp(argv[1], "echo_env")) { + for (int i = 2; i < argc; ++i) { + const char* val = getenv(argv[i]); + printf("%s=%s\n", argv[i], val ? val : ""); + } + fflush(stdout); + return EXIT_SUCCESS; + } + + if (!strcmp(argv[1], "echo_stdin")) { + char buf[1024]; + while (fgets(buf, sizeof(buf), stdin)) { + fputs(buf, stdout); + } + fflush(stdout); + return EXIT_SUCCESS; + } + fflush(stdout); + if (!strcmp(argv[1], "success")) return EXIT_SUCCESS; if (!strcmp(argv[1], "fail")) return EXIT_FAILURE; - if (!strcmp(argv[1], "ret42")) return 42; + + int ret_code = 0; + if (absl::StartsWith(argv[1], "ret") && + absl::SimpleAtoi(argv[1] + 3, &ret_code)) { + return ret_code; + } + if (!strcmp(argv[1], "abort")) abort(); + if (!strcmp(argv[1], "ctrlc")) { +#ifdef _WIN32 + SetConsoleCtrlHandler(NULL, FALSE); + GenerateConsoleCtrlEvent(CTRL_C_EVENT, 0); + Sleep(INFINITE); +#else + raise(SIGINT); +#endif + return EXIT_SUCCESS; + } // Sleep longer than kTimeout in CommandDeathTest_ForkServerHangingBinary. if (!strcmp(argv[1], "sleep")) absl::SleepFor(absl::Seconds(5)); if (!strcmp(argv[1], "hang")) { +#ifndef _WIN32 struct sigaction act{}; act.sa_handler = [](int) {}; sigaction(SIGTERM, &act, nullptr); +#endif absl::SleepFor(absl::Seconds(10)); } diff --git a/centipede/config_file.cc b/centipede/config_file.cc index 89c06131c..e0d59e0b6 100644 --- a/centipede/config_file.cc +++ b/centipede/config_file.cc @@ -162,7 +162,7 @@ AugmentedArgvWithCleanup LocalizeConfigFilesInArgv( if (!path.empty() && !std::filesystem::exists(path)) { // assume remote // Read the remote file. std::string contents; - FUZZTEST_CHECK_OK(RemoteFileGetContents(path.c_str(), contents)); + FUZZTEST_CHECK_OK(RemoteFileGetContents(path.string(), contents)); // Save a temporary local copy. const std::filesystem::path tmp_dir = TemporaryLocalDirPath(); @@ -170,11 +170,11 @@ AugmentedArgvWithCleanup LocalizeConfigFilesInArgv( FUZZTEST_LOG(INFO) << "Localizing remote config: " << VV(path) << VV(local_path); // NOTE: Ignore "Remote" in the API names here: the paths are always local. - FUZZTEST_CHECK_OK(RemoteMkdir(tmp_dir.c_str())); - FUZZTEST_CHECK_OK(RemoteFileSetContents(local_path.c_str(), contents)); + FUZZTEST_CHECK_OK(RemoteMkdir(tmp_dir.string())); + FUZZTEST_CHECK_OK(RemoteFileSetContents(local_path.string(), contents)); // Augment the argv to point at the local copy and ensure it is cleaned up. - replacements.emplace_back(path.c_str(), local_path.c_str()); + replacements.emplace_back(path.string(), local_path.string()); cleanup = [local_path]() { std::filesystem::remove(local_path); }; } @@ -245,7 +245,7 @@ set -x } else { file_contents = flags_str; } - FUZZTEST_CHECK_OK(RemoteFileSetContents(path.c_str(), file_contents)); + FUZZTEST_CHECK_OK(RemoteFileSetContents(path.string(), file_contents)); } return path; diff --git a/centipede/control_flow_test.cc b/centipede/control_flow_test.cc index 0c687c732..c4a8239e6 100644 --- a/centipede/control_flow_test.cc +++ b/centipede/control_flow_test.cc @@ -182,13 +182,15 @@ TEST(ControlFlowGraph, LazyReachability) { // Returns path to test_fuzz_target. static std::string GetTargetPath() { - return GetDataDependencyFilepath("centipede/testing/test_fuzz_target"); + return GetDataDependencyFilepath("centipede/testing/test_fuzz_target") + .string(); } // Returns path to test_fuzz_target_trace_pc. static std::string GetTracePCTargetPath() { return GetDataDependencyFilepath( - "centipede/testing/test_fuzz_target_trace_pc"); + "centipede/testing/test_fuzz_target_trace_pc") + .string(); } // Tests GetCfTableFromBinary() on test_fuzz_target. diff --git a/centipede/corpus_io_test.cc b/centipede/corpus_io_test.cc index f368a3195..3025c04eb 100644 --- a/centipede/corpus_io_test.cc +++ b/centipede/corpus_io_test.cc @@ -51,7 +51,7 @@ std::vector ReadInputsFromFiles(std::string_view dir) { std::vector inputs; for (const auto& file : std::filesystem::directory_iterator(dir)) { ByteArray input; - ReadFromLocalFile(file.path().c_str(), input); + ReadFromLocalFile(file.path().string(), input); inputs.push_back(std::move(input)); } return inputs; @@ -110,16 +110,16 @@ TEST(ExportCorpusTest, ExportsCorpusToIndividualFiles) { const std::filesystem::path temp_dir = GetTestTempDir(test_info_->name()); const std::filesystem::path out_dir = temp_dir / "out_dir"; FUZZTEST_CHECK(std::filesystem::create_directory(out_dir)); - const WorkDir workdir{temp_dir.c_str(), "fake_binary_name", + const WorkDir workdir{temp_dir.string(), "fake_binary_name", "fake_binary_hash", /*my_shard_index=*/0}; const auto corpus_file_paths = workdir.CorpusFilePaths(); WriteBlobsToFile(corpus_file_paths.Shard(0), {ByteArray{1, 2}, ByteArray{3}}); WriteBlobsToFile(corpus_file_paths.Shard(1), {ByteArray{4}, ByteArray{5, 6}}); ExportCorpus({corpus_file_paths.Shard(0), corpus_file_paths.Shard(1)}, - out_dir.c_str()); + out_dir.string()); - EXPECT_THAT(ReadInputsFromFiles(out_dir.c_str()), + EXPECT_THAT(ReadInputsFromFiles(out_dir.string()), UnorderedElementsAre(ByteArray{1, 2}, ByteArray{3}, ByteArray{4}, ByteArray{5, 6})); } diff --git a/centipede/corpus_test.cc b/centipede/corpus_test.cc index 46e4d4e37..bbdaa1722 100644 --- a/centipede/corpus_test.cc +++ b/centipede/corpus_test.cc @@ -66,7 +66,7 @@ TEST(Corpus, PrintStats) { corpus.Add({1, 2, 3}, features1, {}, /*stats=*/{}, fs, coverage_frontier); fs.MergeFeatures(features2); corpus.Add({4, 5}, features2, {}, /*stats=*/{}, fs, coverage_frontier); - const std::string stats_filepath = test_tmpdir / "corpus.txt"; + const std::string stats_filepath = (test_tmpdir / "corpus.txt").string(); corpus.DumpStatsToFile(fs, stats_filepath, "Test corpus"); std::string stats_file_contents; ReadFromLocalFile(stats_filepath, stats_file_contents); diff --git a/centipede/coverage_test.cc b/centipede/coverage_test.cc index 4f3a830b0..b327b56ab 100644 --- a/centipede/coverage_test.cc +++ b/centipede/coverage_test.cc @@ -175,12 +175,14 @@ TEST(Coverage, CoverageLogger) { // Returns path to test_fuzz_target. static std::string GetTargetPath() { - return GetDataDependencyFilepath("centipede/testing/test_fuzz_target"); + return GetDataDependencyFilepath("centipede/testing/test_fuzz_target") + .string(); } // Returns path to threaded_fuzz_target. static std::string GetThreadedTargetPath() { - return GetDataDependencyFilepath("centipede/testing/threaded_fuzz_target"); + return GetDataDependencyFilepath("centipede/testing/threaded_fuzz_target") + .string(); } // Tests coverage collection on test_fuzz_target diff --git a/centipede/crash_deduplication.cc b/centipede/crash_deduplication.cc index 397110564..5cf211ed7 100644 --- a/centipede/crash_deduplication.cc +++ b/centipede/crash_deduplication.cc @@ -112,7 +112,7 @@ absl::StatusOr> ReadExistingCrashes( std::vector existing_crashes; ASSIGN_OR_RETURN_IF_NOT_OK( const std::vector input_files, - RemoteListFiles(crashing_dir.c_str(), /*recursively=*/false)); + RemoteListFiles(crashing_dir.string(), /*recursively=*/false)); existing_crashes.reserve(input_files.size()); for (const std::string& input_file : input_files) { @@ -139,14 +139,14 @@ absl::StatusOr> ReadIncubatingCrashes( std::vector incubating_crashes; ASSIGN_OR_RETURN_IF_NOT_OK( const std::vector input_files, - RemoteListFiles(incubating_dir.c_str(), /*recursively=*/false)); + RemoteListFiles(incubating_dir.string(), /*recursively=*/false)); incubating_crashes.reserve(input_files.size()); for (const std::string& input_file : input_files) { IncubatingCrash incubating; incubating.details.input_path = input_file; incubating.details.input_signature = - std::filesystem::path(input_file).filename().c_str(); + std::filesystem::path(input_file).filename().string(); incubating_crashes.push_back(std::move(incubating)); } return incubating_crashes; @@ -332,9 +332,9 @@ absl::Status WriteCrashToFile(const std::filesystem::path& crashing_dir, GetInputFileName(bug_id, crash_signature, details.input_signature); std::filesystem::path new_input_path = crashing_dir / new_input_file_name; - if (details.input_path != new_input_path.c_str()) { + if (details.input_path != new_input_path.string()) { RETURN_IF_NOT_OK( - RemoteFileCopy(details.input_path, new_input_path.c_str())); + RemoteFileCopy(details.input_path, new_input_path.string())); } crash_summary.AddCrash({/*id=*/new_input_file_name, @@ -379,7 +379,7 @@ absl::Status TouchCrash(const ExistingCrash& existing, crash_summary.AddCrash({ /*id=*/std::filesystem::path(existing.crash_report.details.input_path) .filename() - .c_str(), + .string(), /*category=*/description, existing.crash_report.signature, description, @@ -433,7 +433,7 @@ absl::Status ExecuteExistingCrashDestructiveActions( std::filesystem::path dest_path = incubating_dir / existing.crash_report.details.input_signature; RETURN_IF_NOT_OK(RemoteFileRename( - existing.crash_report.details.input_path, dest_path.c_str())); + existing.crash_report.details.input_path, dest_path.string())); break; } @@ -492,12 +492,12 @@ absl::Status MoveExpiredCrashesToRegression( const std::filesystem::path& source_dir, const std::filesystem::path& regression_dir, absl::Duration ttl, absl::Clock& clock) { - if (!RemotePathExists(source_dir.c_str())) { + if (!RemotePathExists(source_dir.string())) { return absl::OkStatus(); } ASSIGN_OR_RETURN_IF_NOT_OK( const std::vector active_crash_files, - RemoteListFiles(source_dir.c_str(), /*recursively=*/false)); + RemoteListFiles(source_dir.string(), /*recursively=*/false)); absl::Time now = clock.TimeNow(); @@ -510,7 +510,7 @@ absl::Status MoveExpiredCrashesToRegression( if (input_file_components.ok()) { dest_filename = input_file_components->input_signature; } else { - dest_filename = std::filesystem::path(file_path).filename().c_str(); + dest_filename = std::filesystem::path(file_path).filename().string(); } std::filesystem::path dest_path = regression_dir / dest_filename; @@ -518,7 +518,7 @@ absl::Status MoveExpiredCrashesToRegression( << "' -> '" << dest_path.c_str() << "'. Reason: Crash expired (not reproduced for " << (now - mtime) << " > TTL " << ttl << ")."; - RETURN_IF_NOT_OK(RemoteFileRename(file_path, dest_path.c_str())); + RETURN_IF_NOT_OK(RemoteFileRename(file_path, dest_path.string())); } } return absl::OkStatus(); @@ -544,9 +544,10 @@ absl::flat_hash_map GetCrashesFromWorkdir( for (std::string& crashing_input_path : crashing_input_paths) { std::string crashing_input_file_name = - std::filesystem::path(crashing_input_path).filename(); + std::filesystem::path(crashing_input_path).filename().string(); const std::string crash_signature_path = - crash_metadata_dir / absl::StrCat(crashing_input_file_name, ".sig"); + (crash_metadata_dir / absl::StrCat(crashing_input_file_name, ".sig")) + .string(); std::string crash_signature; const absl::Status status = RemoteFileGetContents(crash_signature_path, crash_signature); @@ -568,7 +569,8 @@ absl::flat_hash_map GetCrashesFromWorkdir( if (crashes.contains(crash_signature)) continue; const std::string crash_description_path = - crash_metadata_dir / absl::StrCat(crashing_input_file_name, ".desc"); + (crash_metadata_dir / absl::StrCat(crashing_input_file_name, ".desc")) + .string(); std::string crash_description; const absl::Status description_status = RemoteFileGetContents(crash_description_path, crash_description); @@ -609,9 +611,9 @@ absl::Status OrganizeCrashingInputs( new_crashes_by_signature, CrashSummary& crash_summary, StopCondition& stop_condition, absl::Duration regression_ttl, absl::Clock& clock) { - RETURN_IF_NOT_OK(RemoteMkdir(crashing_dir.c_str())); - RETURN_IF_NOT_OK(RemoteMkdir(regression_dir.c_str())); - RETURN_IF_NOT_OK(RemoteMkdir(incubating_dir.c_str())); + RETURN_IF_NOT_OK(RemoteMkdir(crashing_dir.string())); + RETURN_IF_NOT_OK(RemoteMkdir(regression_dir.string())); + RETURN_IF_NOT_OK(RemoteMkdir(incubating_dir.string())); ASSIGN_OR_RETURN_IF_NOT_OK(auto existing_crashes, ReadExistingCrashes(crashing_dir)); diff --git a/centipede/crash_deduplication_test.cc b/centipede/crash_deduplication_test.cc index b65f5579d..aed70f6c2 100644 --- a/centipede/crash_deduplication_test.cc +++ b/centipede/crash_deduplication_test.cc @@ -78,14 +78,14 @@ class LogCapture : public absl::LogSink { std::string SetContentsAndGetPath(const std::filesystem::path& dir, std::string_view file_name, std::string_view contents) { - const std::string file_path = dir / file_name; + const std::string file_path = (dir / file_name).string(); WriteToLocalFile(file_path, contents); return file_path; } TEST(GetCrashesFromWorkdirTest, ReturnsOneCrashPerCrashSignature) { TempDir test_dir; - const std::string workdir_path = test_dir.path(); + const std::string workdir_path = test_dir.path().string(); WorkDir workdir{workdir_path, "binary_name", "binary_hash", /*my_shard_index=*/0}; @@ -138,7 +138,7 @@ TEST(GetCrashesFromWorkdirTest, ReturnsOneCrashPerCrashSignature) { TEST(GetCrashesFromWorkdirTest, FailsOnEmptyCrashSignatureIfEnvVarSet) { TempDir test_dir; - const std::string workdir_path = test_dir.path(); + const std::string workdir_path = test_dir.path().string(); WorkDir workdir{workdir_path, "binary_name", "binary_hash", /*my_shard_index=*/0}; @@ -161,7 +161,7 @@ TEST(GetCrashesFromWorkdirTest, FailsOnEmptyCrashSignatureIfEnvVarSet) { TEST(GetCrashesFromWorkdirTest, FailsOnEmptyCrashDescriptionIfEnvVarSet) { TempDir test_dir; - const std::string workdir_path = test_dir.path(); + const std::string workdir_path = test_dir.path().string(); WorkDir workdir{workdir_path, "binary_name", "binary_hash", /*my_shard_index=*/0}; @@ -196,9 +196,9 @@ std::vector ReadFiles(const std::filesystem::path& dir) { std::vector files; for (const auto& f : std::filesystem::directory_iterator(dir)) { std::string contents; - ReadFromLocalFile(f.path().c_str(), contents); - files.push_back(FileAndContents{std::filesystem::path(f).filename(), - std::move(contents)}); + ReadFromLocalFile(f.path().string(), contents); + files.push_back(FileAndContents{ + std::filesystem::path(f).filename().string(), std::move(contents)}); } return files; } diff --git a/centipede/distill_test.cc b/centipede/distill_test.cc index 047f67fa3..4b698a5d7 100644 --- a/centipede/distill_test.cc +++ b/centipede/distill_test.cc @@ -102,7 +102,7 @@ std::vector TestDistill( // We need to set at least --binary_hash before `env` is constructed, // so we do this by overriding the flags. absl::FlagSaver flag_saver; - std::string dir = GetTestTempDir(test_name); + std::string dir = GetTestTempDir(test_name).string(); std::filesystem::remove_all(dir); std::filesystem::create_directories(dir); Environment env; diff --git a/centipede/engine_worker.cc b/centipede/engine_worker.cc index a902fa1e2..4bbe8e0d2 100644 --- a/centipede/engine_worker.cc +++ b/centipede/engine_worker.cc @@ -325,8 +325,11 @@ BlobSequence* GetInputsBlobSequence() { } const char* input_path = GetWorkerFlags().GetStringFlag(kWorkerInputsBlobSequencePathFlagHeader); - WorkerCheck(input_path != nullptr, "inputs blob sequence is missing"); - return new SharedMemoryBlobSequence(input_path, shmem_size); + if (input_path == nullptr) { + WorkerCheck(false, "inputs blob sequence is missing"); + __builtin_unreachable(); + } + return OpenSharedMemoryBlobSequence(input_path, shmem_size).release(); }(); return result; } @@ -339,8 +342,11 @@ BlobSequence* GetOutputsBlobSequence() { } const char* output_path = GetWorkerFlags().GetStringFlag( kWorkerOutputsBlobSequencePathFlagHeader); - WorkerCheck(output_path != nullptr, "outputs blob sequence is missing"); - return new SharedMemoryBlobSequence(output_path, shmem_size); + if (output_path == nullptr) { + WorkerCheck(false, "outputs blob sequence is missing"); + __builtin_unreachable(); + } + return OpenSharedMemoryBlobSequence(output_path, shmem_size).release(); }(); return result; } diff --git a/centipede/runner.cc b/centipede/runner.cc index c2d3eefd3..401255bdd 100644 --- a/centipede/runner.cc +++ b/centipede/runner.cc @@ -904,10 +904,10 @@ void GlobalRunnerState::OnTermination() { if (!state->centipede_runner_main_executed && state->run_time_flags.shmem_size_mb != 0) { PostProcessSancov(); // TODO(xinhaoyuan): do we know our exit status? - SharedMemoryBlobSequence outputs_blobseq( + auto outputs_blobseq = OpenSharedMemoryBlobSequence( sancov_state->arg2, state->run_time_flags.shmem_size_mb << 20); - StartSendingOutputsToEngine(outputs_blobseq); - FinishSendingOutputsToEngine(outputs_blobseq); + StartSendingOutputsToEngine(*outputs_blobseq); + FinishSendingOutputsToEngine(*outputs_blobseq); } { LockGuard lock(state->execution_result_override_mu); @@ -1018,16 +1018,16 @@ int RunnerMain(int argc, char** argv, RunnerCallbacks& callbacks) { // Inputs / outputs from shmem. if (state->run_time_flags.shmem_size_mb != 0) { if (!sancov_state->arg1 || !sancov_state->arg2) return EXIT_FAILURE; - SharedMemoryBlobSequence inputs_blobseq( + auto inputs_blobseq = OpenSharedMemoryBlobSequence( sancov_state->arg1, state->run_time_flags.shmem_size_mb << 20); - SharedMemoryBlobSequence outputs_blobseq( + auto outputs_blobseq = OpenSharedMemoryBlobSequence( sancov_state->arg2, state->run_time_flags.shmem_size_mb << 20); // Persistent mode loop. if (state->persistent_mode_socket > 0) { - return HandlePersistentMode(callbacks, inputs_blobseq, outputs_blobseq); + return HandlePersistentMode(callbacks, *inputs_blobseq, *outputs_blobseq); } - return HandleSharedMemoryRequest(callbacks, inputs_blobseq, - outputs_blobseq); + return HandleSharedMemoryRequest(callbacks, *inputs_blobseq, + *outputs_blobseq); } // By default, run every input file one-by-one. diff --git a/centipede/runner_result_test.cc b/centipede/runner_result_test.cc index f8c86cb33..02e917d3c 100644 --- a/centipede/runner_result_test.cc +++ b/centipede/runner_result_test.cc @@ -103,7 +103,8 @@ TEST(ExecutionResult, WriteThenRead) { } TEST(ExecutionResult, WriteIntoFileThenRead) { - const std::string temp_file = GetTestTempDir(test_info_->name()) / "tmp.txt"; + const std::string temp_file = + (GetTestTempDir(test_info_->name()) / "tmp.txt").string(); std::ofstream output_stream(temp_file, std::ios::out); ASSERT_TRUE(output_stream.is_open()); diff --git a/centipede/rusage_stats.cc b/centipede/rusage_stats.cc index c5bf85072..3ce6e83cb 100644 --- a/centipede/rusage_stats.cc +++ b/centipede/rusage_stats.cc @@ -14,14 +14,22 @@ #include "./centipede/rusage_stats.h" +#ifdef _WIN32 +#include "./common/windows_includes.h" +#else // _WIN32 + +#include #ifdef __APPLE__ #include #include #include -#endif // __APPLE__ +#else // __APPLE__ #include #include #include +#endif // __APPLE__ + +#endif // _WIN32 #include #include @@ -47,41 +55,84 @@ namespace fuzztest::internal { +namespace { + +void GetTimeUsage(double& user, double& sys) { +#ifdef _WIN32 + FILETIME creation_time, exit_time, kernel_time, user_time; + FUZZTEST_CHECK(GetProcessTimes(GetCurrentProcess(), &creation_time, + &exit_time, &kernel_time, &user_time) != 0) + << "Failed to get time usage by GetProcessTimes()"; + uint64_t user_100ns = + (static_cast(user_time.dwHighDateTime) << 32) | + user_time.dwLowDateTime; + uint64_t sys_100ns = + (static_cast(kernel_time.dwHighDateTime) << 32) | + kernel_time.dwLowDateTime; + static constexpr double kSecondTo100ns = 1000 * 1000 * 10; + user = user_100ns / kSecondTo100ns; + sys = sys_100ns / kSecondTo100ns; +#else + struct rusage rusage; + FUZZTEST_CHECK(getrusage(RUSAGE_SELF, &rusage) == 0) + << "Failed to get time usage by getrusage()"; + user = absl::ToDoubleSeconds(absl::DurationFromTimeval(rusage.ru_utime)); + sys = absl::ToDoubleSeconds(absl::DurationFromTimeval(rusage.ru_stime)); +#endif +} + +} // namespace + //------------------------------------------------------------------------------ // ProcessTimer //------------------------------------------------------------------------------ -ProcessTimer::ProcessTimer() : start_time_{absl::Now()}, start_rusage_{} { - getrusage(RUSAGE_SELF, &start_rusage_); +ProcessTimer::ProcessTimer() + : start_time_{absl::Now()}, usage_user_{0}, usage_sys_{0} { + GetTimeUsage(usage_user_, usage_sys_); } void ProcessTimer::Get(double& user, double& sys, double& wall) const { - struct rusage curr_rusage = {}; - getrusage(RUSAGE_SELF, &curr_rusage); - // clang-format off - user = absl::ToDoubleSeconds( - absl::DurationFromTimeval(curr_rusage.ru_utime) - - absl::DurationFromTimeval(start_rusage_.ru_utime)); - sys = absl::ToDoubleSeconds( - absl::DurationFromTimeval(curr_rusage.ru_stime) - - absl::DurationFromTimeval(start_rusage_.ru_stime)); + double cur_usage_user = 0; + double cur_usage_sys = 0; + GetTimeUsage(cur_usage_user, cur_usage_sys); + user = cur_usage_user - usage_user_; + sys = cur_usage_sys - usage_sys_; wall = absl::ToDoubleSeconds(absl::Now() - start_time_); - // clang-format on } //------------------------------------------------------------------------------ // RUsageScope //------------------------------------------------------------------------------ -#ifdef __APPLE__ +#if defined(_WIN32) class RUsageScope::PlatformInfo { public: - PlatformInfo(pid_t pid) : pid_(pid) {} + enum ProcFile : size_t { + kSched = 0, + kStatm = 1, + kStatus = 2, + kNumDoNotUseDirectly = 3 + }; + PlatformInfo(ProcessId pid) : pid_(pid) {} + ProcessId pid() const { return pid_; } + const std::string& GetProcFilePath(ProcFile file) const { + static const std::string empty; + return empty; + } + + private: + ProcessId pid_; +}; +#elif defined(__APPLE__) +class RUsageScope::PlatformInfo { + public: + PlatformInfo(ProcessId pid) : pid_(pid) {} - pid_t pid() const { return pid_; } + ProcessId pid() const { return pid_; } private: - pid_t pid_; + ProcessId pid_; }; #else class RUsageScope::PlatformInfo { @@ -93,7 +144,7 @@ class RUsageScope::PlatformInfo { kNumDoNotUseDirectly = 3 }; - PlatformInfo(pid_t pid) + PlatformInfo(ProcessId pid) : proc_file_paths_{ absl::StrFormat("/proc/%d/sched", pid), absl::StrFormat("/proc/%d/statm", pid), @@ -112,15 +163,19 @@ class RUsageScope::PlatformInfo { #endif RUsageScope RUsageScope::ThisProcess() { // +#if defined(_WIN32) + return RUsageScope{GetCurrentProcessId()}; +#else return RUsageScope{getpid()}; +#endif } -RUsageScope RUsageScope::Process(pid_t pid) { // +RUsageScope RUsageScope::Process(ProcessId pid) { // return RUsageScope{pid}; } -RUsageScope::RUsageScope(pid_t pid) - : description_{absl::StrFormat("PID=%d", pid)}, +RUsageScope::RUsageScope(ProcessId pid) + : description_{absl::StrCat("PID=", pid)}, info_(std::make_shared(pid)) {} namespace detail { @@ -362,7 +417,9 @@ RUsageTiming RUsageTiming::Snapshot( // // TODO(b/265480321): This does not honor `scope`. timer.Get(user_time, sys_time, wall_time); double cpu_utilization = 0; -#ifdef __APPLE__ +#if defined(_WIN32) + // Not supported on Windows +#elif defined(__APPLE__) int mib[4] = {CTL_KERN, KERN_PROC, KERN_PROC_PID, scope.info().pid()}; struct kinfo_proc info = {}; size_t size = sizeof(info); @@ -517,7 +574,30 @@ RUsageMemory RUsageMemory::Max() { RUsageMemory RUsageMemory::Snapshot(const RUsageScope& scope) { [[maybe_unused]] MemSize vsize = 0, rss = 0, shared = 0, code = 0, unused = 0, data = 0, vpeak = 0; -#ifdef __APPLE__ +#if defined(_WIN32) + HANDLE h_process = NULL; + if (scope.info().pid() == static_cast(GetCurrentProcessId())) { + h_process = GetCurrentProcess(); + } else { + h_process = OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, FALSE, + scope.info().pid()); + } + if (h_process != NULL) { + PROCESS_MEMORY_COUNTERS_EX pmc; + if (GetProcessMemoryInfo(h_process, + reinterpret_cast(&pmc), + sizeof(pmc))) { + // Getting proper v{size,peak} is too hard. Use rss for now. + vsize = static_cast(pmc.WorkingSetSize); + vpeak = static_cast(pmc.PeakWorkingSetSize); + rss = static_cast(pmc.WorkingSetSize); + data = static_cast(pmc.PrivateUsage); + } + if (h_process != GetCurrentProcess()) { + CloseHandle(h_process); + } + } +#elif defined(__APPLE__) if (scope.info().pid() != getpid()) return {}; struct proc_taskinfo pti = {}; FUZZTEST_CHECK(proc_pidinfo(scope.info().pid(), PROC_PIDTASKINFO, 0, &pti, diff --git a/centipede/rusage_stats.h b/centipede/rusage_stats.h index d0d1cde58..90a70bfcc 100644 --- a/centipede/rusage_stats.h +++ b/centipede/rusage_stats.h @@ -21,8 +21,6 @@ #ifndef THIRD_PARTY_CENTIPEDE_RUSAGE_STATS_H_ #define THIRD_PARTY_CENTIPEDE_RUSAGE_STATS_H_ -#include - #include #include #include @@ -49,6 +47,13 @@ using CpuHyperCores = double; // scheduler's accounting logic. using CpuUtilization = long double; +// Platform dependent process ID type without including platform headers. +#if defined(_WIN32) +using ProcessId = unsigned int; +#else +using ProcessId = int; +#endif + //------------------------------------------------------------------------------ // RUsageScope // @@ -59,7 +64,7 @@ class RUsageScope { // Static ctors for supported use cases. If the same scope is used repeatedly, // callers should prefer caching it, as construction may involve syscalls. static RUsageScope ThisProcess(); - static RUsageScope Process(pid_t pid); + static RUsageScope Process(ProcessId pid); // Copyable and movable. RUsageScope(const RUsageScope&) = default; @@ -83,7 +88,7 @@ class RUsageScope { } private: - explicit RUsageScope(pid_t pid); + explicit RUsageScope(ProcessId pid); std::string description_; // Use shared_ptr to make the class copyable (without actually copying the @@ -108,7 +113,8 @@ class ProcessTimer { private: absl::Time start_time_; - struct rusage start_rusage_; + double usage_user_; + double usage_sys_; }; //------------------------------------------------------------------------------ diff --git a/centipede/rusage_stats_test.cc b/centipede/rusage_stats_test.cc index 7f5e0580f..d34c34b63 100644 --- a/centipede/rusage_stats_test.cc +++ b/centipede/rusage_stats_test.cc @@ -308,7 +308,7 @@ TEST(RUsageMemoryTest, Accuracy) { } TEST(RUsageMemoryTest, BadScope) { - constexpr pid_t kBadPid = 999999999; + constexpr ProcessId kBadPid = 999999999; EXPECT_NO_FATAL_FAILURE( // const auto timing = RUsageTiming::Snapshot(RUsageScope::Process(kBadPid)); FUZZTEST_VLOG(1) << "Timing: " << timing;); @@ -507,11 +507,11 @@ TEST(RUsageTimingTest, Logging) { } TEST(RUsageMemoryTest, Logging) { - RUsageMemory memory{/*mem_vsize=*/1L * 1024 * 1024 * 1024, - /*mem_vpeak=*/2L * 1024 * 1024 * 1024, - /*mem_rss=*/500L * 1024 * 1024, - /*mem_data=*/750L * 1024 * 1024, - /*mem_shared=*/250L * 1024, + RUsageMemory memory{/*mem_vsize=*/1LL * 1024 * 1024 * 1024, + /*mem_vpeak=*/2LL * 1024 * 1024 * 1024, + /*mem_rss=*/500LL * 1024 * 1024, + /*mem_data=*/750LL * 1024 * 1024, + /*mem_shared=*/250LL * 1024, /*is_delta=*/false}; std::stringstream ss; diff --git a/centipede/seed_corpus_maker.cc b/centipede/seed_corpus_maker.cc index 01830d87c..3f2b6b205 100644 --- a/centipede/seed_corpus_maker.cc +++ b/centipede/seed_corpus_maker.cc @@ -32,7 +32,8 @@ int main(int argc, char** absl_nonnull argv) { FUZZTEST_QCHECK(!config.empty()); const std::string override_out_dir = absl::GetFlag(FLAGS_override_out_dir); const std::string binary_path = absl::GetFlag(FLAGS_coverage_binary_path); - const std::string binary_name = std::filesystem::path{binary_path}.filename(); + const std::string binary_name = + std::filesystem::path{binary_path}.filename().string(); FUZZTEST_QCHECK(!binary_name.empty()) << "--coverage_binary_path yields empty basename"; std::string binary_hash = absl::GetFlag(FLAGS_coverage_binary_hash); diff --git a/centipede/seed_corpus_maker_lib.cc b/centipede/seed_corpus_maker_lib.cc index d9171c76d..f9e14ba82 100644 --- a/centipede/seed_corpus_maker_lib.cc +++ b/centipede/seed_corpus_maker_lib.cc @@ -105,29 +105,78 @@ absl::Status SampleSeedCorpusElementsFromSource( // // `source.num_recent_dirs()` most recent ones. std::vector src_dirs; - if (const auto match_status = RemoteGlobMatch(source.dir_glob, src_dirs); - !match_status.ok() && !absl::IsNotFound(match_status)) { - return match_status; + if (!source.dir_glob.empty()) { + if (const auto match_status = RemoteGlobMatch(source.dir_glob, src_dirs); + !match_status.ok() && !absl::IsNotFound(match_status)) { + return match_status; + } + FUZZTEST_LOG(INFO) << "Found " << src_dirs.size() + << " corpus dir(s) candidates matching " + << source.dir_glob; } - FUZZTEST_LOG(INFO) << "Found " << src_dirs.size() - << " corpus dir(s) matching " << source.dir_glob; + src_dirs.insert(src_dirs.end(), source.src_dirs.begin(), + source.src_dirs.end()); // Sort in the ascending lexicographical order. We expect that dir names // contain timestamps and therefore will be sorted from oldest to newest. std::sort(src_dirs.begin(), src_dirs.end(), std::less()); + src_dirs.erase(std::unique(src_dirs.begin(), src_dirs.end()), src_dirs.end()); + src_dirs.erase(std::remove_if(src_dirs.begin(), src_dirs.end(), + [](const auto& path) { + return !RemotePathIsDirectory(path); + }), + src_dirs.end()); if (source.num_recent_dirs < src_dirs.size()) { src_dirs.erase(src_dirs.begin(), src_dirs.end() - source.num_recent_dirs); FUZZTEST_LOG(INFO) << "Selected " << src_dirs.size() << " corpus dir(s)"; } + FUZZTEST_LOG(INFO) + << "Reading/sampling seed corpus elements from source dir(s): "; + for (const auto& src_dir : src_dirs) { + FUZZTEST_LOG(INFO) << " " << src_dir; + } + // Find all the corpus shard and individual input files in the found dirs. std::vector corpus_shard_fnames; std::vector individual_input_fnames; for (const auto& dir : src_dirs) { absl::flat_hash_set current_corpus_shard_fnames; + if (source.shard_rel_prefix.has_value()) { + if (!source.shard_rel_glob.empty()) { + return absl::InvalidArgumentError( + "Must not specify both shard_rel_prefix and shard_rel_glob"); + } + const auto shard_prefix = + std::filesystem::path{fs::path{dir} / *source.shard_rel_prefix}; + const auto candidates = + RemoteListFiles(shard_prefix.parent_path().string(), + /*recursively=*/false); + if (candidates.ok()) { + const auto shard_prefix_filename = shard_prefix.filename().string(); + size_t num_added_shards = 0; + for (const auto& candidate : *candidates) { + if (!absl::StartsWith( + std::filesystem::path{candidate}.filename().string(), + shard_prefix_filename)) { + continue; + } + ++num_added_shards; + corpus_shard_fnames.push_back(candidate); + current_corpus_shard_fnames.insert(candidate); + } + FUZZTEST_LOG(INFO) << "Found " << num_added_shards + << " shard(s) matching prefix [" + << *source.shard_rel_prefix << "]"; + } else { + FUZZTEST_LOG(ERROR) + << "Got error when listing with " << VV(*source.shard_rel_prefix) + << ": " << candidates.status(); + } + } if (!source.shard_rel_glob.empty()) { std::vector matched_fnames; - const std::string glob = fs::path{dir} / source.shard_rel_glob; + const std::string glob = (fs::path{dir} / source.shard_rel_glob).string(); const auto match_status = RemoteGlobMatch(glob, matched_fnames); if (!match_status.ok() && !absl::IsNotFound(match_status)) { FUZZTEST_LOG(ERROR) << "Got error when glob-matching in " << dir << ": " @@ -142,9 +191,47 @@ absl::Status SampleSeedCorpusElementsFromSource( // << " shard(s) matching " << glob; } } + if (source.individual_input_rel_prefix.has_value()) { + if (!source.individual_input_rel_glob.empty()) { + return absl::InvalidArgumentError( + "Must not specify both individual_input_rel_prefix and " + "individual_input_rel_glob"); + } + const auto individual_input_prefix = std::filesystem::path{ + fs::path{dir} / *source.individual_input_rel_prefix}; + const auto candidates = + RemoteListFiles(individual_input_prefix.parent_path().string(), + /*recursively=*/false); + if (candidates.ok()) { + const auto individual_input_prefix_filename = + individual_input_prefix.filename().string(); + size_t num_added_individual_inputs = 0; + for (const auto& candidate : *candidates) { + if (RemotePathIsDirectory(candidate)) { + continue; + } + if (!absl::StartsWith( + std::filesystem::path{candidate}.filename().string(), + individual_input_prefix_filename)) { + continue; + } + if (current_corpus_shard_fnames.contains(candidate)) continue; + ++num_added_individual_inputs; + individual_input_fnames.push_back(candidate); + } + FUZZTEST_LOG(INFO) << "Found " << num_added_individual_inputs + << " individual input(s) with prefix: [" + << *source.individual_input_rel_prefix << "]"; + } else { + FUZZTEST_LOG(ERROR) << "Got error when listing with " + << VV(*source.individual_input_rel_prefix) << ": " + << candidates.status(); + } + } if (!source.individual_input_rel_glob.empty()) { std::vector matched_fnames; - const std::string glob = fs::path{dir} / source.individual_input_rel_glob; + const std::string glob = + (fs::path{dir} / source.individual_input_rel_glob).string(); const auto match_status = RemoteGlobMatch(glob, matched_fnames); if (!match_status.ok() && !absl::IsNotFound(match_status)) { FUZZTEST_LOG(ERROR) << "Got error when glob-matching in " << dir << ": " @@ -378,10 +465,19 @@ absl::Status WriteSeedCorpusElementsToDestination( // return absl::InvalidArgumentError( "Requested number of destination shards must be > 0"); } - if (!absl::StrContains(destination.shard_rel_glob, "*")) { + if (!destination.shard_rel_glob.empty()) { + if (!absl::StrContains(destination.shard_rel_glob, "*")) { + return absl::InvalidArgumentError( + absl::StrCat("Destination shard pattern must contain '*', got ", + destination.shard_rel_glob)); + } + if (destination.shard_rel_prefix.has_value()) { + return absl::InvalidArgumentError( + "Cannot have both shard_rel_glob and shard_rel_prefix"); + } + } else if (!destination.shard_rel_prefix.has_value()) { return absl::InvalidArgumentError( - absl::StrCat("Destination shard pattern must contain '*', got ", - destination.shard_rel_glob)); + "Missing shard_rel_glob and shard_rel_prefix"); } // Compute shard sizes. If the elements can't be evenly divided between the @@ -424,9 +520,12 @@ absl::Status WriteSeedCorpusElementsToDestination( // const std::string shard_idx = absl::StrFormat("%0*d", destination.shard_index_digits, shard); const std::string corpus_rel_fname = - absl::StrReplaceAll(destination.shard_rel_glob, {{"*", shard_idx}}); + !destination.shard_rel_glob.empty() + ? absl::StrReplaceAll(destination.shard_rel_glob, + {{"*", shard_idx}}) + : absl::StrCat(*destination.shard_rel_prefix, shard_idx); const std::string corpus_fname = - fs::path{destination.dir_path} / corpus_rel_fname; + (fs::path{destination.dir_path} / corpus_rel_fname).string(); const auto work_dir = WorkDir::FromCorpusShardPath( // corpus_fname, coverage_binary_name, coverage_binary_hash); diff --git a/centipede/seed_corpus_maker_lib.h b/centipede/seed_corpus_maker_lib.h index a20e121fa..e4d15c44b 100644 --- a/centipede/seed_corpus_maker_lib.h +++ b/centipede/seed_corpus_maker_lib.h @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -37,12 +38,18 @@ namespace fuzztest::internal { // difference is commented below. struct SeedCorpusSource { std::string dir_glob; + // Source directory in addition to scanning `dir_glob`. + std::vector src_dirs; uint32_t num_recent_dirs; std::string shard_rel_glob; + // A path prefix relative to the source dirs used to find corpus shards. + std::optional shard_rel_prefix; // If non-empty, will be used to glob the individual input files (with one // input in each file) in the source dirs. Any files matching `shard_rel_glob` // will be skipped. std::string individual_input_rel_glob; + // A path prefix relative to the source dirs to find individual inputs. + std::optional individual_input_rel_prefix; std::variant sampled_fraction_or_count; std::string features_start_point; @@ -56,6 +63,8 @@ struct SeedCorpusSource { struct SeedCorpusDestination { std::string dir_path; std::string shard_rel_glob; + // A path prefix relative to `dir_path` to generate output shards. + std::optional shard_rel_prefix; uint32_t shard_index_digits; uint32_t num_shards; }; diff --git a/centipede/seed_corpus_maker_lib_test.cc b/centipede/seed_corpus_maker_lib_test.cc index f7a15014b..a98ac4983 100644 --- a/centipede/seed_corpus_maker_lib_test.cc +++ b/centipede/seed_corpus_maker_lib_test.cc @@ -14,11 +14,10 @@ #include "./centipede/seed_corpus_maker_lib.h" -#include - #include #include #include // NOLINT +#include #include #include #include @@ -40,6 +39,7 @@ namespace fs = std::filesystem; using ::testing::IsSubsetOf; using ::testing::IsSupersetOf; +using ::testing::UnitTest; inline constexpr auto kIdxDigits = WorkDir::kDigitsInShardIndex; @@ -93,9 +93,17 @@ void VerifyDumpedConfig( // << VV(workdir); } -TEST(SeedCorpusMakerLibTest, RoundTripWriteReadWrite) { - const fs::path test_dir = GetTestTempDir(test_info_->name()); - chdir(test_dir.c_str()); +struct SeedCorpusMakerLibTestParam { + bool use_globs; +}; + +class SeedCorpusMakerLibTest + : public testing::TestWithParam {}; + +TEST_P(SeedCorpusMakerLibTest, RoundTripWriteReadWrite) { + const fs::path test_dir = + GetTestTempDir(UnitTest::GetInstance()->current_test_info()->name()); + fs::current_path(test_dir); const InputAndFeaturesVec kElements = { {{0}, {}}, @@ -117,13 +125,18 @@ TEST(SeedCorpusMakerLibTest, RoundTripWriteReadWrite) { constexpr size_t kNumShards = 2; const SeedCorpusDestination destination = { /*dir_path=*/std::string(kRelDir1), - /*shard_rel_glob=*/absl::StrCat("distilled-", kCovBin, ".*"), + /*shard_rel_glob=*/GetParam().use_globs + ? absl::StrCat("distilled-", kCovBin, ".*") + : "", + /*shard_rel_prefix=*/GetParam().use_globs + ? std::nullopt + : std::make_optional(absl::StrCat("distilled-", kCovBin, ".")), /*shard_index_digits=*/kIdxDigits, /*num_shards=*/kNumShards, }; ASSERT_OK(WriteSeedCorpusElementsToDestination( // kElements, kCovBin, kCovHash, destination)); - const std::string workdir = (test_dir / kRelDir1).c_str(); + const std::string workdir = (test_dir / kRelDir1).string(); ASSERT_NO_FATAL_FAILURE(VerifyShardsExist( // workdir, kCovBin, kCovHash, kNumShards, ShardType::kDistilled)); } @@ -133,9 +146,14 @@ TEST(SeedCorpusMakerLibTest, RoundTripWriteReadWrite) { { for (const float fraction : {1.0, 0.5, 0.2}) { SeedCorpusSource source; - source.dir_glob = std::string(kRelDir1); source.num_recent_dirs = 2; - source.shard_rel_glob = absl::StrCat("distilled-", kCovBin, ".*"); + if (GetParam().use_globs) { + source.dir_glob = std::string(kRelDir1); + source.shard_rel_glob = absl::StrCat("distilled-", kCovBin, ".*"); + } else { + source.src_dirs = {std::string(kRelDir1)}; + source.shard_rel_prefix = absl::StrCat("distilled-", kCovBin, "."); + } source.sampled_fraction_or_count = fraction; InputAndFeaturesVec elements; @@ -154,16 +172,24 @@ TEST(SeedCorpusMakerLibTest, RoundTripWriteReadWrite) { constexpr size_t kNumShards = 3; SeedCorpusSource source; - source.dir_glob = std::string(kRelDir1); source.num_recent_dirs = 1; - source.shard_rel_glob = absl::StrCat("distilled-", kCovBin, ".*"); + if (GetParam().use_globs) { + source.dir_glob = std::string(kRelDir1); + source.shard_rel_glob = absl::StrCat("distilled-", kCovBin, ".*"); + } else { + source.src_dirs = {std::string(kRelDir1)}; + source.shard_rel_prefix = absl::StrCat("distilled-", kCovBin, "."); + } source.sampled_fraction_or_count = 1.0f; const SeedCorpusConfig config = { /*sources=*/{{source}}, /*destination=*/ { /*dir_path=*/std::string(kRelDir2), - /*shard_rel_glob=*/"corpus.*", + /*shard_rel_glob=*/GetParam().use_globs ? "corpus.*" : "", + /*shard_rel_prefix=*/GetParam().use_globs + ? std::nullopt + : std::optional("corpus."), /*shard_index_digits=*/kIdxDigits, /*num_shards=*/kNumShards, }, @@ -172,16 +198,17 @@ TEST(SeedCorpusMakerLibTest, RoundTripWriteReadWrite) { { ASSERT_OK(GenerateSeedCorpusFromConfig( // config, kCovBin, kCovHash)); - const std::string workdir = (test_dir / kRelDir2).c_str(); + const std::string workdir = (test_dir / kRelDir2).string(); ASSERT_NO_FATAL_FAILURE(VerifyShardsExist( // workdir, kCovBin, kCovHash, kNumShards, ShardType::kNormal)); } } } -TEST(SeedCorpusMakerLibTest, LoadsBothIndividualInputsAndShardsFromSource) { - const fs::path test_dir = GetTestTempDir(test_info_->name()); - chdir(test_dir.c_str()); +TEST_P(SeedCorpusMakerLibTest, LoadsBothIndividualInputsAndShardsFromSource) { + const fs::path test_dir = + GetTestTempDir(UnitTest::GetInstance()->current_test_info()->name()); + fs::current_path(test_dir); const InputAndFeaturesVec kShardedInputs = { {{0}, {}}, @@ -202,13 +229,18 @@ TEST(SeedCorpusMakerLibTest, LoadsBothIndividualInputsAndShardsFromSource) { constexpr size_t kNumShards = 2; const SeedCorpusDestination destination = { /*dir_path=*/std::string(kRelDir), - /*shard_rel_glob=*/absl::StrCat("distilled-", kCovBin, ".*"), + /*shard_rel_glob=*/GetParam().use_globs + ? absl::StrCat("distilled-", kCovBin, ".*") + : "", + /*shard_rel_prefix=*/GetParam().use_globs + ? std::nullopt + : std::make_optional(absl::StrCat("distilled-", kCovBin, ".")), /*shard_index_digits=*/kIdxDigits, /*num_shards=*/kNumShards, }; FUZZTEST_CHECK_OK(WriteSeedCorpusElementsToDestination( // kShardedInputs, kCovBin, kCovHash, destination)); - const std::string workdir = (test_dir / kRelDir).c_str(); + const std::string workdir = (test_dir / kRelDir).string(); ASSERT_NO_FATAL_FAILURE(VerifyShardsExist( // workdir, kCovBin, kCovHash, kNumShards, ShardType::kDistilled)); } @@ -226,12 +258,23 @@ TEST(SeedCorpusMakerLibTest, LoadsBothIndividualInputsAndShardsFromSource) { InputAndFeaturesVec elements; ASSERT_OK(SampleSeedCorpusElementsFromSource( // SeedCorpusSource{ - /*dir_glob=*/std::string(kRelDir), + /*dir_glob=*/GetParam().use_globs ? std::string(kRelDir) : "", + /*src_dirs=*/GetParam().use_globs + ? std::vector{} + : std::vector{std::string{kRelDir}}, /*num_recent_dirs=*/1, - /*shard_rel_glob=*/absl::StrCat("distilled-", kCovBin, ".*"), + /*shard_rel_glob=*/GetParam().use_globs + ? absl::StrCat("distilled-", kCovBin, ".*") + : "", + /*shard_rel_prefix=*/GetParam().use_globs + ? std::nullopt + : std::make_optional(absl::StrCat("distilled-", kCovBin, ".")), // Intentionally try to match the shard files and test if they will // be read as individual inputs. - /*individual_input_rel_glob=*/"*", + /*individual_input_rel_glob=*/GetParam().use_globs ? "*" : "", + /*individual_input_rel_prefix=*/GetParam().use_globs + ? std::nullopt + : std::optional{""}, /*sampled_fraction_or_count=*/1.0f, }, kCovBin, kCovHash, elements)); @@ -244,9 +287,10 @@ TEST(SeedCorpusMakerLibTest, LoadsBothIndividualInputsAndShardsFromSource) { } } -TEST(SeedCorpusMakerLibTest, FeaturesStartPointCanDropFeatures) { - const fs::path test_dir = GetTestTempDir(test_info_->name()); - chdir(test_dir.c_str()); +TEST_P(SeedCorpusMakerLibTest, FeaturesStartPointCanDropFeatures) { + const fs::path test_dir = + GetTestTempDir(UnitTest::GetInstance()->current_test_info()->name()); + fs::current_path(test_dir); const InputAndFeaturesVec kElementsSrc1 = { {{0}, {}}, @@ -269,7 +313,12 @@ TEST(SeedCorpusMakerLibTest, FeaturesStartPointCanDropFeatures) { constexpr size_t kNumShards = 2; const SeedCorpusDestination destination_src1 = { /*dir_path=*/std::string(kSrcDir1), - /*shard_rel_glob=*/absl::StrCat("distilled-", kCovBin, ".*"), + /*shard_rel_glob=*/GetParam().use_globs + ? absl::StrCat("distilled-", kCovBin, ".*") + : "", + /*shard_rel_prefix=*/GetParam().use_globs + ? std::nullopt + : std::make_optional(absl::StrCat("distilled-", kCovBin, ".")), /*shard_index_digits=*/kIdxDigits, /*num_shards=*/kNumShards, }; @@ -278,7 +327,12 @@ TEST(SeedCorpusMakerLibTest, FeaturesStartPointCanDropFeatures) { const SeedCorpusDestination destination_src2 = { /*dir_path=*/std::string(kSrcDir2), - /*shard_rel_glob=*/absl::StrCat("distilled-", kCovBin, ".*"), + /*shard_rel_glob=*/GetParam().use_globs + ? absl::StrCat("distilled-", kCovBin, ".*") + : "", + /*shard_rel_prefix=*/GetParam().use_globs + ? std::nullopt + : std::make_optional(absl::StrCat("distilled-", kCovBin, ".")), /*shard_index_digits=*/kIdxDigits, /*num_shards=*/kNumShards, }; @@ -287,9 +341,17 @@ TEST(SeedCorpusMakerLibTest, FeaturesStartPointCanDropFeatures) { } SeedCorpusSource source; - source.dir_glob = std::string("dir/src/*"); + if (GetParam().use_globs) { + source.dir_glob = std::string("dir/src/*"); + } else { + source.src_dirs = {std::string{kSrcDir1}, std::string{kSrcDir2}}; + } source.num_recent_dirs = 2; - source.shard_rel_glob = absl::StrCat("distilled-", kCovBin, ".*"); + if (GetParam().use_globs) { + source.shard_rel_glob = absl::StrCat("distilled-", kCovBin, ".*"); + } else { + source.shard_rel_prefix = absl::StrCat("distilled-", kCovBin, "."); + } source.sampled_fraction_or_count = 1.0f; InputAndFeaturesVec elements; @@ -316,5 +378,10 @@ TEST(SeedCorpusMakerLibTest, FeaturesStartPointCanDropFeatures) { ASSERT_EQ(get_num_features(elements), get_num_features(kElementsSrc2)); } +INSTANTIATE_TEST_SUITE_P( + SeedCorpusMakerLibTestWithConfigurations, SeedCorpusMakerLibTest, + testing::Values(SeedCorpusMakerLibTestParam{/*use_globs=*/false}, + SeedCorpusMakerLibTestParam{/*use_globs=*/true})); + } // namespace } // namespace fuzztest::internal diff --git a/centipede/seed_corpus_maker_proto_lib.cc b/centipede/seed_corpus_maker_proto_lib.cc index bbf809175..50b30ac39 100644 --- a/centipede/seed_corpus_maker_proto_lib.cc +++ b/centipede/seed_corpus_maker_proto_lib.cc @@ -53,14 +53,14 @@ absl::StatusOr ResolveSeedCorpusConfigProto( // << VV(config_spec); RETURN_IF_NOT_OK(RemoteFileGetContents(config_spec, config_str)); FUZZTEST_LOG(INFO) << "Raw config read from file:\n" << config_str; - base_dir = std::filesystem::path{config_spec}.parent_path(); + base_dir = std::filesystem::path{config_spec}.parent_path().string(); } else { FUZZTEST_LOG(INFO) << "Config spec is not a file, or file doesn't exist; trying to " "parse textproto config verbatim: " << VV(config_spec); config_str = config_spec; - base_dir = fs::current_path(); + base_dir = fs::current_path().string(); } proto::SeedCorpusConfig config; @@ -80,7 +80,7 @@ absl::StatusOr ResolveSeedCorpusConfigProto( // for (auto& src : *config.mutable_sources()) { auto* dir = src.mutable_dir_glob(); if (dir->empty() || !fs::path{*dir}.is_absolute()) { - *dir = fs::path{base_dir} / *dir; + *dir = (fs::path{base_dir} / *dir).string(); } } @@ -91,7 +91,7 @@ absl::StatusOr ResolveSeedCorpusConfigProto( // if (!override_out_dir.empty()) { *dir = override_out_dir; } else if (dir->empty() || !fs::path{*dir}.is_absolute()) { - *dir = fs::path{base_dir} / *dir; + *dir = (fs::path{base_dir} / *dir).string(); } } @@ -148,9 +148,9 @@ absl::Status DumpConfigProtoToDebugDir(const proto::SeedCorpusConfig& config, /*my_shard_index=*/0, }; const std::filesystem::path debug_info_dir = workdir.DebugInfoDirPath(); - RETURN_IF_NOT_OK(RemoteMkdir(debug_info_dir.c_str())); + RETURN_IF_NOT_OK(RemoteMkdir(debug_info_dir.string())); RETURN_IF_NOT_OK(RemoteFileSetContents( - (debug_info_dir / "seeding.cfg").c_str(), absl::StrCat(config))); + (debug_info_dir / "seeding.cfg").string(), absl::StrCat(config))); return absl::OkStatus(); } diff --git a/centipede/seed_corpus_maker_proto_lib_test.cc b/centipede/seed_corpus_maker_proto_lib_test.cc index 29ab4f683..1e884657a 100644 --- a/centipede/seed_corpus_maker_proto_lib_test.cc +++ b/centipede/seed_corpus_maker_proto_lib_test.cc @@ -16,6 +16,7 @@ #include #include // NOLINT +#include #include #include @@ -111,6 +112,7 @@ TEST(SeedCorpusMakerProtoLibTest, MakesSeedUsingConfigProto) { const SeedCorpusDestination destination = { /*dir_path=*/std::string(kRelDir1), /*shard_rel_glob=*/absl::StrCat("distilled-", kCovBin, ".*"), + /*shard_rel_prefix=*/std::nullopt, /*shard_index_digits=*/kIdxDigits, /*num_shards=*/2, }; @@ -142,7 +144,7 @@ TEST(SeedCorpusMakerProtoLibTest, MakesSeedUsingConfigProto) { ASSERT_OK(GenerateSeedCorpusFromConfigProto( // config_str, kCovBin, kCovHash, "")); - const std::string workdir = (test_dir / kRelDir2).c_str(); + const std::string workdir = (test_dir / kRelDir2).string(); ASSERT_NO_FATAL_FAILURE(VerifyDumpedConfig(workdir, kCovBin, kCovHash)); ASSERT_NO_FATAL_FAILURE(VerifyShardsExist( // workdir, kCovBin, kCovHash, kNumShards, ShardType::kNormal)); diff --git a/centipede/shared_memory_blob_sequence.cc b/centipede/shared_memory_blob_sequence.cc index 51415df13..b3e6da1f7 100644 --- a/centipede/shared_memory_blob_sequence.cc +++ b/centipede/shared_memory_blob_sequence.cc @@ -14,27 +14,39 @@ #include "./centipede/shared_memory_blob_sequence.h" +#if defined(_WIN32) +#include "./common/windows_includes.h" +#else #include +#include #include #include #include #include #include +#endif #include #include +#include +#include +#include #include "absl/base/nullability.h" namespace fuzztest::internal { +namespace { + // TODO(ussuri): Refactor `char *` into a `string_view`. -static void ErrorOnFailure(bool condition, const char *absl_nonnull text) { +void ErrorOnFailure(bool condition, const char* absl_nonnull text) { if (!condition) return; std::perror(text); abort(); } +} // namespace + BlobSequence::BlobSequence(uint8_t *data, size_t size) : data_(data), size_(size) { ErrorOnFailure(size < sizeof(Blob::size), "Size too small"); @@ -50,6 +62,12 @@ bool BlobSequence::Write(Blob blob) { return false; } + size_t write_end = offset_ + sizeof(blob.tag) + sizeof(blob.size) + blob.size; + if (write_end + sizeof(blob.size) + sizeof(blob.tag) <= size_) { + write_end += sizeof(blob.size) + sizeof(blob.tag); + } + if (!CommitMemory(write_end)) return false; + // Write tag. memcpy(data_ + offset_, &blob.tag, sizeof(blob.tag)); offset_ += sizeof(blob.tag); @@ -104,85 +122,204 @@ void BlobSequence::Reset() { had_writes_after_reset_ = false; } -SharedMemoryBlobSequence::SharedMemoryBlobSequence(const char *name, - size_t size, - bool use_posix_shmem) { - ErrorOnFailure(size < sizeof(Blob::size), "Size too small"); - size_ = size; - if (use_posix_shmem) { - fd_ = shm_open(name, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); - ErrorOnFailure(fd_ < 0, "shm_open() failed"); - strncpy(path_, name, PATH_MAX); - ErrorOnFailure(path_[PATH_MAX - 1] != 0, - "shm_open() path length exceeds PATH_MAX."); - path_is_owned_ = true; - } else { -#ifdef __APPLE__ - ErrorOnFailure(true, "must use POSIX shmem"); -#else // __APPLE__ - fd_ = memfd_create(name, MFD_CLOEXEC); - ErrorOnFailure(fd_ < 0, "memfd_create() failed"); - const size_t path_size = - snprintf(path_, PATH_MAX, "/proc/%d/fd/%d", getpid(), fd_); - ErrorOnFailure(path_size >= PATH_MAX, - "internal fd path length exceeds PATH_MAX."); - // memfd_create descriptors are automatically freed on close(). - path_is_owned_ = false; -#endif // __APPLE__ +#if defined(_WIN32) + +namespace { + +class WindowsSharedMemoryBlobSequence : public SharedMemoryBlobSequence { + public: + WindowsSharedMemoryBlobSequence(const char* name, size_t size, + bool /*use_posix_shmem*/) { + ErrorOnFailure(size < sizeof(Blob::size), "Size too small"); + size_ = size; + strncpy(path_, name, sizeof(path_) - 1); + path_[sizeof(path_) - 1] = '\0'; + mapping_handle_ = CreateFileMappingA( + INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE | SEC_RESERVE, + static_cast(size_ >> 32), static_cast(size_ & 0xFFFFFFFF), + path_); + ErrorOnFailure(mapping_handle_ == NULL, "CreateFileMappingA() failed"); + data_ = static_cast(MapViewOfFile( + mapping_handle_, FILE_MAP_READ | FILE_MAP_WRITE, 0, 0, size_)); + ErrorOnFailure(data_ == NULL, "MapViewOfFile() failed"); } - ErrorOnFailure(ftruncate(fd_, static_cast(size_)), - "ftruncate() failed)"); - MmapData(); -} -SharedMemoryBlobSequence::SharedMemoryBlobSequence(const char* path, - size_t size) { - ErrorOnFailure(size < sizeof(Blob::size), "Size too small"); - size_ = size; - // This is a quick way to tell shm-allocated paths from memfd paths without - // requiring the caller to specify. - if (strncmp(path, "/proc/", 6) == 0) { - fd_ = open(path, O_RDWR, O_CLOEXEC); - } else { - fd_ = shm_open(path, O_RDWR, 0); + WindowsSharedMemoryBlobSequence(const char* path, size_t size) { + ErrorOnFailure(size < sizeof(Blob::size), "Size too small"); + size_ = size; + strncpy(path_, path, sizeof(path_) - 1); + path_[sizeof(path_) - 1] = '\0'; + mapping_handle_ = + OpenFileMappingA(FILE_MAP_READ | FILE_MAP_WRITE, FALSE, path_); + ErrorOnFailure(mapping_handle_ == NULL, "OpenFileMappingA() failed"); + data_ = static_cast(MapViewOfFile( + mapping_handle_, FILE_MAP_READ | FILE_MAP_WRITE, 0, 0, 0)); + ErrorOnFailure(data_ == NULL, "MapViewOfFile() failed"); + } + + ~WindowsSharedMemoryBlobSequence() override { + if (data_ != nullptr) { + UnmapViewOfFile(data_); + } + if (mapping_handle_ != nullptr) { + CloseHandle(mapping_handle_); + } } - ErrorOnFailure(fd_ < 0, "open() failed"); - strncpy(path_, path, PATH_MAX); - ErrorOnFailure(path_[PATH_MAX - 1] != 0, "path length exceeds PATH_MAX."); - MmapData(); + + void ReleaseSharedMemory() override {} + + size_t NumBytesUsed() const override { return committed_; } + + const char* absl_nonnull path() const override { return path_; } + + protected: + bool CommitMemory(size_t write_end) override { + if (write_end <= committed_) return true; + + constexpr size_t kCommitGranularity = 4 * 1024; + const size_t target_committed = + std::min((write_end + kCommitGranularity - 1) / kCommitGranularity * + kCommitGranularity, + size_); + void* res = VirtualAlloc(data_ + committed_, target_committed - committed_, + MEM_COMMIT, PAGE_READWRITE); + ErrorOnFailure(res == nullptr, "VirtualAlloc() failed"); + committed_ = target_committed; + return true; + } + + private: + char path_[MAX_PATH] = {0}; + HANDLE mapping_handle_ = nullptr; + size_t committed_ = 0; +}; + +} // namespace + +std::unique_ptr absl_nonnull +CreateSharedMemoryBlobSequence(const char* absl_nonnull name, size_t size, + bool use_posix_shmem) { + return std::make_unique(name, size, + use_posix_shmem); } -void SharedMemoryBlobSequence::MmapData() { - data_ = static_cast( - mmap(nullptr, size_, PROT_READ | PROT_WRITE, MAP_SHARED, fd_, 0)); - ErrorOnFailure(data_ == MAP_FAILED, "mmap() failed"); +std::unique_ptr absl_nonnull +OpenSharedMemoryBlobSequence(const char* absl_nonnull path, size_t size) { + return std::make_unique(path, size); } -SharedMemoryBlobSequence::~SharedMemoryBlobSequence() { - if (path_is_owned_) { - ErrorOnFailure(shm_unlink(path_), "shm_unlink() failed"); +#else // !_WIN32 + +namespace { + +class PosixSharedMemoryBlobSequence : public SharedMemoryBlobSequence { + public: + PosixSharedMemoryBlobSequence(const char* name, size_t size, + bool use_posix_shmem) { + ErrorOnFailure(size < sizeof(Blob::size), "Size too small"); + size_ = size; + if (use_posix_shmem) { + fd_ = shm_open(name, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); + ErrorOnFailure(fd_ < 0, "shm_open() failed"); + strncpy(path_, name, sizeof(path_)); + ErrorOnFailure(path_[sizeof(path_) - 1] != 0, + "shm_open() path length exceeds PATH_MAX."); + path_is_owned_ = true; + } else { +#ifdef __APPLE__ + ErrorOnFailure(true, "must use POSIX shmem"); +#else // __APPLE__ + fd_ = memfd_create(name, MFD_CLOEXEC); + ErrorOnFailure(fd_ < 0, "memfd_create() failed"); + const size_t path_size = + snprintf(path_, sizeof(path_), "/proc/%d/fd/%d", getpid(), fd_); + ErrorOnFailure(path_size >= sizeof(path_), + "internal fd path length exceeds PATH_MAX."); + // memfd_create descriptors are automatically freed on close(). + path_is_owned_ = false; +#endif // __APPLE__ + } + ErrorOnFailure(ftruncate(fd_, static_cast(size_)), + "ftruncate() failed)"); + MmapData(); + } + + PosixSharedMemoryBlobSequence(const char* path, size_t size) { + ErrorOnFailure(size < sizeof(Blob::size), "Size too small"); + size_ = size; + // This is a quick way to tell shm-allocated paths from memfd paths without + // requiring the caller to specify. + if (strncmp(path, "/proc/", 6) == 0) { + fd_ = open(path, O_RDWR, O_CLOEXEC); + } else { + fd_ = shm_open(path, O_RDWR, 0); + } + ErrorOnFailure(fd_ < 0, "open() failed"); + strncpy(path_, path, sizeof(path_)); + ErrorOnFailure(path_[sizeof(path_) - 1] != 0, + "path length exceeds PATH_MAX."); + MmapData(); + } + + ~PosixSharedMemoryBlobSequence() override { + if (data_ != nullptr) { + ErrorOnFailure(munmap(data_, size_), "munmap() failed"); + data_ = nullptr; + size_ = 0; + } + if (path_is_owned_) { + ErrorOnFailure(shm_unlink(path_), "shm_unlink() failed"); + } + ErrorOnFailure(close(fd_), "close() failed"); } - ErrorOnFailure(munmap(data_, size_), "munmap() failed"); - ErrorOnFailure(close(fd_), "close() failed"); -} -void SharedMemoryBlobSequence::ReleaseSharedMemory() { + void ReleaseSharedMemory() override { #ifdef __APPLE__ - // MacOS only allows ftruncate shm once - // (https://stackoverflow.com/questions/25502229/ftruncate-not-working-on-posix-shared-memory-in-mac-os-x). - // So nothing we can do here. + // MacOS only allows ftruncate shm once + // (https://stackoverflow.com/questions/25502229/ftruncate-not-working-on-posix-shared-memory-in-mac-os-x). + // So nothing we can do here. #else // __APPLE__ - // Setting size to 0 releases the memory to OS. - ErrorOnFailure(ftruncate(fd_, 0) != 0, "ftruncate(0) failed)"); - // Set the size back to `size`. The memory is not actually reserved. - ErrorOnFailure(ftruncate(fd_, size_) != 0, "ftruncate(size_) failed)"); + // Setting size to 0 releases the memory to OS. + ErrorOnFailure(ftruncate(fd_, 0) != 0, "ftruncate(0) failed)"); + // Set the size back to `size`. The memory is not actually reserved. + ErrorOnFailure(ftruncate(fd_, size_) != 0, "ftruncate(size_) failed)"); #endif // __APPLE__ + } + + size_t NumBytesUsed() const override { + struct stat statbuf; + ErrorOnFailure(fstat(fd_, &statbuf), "fstat() failed)"); + return statbuf.st_blocks * S_BLKSIZE; + } + + const char* absl_nonnull path() const override { return path_; } + + private: + void MmapData() { + data_ = static_cast( + mmap(nullptr, size_, PROT_READ | PROT_WRITE, MAP_SHARED, fd_, 0)); + ErrorOnFailure(data_ == MAP_FAILED, "mmap() failed"); + } + + char path_[PATH_MAX] = {0}; + bool path_is_owned_ = false; + int fd_ = -1; +}; + +} // namespace + +std::unique_ptr absl_nonnull +CreateSharedMemoryBlobSequence(const char* absl_nonnull name, size_t size, + bool use_posix_shmem) { + return std::make_unique(name, size, + use_posix_shmem); } -size_t SharedMemoryBlobSequence::NumBytesUsed() const { - struct stat statbuf; - ErrorOnFailure(fstat(fd_, &statbuf), "fstat() failed)"); - return statbuf.st_blocks * S_BLKSIZE; +std::unique_ptr absl_nonnull +OpenSharedMemoryBlobSequence(const char* absl_nonnull path, size_t size) { + return std::make_unique(path, size); } +#endif // _WIN32 + } // namespace fuzztest::internal diff --git a/centipede/shared_memory_blob_sequence.h b/centipede/shared_memory_blob_sequence.h index b5d781e3c..e2a32ff9c 100644 --- a/centipede/shared_memory_blob_sequence.h +++ b/centipede/shared_memory_blob_sequence.h @@ -15,9 +15,9 @@ #ifndef THIRD_PARTY_CENTIPEDE_SHARED_MEMORY_BLOB_SEQUENCE_H_ #define THIRD_PARTY_CENTIPEDE_SHARED_MEMORY_BLOB_SEQUENCE_H_ -#include #include #include +#include #include #include "absl/base/nullability.h" @@ -57,6 +57,7 @@ class BlobSequence { // must be >= 8. Aborts on any failure. The amount of actual data that can be // written is slightly less. explicit BlobSequence(uint8_t *data, size_t size); + virtual ~BlobSequence() = default; // Writes the contents of `blob` to the blob sequence. // Returns true on success. @@ -93,6 +94,9 @@ class BlobSequence { // constructors of child classes. explicit BlobSequence() = default; + // Commits the memory for `size` bytes starting from `data_`. + virtual bool CommitMemory(size_t size) { return true; } + // data_ contains a sequence of {size, payload} pairs, // where size is 8 bytes and payload is size bytes. // After writing a blob, we also write 0 in place of the next blob's size, @@ -122,11 +126,12 @@ class BlobSequence { // Usage example: // void ParentProcess() { // // Create a new blob sequence. -// SharedMemoryBlobSequence parent("/foo", 1000); +// auto parent = CreateSharedMemoryBlobSequence( +// "/foo", 1000, /*use_posix_shmem=*/true); // // // Parent process writes some data to the shared blob: -// parent.Write({some_data, some_data_size}); -// parent.Write({some_other_data, some_other_data_size}); +// parent->Write({some_data, some_data_size}); +// parent->Write({some_other_data, some_other_data_size}); // // // Run the child process. // ExecuteChildProcessAndWaitUntilItIsDone(); @@ -134,11 +139,11 @@ class BlobSequence { // // void Child() { // // Open an existing blob sequence. -// SharedMemoryBlobSequence child("/foo", 1000); +// auto child = OpenSharedMemoryBlobSequence("/foo", 1000); // // // Read the data written by parent. // while (true) { -// auto blob = parent.Read(); +// auto blob = child->Read(); // if (!blob.size) break; // Use({blob.data, blob.size}); // } @@ -146,47 +151,41 @@ class BlobSequence { // class SharedMemoryBlobSequence : public BlobSequence { public: - // Creates a new shared blob sequence with `name` (for debugging only, not an - // actual path). Aborts on any failure. `size` is the size of the shared - // memory region in bytes, must be >= 8. The amount of actual data that can be - // written is slightly less. - // The `use_posix_shmem` argument specifies which API to use to allocate the - // shared memory. When true, shm_open(2) will be used, otherwise - // memfd_create(2). - SharedMemoryBlobSequence(const char *name, size_t size, bool use_posix_shmem); - - // Opens an existing shared blob sequence with the file `path` and `size`. - // Aborts on any failure. - SharedMemoryBlobSequence(const char* path, size_t size); - // Releases all resources. - ~SharedMemoryBlobSequence(); + ~SharedMemoryBlobSequence() override = default; // Releases shared memory used by `this`. - void ReleaseSharedMemory(); + virtual void ReleaseSharedMemory() = 0; // Returns the number of bytes used by the shared mapping. // It will be zero just after creation and after the call to // ReleaseSharedMemory(). - size_t NumBytesUsed() const; + virtual size_t NumBytesUsed() const = 0; // Gets the file path that can be used to create new instances. // TODO(ussuri): Refactor `char *` into a `string_view`. - const char *absl_nonnull path() const { return path_; } + virtual const char* absl_nonnull path() const = 0; - private: - // mmaps `size_` bytes from `fd_`, assigns to `data_`. Crashes if mmap failed. - void MmapData(); - - // Will be initialized as a generated internal path or a copy of `path` - // passed in. - char path_[PATH_MAX] = {0}; - int fd_ = -1; // file descriptor used to mmap the shared memory region. - // Whether the file pointed to by path_ is owned by this and needs to be - // deallocated on destruction. - bool path_is_owned_ = false; + protected: + SharedMemoryBlobSequence() = default; }; +// Creates a new shared blob sequence with `name` (for debugging only, not an +// actual path). Aborts on any failure. `size` is the size of the shared +// memory region in bytes, must be >= 8. The amount of actual data that can be +// written is slightly less. +// The `use_posix_shmem` argument specifies which API to use to allocate the +// shared memory. When true, shm_open(2) will be used, otherwise +// memfd_create(2). +std::unique_ptr absl_nonnull +CreateSharedMemoryBlobSequence(const char* absl_nonnull name, size_t size, + bool use_posix_shmem); + +// Opens an existing shared blob sequence with the file `path` and `size`. +// Aborts on any failure. +std::unique_ptr absl_nonnull +OpenSharedMemoryBlobSequence(const char* absl_nonnull path, size_t size); + } // namespace fuzztest::internal #endif // THIRD_PARTY_CENTIPEDE_SHARED_MEMORY_BLOB_SEQUENCE_H_ diff --git a/centipede/shared_memory_blob_sequence_test.cc b/centipede/shared_memory_blob_sequence_test.cc index f5dd4fc45..7f92c8908 100644 --- a/centipede/shared_memory_blob_sequence_test.cc +++ b/centipede/shared_memory_blob_sequence_test.cc @@ -14,7 +14,11 @@ #include "./centipede/shared_memory_blob_sequence.h" +#if !defined(_WIN32) #include +#else +#include "./common/windows_includes.h" +#endif #include #include @@ -30,7 +34,12 @@ namespace fuzztest::internal { std::string ShmemName() { std::ostringstream oss; +#if defined(_WIN32) + oss << "/shm_test-" << GetCurrentProcessId() << "-" + << std::this_thread::get_id(); +#else oss << "/shm_test-" << getpid() << "-" << std::this_thread::get_id(); +#endif return oss.str(); } @@ -85,20 +94,18 @@ TEST(BlobSequence, ReadReturnErrorOnSizeOverflow) { class SharedMemoryBlobSequenceTest : public testing::TestWithParam { - public: - void SetUp() override { -#ifdef __APPLE__ - const bool use_shm = GetParam(); - if (!use_shm) { - GTEST_SKIP() << "Skipping test that does not use POSIX shmem on MacOS"; - } -#endif // __APPLE__ - } }; INSTANTIATE_TEST_SUITE_P(SharedMemoryBlobSequenceParametrizedTest, SharedMemoryBlobSequenceTest, - testing::Values(true, false)); + testing::ValuesIn({ +#ifndef _WIN32 + true, +#endif +#ifndef __APPLE__ + false, +#endif + })); TEST_P(SharedMemoryBlobSequenceTest, ParentChild) { std::vector kTestData1 = {1, 2, 3}; @@ -106,32 +113,33 @@ TEST_P(SharedMemoryBlobSequenceTest, ParentChild) { std::vector kTestData3 = {8, 9}; std::vector kTestData4 = {'a', 'b', 'c', 'd', 'e'}; - SharedMemoryBlobSequence parent(ShmemName().c_str(), 1000, GetParam()); + auto parent = + CreateSharedMemoryBlobSequence(ShmemName().c_str(), 1000, GetParam()); // Parent writes data. - EXPECT_TRUE(parent.Write(BlobFromVec(kTestData1, 123))); - EXPECT_TRUE(parent.Write(BlobFromVec(kTestData2, 456))); + EXPECT_TRUE(parent->Write(BlobFromVec(kTestData1, 123))); + EXPECT_TRUE(parent->Write(BlobFromVec(kTestData2, 456))); // Child created. - SharedMemoryBlobSequence child(parent.path(), 1000); + auto child = OpenSharedMemoryBlobSequence(parent->path(), 1000); // Child reads data. - auto blob1 = child.Read(); + auto blob1 = child->Read(); EXPECT_EQ(kTestData1, Vec(blob1)); EXPECT_EQ(blob1.tag, 123); - auto blob2 = child.Read(); + auto blob2 = child->Read(); EXPECT_EQ(kTestData2, Vec(blob2)); EXPECT_EQ(blob2.tag, 456); - EXPECT_FALSE(child.Read().IsValid()); + EXPECT_FALSE(child->Read().IsValid()); // Child writes data. - child.Reset(); - EXPECT_TRUE(child.Write(BlobFromVec(kTestData3))); - EXPECT_TRUE(child.Write(BlobFromVec(kTestData4))); + child->Reset(); + EXPECT_TRUE(child->Write(BlobFromVec(kTestData3))); + EXPECT_TRUE(child->Write(BlobFromVec(kTestData4))); // Parent reads data. - parent.Reset(); - EXPECT_EQ(kTestData3, Vec(parent.Read())); - EXPECT_EQ(kTestData4, Vec(parent.Read())); - EXPECT_FALSE(parent.Read().IsValid()); + parent->Reset(); + EXPECT_EQ(kTestData3, Vec(parent->Read())); + EXPECT_EQ(kTestData4, Vec(parent->Read())); + EXPECT_FALSE(parent->Read().IsValid()); } TEST_P(SharedMemoryBlobSequenceTest, CheckForResourceLeaks) { @@ -139,117 +147,123 @@ TEST_P(SharedMemoryBlobSequenceTest, CheckForResourceLeaks) { const int kBlobSize = 1 << 30; // Some large blob size. // Create and destroy lots of parent/child blob pairs. for (int iter = 0; iter < kNumIters; iter++) { - SharedMemoryBlobSequence parent(ShmemName().c_str(), kBlobSize, GetParam()); - parent.Write(BlobFromVec({1, 2, 3})); - SharedMemoryBlobSequence child(parent.path(), kBlobSize); - EXPECT_EQ(child.Read().size, 3); + auto parent = CreateSharedMemoryBlobSequence(ShmemName().c_str(), kBlobSize, + GetParam()); + parent->Write(BlobFromVec({1, 2, 3})); + auto child = OpenSharedMemoryBlobSequence(parent->path(), kBlobSize); + EXPECT_EQ(child->Read().size, 3); } // Create a parent blob, then create and destroy lots of child blobs. - SharedMemoryBlobSequence parent(ShmemName().c_str(), kBlobSize, GetParam()); - parent.Write(BlobFromVec({1, 2, 3, 4})); + auto parent = CreateSharedMemoryBlobSequence(ShmemName().c_str(), kBlobSize, + GetParam()); + parent->Write(BlobFromVec({1, 2, 3, 4})); for (int iter = 0; iter < kNumIters; iter++) { - SharedMemoryBlobSequence child(parent.path(), kBlobSize); - EXPECT_EQ(child.Read().size, 4); + auto child = OpenSharedMemoryBlobSequence(parent->path(), kBlobSize); + EXPECT_EQ(child->Read().size, 4); } } // Tests that Read-after-Write or Write-after-Read w/o Reset crashes. TEST_P(SharedMemoryBlobSequenceTest, ReadVsWriteWithoutReset) { - SharedMemoryBlobSequence blobseq(ShmemName().c_str(), 1000, GetParam()); - blobseq.Write(BlobFromVec({1, 2, 3})); - EXPECT_DEATH(blobseq.Read(), "Had writes after reset"); - blobseq.Reset(); - EXPECT_EQ(blobseq.Read().size, 3); - EXPECT_DEATH(blobseq.Write(BlobFromVec({1, 2, 3, 4})), + auto blobseq = + CreateSharedMemoryBlobSequence(ShmemName().c_str(), 1000, GetParam()); + blobseq->Write(BlobFromVec({1, 2, 3})); + EXPECT_DEATH(blobseq->Read(), "Had writes after reset"); + blobseq->Reset(); + EXPECT_EQ(blobseq->Read().size, 3); + EXPECT_DEATH(blobseq->Write(BlobFromVec({1, 2, 3, 4})), "Had reads after reset"); - blobseq.Reset(); - blobseq.Write(BlobFromVec({1, 2, 3, 4})); + blobseq->Reset(); + blobseq->Write(BlobFromVec({1, 2, 3, 4})); } // Check cases when SharedMemoryBlobSequence is nearly full. TEST_P(SharedMemoryBlobSequenceTest, WriteToFullSequence) { // Can't create SharedMemoryBlobSequence with sizes < 8. EXPECT_DEATH( - SharedMemoryBlobSequence blobseq(ShmemName().c_str(), 7, GetParam()), + CreateSharedMemoryBlobSequence(ShmemName().c_str(), 7, GetParam()), "Size too small"); // Allocate a blob sequence with 28 bytes of storage. - SharedMemoryBlobSequence blobseq(ShmemName().c_str(), 28, GetParam()); + auto blobseq = + CreateSharedMemoryBlobSequence(ShmemName().c_str(), 28, GetParam()); // 17 bytes: 8 bytes size, 8 bytes tag, 1 byte payload. - EXPECT_TRUE(blobseq.Write(BlobFromVec({1}))); - blobseq.Reset(); - EXPECT_EQ(blobseq.Read().size, 1); - EXPECT_FALSE(blobseq.Read().IsValid()); + EXPECT_TRUE(blobseq->Write(BlobFromVec({1}))); + blobseq->Reset(); + EXPECT_EQ(blobseq->Read().size, 1); + EXPECT_FALSE(blobseq->Read().IsValid()); // 20 bytes: 4-byte payload. - blobseq.Reset(); - EXPECT_TRUE(blobseq.Write(BlobFromVec({1, 2, 3, 4}))); - blobseq.Reset(); - EXPECT_EQ(blobseq.Read().size, 4); - EXPECT_FALSE(blobseq.Read().IsValid()); + blobseq->Reset(); + EXPECT_TRUE(blobseq->Write(BlobFromVec({1, 2, 3, 4}))); + blobseq->Reset(); + EXPECT_EQ(blobseq->Read().size, 4); + EXPECT_FALSE(blobseq->Read().IsValid()); // 23 bytes: 7-byte payload. - blobseq.Reset(); - EXPECT_TRUE(blobseq.Write(BlobFromVec({1, 2, 3, 4, 5, 6, 7}))); - blobseq.Reset(); - EXPECT_EQ(blobseq.Read().size, 7); - EXPECT_FALSE(blobseq.Read().IsValid()); + blobseq->Reset(); + EXPECT_TRUE(blobseq->Write(BlobFromVec({1, 2, 3, 4, 5, 6, 7}))); + blobseq->Reset(); + EXPECT_EQ(blobseq->Read().size, 7); + EXPECT_FALSE(blobseq->Read().IsValid()); // 28 bytes: 12-byte payload. - blobseq.Reset(); + blobseq->Reset(); EXPECT_TRUE( - blobseq.Write(BlobFromVec({1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}))); - blobseq.Reset(); - EXPECT_EQ(blobseq.Read().size, 12); - EXPECT_FALSE(blobseq.Read().IsValid()); + blobseq->Write(BlobFromVec({1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}))); + blobseq->Reset(); + EXPECT_EQ(blobseq->Read().size, 12); + EXPECT_FALSE(blobseq->Read().IsValid()); // 13-byte payload - there is not enough space (for 13+8 bytes). - blobseq.Reset(); + blobseq->Reset(); EXPECT_FALSE( - blobseq.Write(BlobFromVec({1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}))); - blobseq.Reset(); - EXPECT_EQ(blobseq.Read().size, 12); // State remained the same. + blobseq->Write(BlobFromVec({1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}))); + blobseq->Reset(); + EXPECT_EQ(blobseq->Read().size, 12); // State remained the same. // 1-, and 2- byte payloads. The last one fails. - blobseq.Reset(); - EXPECT_TRUE(blobseq.Write(BlobFromVec({1}))); - EXPECT_FALSE(blobseq.Write(BlobFromVec({1, 2}))); - blobseq.Reset(); - EXPECT_EQ(blobseq.Read().size, 1); - EXPECT_FALSE(blobseq.Read().IsValid()); + blobseq->Reset(); + EXPECT_TRUE(blobseq->Write(BlobFromVec({1}))); + EXPECT_FALSE(blobseq->Write(BlobFromVec({1, 2}))); + blobseq->Reset(); + EXPECT_EQ(blobseq->Read().size, 1); + EXPECT_FALSE(blobseq->Read().IsValid()); } // Test Write-Reset-Write-Read scenario. TEST_P(SharedMemoryBlobSequenceTest, WriteAfterReset) { // Allocate a blob sequence with 28 bytes of storage. - SharedMemoryBlobSequence blobseq(ShmemName().c_str(), 100, GetParam()); + auto blobseq = + CreateSharedMemoryBlobSequence(ShmemName().c_str(), 100, GetParam()); const std::vector kFirstWriteData(/*count=*/64, /*value=*/255); - EXPECT_TRUE(blobseq.Write(BlobFromVec(kFirstWriteData))); - blobseq.Reset(); // The data in shmem is unchanged. + EXPECT_TRUE(blobseq->Write(BlobFromVec(kFirstWriteData))); + blobseq->Reset(); // The data in shmem is unchanged. const std::vector kSecondWriteData{42, 43}; - EXPECT_TRUE(blobseq.Write(BlobFromVec(kSecondWriteData))); - blobseq.Reset(); // The data in shmem is unchanged. - auto blob1 = blobseq.Read(); + EXPECT_TRUE(blobseq->Write(BlobFromVec(kSecondWriteData))); + blobseq->Reset(); // The data in shmem is unchanged. + auto blob1 = blobseq->Read(); EXPECT_TRUE(blob1.IsValid()); EXPECT_EQ(Vec(blob1), kSecondWriteData); - auto blob2 = blobseq.Read(); // must be invalid. + auto blob2 = blobseq->Read(); // must be invalid. EXPECT_FALSE(blob2.IsValid()); } -// MacOS does not support releasing the shm memory. -#ifndef __APPLE__ +// MacOS and Windows do not support releasing the shm memory. +#if !defined(__APPLE__) && !defined(_WIN32) // Test ReleaseSharedMemory and NumBytesUsed. TEST_P(SharedMemoryBlobSequenceTest, ReleaseSharedMemory) { // Allocate a blob sequence with 1M bytes of storage. - SharedMemoryBlobSequence blobseq(ShmemName().c_str(), 1 << 20, GetParam()); - EXPECT_EQ(blobseq.NumBytesUsed(), 0); - EXPECT_TRUE(blobseq.Write(BlobFromVec({1, 2, 3, 4}))); - EXPECT_GT(blobseq.NumBytesUsed(), 5); - blobseq.ReleaseSharedMemory(); - EXPECT_EQ(blobseq.NumBytesUsed(), 0); - EXPECT_TRUE(blobseq.Write(BlobFromVec({1, 2, 3, 4}))); - EXPECT_GT(blobseq.NumBytesUsed(), 5); + auto blobseq = + CreateSharedMemoryBlobSequence(ShmemName().c_str(), 1 << 20, GetParam()); + EXPECT_EQ(blobseq->NumBytesUsed(), 0); + EXPECT_TRUE(blobseq->Write(BlobFromVec({1, 2, 3, 4}))); + EXPECT_GT(blobseq->NumBytesUsed(), 5); + blobseq->ReleaseSharedMemory(); + EXPECT_EQ(blobseq->NumBytesUsed(), 0); + EXPECT_TRUE(blobseq->Write(BlobFromVec({1, 2, 3, 4}))); + EXPECT_GT(blobseq->NumBytesUsed(), 5); } #endif diff --git a/centipede/stats_test.cc b/centipede/stats_test.cc index 491a428d9..82ce3d5ac 100644 --- a/centipede/stats_test.cc +++ b/centipede/stats_test.cc @@ -451,13 +451,13 @@ TEST(Stats, DumpStatsToCsvFile) { } const std::vector env_vec = { - CreateEnv(workdir.c_str(), /*experiment_name=*/"ExperimentA", + CreateEnv(workdir.string(), /*experiment_name=*/"ExperimentA", /*experiment_flags=*/"AAA"), - CreateEnv(workdir.c_str(), /*experiment_name=*/"ExperimentB", + CreateEnv(workdir.string(), /*experiment_name=*/"ExperimentB", /*experiment_flags=*/"BBB"), - CreateEnv(workdir.c_str(), /*experiment_name=*/"ExperimentA", + CreateEnv(workdir.string(), /*experiment_name=*/"ExperimentA", /*experiment_flags=*/"AAA"), - CreateEnv(workdir.c_str(), /*experiment_name=*/"ExperimentB", + CreateEnv(workdir.string(), /*experiment_name=*/"ExperimentB", /*experiment_flags=*/"BBB")}; { @@ -507,8 +507,8 @@ TEST(Stats, DumpStatsToCsvFile) { } const std::vector kExpectedCsvs = { - workdir / "fuzzing-stats-.000000.ExperimentA.csv", - workdir / "fuzzing-stats-.000000.ExperimentB.csv", + (workdir / "fuzzing-stats-.000000.ExperimentA.csv").string(), + (workdir / "fuzzing-stats-.000000.ExperimentB.csv").string(), }; const std::vector> kExpectedCsvLines = { // CSV #1. @@ -771,15 +771,17 @@ TEST(Stats, DumpStatsToExistingCsvFile) { }; const std::filesystem::path workdir = GetTestTempDir(test_info_->name()); - const std::vector env_vec = {CreateEnv(workdir.c_str()), - CreateEnv(workdir.c_str())}; + const std::vector env_vec = {CreateEnv(workdir.string()), + CreateEnv(workdir.string())}; std::vector> stats_vec(2); stats_vec[0].store({StatsMeta{/*timestamp_unix_micros=*/1000000}}); stats_vec[1].store({StatsMeta{/*timestamp_unix_micros=*/2000000}}); - const std::string kExpectedCsv = workdir / "fuzzing-stats-.000000.csv"; - const std::string kExpectedCsvBak = workdir / "fuzzing-stats-.000000.csv.bak"; + const std::string kExpectedCsv = + (workdir / "fuzzing-stats-.000000.csv").string(); + const std::string kExpectedCsvBak = + (workdir / "fuzzing-stats-.000000.csv.bak").string(); // `StatsCsvFileAppender` creates a brand-new fuzzing-stats file and writes // the CSV header and a 1st stats line to it. diff --git a/centipede/symbol_table.cc b/centipede/symbol_table.cc index e693b4817..6e58ebdfe 100644 --- a/centipede/symbol_table.cc +++ b/centipede/symbol_table.cc @@ -120,10 +120,9 @@ void SymbolTable::GetSymbolsFromOneDso(absl::Span pc_infos, "--no-inlines", "-e", std::string(dso_path), - "<", - std::string(pcs_file.path()), }; - cmd_options.stdout_file_prefix = symbols_file_prefix; + cmd_options.stdin_file_path = std::string(pcs_file.path()); + cmd_options.stdout_file_prefix = symbols_file_prefix.string(); Command cmd{symbolizer_path, std::move(cmd_options)}; int exit_code = cmd.Execute(); if (exit_code != EXIT_SUCCESS) { diff --git a/centipede/util.cc b/centipede/util.cc index 583cc1f2c..8a9064e02 100644 --- a/centipede/util.cc +++ b/centipede/util.cc @@ -17,8 +17,12 @@ #include "./centipede/util.h" +#ifdef _WIN32 +#include "./common/windows_includes.h" +#else #include #include +#endif #include #include @@ -47,6 +51,7 @@ #include "absl/base/const_init.h" #include "absl/base/nullability.h" #include "absl/base/thread_annotations.h" +#include "absl/strings/match.h" #include "absl/strings/str_format.h" #include "absl/strings/str_replace.h" #include "absl/strings/str_split.h" @@ -63,8 +68,13 @@ namespace fuzztest::internal { size_t GetRandomSeed(size_t seed) { if (seed != 0) return seed; +#ifdef _WIN32 + return time(nullptr) + GetCurrentProcessId() + + std::hash{}(std::this_thread::get_id()); +#else return time(nullptr) + getpid() + std::hash{}(std::this_thread::get_id()); +#endif } std::string AsPrintableString(ByteSpan data, size_t max_len) { @@ -83,7 +93,7 @@ std::string AsPrintableString(ByteSpan data, size_t max_len) { template void ReadFromLocalFile(std::string_view file_path, Container &data) { - std::ifstream f(std::string{file_path}); + std::ifstream f(std::string{file_path}, std::ios::in | std::ios::binary); if (!f) return; f.seekg(0, std::ios_base::end); auto size = f.tellg(); @@ -112,12 +122,13 @@ void ReadFromLocalFile(std::string_view file_path, } void ClearLocalFileContents(std::string_view file_path) { - std::ofstream f(std::string{file_path}, std::ios::out | std::ios::trunc); + std::ofstream f(std::string{file_path}, + std::ios::out | std::ios::trunc | std::ios::binary); FUZZTEST_CHECK(f) << "Failed to clear the file: " << file_path; } void WriteToLocalFile(std::string_view file_path, ByteSpan data) { - std::ofstream f(std::string{file_path}); + std::ofstream f(std::string{file_path}, std::ios::out | std::ios::binary); FUZZTEST_CHECK(f) << "Failed to open local file: " << file_path; f.write(reinterpret_cast(data.data()), static_cast(data.size())); @@ -136,13 +147,15 @@ void WriteToLocalFile(std::string_view file_path, const FeatureVec &data) { void WriteToLocalHashedFileInDir(std::string_view dir_path, ByteSpan data) { if (dir_path.empty()) return; - std::string file_path = std::filesystem::path(dir_path).append(Hash(data)); + std::string file_path = + std::filesystem::path(dir_path).append(Hash(data)).string(); WriteToLocalFile(file_path, data); } void WriteToRemoteHashedFileInDir(std::string_view dir_path, ByteSpan data) { if (dir_path.empty()) return; - std::string file_path = std::filesystem::path(dir_path).append(Hash(data)); + std::string file_path = + std::filesystem::path(dir_path).append(Hash(data)).string(); FUZZTEST_CHECK_OK( RemoteFileSetContents(file_path, std::string(data.begin(), data.end()))); } @@ -155,17 +168,24 @@ std::string HashOfFileContents(std::string_view file_path) { } std::string ProcessAndThreadUniqueID(std::string_view prefix) { - // operator << is the only way to serialize std::this_thread::get_id(). std::ostringstream oss; +#ifdef _WIN32 + oss << prefix << GetCurrentProcessId() << "-" << GetCurrentThreadId(); +#else + // operator << is the only way to serialize std::this_thread::get_id(). oss << prefix << getpid() << "-" << std::this_thread::get_id(); +#endif return oss.str(); } std::string TemporaryLocalDirPath() { const char *TMPDIR = getenv("TMPDIR"); + if (!TMPDIR) TMPDIR = getenv("TEMP"); + if (!TMPDIR) TMPDIR = getenv("TMP"); std::string tmp = TMPDIR ? TMPDIR : "/tmp"; - return std::filesystem::path(tmp).append( - ProcessAndThreadUniqueID("centipede-")); + return std::filesystem::path(tmp) + .append(ProcessAndThreadUniqueID("centipede-")) + .string(); } // We need to maintain a global set of dirs that CreateLocalDirRemovedAtExit() @@ -189,13 +209,19 @@ static void RemoveDirsAtExit() { void CreateLocalDirRemovedAtExit(std::string_view path) { // Safeguard against removing dirs not created by TemporaryLocalDirPath(). - FUZZTEST_CHECK_NE(path.find("/centipede-"), std::string::npos); + FUZZTEST_CHECK(absl::StrContains(path, "/centipede-") || + absl::StrContains(path, "\\centipede-")); // Create the dir. std::error_code error; - std::filesystem::remove_all(path, error); - FUZZTEST_LOG_IF(ERROR, error) - << "Unable to clean up existing dir " << path << ": " << error.message(); - std::filesystem::create_directories(path); + std::filesystem::path p(path); + if (std::filesystem::exists(p, error)) { + std::filesystem::remove_all(p, error); + FUZZTEST_LOG_IF(ERROR, error) + << "Unable to clean up existing dir " << p << ": " << error.message(); + } + std::filesystem::create_directories(p, error); + FUZZTEST_CHECK(!error) << "Failed to create local dir " << p << ": " + << error.message(); // Add to dirs_to_delete_at_exit. absl::MutexLock lock(dirs_to_delete_at_exit_mutex); if (!dirs_to_delete_at_exit) { @@ -206,7 +232,7 @@ void CreateLocalDirRemovedAtExit(std::string_view path) { } ScopedFile::ScopedFile(std::string_view dir_path, std::string_view name) - : my_path_(std::filesystem::path(dir_path) / name) {} + : my_path_((std::filesystem::path(dir_path) / name).string()) {} ScopedFile::~ScopedFile() { std::error_code error; @@ -359,16 +385,85 @@ std::vector RandomWeightedSubset(absl::Span set, return res; } +#ifdef _WIN32 +// On Windows, we use the first page for the magic cookies of mmapped regions so +// that our VEH can handle it properly. +static constexpr std::string_view kMmapMagicCookie = "CENTIPED"; + +static const auto page_size = []() { + SYSTEM_INFO si; + GetSystemInfo(&si); + return static_cast(si.dwPageSize); +}(); + +static LONG CALLBACK +AutoCommitPageFaultHandler(PEXCEPTION_POINTERS exception_info) { + auto record = exception_info->ExceptionRecord; + if (record->ExceptionCode != EXCEPTION_ACCESS_VIOLATION || + record->NumberParameters < 2) { + return EXCEPTION_CONTINUE_SEARCH; + } + auto fault_addr = reinterpret_cast(record->ExceptionInformation[1]); + MEMORY_BASIC_INFORMATION mbi; + if (VirtualQuery(fault_addr, &mbi, sizeof(mbi)) != sizeof(mbi)) { + return EXCEPTION_CONTINUE_SEARCH; + } + if (mbi.State != MEM_RESERVE) { + return EXCEPTION_CONTINUE_SEARCH; + } + auto cookie_addr = reinterpret_cast(mbi.AllocationBase); + if (VirtualQuery(cookie_addr, &mbi, sizeof(mbi)) != sizeof(mbi)) { + return EXCEPTION_CONTINUE_SEARCH; + } + if (mbi.State != MEM_COMMIT) { + return EXCEPTION_CONTINUE_SEARCH; + } + if (std::memcmp(cookie_addr, kMmapMagicCookie.data(), + kMmapMagicCookie.size()) != 0) { + return EXCEPTION_CONTINUE_SEARCH; + } + if (VirtualAlloc(fault_addr, 1, MEM_COMMIT, PAGE_READWRITE) == nullptr) { + return EXCEPTION_CONTINUE_SEARCH; + } + return EXCEPTION_CONTINUE_EXECUTION; +} +#endif + uint8_t *MmapNoReserve(size_t size) { +#ifdef _WIN32 + // Set up page fault handler to commit page on demand. + [[maybe_unused]] static bool installed_veh = []() { + // Must use `First=0` as it could otherwise conflict with e.g. sanitizers. + AddVectoredExceptionHandler(/*First=*/0, AutoCommitPageFaultHandler); + return true; + }(); + // MEM_RESERVE has different semantics and does not contradict with + // MAP_NORESERVE for mmap. + auto result = reinterpret_cast( + VirtualAlloc(nullptr, size + page_size, MEM_RESERVE, PAGE_READWRITE)); + FUZZTEST_CHECK(result != nullptr) + << "VirtualAlloc failed for size " << size << " err=" << GetLastError(); + FUZZTEST_CHECK(VirtualAlloc(result, kMmapMagicCookie.size(), MEM_COMMIT, + PAGE_READWRITE) != nullptr) + << "VirtualAlloc failed to commit the memory region cookie"; + std::memcpy(result, kMmapMagicCookie.data(), kMmapMagicCookie.size()); + return result + page_size; +#else auto result = mmap(0, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON | MAP_NORESERVE, -1, 0); FUZZTEST_CHECK(result != MAP_FAILED); return reinterpret_cast(result); +#endif } void Munmap(uint8_t *ptr, size_t size) { +#ifdef _WIN32 + BOOL result = VirtualFree(ptr - page_size, 0, MEM_RELEASE); + FUZZTEST_CHECK(result != 0); +#else auto result = munmap(ptr, size); FUZZTEST_CHECK_EQ(result, 0); +#endif } int PollTimeoutMs(absl::Duration timeout) { diff --git a/centipede/util_test.cc b/centipede/util_test.cc index 6f26650ec..52041aa27 100644 --- a/centipede/util_test.cc +++ b/centipede/util_test.cc @@ -32,6 +32,10 @@ #include "./common/hash.h" #include "./common/logging.h" +#ifdef _WIN32 +#define setenv(n, v, _r) _putenv_s(n, v) +#endif + namespace fuzztest::internal { TEST(UtilTest, AsString) { @@ -116,7 +120,8 @@ TEST(UtilTest, TemporaryLocalDirPath) { auto temp_dir = TemporaryLocalDirPath(); // Create dir, create a file there, write to file, read from it, remove dir. std::filesystem::create_directories(temp_dir); - std::string temp_file_path = std::filesystem::path(temp_dir).append("blah"); + std::string temp_file_path = + std::filesystem::path(temp_dir).append("blah").string(); ByteArray written_data{1, 2, 3}; WriteToLocalFile(temp_file_path, written_data); ByteArray read_data; @@ -162,8 +167,8 @@ TEST(UtilTest, CreateLocalDirRemovedAtExit) { EXPECT_TRUE(std::filesystem::exists(tmpdir)); setenv("CENTIPEDE_UTIL_TEST_TEMP_DIR", tmpdir.c_str(), 1); // Create two subdirs via CreateLocalDirRemovedAtExit. - std::string subdir1 = std::filesystem::path(tmpdir).append("1"); - std::string subdir2 = std::filesystem::path(tmpdir).append("2"); + std::string subdir1 = std::filesystem::path(tmpdir).append("1").string(); + std::string subdir2 = std::filesystem::path(tmpdir).append("2").string(); CreateLocalDirRemovedAtExit(subdir1); CreateLocalDirRemovedAtExit(subdir2); EXPECT_TRUE(std::filesystem::exists(subdir1)); @@ -293,6 +298,15 @@ TEST(UtilTest, RemoveSubset) { testing::ElementsAre(std::vector{1}, std::vector{3})); } +TEST(UtilTest, MmapTest) { + static constexpr size_t kBufSize = 1 << 30; // 1 GiB + auto* buf = MmapNoReserve(kBufSize); + ASSERT_NE(buf, nullptr); + EXPECT_EQ(buf[1234], 0); + EXPECT_EQ(buf[567890], 0); + Munmap(buf, kBufSize); +} + TEST(UtilTest, PollTimeoutMsWorks) { EXPECT_GT(PollTimeoutMs(absl::ZeroDuration()), 0); EXPECT_GT(PollTimeoutMs(-absl::InfiniteDuration()), 0); diff --git a/centipede/workdir.cc b/centipede/workdir.cc index c0f4090d6..676f8b69e 100644 --- a/centipede/workdir.cc +++ b/centipede/workdir.cc @@ -58,7 +58,7 @@ std::string NormalizeAnnotation(std::string_view annotation) { WorkDir::PathShards::PathShards(std::string_view base_dir, std::string_view rel_prefix, size_t my_shard_index) - : prefix_{std::filesystem::path(base_dir) / rel_prefix}, + : prefix_{(std::filesystem::path(base_dir) / rel_prefix).string()}, my_shard_index_{my_shard_index} {} std::string WorkDir::PathShards::Shard(size_t shard_index) const { @@ -97,12 +97,12 @@ WorkDir WorkDir::FromCorpusShardPath( // std::string_view binary_name, // std::string_view binary_hash) { const std::filesystem::path path{corpus_shard_path}; - const std::string dir = path.parent_path(); - const std::string stem = path.stem(); + const std::string dir = path.parent_path().string(); + const std::string stem = path.stem().string(); FUZZTEST_CHECK(stem == kCorpusShardStem || absl::StartsWith(stem, kDistilledCorpusShardStemPrefix)) << VV(corpus_shard_path); - const std::string dot_ext = path.extension(); + const std::string dot_ext = path.extension().string(); FUZZTEST_CHECK(!dot_ext.empty() && dot_ext[0] == '.') << VV(corpus_shard_path); const std::string ext = dot_ext.substr(1); @@ -142,8 +142,9 @@ WorkDir::PathShards WorkDir::CorpusFilePaths() const { } std::string WorkDir::CoverageDirPath() const { - return std::filesystem::path(workdir_) / - absl::StrCat(binary_name_, "-", binary_hash_); + return (std::filesystem::path(workdir_) / + absl::StrCat(binary_name_, "-", binary_hash_)) + .string(); } WorkDir::PathShards WorkDir::CrashReproducerDirPaths() const { @@ -155,11 +156,11 @@ WorkDir::PathShards WorkDir::CrashMetadataDirPaths() const { } std::string WorkDir::BinaryInfoDirPath() const { - return std::filesystem::path(CoverageDirPath()) / "binary-info"; + return (std::filesystem::path(CoverageDirPath()) / "binary-info").string(); } std::string WorkDir::DebugInfoDirPath() const { - return std::filesystem::path(workdir_) / "debug"; + return (std::filesystem::path(workdir_) / "debug").string(); } WorkDir::PathShards WorkDir::DistilledCorpusFilePaths() const { @@ -179,53 +180,60 @@ WorkDir::PathShards WorkDir::DistilledFeaturesFilePaths() const { } std::string WorkDir::CoverageReportPath(std::string_view annotation) const { - return std::filesystem::path(workdir_) / - absl::StrFormat("coverage-report-%s.%0*d%s.txt", binary_name_, - kDigitsInShardIndex, my_shard_index_, - NormalizeAnnotation(annotation)); + return (std::filesystem::path(workdir_) / + absl::StrFormat("coverage-report-%s.%0*d%s.txt", binary_name_, + kDigitsInShardIndex, my_shard_index_, + NormalizeAnnotation(annotation))) + .string(); } std::string WorkDir::CorpusStatsPath(std::string_view annotation) const { - return std::filesystem::path(workdir_) / - absl::StrFormat("corpus-stats-%s.%0*d%s.json", binary_name_, - kDigitsInShardIndex, my_shard_index_, - NormalizeAnnotation(annotation)); + return (std::filesystem::path(workdir_) / + absl::StrFormat("corpus-stats-%s.%0*d%s.json", binary_name_, + kDigitsInShardIndex, my_shard_index_, + NormalizeAnnotation(annotation))) + .string(); } std::string WorkDir::FuzzingStatsPath(std::string_view annotation) const { - return std::filesystem::path(workdir_) / - absl::StrFormat("fuzzing-stats-%s.%0*d%s.csv", binary_name_, - kDigitsInShardIndex, my_shard_index_, - NormalizeAnnotation(annotation)); + return (std::filesystem::path(workdir_) / + absl::StrFormat("fuzzing-stats-%s.%0*d%s.csv", binary_name_, + kDigitsInShardIndex, my_shard_index_, + NormalizeAnnotation(annotation))) + .string(); } std::string WorkDir::SourceBasedCoverageRawProfilePath() const { // Pass %m to enable online merge mode: updates file in place instead of // replacing it %m is replaced by lprofGetLoadModuleSignature(void) which // should be consistent for a fixed binary - return std::filesystem::path(CoverageDirPath()) / - absl::StrFormat("clang_coverage.%0*d.%s.profraw", kDigitsInShardIndex, - my_shard_index_, "%m"); + return (std::filesystem::path(CoverageDirPath()) / + absl::StrFormat("clang_coverage.%0*d.%s.profraw", kDigitsInShardIndex, + my_shard_index_, "%m")) + .string(); } std::string WorkDir::SourceBasedCoverageIndexedProfilePath() const { - return std::filesystem::path(CoverageDirPath()) / - absl::StrFormat("clang_coverage.profdata"); + return (std::filesystem::path(CoverageDirPath()) / + absl::StrFormat("clang_coverage.profdata")) + .string(); } std::string WorkDir::SourceBasedCoverageReportPath( std::string_view annotation) const { - return std::filesystem::path(workdir_) / - absl::StrFormat("source-coverage-report-%s.%0*d%s", binary_name_, - kDigitsInShardIndex, my_shard_index_, - NormalizeAnnotation(annotation)); + return (std::filesystem::path(workdir_) / + absl::StrFormat("source-coverage-report-%s.%0*d%s", binary_name_, + kDigitsInShardIndex, my_shard_index_, + NormalizeAnnotation(annotation))) + .string(); } std::string WorkDir::RUsageReportPath(std::string_view annotation) const { - return std::filesystem::path(workdir_) / - (absl::StrFormat("rusage-report-%s.%0*d%s.txt", binary_name_, - kDigitsInShardIndex, my_shard_index_, - NormalizeAnnotation(annotation))); + return (std::filesystem::path(workdir_) / + (absl::StrFormat("rusage-report-%s.%0*d%s.txt", binary_name_, + kDigitsInShardIndex, my_shard_index_, + NormalizeAnnotation(annotation)))) + .string(); } std::vector WorkDir::EnumerateRawCoverageProfiles() const { @@ -247,7 +255,7 @@ std::vector WorkDir::EnumerateRawCoverageProfiles() const { std::vector raw_profiles; for (const auto &entry : dir_iter) { if (entry.is_regular_file() && entry.path().extension() == ".profraw") - raw_profiles.push_back(entry.path()); + raw_profiles.push_back(entry.path().string()); } return raw_profiles; } diff --git a/centipede/workdir.h b/centipede/workdir.h index 5192b939f..d8f6a1d2f 100644 --- a/centipede/workdir.h +++ b/centipede/workdir.h @@ -51,6 +51,8 @@ class WorkDir { // Returns the shard index of `path` if it is a shard parth, `nullopt` // otherwise. std::optional GetShardIndex(std::string_view path) const; + // Gets the common prefix of the sharded path. + const std::string& prefix() const { return prefix_; } private: friend class WorkDir; diff --git a/common/BUILD b/common/BUILD index bbdb792b6..296b1406f 100644 --- a/common/BUILD +++ b/common/BUILD @@ -208,6 +208,7 @@ cc_library( ":blob_file", ":defs", ":logging", + ":windows_includes", "@abseil-cpp//absl/strings", "@abseil-cpp//absl/strings:str_format", "@googletest//:gtest", @@ -225,6 +226,11 @@ cc_library( ], ) +cc_library( + name = "windows_includes", + hdrs = ["windows_includes.h"], +) + ### Tests cc_test( diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index 671195e07..b77576334 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -148,6 +148,13 @@ fuzztest_cc_library( absl::core_headers ) +fuzztest_cc_library( + NAME + windows_includes + HDRS + "windows_includes.h" +) + fuzztest_cc_library( NAME test_util @@ -159,6 +166,7 @@ fuzztest_cc_library( fuzztest::common_logging fuzztest::blob_file fuzztest::defs + fuzztest::windows_includes absl::strings absl::str_format GTest::gtest diff --git a/common/test_util.cc b/common/test_util.cc index 3232d551f..8bc70636c 100644 --- a/common/test_util.cc +++ b/common/test_util.cc @@ -18,10 +18,20 @@ #include #include // NOLINT +#ifdef _WIN32 +#include "./common/windows_includes.h" +#else +#include +#endif + #include "gtest/gtest.h" #include "absl/strings/str_cat.h" #include "./common/logging.h" +#ifdef _WIN32 +#define setenv(n, v, _r) _putenv_s(n, v) +#endif + namespace fuzztest::internal { std::filesystem::path GetTestTempDir(std::string_view subdir) { @@ -35,11 +45,18 @@ std::filesystem::path GetTestTempDir(std::string_view subdir) { FUZZTEST_CHECK(!error) << "Failed to create dir: " VV(dir) << error.message(); } - return std::filesystem::canonical(dir); + return std::filesystem::absolute(dir); } std::string GetTempFilePath(std::string_view subdir, size_t i) { - return GetTestTempDir(subdir) / absl::StrCat("tmp.", getpid(), ".", i); + return (GetTestTempDir(subdir) / absl::StrCat("tmp.", +#ifdef _WIN32 + GetCurrentProcessId(), +#else + getpid(), +#endif + ".", i)) + .string(); } std::filesystem::path GetTestRunfilesDir() { @@ -59,8 +76,15 @@ std::filesystem::path GetDataDependencyFilepath(std::string_view rel_path) { const auto runfiles_dir = GetTestRunfilesDir(); auto path = runfiles_dir; path.append(rel_path); - FUZZTEST_CHECK(std::filesystem::exists(path)) // - << "No such path: " << VV(path) << VV(runfiles_dir) << VV(rel_path); + std::error_code ec; +#ifdef _WIN32 + auto win_path = path; + win_path += ".exe"; + if (std::filesystem::exists(win_path, ec)) return win_path; +#endif + FUZZTEST_CHECK(std::filesystem::exists(path, ec)) // + << "No such path: " << VV(path) << VV(runfiles_dir) << VV(rel_path) + << VV(ec); return path; } diff --git a/common/test_util.h b/common/test_util.h index c859bff94..e363f1868 100644 --- a/common/test_util.h +++ b/common/test_util.h @@ -79,7 +79,7 @@ class TempDir { const std::filesystem::path& path() const { return path_; } std::string GetFilePath(std::string_view file_name) const { - return path_ / file_name; + return (path_ / file_name).string(); } std::string CreateSubdir(std::string_view name) const { diff --git a/common/windows_includes.h b/common/windows_includes.h new file mode 100644 index 000000000..f08f63a06 --- /dev/null +++ b/common/windows_includes.h @@ -0,0 +1,40 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef FUZZTEST_COMMON_WINDOWS_INCLUDES_H_ +#define FUZZTEST_COMMON_WINDOWS_INCLUDES_H_ + +#ifdef _WIN32 + +// clang-format off +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif + +#ifndef NOMINMAX +#define NOMINMAX +#endif + +#ifndef NOGDI +#define NOGDI +#endif + +#include +#include + +// clang-format on + +#endif // _WIN32 + +#endif // FUZZTEST_COMMON_WINDOWS_INCLUDES_H_