Skip to content

Preserve include-only files in FileDeck COPY output #5333

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
kriben wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Preserve include-only files in FileDeck COPY output #5333

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
91 changes: 74 additions & 17 deletions opm/input/eclipse/Deck/FileDeck.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -410,23 +410,21 @@ std::ofstream&
FileDeck::DumpContext::open_file(const std::string& deck_name,
const fs::path& output_file)
{
const auto& [filePos, fileInserted] = this->file_map_
.try_emplace(deck_name, output_file.generic_string());
const auto output_name = output_file.generic_string();
this->file_map_.insert_or_assign(deck_name, output_name);

if (fileInserted) {
if (! fs::is_directory(output_file.parent_path())) {
fs::create_directories(output_file.parent_path());
}
if (! fs::is_directory(output_file.parent_path())) {
fs::create_directories(output_file.parent_path());
}

const auto& [streamPos, streamInserted] = this->stream_map_
.try_emplace(output_file.generic_string(), output_file);
const auto& [streamPos, streamInserted] = this->stream_map_
.try_emplace(output_name, output_file);

if (! streamInserted) {
streamPos->second.open(output_file);
}
if (! streamInserted && ! streamPos->second.is_open()) {
streamPos->second.open(output_file);
}

return this->stream_map_.at(output_file.generic_string());
return streamPos->second;
}

// ---------------------------------------------------------------------------
Expand Down Expand Up @@ -490,20 +488,44 @@ void FileDeck::include_block(const std::string& input_file,
const std::string& output_dir,
FileDeck::DumpContext& context) const
{
const auto input_root = fs::canonical(this->input_directory);
const auto output_root = fs::canonical(output_dir);

auto current_file = input_file;
auto current_output = fs::path(output_file);

while (true) {
const auto& parent = this->deck_tree.parent(current_file);

auto* stream = context.get_stream(parent);
if (stream != nullptr) {
// Should ideally use fs::relative()
INCLUDE(*stream, fs::proximate(output_file, output_dir).generic_string());
// INCLUDE paths are resolved relative to the root deck directory.
INCLUDE(*stream, fs::proximate(current_output, output_root).generic_string());

break;
}

// The parent file contains only INCLUDE statements and therefore has
// no keyword block of its own. Create it in the output directory to
// preserve the include hierarchy, write the include statement into
// it, and continue up the tree to include the parent itself.
const auto rel_path = fs::path(parent).lexically_relative(input_root);
if (rel_path.empty() || rel_path.is_absolute()
|| (std::find(rel_path.begin(), rel_path.end(), fs::path{".."}) != rel_path.end()))
{
// Preserve the previous flattening behavior for include-only files
// outside the root deck directory rather than writing outside the
// requested output directory.
current_file = parent;
continue;
}

const auto parent_output = output_root / rel_path;
auto& parent_stream = context.open_file(parent, parent_output);
INCLUDE(parent_stream, fs::proximate(current_output, output_root).generic_string());

current_file = parent;
current_output = parent_output;
}
}

Expand Down Expand Up @@ -532,10 +554,45 @@ void FileDeck::dump(const std::string& output_dir,

for (std::size_t block_index = 1; block_index < this->blocks.size(); ++block_index) {
const auto& block = this->blocks[block_index];
// For now, originally binary files will be written as GRDECL
const auto& include_file = this->dump_block(block, output_dir, {}, context);

if (block.fname != this->deck_tree.root()) {
auto continues_after_nested_include = false;
auto previous_file = this->blocks[block_index - 1].fname;
while (previous_file != this->deck_tree.root()) {
previous_file = this->deck_tree.parent(previous_file);
if (previous_file == block.fname) {
continues_after_nested_include = true;
break;
}
}

std::string include_file;
if ((block.fname != this->deck_tree.root())
&& context.has_file(block.fname)
&& !continues_after_nested_include)
{
// This is a repeated INCLUDE, not a continuation after a nested
// include. Write it to a separate file so another INCLUDE can
// be emitted at the original position without replaying the
// earlier occurrence's keywords.
auto rel_path = fs::proximate(block.fname, this->input_directory);
rel_path += fmt::format(".{}", block_index);
auto output_file = fs::path(output_dir) / rel_path;
touch_file(output_file);
output_file = fs::canonical(output_file);

auto& stream = context.open_file(block.fname, output_file);
DeckOutput out(stream, 10);
block.dump(out);
include_file = output_file.generic_string();
}
else {
// For now, originally binary files will be written as GRDECL.
// An empty include_file identifies a block which continues in
// an output file already open for its source file.
include_file = this->dump_block(block, output_dir, {}, context);
}

if (!include_file.empty() && (block.fname != this->deck_tree.root())) {
this->include_block(block.fname, include_file, output_dir, context);
Comment thread
kriben marked this conversation as resolved.
}
}
Expand Down
206 changes: 197 additions & 9 deletions tests/parser/ScheduleRestartTests.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,12 +68,52 @@
#include <utility>
#include <vector>

#include <fmt/format.h>

namespace fs = std::filesystem;

using namespace Opm;

namespace {

void write_file(const fs::path& file, const std::string& contents)
{
if (const auto& dir = file.parent_path(); !dir.empty()) {
fs::create_directories(dir);
}

std::ofstream stream {file};
BOOST_REQUIRE_MESSAGE(stream.is_open(), "Unable to open " << file.generic_string());

stream << contents;
stream.close();
BOOST_REQUIRE_MESSAGE(stream, "Unable to write " << file.generic_string());
}

std::string read_file(const fs::path& file)
{
BOOST_REQUIRE_MESSAGE(fs::exists(file), "Missing file " << file.generic_string());

std::ifstream stream {file};
BOOST_REQUIRE_MESSAGE(stream.is_open(), "Unable to open " << file.generic_string());

return { std::istreambuf_iterator<char> {stream},
std::istreambuf_iterator<char> {} };
}

std::string vfpprod(const int table)
{
return fmt::format(R"(VFPPROD
{} 2000.0 'LIQ' 'WCT' 'GOR' 'THP' ' ' 'METRIC' 'BHP' /
100.0 /
10.0 /
0.0 /
0.0 /
0.0 /
1 1 1 1 200.0 /
)", table);
}

void compare_connections(const RestartIO::RstConnection& rst_conn,
const Connection& sched_conn)
{
Expand Down Expand Up @@ -385,6 +425,163 @@ BOOST_AUTO_TEST_CASE(FileDeckIterationSkipsEmptyIncludeBlock)
BOOST_CHECK_EQUAL(num_kw, 0);
}

BOOST_AUTO_TEST_CASE(FileDeckCopyIncludeOnlyFiles)
{
WorkArea work_area {"file_deck_copy"};

// The deck includes a file which holds INCLUDE statements only, e.g. a
// wrapper collecting the lift curve files of a model. That wrapper
// includes one file with keywords, and a second wrapper which in turn
// includes another file with keywords.
write_file("CASE.DATA", R"(RUNSPEC
DIMENS
2 2 2 /
OIL
WATER
GAS
METRIC
START
1 'JAN' 2000 /
GRID
DX
8*100.0 /
DY
8*100.0 /
DZ
8*10.0 /
TOPS
4*2000.0 /
PORO
8*0.2 /
PERMX
8*100.0 /
SCHEDULE
INCLUDE
'include/lift_curves.inc' /
END
)");

write_file("include/lift_curves.inc", R"(INCLUDE
'include/vfp_curve_1.ecl' /
INCLUDE
'include/more_curves.inc' /
)");

write_file("include/more_curves.inc", R"(INCLUDE
'include/vfp_curve_2.ecl' /
)");

write_file("include/vfp_curve_1.ecl", vfpprod(1));
write_file("include/vfp_curve_2.ecl", vfpprod(2));

const auto deck = Parser{}.parseFile("CASE.DATA");
FileDeck fd(deck);

fd.dump("out", "CASE.DATA", FileDeck::OutputMode::COPY);

// The main deck must include the wrapper, not the files below it.
const auto main_deck = read_file("out/CASE.DATA");
BOOST_CHECK(main_deck.find("include/lift_curves.inc") != std::string::npos);
BOOST_CHECK(main_deck.find("vfp_curve") == std::string::npos);
BOOST_CHECK(main_deck.find("more_curves") == std::string::npos);

// The wrappers must be recreated with their own include statements.
const auto wrapper = read_file("out/include/lift_curves.inc");
BOOST_CHECK(wrapper.find("include/vfp_curve_1.ecl") != std::string::npos);
BOOST_CHECK(wrapper.find("include/more_curves.inc") != std::string::npos);

const auto nested_wrapper = read_file("out/include/more_curves.inc");
BOOST_CHECK(nested_wrapper.find("include/vfp_curve_2.ecl") != std::string::npos);

// Reloading the dumped deck must give the same keywords back.
const auto out_deck = Parser{}.parseFile("out/CASE.DATA");
BOOST_CHECK_EQUAL(out_deck.count("VFPPROD"), 2);
BOOST_CHECK_EQUAL(out_deck.size(), deck.size());
}

BOOST_AUTO_TEST_CASE(FileDeckCopyFlattensExternalIncludeOnlyFile)
{
WorkArea work_area {"file_deck_copy_external_wrapper"};

write_file("case/CASE.DATA", R"(RUNSPEC
DIMENS
1 1 1 /
OIL
WATER
GAS
METRIC
START
1 'JAN' 2000 /
SCHEDULE
INCLUDE
'../wrapper.inc' /
END
)");
write_file("wrapper.inc", R"(INCLUDE
'leaf.inc' /
)");
write_file("case/leaf.inc", vfpprod(1));

const auto deck = Parser{}.parseFile("case/CASE.DATA");
FileDeck fd(deck);

fd.dump("case/out", "CASE.DATA", FileDeck::OutputMode::COPY);

// Preserve the old flattening behavior for an include-only wrapper outside
// the root deck directory, but do not recreate it outside the output root.
const auto main_deck = read_file("case/out/CASE.DATA");
BOOST_CHECK(main_deck.find("leaf.inc") != std::string::npos);
BOOST_CHECK(main_deck.find("wrapper.inc") == std::string::npos);
BOOST_CHECK(!fs::exists("case/wrapper.inc"));

const auto out_deck = Parser{}.parseFile("case/out/CASE.DATA");
BOOST_CHECK_EQUAL(out_deck.count("VFPPROD"), 1);
BOOST_CHECK_EQUAL(out_deck.size(), deck.size());
}

BOOST_AUTO_TEST_CASE(FileDeckCopyPreservesRepeatedIncludeOrder)
{
WorkArea work_area {"file_deck_copy_repeated_include"};

write_file("CASE.DATA", R"(RUNSPEC
DIMENS
1 1 1 /
OIL
WATER
GAS
METRIC
START
1 'JAN' 2000 /
SCHEDULE
INCLUDE
'vfp.inc' /
TUNING
/
/
/
INCLUDE
'vfp.inc' /
END
)");
write_file("vfp.inc", vfpprod(1));

const auto deck = Parser{}.parseFile("CASE.DATA");
FileDeck fd(deck);

fd.dump("out", "CASE.DATA", FileDeck::OutputMode::COPY);

const auto main_deck = read_file("out/CASE.DATA");
const auto first_include = main_deck.find("\nINCLUDE");
BOOST_REQUIRE(first_include != std::string::npos);
BOOST_CHECK(main_deck.find("\nINCLUDE", first_include + 1) != std::string::npos);

const auto out_deck = Parser{}.parseFile("out/CASE.DATA");
BOOST_REQUIRE_EQUAL(out_deck.size(), deck.size());
for (std::size_t index = 0; index < deck.size(); ++index) {
BOOST_CHECK_EQUAL(out_deck[index].name(), deck[index].name());
}
}

BOOST_AUTO_TEST_CASE(RestartTest2)
{
const auto deck = Parser{}.parseFile("UDQ_WCONPROD.DATA");
Expand Down Expand Up @@ -491,15 +688,6 @@ BOOST_AUTO_TEST_CASE(RestartTest)

namespace {

void write_file(const fs::path& fname, const std::string& contents)
{
if (fname.has_parent_path()) {
fs::create_directories(fname.parent_path());
}

std::ofstream { fname } << contents;
}

std::string case_deck(const std::string& solution_body,
const std::string& schedule_body)
{
Expand Down