diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index ec1d08f1..4edd7f49 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -3,7 +3,8 @@
add_executable(calc_server
calculator/calc_server.cxx
logger.cc
- in_memory_log_store.cxx)
+ in_memory_log_store.cxx
+ file_based_log_store.cxx)
# add_dependencies(calc_server
# ${LIBRARY_NAME})
@@ -14,7 +15,8 @@ target_link_libraries(calc_server
add_executable(echo_server
echo/echo_server.cxx
logger.cc
- in_memory_log_store.cxx)
+ in_memory_log_store.cxx
+ file_based_log_store.cxx)
# add_dependencies(echo_server
# ${LIBRARY_NAME})
target_link_libraries(echo_server
@@ -24,7 +26,8 @@ target_link_libraries(echo_server
add_executable(quick_start
quick_start.cxx
logger.cc
- in_memory_log_store.cxx)
+ in_memory_log_store.cxx
+ file_based_log_store.cxx)
# add_dependencies(quick_start
# ${LIBRARY_NAME})
target_link_libraries(quick_start
diff --git a/examples/calculator/calc_server.cxx b/examples/calculator/calc_server.cxx
index 256f8aeb..82c86410 100644
--- a/examples/calculator/calc_server.cxx
+++ b/examples/calculator/calc_server.cxx
@@ -17,6 +17,7 @@ limitations under the License.
#include "calc_state_machine.hxx"
#include "in_memory_state_mgr.hxx"
+#include "file_based_state_mgr.hxx"
#include "logger_wrapper.hxx"
#include "nuraft.hxx"
@@ -220,6 +221,11 @@ void check_additional_flags(int argc, char** argv) {
CALL_TYPE = raft_params::async_handler;
} else if (strcmp(argv[ii], "--async-snapshot-creation") == 0) {
ASYNC_SNAPSHOT_CREATION = true;
+ } else if (strcmp(argv[ii], "--persistent") == 0) {
+ set_persistent_storage(true);
+ } else if (strncmp(argv[ii], "--persistent-dir=", 17) == 0) {
+ std::string dir = argv[ii] + 17;
+ set_persistent_storage(true, dir);
}
}
}
@@ -231,8 +237,11 @@ void calc_usage(int argc, char** argv) {
ss << std::endl << std::endl;
ss << " options:" << std::endl;
ss << " --async-handler: use async type handler." << std::endl;
- ss << " --async-snapshot-creation: create snapshots asynchronously."
- << std::endl << std::endl;
+ ss << " --async-snapshot-creation: create snapshots asynchronously." << std::endl;
+ ss << " --persistent: enable file-based persistent storage." << std::endl;
+ ss << " --persistent-dir=
: specify directory for persistent storage" << std::endl;
+ ss << " (default: ./nuraft_data)." << std::endl
+ << std::endl;
std::cout << ss.str();
exit(0);
diff --git a/examples/example_common.hxx b/examples/example_common.hxx
index e6486a33..deaa58f0 100644
--- a/examples/example_common.hxx
+++ b/examples/example_common.hxx
@@ -21,6 +21,21 @@ using namespace nuraft;
using raft_result = cmd_result< ptr >;
+// Global flag to enable persistent storage.
+// If true, file-based storage will be used instead of in-memory.
+static bool USE_PERSISTENT_STORAGE = false;
+
+// Directory for persistent storage.
+static std::string PERSISTENT_STORAGE_DIR = "./nuraft_data";
+
+// Helper to set persistent storage options.
+inline void set_persistent_storage(bool enable, const std::string& dir = "./nuraft_data") {
+ USE_PERSISTENT_STORAGE = enable;
+ if (enable) {
+ PERSISTENT_STORAGE_DIR = dir;
+ }
+}
+
struct server_stuff {
server_stuff()
: server_id_(1)
@@ -155,10 +170,18 @@ void init_raft(ptr sm_instance) {
ptr log_wrap = cs_new( log_file_name, 4 );
stuff.raft_logger_ = log_wrap;
+ // State manager: use persistent or in-memory based on flag.
+ if (USE_PERSISTENT_STORAGE) {
+ std::cout << "using persistent storage: " << PERSISTENT_STORAGE_DIR << std::endl;
+ stuff.smgr_ = cs_new( stuff.server_id_,
+ stuff.endpoint_,
+ PERSISTENT_STORAGE_DIR );
+ } else {
+ std::cout << "using in-memory storage" << std::endl;
+ stuff.smgr_ = cs_new( stuff.server_id_,
+ stuff.endpoint_ );
+ }
// State machine.
- stuff.smgr_ = cs_new( stuff.server_id_,
- stuff.endpoint_ );
- // State manager.
stuff.sm_ = sm_instance;
// ASIO options.
diff --git a/examples/file_based_log_store.cxx b/examples/file_based_log_store.cxx
new file mode 100644
index 00000000..f25e1ed5
--- /dev/null
+++ b/examples/file_based_log_store.cxx
@@ -0,0 +1,403 @@
+/************************************************************************
+Copyright 2017-2019 eBay Inc.
+Author/Developer(s): Jung-Sang Ahn
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+**************************************************************************/
+
+#include "file_based_log_store.hxx"
+
+#include "nuraft.hxx"
+
+#include
+#include
+#include
+#include
+#include
+
+namespace nuraft {
+
+file_based_log_store::file_based_log_store(const std::string& log_dir, int server_id)
+ : log_dir_(log_dir)
+ , server_id_(server_id)
+ , start_idx_(1)
+ , last_durable_idx_(0)
+{
+ // Create server-specific subdirectory
+ std::string server_dir = log_dir_ + "/server_" + std::to_string(server_id_);
+ mkdir(server_dir.c_str(), 0755);
+
+ // Load existing log entries from disk
+ load_from_disk();
+
+ // Ensure we have at least the dummy entry at index 0
+ {
+ std::lock_guard l(logs_lock_);
+ if (logs_.find(0) == logs_.end()) {
+ ptr buf = buffer::alloc(sz_ulong);
+ logs_[0] = cs_new(0, buf);
+ }
+ }
+}
+
+file_based_log_store::~file_based_log_store() {
+ close();
+}
+
+ptr file_based_log_store::make_clone(const ptr& entry) {
+ // NOTE:
+ // Timestamp is used only when `replicate_log_timestamp_` option is on.
+ // Otherwise, log store does not need to store or load it.
+ ptr clone = cs_new
+ ( entry->get_term(),
+ buffer::clone( entry->get_buf() ),
+ entry->get_val_type(),
+ entry->get_timestamp(),
+ entry->has_crc32(),
+ entry->get_crc32(),
+ false );
+ return clone;
+}
+
+std::string file_based_log_store::get_file_path(ulong index) const {
+ // Format: /server_/.log
+ return log_dir_ + "/server_" + std::to_string(server_id_) + "/" +
+ std::to_string(index) + ".log";
+}
+
+void file_based_log_store::load_from_disk() {
+ std::string server_dir = log_dir_ + "/server_" + std::to_string(server_id_);
+ DIR* dir = opendir(server_dir.c_str());
+ if (!dir) {
+ // Directory doesn't exist yet, nothing to load
+ return;
+ }
+
+ struct dirent* entry;
+ std::vector indices;
+
+ // Scan directory for log files
+ while ((entry = readdir(dir)) != nullptr) {
+ std::string name(entry->d_name);
+ if (name.length() > 4 && name.substr(name.length() - 4) == ".log") {
+ try {
+ ulong index = std::stoull(name.substr(0, name.length() - 4));
+ indices.push_back(index);
+ } catch (...) {
+ // Skip invalid file names
+ continue;
+ }
+ }
+ }
+ closedir(dir);
+
+ // Sort indices
+ std::sort(indices.begin(), indices.end());
+
+ // Load each log entry
+ for (ulong index : indices) {
+ ptr entry = read_from_disk(index);
+ if (entry) {
+ std::lock_guard l(logs_lock_);
+ logs_[index] = entry;
+ }
+ }
+
+ // Set start_idx_ to the first non-zero index
+ {
+ std::lock_guard l(logs_lock_);
+ if (logs_.size() > 1) { // More than just dummy entry
+ auto entry = logs_.upper_bound(0);
+ if (entry != logs_.end()) {
+ start_idx_ = entry->first;
+ }
+ }
+ last_durable_idx_ = logs_.size() > 0 ? logs_.rbegin()->first : 0;
+ }
+}
+
+bool file_based_log_store::write_to_disk(ulong index, ptr& entry) {
+ std::string file_path = get_file_path(index);
+ std::ofstream outfile(file_path, std::ios::binary);
+ if (!outfile.is_open()) {
+ return false;
+ }
+
+ try {
+ ptr buf = entry->serialize();
+ outfile.write(reinterpret_cast(buf->data_begin()), buf->size());
+ outfile.close();
+ return true;
+ } catch (...) {
+ return false;
+ }
+}
+
+void file_based_log_store::delete_from_disk(ulong index) {
+ std::string file_path = get_file_path(index);
+ unlink(file_path.c_str());
+}
+
+ptr file_based_log_store::read_from_disk(ulong index) {
+ std::string file_path = get_file_path(index);
+ std::ifstream infile(file_path, std::ios::binary | std::ios::ate);
+ if (!infile.is_open()) {
+ return nullptr;
+ }
+
+ try {
+ std::streamsize size = infile.tellg();
+ infile.seekg(0, std::ios::beg);
+
+ ptr buf = buffer::alloc(size);
+ infile.read(reinterpret_cast(buf->data_begin()), size);
+ infile.close();
+
+ return log_entry::deserialize(*buf);
+ } catch (...) {
+ return nullptr;
+ }
+}
+
+ulong file_based_log_store::next_slot() const {
+ std::lock_guard l(logs_lock_);
+ // Exclude the dummy entry.
+ return start_idx_ + logs_.size() - 1;
+}
+
+ulong file_based_log_store::start_index() const {
+ return start_idx_;
+}
+
+ptr file_based_log_store::last_entry() const {
+ ulong next_idx = next_slot();
+ std::lock_guard l(logs_lock_);
+ auto entry = logs_.find( next_idx - 1 );
+ if (entry == logs_.end()) {
+ entry = logs_.find(0);
+ }
+
+ return make_clone(entry->second);
+}
+
+ulong file_based_log_store::append(ptr& entry) {
+ ptr clone = make_clone(entry);
+
+ std::lock_guard l(logs_lock_);
+ size_t idx = start_idx_ + logs_.size() - 1;
+ logs_[idx] = clone;
+
+ // Write to disk
+ write_to_disk(idx, clone);
+ last_durable_idx_ = idx;
+
+ return idx;
+}
+
+void file_based_log_store::write_at(ulong index, ptr& entry) {
+ ptr clone = make_clone(entry);
+
+ // Discard all logs equal to or greater than `index`.
+ std::lock_guard l(logs_lock_);
+ auto itr = logs_.lower_bound(index);
+ std::vector to_delete;
+ while (itr != logs_.end()) {
+ to_delete.push_back(itr->first);
+ itr = logs_.erase(itr);
+ }
+ logs_[index] = clone;
+
+ // Update disk
+ write_to_disk(index, clone);
+ for (ulong idx : to_delete) {
+ delete_from_disk(idx);
+ }
+ last_durable_idx_ = index;
+}
+
+ptr< std::vector< ptr > >
+ file_based_log_store::log_entries(ulong start, ulong end)
+{
+ ptr< std::vector< ptr > > ret =
+ cs_new< std::vector< ptr > >();
+
+ ret->resize(end - start);
+ ulong cc=0;
+ for (ulong ii = start ; ii < end ; ++ii) {
+ ptr src = nullptr;
+ { std::lock_guard l(logs_lock_);
+ auto entry = logs_.find(ii);
+ if (entry == logs_.end()) {
+ entry = logs_.find(0);
+ assert(0);
+ }
+ src = entry->second;
+ }
+ (*ret)[cc++] = make_clone(src);
+ }
+ return ret;
+}
+
+ptr>>
+ file_based_log_store::log_entries_ext(ulong start,
+ ulong end,
+ int64 batch_size_hint_in_bytes)
+{
+ ptr< std::vector< ptr > > ret =
+ cs_new< std::vector< ptr > >();
+
+ if (batch_size_hint_in_bytes < 0) {
+ return ret;
+ }
+
+ size_t accum_size = 0;
+ for (ulong ii = start ; ii < end ; ++ii) {
+ ptr src = nullptr;
+ { std::lock_guard l(logs_lock_);
+ auto entry = logs_.find(ii);
+ if (entry == logs_.end()) {
+ entry = logs_.find(0);
+ assert(0);
+ }
+ src = entry->second;
+ }
+ ret->push_back(make_clone(src));
+ accum_size += src->get_buf().size();
+ if (batch_size_hint_in_bytes &&
+ accum_size >= (ulong)batch_size_hint_in_bytes) break;
+ }
+ return ret;
+}
+
+ptr file_based_log_store::entry_at(ulong index) {
+ ptr src = nullptr;
+ { std::lock_guard l(logs_lock_);
+ auto entry = logs_.find(index);
+ if (entry == logs_.end()) {
+ entry = logs_.find(0);
+ }
+ src = entry->second;
+ }
+ return make_clone(src);
+}
+
+ulong file_based_log_store::term_at(ulong index) {
+ ulong term = 0;
+ { std::lock_guard l(logs_lock_);
+ auto entry = logs_.find(index);
+ if (entry == logs_.end()) {
+ entry = logs_.find(0);
+ }
+ term = entry->second->get_term();
+ }
+ return term;
+}
+
+ptr file_based_log_store::pack(ulong index, int32 cnt) {
+ std::vector< ptr > logs;
+
+ size_t size_total = 0;
+ for (ulong ii=index; ii le = nullptr;
+ { std::lock_guard l(logs_lock_);
+ le = logs_[ii];
+ }
+ assert(le.get());
+ ptr buf = le->serialize();
+ size_total += buf->size();
+ logs.push_back( buf );
+ }
+
+ ptr buf_out = buffer::alloc
+ ( sizeof(int32) +
+ cnt * sizeof(int32) +
+ size_total );
+ buf_out->pos(0);
+ buf_out->put((int32)cnt);
+
+ for (auto& entry: logs) {
+ ptr& bb = entry;
+ buf_out->put((int32)bb->size());
+ buf_out->put(*bb);
+ }
+ return buf_out;
+}
+
+void file_based_log_store::apply_pack(ulong index, buffer& pack) {
+ pack.pos(0);
+ int32 num_logs = pack.get_int();
+
+ for (int32 ii=0; ii buf_local = buffer::alloc(buf_size);
+ pack.get(buf_local);
+
+ ptr le = log_entry::deserialize(*buf_local);
+ { std::lock_guard l(logs_lock_);
+ logs_[cur_idx] = le;
+ }
+ // Write to disk
+ write_to_disk(cur_idx, le);
+ }
+
+ { std::lock_guard l(logs_lock_);
+ auto entry = logs_.upper_bound(0);
+ if (entry != logs_.end()) {
+ start_idx_ = entry->first;
+ } else {
+ start_idx_ = 1;
+ }
+ // Update last durable index
+ if (!logs_.empty()) {
+ last_durable_idx_ = logs_.rbegin()->first;
+ }
+ }
+}
+
+bool file_based_log_store::compact(ulong last_log_index) {
+ std::lock_guard l(logs_lock_);
+ for (ulong ii = start_idx_; ii <= last_log_index; ++ii) {
+ auto entry = logs_.find(ii);
+ if (entry != logs_.end()) {
+ logs_.erase(entry);
+ // Delete from disk
+ delete_from_disk(ii);
+ }
+ }
+
+ // WARNING:
+ // Even though nothing has been erased,
+ // we should set `start_idx_` to new index.
+ if (start_idx_ <= last_log_index) {
+ start_idx_ = last_log_index + 1;
+ }
+ return true;
+}
+
+bool file_based_log_store::flush() {
+ // Already flushed on each write in this simple implementation
+ last_durable_idx_ = next_slot() - 1;
+ return true;
+}
+
+void file_based_log_store::close() {
+ flush();
+}
+
+ulong file_based_log_store::last_durable_index() {
+ return last_durable_idx_.load();
+}
+
+}
diff --git a/examples/file_based_log_store.hxx b/examples/file_based_log_store.hxx
new file mode 100644
index 00000000..505379e8
--- /dev/null
+++ b/examples/file_based_log_store.hxx
@@ -0,0 +1,165 @@
+/************************************************************************
+Copyright 2017-2019 eBay Inc.
+Author/Developer(s): Jung-Sang Ahn
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+**************************************************************************/
+
+#pragma once
+
+#include "event_awaiter.hxx"
+#include "internal_timer.hxx"
+#include "log_store.hxx"
+
+#include
+#include
+#include