diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index ec1d08f1..4edd7f49 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -3,7 +3,8 @@ add_executable(calc_server calculator/calc_server.cxx logger.cc - in_memory_log_store.cxx) + in_memory_log_store.cxx + file_based_log_store.cxx) # add_dependencies(calc_server # ${LIBRARY_NAME}) @@ -14,7 +15,8 @@ target_link_libraries(calc_server add_executable(echo_server echo/echo_server.cxx logger.cc - in_memory_log_store.cxx) + in_memory_log_store.cxx + file_based_log_store.cxx) # add_dependencies(echo_server # ${LIBRARY_NAME}) target_link_libraries(echo_server @@ -24,7 +26,8 @@ target_link_libraries(echo_server add_executable(quick_start quick_start.cxx logger.cc - in_memory_log_store.cxx) + in_memory_log_store.cxx + file_based_log_store.cxx) # add_dependencies(quick_start # ${LIBRARY_NAME}) target_link_libraries(quick_start diff --git a/examples/calculator/calc_server.cxx b/examples/calculator/calc_server.cxx index 256f8aeb..82c86410 100644 --- a/examples/calculator/calc_server.cxx +++ b/examples/calculator/calc_server.cxx @@ -17,6 +17,7 @@ limitations under the License. #include "calc_state_machine.hxx" #include "in_memory_state_mgr.hxx" +#include "file_based_state_mgr.hxx" #include "logger_wrapper.hxx" #include "nuraft.hxx" @@ -220,6 +221,11 @@ void check_additional_flags(int argc, char** argv) { CALL_TYPE = raft_params::async_handler; } else if (strcmp(argv[ii], "--async-snapshot-creation") == 0) { ASYNC_SNAPSHOT_CREATION = true; + } else if (strcmp(argv[ii], "--persistent") == 0) { + set_persistent_storage(true); + } else if (strncmp(argv[ii], "--persistent-dir=", 17) == 0) { + std::string dir = argv[ii] + 17; + set_persistent_storage(true, dir); } } } @@ -231,8 +237,11 @@ void calc_usage(int argc, char** argv) { ss << std::endl << std::endl; ss << " options:" << std::endl; ss << " --async-handler: use async type handler." << std::endl; - ss << " --async-snapshot-creation: create snapshots asynchronously." - << std::endl << std::endl; + ss << " --async-snapshot-creation: create snapshots asynchronously." << std::endl; + ss << " --persistent: enable file-based persistent storage." << std::endl; + ss << " --persistent-dir=: specify directory for persistent storage" << std::endl; + ss << " (default: ./nuraft_data)." << std::endl + << std::endl; std::cout << ss.str(); exit(0); diff --git a/examples/example_common.hxx b/examples/example_common.hxx index e6486a33..deaa58f0 100644 --- a/examples/example_common.hxx +++ b/examples/example_common.hxx @@ -21,6 +21,21 @@ using namespace nuraft; using raft_result = cmd_result< ptr >; +// Global flag to enable persistent storage. +// If true, file-based storage will be used instead of in-memory. +static bool USE_PERSISTENT_STORAGE = false; + +// Directory for persistent storage. +static std::string PERSISTENT_STORAGE_DIR = "./nuraft_data"; + +// Helper to set persistent storage options. +inline void set_persistent_storage(bool enable, const std::string& dir = "./nuraft_data") { + USE_PERSISTENT_STORAGE = enable; + if (enable) { + PERSISTENT_STORAGE_DIR = dir; + } +} + struct server_stuff { server_stuff() : server_id_(1) @@ -155,10 +170,18 @@ void init_raft(ptr sm_instance) { ptr log_wrap = cs_new( log_file_name, 4 ); stuff.raft_logger_ = log_wrap; + // State manager: use persistent or in-memory based on flag. + if (USE_PERSISTENT_STORAGE) { + std::cout << "using persistent storage: " << PERSISTENT_STORAGE_DIR << std::endl; + stuff.smgr_ = cs_new( stuff.server_id_, + stuff.endpoint_, + PERSISTENT_STORAGE_DIR ); + } else { + std::cout << "using in-memory storage" << std::endl; + stuff.smgr_ = cs_new( stuff.server_id_, + stuff.endpoint_ ); + } // State machine. - stuff.smgr_ = cs_new( stuff.server_id_, - stuff.endpoint_ ); - // State manager. stuff.sm_ = sm_instance; // ASIO options. diff --git a/examples/file_based_log_store.cxx b/examples/file_based_log_store.cxx new file mode 100644 index 00000000..f25e1ed5 --- /dev/null +++ b/examples/file_based_log_store.cxx @@ -0,0 +1,403 @@ +/************************************************************************ +Copyright 2017-2019 eBay Inc. +Author/Developer(s): Jung-Sang Ahn + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +**************************************************************************/ + +#include "file_based_log_store.hxx" + +#include "nuraft.hxx" + +#include +#include +#include +#include +#include + +namespace nuraft { + +file_based_log_store::file_based_log_store(const std::string& log_dir, int server_id) + : log_dir_(log_dir) + , server_id_(server_id) + , start_idx_(1) + , last_durable_idx_(0) +{ + // Create server-specific subdirectory + std::string server_dir = log_dir_ + "/server_" + std::to_string(server_id_); + mkdir(server_dir.c_str(), 0755); + + // Load existing log entries from disk + load_from_disk(); + + // Ensure we have at least the dummy entry at index 0 + { + std::lock_guard l(logs_lock_); + if (logs_.find(0) == logs_.end()) { + ptr buf = buffer::alloc(sz_ulong); + logs_[0] = cs_new(0, buf); + } + } +} + +file_based_log_store::~file_based_log_store() { + close(); +} + +ptr file_based_log_store::make_clone(const ptr& entry) { + // NOTE: + // Timestamp is used only when `replicate_log_timestamp_` option is on. + // Otherwise, log store does not need to store or load it. + ptr clone = cs_new + ( entry->get_term(), + buffer::clone( entry->get_buf() ), + entry->get_val_type(), + entry->get_timestamp(), + entry->has_crc32(), + entry->get_crc32(), + false ); + return clone; +} + +std::string file_based_log_store::get_file_path(ulong index) const { + // Format: /server_/.log + return log_dir_ + "/server_" + std::to_string(server_id_) + "/" + + std::to_string(index) + ".log"; +} + +void file_based_log_store::load_from_disk() { + std::string server_dir = log_dir_ + "/server_" + std::to_string(server_id_); + DIR* dir = opendir(server_dir.c_str()); + if (!dir) { + // Directory doesn't exist yet, nothing to load + return; + } + + struct dirent* entry; + std::vector indices; + + // Scan directory for log files + while ((entry = readdir(dir)) != nullptr) { + std::string name(entry->d_name); + if (name.length() > 4 && name.substr(name.length() - 4) == ".log") { + try { + ulong index = std::stoull(name.substr(0, name.length() - 4)); + indices.push_back(index); + } catch (...) { + // Skip invalid file names + continue; + } + } + } + closedir(dir); + + // Sort indices + std::sort(indices.begin(), indices.end()); + + // Load each log entry + for (ulong index : indices) { + ptr entry = read_from_disk(index); + if (entry) { + std::lock_guard l(logs_lock_); + logs_[index] = entry; + } + } + + // Set start_idx_ to the first non-zero index + { + std::lock_guard l(logs_lock_); + if (logs_.size() > 1) { // More than just dummy entry + auto entry = logs_.upper_bound(0); + if (entry != logs_.end()) { + start_idx_ = entry->first; + } + } + last_durable_idx_ = logs_.size() > 0 ? logs_.rbegin()->first : 0; + } +} + +bool file_based_log_store::write_to_disk(ulong index, ptr& entry) { + std::string file_path = get_file_path(index); + std::ofstream outfile(file_path, std::ios::binary); + if (!outfile.is_open()) { + return false; + } + + try { + ptr buf = entry->serialize(); + outfile.write(reinterpret_cast(buf->data_begin()), buf->size()); + outfile.close(); + return true; + } catch (...) { + return false; + } +} + +void file_based_log_store::delete_from_disk(ulong index) { + std::string file_path = get_file_path(index); + unlink(file_path.c_str()); +} + +ptr file_based_log_store::read_from_disk(ulong index) { + std::string file_path = get_file_path(index); + std::ifstream infile(file_path, std::ios::binary | std::ios::ate); + if (!infile.is_open()) { + return nullptr; + } + + try { + std::streamsize size = infile.tellg(); + infile.seekg(0, std::ios::beg); + + ptr buf = buffer::alloc(size); + infile.read(reinterpret_cast(buf->data_begin()), size); + infile.close(); + + return log_entry::deserialize(*buf); + } catch (...) { + return nullptr; + } +} + +ulong file_based_log_store::next_slot() const { + std::lock_guard l(logs_lock_); + // Exclude the dummy entry. + return start_idx_ + logs_.size() - 1; +} + +ulong file_based_log_store::start_index() const { + return start_idx_; +} + +ptr file_based_log_store::last_entry() const { + ulong next_idx = next_slot(); + std::lock_guard l(logs_lock_); + auto entry = logs_.find( next_idx - 1 ); + if (entry == logs_.end()) { + entry = logs_.find(0); + } + + return make_clone(entry->second); +} + +ulong file_based_log_store::append(ptr& entry) { + ptr clone = make_clone(entry); + + std::lock_guard l(logs_lock_); + size_t idx = start_idx_ + logs_.size() - 1; + logs_[idx] = clone; + + // Write to disk + write_to_disk(idx, clone); + last_durable_idx_ = idx; + + return idx; +} + +void file_based_log_store::write_at(ulong index, ptr& entry) { + ptr clone = make_clone(entry); + + // Discard all logs equal to or greater than `index`. + std::lock_guard l(logs_lock_); + auto itr = logs_.lower_bound(index); + std::vector to_delete; + while (itr != logs_.end()) { + to_delete.push_back(itr->first); + itr = logs_.erase(itr); + } + logs_[index] = clone; + + // Update disk + write_to_disk(index, clone); + for (ulong idx : to_delete) { + delete_from_disk(idx); + } + last_durable_idx_ = index; +} + +ptr< std::vector< ptr > > + file_based_log_store::log_entries(ulong start, ulong end) +{ + ptr< std::vector< ptr > > ret = + cs_new< std::vector< ptr > >(); + + ret->resize(end - start); + ulong cc=0; + for (ulong ii = start ; ii < end ; ++ii) { + ptr src = nullptr; + { std::lock_guard l(logs_lock_); + auto entry = logs_.find(ii); + if (entry == logs_.end()) { + entry = logs_.find(0); + assert(0); + } + src = entry->second; + } + (*ret)[cc++] = make_clone(src); + } + return ret; +} + +ptr>> + file_based_log_store::log_entries_ext(ulong start, + ulong end, + int64 batch_size_hint_in_bytes) +{ + ptr< std::vector< ptr > > ret = + cs_new< std::vector< ptr > >(); + + if (batch_size_hint_in_bytes < 0) { + return ret; + } + + size_t accum_size = 0; + for (ulong ii = start ; ii < end ; ++ii) { + ptr src = nullptr; + { std::lock_guard l(logs_lock_); + auto entry = logs_.find(ii); + if (entry == logs_.end()) { + entry = logs_.find(0); + assert(0); + } + src = entry->second; + } + ret->push_back(make_clone(src)); + accum_size += src->get_buf().size(); + if (batch_size_hint_in_bytes && + accum_size >= (ulong)batch_size_hint_in_bytes) break; + } + return ret; +} + +ptr file_based_log_store::entry_at(ulong index) { + ptr src = nullptr; + { std::lock_guard l(logs_lock_); + auto entry = logs_.find(index); + if (entry == logs_.end()) { + entry = logs_.find(0); + } + src = entry->second; + } + return make_clone(src); +} + +ulong file_based_log_store::term_at(ulong index) { + ulong term = 0; + { std::lock_guard l(logs_lock_); + auto entry = logs_.find(index); + if (entry == logs_.end()) { + entry = logs_.find(0); + } + term = entry->second->get_term(); + } + return term; +} + +ptr file_based_log_store::pack(ulong index, int32 cnt) { + std::vector< ptr > logs; + + size_t size_total = 0; + for (ulong ii=index; ii le = nullptr; + { std::lock_guard l(logs_lock_); + le = logs_[ii]; + } + assert(le.get()); + ptr buf = le->serialize(); + size_total += buf->size(); + logs.push_back( buf ); + } + + ptr buf_out = buffer::alloc + ( sizeof(int32) + + cnt * sizeof(int32) + + size_total ); + buf_out->pos(0); + buf_out->put((int32)cnt); + + for (auto& entry: logs) { + ptr& bb = entry; + buf_out->put((int32)bb->size()); + buf_out->put(*bb); + } + return buf_out; +} + +void file_based_log_store::apply_pack(ulong index, buffer& pack) { + pack.pos(0); + int32 num_logs = pack.get_int(); + + for (int32 ii=0; ii buf_local = buffer::alloc(buf_size); + pack.get(buf_local); + + ptr le = log_entry::deserialize(*buf_local); + { std::lock_guard l(logs_lock_); + logs_[cur_idx] = le; + } + // Write to disk + write_to_disk(cur_idx, le); + } + + { std::lock_guard l(logs_lock_); + auto entry = logs_.upper_bound(0); + if (entry != logs_.end()) { + start_idx_ = entry->first; + } else { + start_idx_ = 1; + } + // Update last durable index + if (!logs_.empty()) { + last_durable_idx_ = logs_.rbegin()->first; + } + } +} + +bool file_based_log_store::compact(ulong last_log_index) { + std::lock_guard l(logs_lock_); + for (ulong ii = start_idx_; ii <= last_log_index; ++ii) { + auto entry = logs_.find(ii); + if (entry != logs_.end()) { + logs_.erase(entry); + // Delete from disk + delete_from_disk(ii); + } + } + + // WARNING: + // Even though nothing has been erased, + // we should set `start_idx_` to new index. + if (start_idx_ <= last_log_index) { + start_idx_ = last_log_index + 1; + } + return true; +} + +bool file_based_log_store::flush() { + // Already flushed on each write in this simple implementation + last_durable_idx_ = next_slot() - 1; + return true; +} + +void file_based_log_store::close() { + flush(); +} + +ulong file_based_log_store::last_durable_index() { + return last_durable_idx_.load(); +} + +} diff --git a/examples/file_based_log_store.hxx b/examples/file_based_log_store.hxx new file mode 100644 index 00000000..505379e8 --- /dev/null +++ b/examples/file_based_log_store.hxx @@ -0,0 +1,165 @@ +/************************************************************************ +Copyright 2017-2019 eBay Inc. +Author/Developer(s): Jung-Sang Ahn + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +**************************************************************************/ + +#pragma once + +#include "event_awaiter.hxx" +#include "internal_timer.hxx" +#include "log_store.hxx" + +#include +#include +#include +#include +#include + +namespace nuraft { + +class raft_server; + +/** + * Simple file-based persistent log store implementation. + * + * This is a naive implementation that persists log entries to individual files + * in a specified directory. Each log entry is stored as a separate file named + * by its index (e.g., "1.log", "2.log", etc.). + * + * This is intended as a minimal example to demonstrate how to make a log store + * persistent. For production use, consider using more sophisticated storage + * engines (e.g., RocksDB, LevelDB) or implementing WAL (Write-Ahead Log) with + * compaction. + */ +class file_based_log_store : public log_store { +public: + /** + * Constructor. + * + * @param log_dir Directory where log files will be stored. + * @param server_id Server ID (used to create unique subdirectory). + */ + file_based_log_store(const std::string& log_dir, int server_id); + + ~file_based_log_store(); + + __nocopy__(file_based_log_store); + +public: + ulong next_slot() const; + + ulong start_index() const; + + ptr last_entry() const; + + ulong append(ptr& entry); + + void write_at(ulong index, ptr& entry); + + ptr>> log_entries(ulong start, ulong end); + + ptr>> log_entries_ext( + ulong start, ulong end, int64 batch_size_hint_in_bytes = 0); + + ptr entry_at(ulong index); + + ulong term_at(ulong index); + + ptr pack(ulong index, int32 cnt); + + void apply_pack(ulong index, buffer& pack); + + bool compact(ulong last_log_index); + + bool flush(); + + void close(); + + ulong last_durable_index(); + +private: + /** + * Load log entries from disk on startup. + * Rebuilds the in-memory index and determines the start index. + */ + void load_from_disk(); + + /** + * Write a log entry to disk. + * @param index Log index. + * @param entry Log entry to write. + * @return true if successful, false otherwise. + */ + bool write_to_disk(ulong index, ptr& entry); + + /** + * Delete a log entry file from disk. + * @param index Log index. + */ + void delete_from_disk(ulong index); + + /** + * Read a log entry from disk. + * @param index Log index. + * @return Log entry, or nullptr if not found. + */ + ptr read_from_disk(ulong index); + + /** + * Get the file path for a given log index. + * @param index Log index. + * @return Full file path. + */ + std::string get_file_path(ulong index) const; + + /** + * Make a clone of a log entry (same as inmem version). + */ + static ptr make_clone(const ptr& entry); + +private: + /** + * Directory where log files are stored. + */ + std::string log_dir_; + + /** + * Server ID. + */ + int server_id_; + + /** + * In-memory cache of log entries for faster access. + * Map of . + */ + std::map> logs_; + + /** + * Lock for `logs_`. + */ + mutable std::mutex logs_lock_; + + /** + * The index of the first log. + */ + std::atomic start_idx_; + + /** + * Last flushed index to disk. + */ + std::atomic last_durable_idx_; +}; + +} diff --git a/examples/file_based_state_mgr.hxx b/examples/file_based_state_mgr.hxx new file mode 100644 index 00000000..ca16efb2 --- /dev/null +++ b/examples/file_based_state_mgr.hxx @@ -0,0 +1,206 @@ +/************************************************************************ +Copyright 2017-2019 eBay Inc. +Author/Developer(s): Jung-Sang Ahn + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +**************************************************************************/ + +#pragma once + +#include "file_based_log_store.hxx" + +#include "nuraft.hxx" +#include +#include +#include +#include + +namespace nuraft { + +/** + * Simple file-based persistent state manager implementation. + * + * This is a naive implementation that persists cluster configuration and + * server state to files in a specified directory. + * + * This is intended as a minimal example to demonstrate how to make a state + * manager persistent. For production use, consider using more robust + * serialization and storage mechanisms. + */ +class file_based_state_mgr: public state_mgr { +public: + file_based_state_mgr(int srv_id, + const std::string& endpoint, + const std::string& state_dir) + : my_id_(srv_id) + , my_endpoint_(endpoint) + , state_dir_(state_dir) + , cur_log_store_( cs_new(state_dir, srv_id) ) + { + // Log to stdout since logger not yet available + std::cout << "[TRACE][file_based_state_mgr_ctor] Creating state manager for server " + << srv_id << " at " << endpoint << ", state dir: " << state_dir << std::endl; + + // Create state directory if it doesn't exist + mkdir(state_dir.c_str(), 0755); + + my_srv_config_ = cs_new( srv_id, endpoint ); + + // Try to load existing state from disk + saved_config_ = load_config(); + if (!saved_config_) { + // No existing config, create initial one + saved_config_ = cs_new(); + saved_config_->get_servers().push_back(my_srv_config_); + save_config(*saved_config_); + std::cout << "[TRACE][file_based_state_mgr_ctor] Initial cluster config created with 1 server" << std::endl; + } else { + std::cout << "[TRACE][file_based_state_mgr_ctor] Loaded existing cluster config with " + << saved_config_->get_servers().size() << " servers" << std::endl; + } + + // Try to load existing server state from disk + saved_state_ = read_state(); + } + + ~file_based_state_mgr() {} + + ptr load_config() { + // Try to read from disk + std::string config_path = get_config_path(); + std::ifstream infile(config_path, std::ios::binary | std::ios::ate); + if (!infile.is_open()) { + // File doesn't exist yet, return nullptr + return nullptr; + } + + try { + std::streamsize size = infile.tellg(); + infile.seekg(0, std::ios::beg); + + ptr buf = buffer::alloc(size); + infile.read(reinterpret_cast(buf->data_begin()), size); + infile.close(); + + std::cout << "[TRACE][file_based_state_mgr] Loaded cluster config from disk" << std::endl; + return cluster_config::deserialize(*buf); + } catch (...) { + std::cerr << "[ERROR][file_based_state_mgr] Failed to load cluster config from disk" << std::endl; + return nullptr; + } + } + + void save_config(const cluster_config& config) { + // Write to disk + std::string config_path = get_config_path(); + std::ofstream outfile(config_path, std::ios::binary); + if (!outfile.is_open()) { + std::cerr << "[ERROR][file_based_state_mgr] Failed to open config file for writing: " + << config_path << std::endl; + return; + } + + try { + ptr buf = config.serialize(); + outfile.write(reinterpret_cast(buf->data_begin()), buf->size()); + outfile.close(); + + // Also keep in memory + saved_config_ = cluster_config::deserialize(*buf); + std::cout << "[TRACE][file_based_state_mgr] Saved cluster config to disk" << std::endl; + } catch (...) { + std::cerr << "[ERROR][file_based_state_mgr] Failed to save cluster config to disk" << std::endl; + } + } + + void save_state(const srv_state& state) { + // Write to disk + std::string state_path = get_state_path(); + std::ofstream outfile(state_path, std::ios::binary); + if (!outfile.is_open()) { + std::cerr << "[ERROR][file_based_state_mgr] Failed to open state file for writing: " + << state_path << std::endl; + return; + } + + try { + ptr buf = state.serialize(); + outfile.write(reinterpret_cast(buf->data_begin()), buf->size()); + outfile.close(); + + // Also keep in memory + saved_state_ = srv_state::deserialize(*buf); + std::cout << "[TRACE][file_based_state_mgr] Saved server state to disk" << std::endl; + } catch (...) { + std::cerr << "[ERROR][file_based_state_mgr] Failed to save server state to disk" << std::endl; + } + } + + ptr read_state() { + // Try to read from disk + std::string state_path = get_state_path(); + std::ifstream infile(state_path, std::ios::binary | std::ios::ate); + if (!infile.is_open()) { + // File doesn't exist yet, return default state + return cs_new(); + } + + try { + std::streamsize size = infile.tellg(); + infile.seekg(0, std::ios::beg); + + ptr buf = buffer::alloc(size); + infile.read(reinterpret_cast(buf->data_begin()), size); + infile.close(); + + std::cout << "[TRACE][file_based_state_mgr] Loaded server state from disk" << std::endl; + return srv_state::deserialize(*buf); + } catch (...) { + std::cerr << "[ERROR][file_based_state_mgr] Failed to load server state from disk" << std::endl; + return cs_new(); + } + } + + ptr load_log_store() { + return cur_log_store_; + } + + int32 server_id() { + return my_id_; + } + + void system_exit(const int exit_code) { + } + + ptr get_srv_config() const { return my_srv_config_; } + +private: + std::string get_config_path() const { + return state_dir_ + "/server_" + std::to_string(my_id_) + "_cluster.config"; + } + + std::string get_state_path() const { + return state_dir_ + "/server_" + std::to_string(my_id_) + "_state.bin"; + } + +private: + int my_id_; + std::string my_endpoint_; + std::string state_dir_; + ptr cur_log_store_; + ptr my_srv_config_; + ptr saved_config_; + ptr saved_state_; +}; + +}