File I/O Utilities#
2026-09-01
2 min read time
#include <cuvs/util/file_io.hpp>
namespace cuvs::util
File I/O Utilities#
-
template<typename T>
std::pair<file_descriptor, size_t> create_numpy_file( - const std::string &path,
- const std::vector<size_t> &shape
Create a numpy file with pre-allocated space and write the header.
Opens a file, writes a numpy header for the given shape/dtype, and pre-allocates space for the data. This is useful for memory-mapped or streaming writes.
- Template Parameters:
T – Data type for the numpy array
- Parameters:
path – File path to create
shape – Shape of the numpy array (e.g., {rows, cols} for 2D)
- Returns:
Pair of (file_descriptor, header_size)
- void read_large_file(
- const file_descriptor &fd,
- void *dest_ptr,
- const size_t total_bytes,
- const uint64_t file_offset
Read large file in chunks using pread.
Reads a file in chunks to avoid issues with very large reads. Uses pread for thread-safe, offset-based reading.
- Parameters:
fd – File descriptor to read from
dest_ptr – Destination buffer
total_bytes – Total bytes to read
file_offset – Starting offset in file
- void write_large_file(
- const file_descriptor &fd,
- const void *data_ptr,
- const size_t total_bytes,
- const uint64_t file_offset
Write large file in chunks using pwrite.
Writes data to a file in chunks to avoid issues with very large writes. Uses pwrite for thread-safe, offset-based writing.
- Parameters:
fd – File descriptor to write to
data_ptr – Source data buffer
total_bytes – Total bytes to write
file_offset – Starting offset in file
-
class fd_streambuf : public std::streambuf#
- #include <file_io.hpp>
Streambuf that reads from a POSIX file descriptor.
-
class fd_istream : public std::istream#
- #include <file_io.hpp>
Istream that reads from a POSIX file descriptor.
-
class file_descriptor#
- #include <file_io.hpp>
RAII wrapper for POSIX file descriptors.
Manages file descriptor lifecycle with automatic cleanup. Non-copyable, move-only.
Public Functions
- inline fd_istream make_istream(
Create an input stream from this file descriptor.
Creates an istream that reads directly from the file descriptor using POSIX read(). The original descriptor remains valid and unchanged (we duplicate it internally). Returns the stream by value (uses move semantics).
- Returns:
fd_istream (movable istream)
-
class buffered_ofstream#
- #include <file_io.hpp>
Buffered output stream wrapper.
Wraps an std::ostream with a buffer to improve write performance by reducing the number of system calls. Automatically flushes on destruction. Non-copyable, non-movable.