From bf501bc2dea36af3095d3bdd757ff72c8a1de087 Mon Sep 17 00:00:00 2001 From: moonshadow565 Date: Sun, 3 Jul 2022 18:09:06 +0200 Subject: [PATCH] make IO an interface --- lib/rlib/iofile.cpp | 39 ++++++++++++------- lib/rlib/iofile.hpp | 88 ++++++++++++++++++++++++++++++++---------- lib/rlib/rbundle.cpp | 8 ++-- lib/rlib/rbundle.hpp | 2 +- lib/rlib/rcache.cpp | 13 ++----- lib/rlib/rcache.hpp | 5 ++- lib/rlib/rmanifest.cpp | 2 +- src/rbun_chk.cpp | 2 +- src/rbun_ex.cpp | 8 ++-- src/rbun_ls.cpp | 2 +- src/rbun_merge.cpp | 2 +- src/rbun_usage.cpp | 2 +- src/rman_bl.cpp | 2 +- src/rman_dl.cpp | 10 ++--- src/rman_ls.cpp | 2 +- 15 files changed, 121 insertions(+), 66 deletions(-) diff --git a/lib/rlib/iofile.cpp b/lib/rlib/iofile.cpp index d5733be..6a79d7e 100644 --- a/lib/rlib/iofile.cpp +++ b/lib/rlib/iofile.cpp @@ -53,7 +53,7 @@ private: bool no_interupt; }; -IOFile::IOFile(fs::path const& path, Flags flags) { +IO::File::File(fs::path const& path, Flags flags) { rlib_trace("path: %s\n", path.generic_string().c_str()); if ((flags & WRITE) && path.has_parent_path()) { fs::create_directories(path.parent_path()); @@ -69,7 +69,7 @@ IOFile::IOFile(fs::path const& path, Flags flags) { attributes |= FILE_FLAG_RANDOM_ACCESS; } auto const fd = ::CreateFile(path.string().c_str(), access, share, 0, disposition, attributes, 0); - if ((std::intptr_t)fd == 0 || (std::intptr_t)fd == -1) [[unlikely]] { + if (!fd || fd == INVALID_HANDLE_VALUE) [[unlikely]] { auto ec = std::error_code((int)GetLastError(), std::system_category()); throw_error("CreateFile: ", ec); } @@ -82,15 +82,28 @@ IOFile::IOFile(fs::path const& path, Flags flags) { impl_ = {.fd = (std::intptr_t)fd, .size = (std::size_t)size.QuadPart, .flags = flags}; } -IOFile::~IOFile() noexcept { +IO::File::~File() noexcept { if (auto impl = std::exchange(impl_, {}); impl.fd) { ::CloseHandle((HANDLE)impl.fd); } } -auto IOFile::resize(std::size_t offset, std::size_t count) noexcept -> bool { - constexpr std::uint64_t MASK = 1ull << 63; - if (!impl_.fd) { +auto IO::File::shrink_to_fit() noexcept -> bool { + if (!impl_.fd || !(impl_.flags & WRITE)) { + return false; + } + return true; +} + +auto IO::File::reserve(std::size_t offset, std::size_t count) noexcept -> bool { + if (!impl_.fd || !(impl_.flags & WRITE)) { + return false; + } + return true; +} + +auto IO::File::resize(std::size_t offset, std::size_t count) noexcept -> bool { + if (!impl_.fd || !(impl_.flags & WRITE)) { return false; } std::uint64_t const total = (std::uint64_t)offset + count; @@ -108,9 +121,9 @@ auto IOFile::resize(std::size_t offset, std::size_t count) noexcept -> bool { return true; } -auto IOFile::read(std::size_t offset, std::span dst) const noexcept -> bool { +auto IO::File::read(std::size_t offset, std::span dst) const noexcept -> bool { constexpr std::size_t CHUNK = 0x1000'0000; - if (!impl_.fd) { + if (!impl_.fd || !(impl_.flags & WRITE)) { return false; } while (!dst.empty()) { @@ -127,16 +140,16 @@ auto IOFile::read(std::size_t offset, std::span dst) const noexcept -> boo return true; } -auto IOFile::write(std::uint64_t offset, std::span src, bool no_interupt) noexcept -> bool { +auto IO::File::write(std::uint64_t offset, std::span src) noexcept -> bool { constexpr std::size_t CHUNK = 0x4000'0000; - if (!impl_.fd) { + if (!impl_.fd || !(impl_.flags & WRITE)) { return false; } std::size_t const write_end = offset + src.size(); if (write_end < offset || write_end < src.size()) { return false; } - NoInterupt no_interupt_lock(no_interupt); + NoInterupt no_interupt_lock(impl_.flags & NO_INTERUPT); while (!src.empty()) { DWORD wanted = (DWORD)std::min(CHUNK, src.size()); OVERLAPPED off = {.Offset = (std::uint32_t)offset, .OffsetHigh = (std::uint32_t)(offset >> 32)}; @@ -157,7 +170,7 @@ auto IOFile::write(std::uint64_t offset, std::span src, bool no_inte return true; } -auto IOFile::copy(std::size_t offset, std::size_t count) const -> std::span { +auto IO::File::copy(std::size_t offset, std::size_t count) const -> std::span { thread_local auto result = std::vector(); if (result.size() < count) { result.clear(); @@ -169,4 +182,4 @@ auto IOFile::copy(std::size_t offset, std::size_t count) const -> std::span std::intptr_t = 0; - constexpr IOFile& operator=(IOFile&& other) noexcept { + virtual auto flags() const noexcept -> Flags = 0; + + virtual auto size() const noexcept -> std::size_t = 0; + + virtual auto shrink_to_fit() noexcept -> bool = 0; + + virtual auto reserve(std::size_t offset, std::size_t count) noexcept -> bool = 0; + + virtual auto resize(std::size_t offset, std::size_t count) noexcept -> bool = 0; + + virtual auto read(std::size_t offset, std::span dst) const noexcept -> bool = 0; + + virtual auto write(std::size_t offset, std::span src) noexcept -> bool = 0; + + virtual auto copy(std::size_t offset, std::size_t count) const -> std::span = 0; + + private: + constexpr IO() noexcept = default; + constexpr IO(IO&& other) noexcept = default; + constexpr IO(IO const& other) noexcept = delete; + constexpr IO& operator=(IO const& other) noexcept = delete; + constexpr IO& operator=(IO&& other) noexcept = default; + }; + + enum IO::Flags : unsigned { + READ = 0, + WRITE = 1 << 0, + SEQUENTIAL = 1 << 1, + RANDOM_ACCESS = 1 << 2, + NO_INTERUPT = 1 << 3, + NO_OVERGROW = 1 << 4, + }; + + constexpr auto operator|(IO::Flags lhs, IO::Flags rhs) noexcept -> IO::Flags { + return (IO::Flags)((unsigned)lhs | (unsigned)rhs); + } + + constexpr auto operator^(IO::Flags lhs, IO::Flags rhs) noexcept -> IO::Flags { + return (IO::Flags)((unsigned)lhs ^ (unsigned)rhs); + } + + constexpr auto operator&(IO::Flags lhs, IO::Flags rhs) noexcept -> IO::Flags { + return (IO::Flags)((unsigned)lhs & (unsigned)rhs); + } + + struct IO::File final : IO { + constexpr File() noexcept = default; + + constexpr File(File&& other) noexcept : impl_(std::exchange(other.impl_, {})) {} + + constexpr File& operator=(File&& other) noexcept { impl_ = std::exchange(other.impl_, {}); return *this; } - constexpr IOFile& operator=(IOFile const& other) noexcept = delete; + File(fs::path const& path, Flags flags); - constexpr explicit operator bool() const noexcept { return impl_.fd; } + ~File() noexcept; - constexpr bool operator!() const noexcept { return !impl_.fd; } + auto fd() const noexcept -> std::intptr_t override { return impl_.fd; } - IOFile(fs::path const& path, bool write) : IOFile(path, WRITE) {} + auto flags() const noexcept -> Flags override { return impl_.flags; } - IOFile(fs::path const& path, Flags flags); + auto size() const noexcept -> std::size_t override { return impl_.size; } - ~IOFile() noexcept; + auto shrink_to_fit() noexcept -> bool override; - auto size() const noexcept -> std::size_t { return impl_.size; } + auto reserve(std::size_t offset, std::size_t count) noexcept -> bool override; - auto resize(std::size_t offset, std::size_t count) noexcept -> bool; + auto resize(std::size_t offset, std::size_t count) noexcept -> bool override; - auto read(std::size_t offset, std::span dst) const noexcept -> bool; + auto read(std::size_t offset, std::span dst) const noexcept -> bool override; - auto write(std::size_t offset, std::span src, bool no_interupt = false) noexcept -> bool; + auto write(std::size_t offset, std::span src) noexcept -> bool override; - auto copy(std::size_t offset, std::size_t count) const -> std::span; + auto copy(std::size_t offset, std::size_t count) const -> std::span override; private: struct Impl { diff --git a/lib/rlib/rbundle.cpp b/lib/rlib/rbundle.cpp index 63901b1..4acb18b 100644 --- a/lib/rlib/rbundle.cpp +++ b/lib/rlib/rbundle.cpp @@ -10,13 +10,13 @@ using namespace rlib; -auto RBUN::read(IOFile const& file, bool no_lookup) -> RBUN { +auto RBUN::read(IO const& io, bool no_lookup) -> RBUN { auto result = RBUN{}; auto footer = Footer{}; - auto file_size = file.size(); + auto file_size = io.size(); rlib_assert(file_size >= sizeof(Footer)); - rlib_assert(file.read(file_size - sizeof(footer), {(char*)&footer, sizeof(footer)})); + rlib_assert(io.read(file_size - sizeof(footer), {(char*)&footer, sizeof(footer)})); rlib_assert(footer.magic == Footer::MAGIC); rlib_assert(footer.version == Footer::VERSION || footer.version == 1); @@ -27,7 +27,7 @@ auto RBUN::read(IOFile const& file, bool no_lookup) -> RBUN { result.toc_offset = file_size - sizeof(footer) - toc_size; result.chunks.resize(footer.entry_count); - rlib_assert(file.read(result.toc_offset, {(char*)result.chunks.data(), toc_size})); + rlib_assert(io.read(result.toc_offset, {(char*)result.chunks.data(), toc_size})); if (footer.version == Footer::VERSION) { if (!no_lookup) { diff --git a/lib/rlib/rbundle.hpp b/lib/rlib/rbundle.hpp index 466bf1e..ca2469c 100644 --- a/lib/rlib/rbundle.hpp +++ b/lib/rlib/rbundle.hpp @@ -26,6 +26,6 @@ namespace rlib { std::vector chunks; std::unordered_map lookup; - static auto read(IOFile const& file, bool no_lookup = false) -> RBUN; + static auto read(IO const& io, bool no_lookup = false) -> RBUN; }; } \ No newline at end of file diff --git a/lib/rlib/rcache.cpp b/lib/rlib/rcache.cpp index add820f..88a594b 100644 --- a/lib/rlib/rcache.cpp +++ b/lib/rlib/rcache.cpp @@ -8,16 +8,11 @@ using namespace rlib; -static IOFile::Flags make_flags(bool readonly) { - auto flags = IOFile::Flags{}; - if (!readonly) { - flags = (IOFile::Flags)(flags | IOFile::WRITE); - } - // flags = (IOFile::Flags)(flags | IOFile::RANDOM_ACCESS); - return flags; +static constexpr auto rcache_file_flags(RCache::Options const& options) -> IO::Flags { + return (options.readonly ? IO::READ : IO::WRITE) | IO::NO_INTERUPT | IO::NO_OVERGROW; } -RCache::RCache(Options const& options) : file_(options.path, make_flags(options.readonly)), options_(options) { +RCache::RCache(Options const& options) : file_(options.path, rcache_file_flags(options)), options_(options) { auto file_size = file_.size(); if (file_size == 0 && !options.readonly) { flush(); @@ -102,7 +97,7 @@ auto RCache::flush() -> bool { auto new_toc_offset = bundle_.toc_offset + buffer_.size(); buffer_.insert(buffer_.end(), (char const*)bundle_.chunks.data(), (char const*)bundle_.chunks.data() + toc_size); buffer_.insert(buffer_.end(), (char const*)&footer, (char const*)&footer + sizeof(footer)); - rlib_assert(file_.write(bundle_.toc_offset, buffer_, true)); + rlib_assert(file_.write(bundle_.toc_offset, buffer_)); buffer_.clear(); bundle_.toc_offset = new_toc_offset; return true; diff --git a/lib/rlib/rcache.hpp b/lib/rlib/rcache.hpp index 9fcd891..8d38fd3 100644 --- a/lib/rlib/rcache.hpp +++ b/lib/rlib/rcache.hpp @@ -26,9 +26,10 @@ namespace rlib { auto flush() -> bool; - auto can_write() const noexcept -> bool { return file_ && !options_.readonly; } + auto can_write() const noexcept -> bool { return file_.fd() && !options_.readonly; } + private: - IOFile file_; + IO::File file_; Options options_; std::vector buffer_; RBUN bundle_; diff --git a/lib/rlib/rmanifest.cpp b/lib/rlib/rmanifest.cpp index 8de7708..4afa6c0 100644 --- a/lib/rlib/rmanifest.cpp +++ b/lib/rlib/rmanifest.cpp @@ -366,7 +366,7 @@ auto RMAN::File::verify(fs::path const& path, RChunk::Dst::data_cb on_data) cons if (!fs::exists(path)) { return chunks; } - auto infile = IOFile(path, false); + auto infile = IO::File(path, IO::READ); auto result = chunks; remove_if(result, [&, failfast = false](RChunk::Dst const& chunk) mutable -> bool { if (failfast) { diff --git a/src/rbun_chk.cpp b/src/rbun_chk.cpp index 53b1121..547bac0 100644 --- a/src/rbun_chk.cpp +++ b/src/rbun_chk.cpp @@ -51,7 +51,7 @@ struct Main { try { rlib_trace("path: %s", path.generic_string().c_str()); std::cout << "START:" << path.filename().generic_string() << std::endl; - auto infile = IOFile(path, false); + auto infile = IO::File(path, IO::READ); auto bundle = RBUN::read(infile, true); { std::uint64_t offset = 0; diff --git a/src/rbun_ex.cpp b/src/rbun_ex.cpp index 9cb19fd..f7b3693 100644 --- a/src/rbun_ex.cpp +++ b/src/rbun_ex.cpp @@ -73,7 +73,7 @@ struct Main { try { rlib_trace("path: %s", path.generic_string().c_str()); std::cout << "START:" << path.filename().generic_string() << std::endl; - auto infile = IOFile(path, false); + auto infile = IO::File(path, IO::READ); auto bundle = RBUN::read(infile, true); { std::uint64_t offset = 0; @@ -90,9 +90,9 @@ struct Main { auto hash_type = RChunk::hash_type(dst, chunk.chunkId); rlib_assert(hash_type != HashType::None); } - auto outfile = IOFile(fs::path(cli.output) / name, true); - outfile.resize(0, dst.size()); - outfile.write(0, dst, true); + auto outpath = fs::path(cli.output) / name; + auto outfile = IO::File(outpath, IO::WRITE | IO::NO_INTERUPT | IO::NO_OVERGROW); + outfile.write(0, dst); seen.insert(std::move(name)); } offset += chunk.compressed_size; diff --git a/src/rbun_ls.cpp b/src/rbun_ls.cpp index 02abd7f..9c64517 100644 --- a/src/rbun_ls.cpp +++ b/src/rbun_ls.cpp @@ -40,7 +40,7 @@ struct Main { auto list_bundle(fs::path const& path) noexcept -> void { try { rlib_trace("path: %s", path.generic_string().c_str()); - auto infile = IOFile(path, true); + auto infile = IO::File(path, IO::WRITE); auto bundle = RBUN::read(infile); for (std::uint64_t offset = 0; auto const& chunk : bundle.chunks) { fmt::dynamic_format_arg_store store{}; diff --git a/src/rbun_merge.cpp b/src/rbun_merge.cpp index 88ced23..cf93d69 100644 --- a/src/rbun_merge.cpp +++ b/src/rbun_merge.cpp @@ -68,7 +68,7 @@ struct Main { try { rlib_trace("path: %s", path.generic_string().c_str()); std::cout << "START:" << path.filename().generic_string() << std::endl; - auto infile = IOFile(path, false); + auto infile = IO::File(path, IO::READ); auto bundle = RBUN::read(infile, true); { std::uint64_t offset = 0; diff --git a/src/rbun_usage.cpp b/src/rbun_usage.cpp index 0e3c81b..cfd943b 100644 --- a/src/rbun_usage.cpp +++ b/src/rbun_usage.cpp @@ -91,7 +91,7 @@ struct Main { auto process_bundle(fs::path const& path) noexcept -> void { try { rlib_trace("path: %s", path.generic_string().c_str()); - auto infile = IOFile(path, false); + auto infile = IO::File(path, IO::READ); auto bundle = RBUN::read(infile, true); for (std::uint64_t offset = 0; auto const& chunk : bundle.chunks) { auto& usage = usage_per_chunk[chunk.chunkId]; diff --git a/src/rman_bl.cpp b/src/rman_bl.cpp index 882bcdb..cd6dc5d 100644 --- a/src/rman_bl.cpp +++ b/src/rman_bl.cpp @@ -29,7 +29,7 @@ struct Main { auto run() -> void { rlib_trace("Manifest file: %s", cli.manifest.c_str()); - auto infile = IOFile(cli.manifest, false); + auto infile = IO::File(cli.manifest, IO::READ); auto manifest = RMAN::read(infile.copy(0, infile.size())); for (auto const& bundle : manifest.bundles) { diff --git a/src/rman_dl.cpp b/src/rman_dl.cpp index 52eaf94..b91fe2b 100644 --- a/src/rman_dl.cpp +++ b/src/rman_dl.cpp @@ -136,7 +136,7 @@ struct Main { auto run() -> void { rlib_trace("Manifest file: %s", cli.manifest.c_str()); - auto infile = IOFile(cli.manifest, false); + auto infile = IO::File(cli.manifest, IO::READ); auto manifest = RMAN::read(infile.copy(0, infile.size())); if (!cli.no_write) { @@ -178,9 +178,9 @@ struct Main { bad_chunks = rfile.chunks; } - auto outfile = IOFile(); + auto outfile = IO::File(); if (!cli.no_write) { - outfile = IOFile(path, true); + outfile = IO::File(path, IO::WRITE); rlib_assert(outfile.resize(0, rfile.size)); } @@ -188,7 +188,7 @@ struct Main { progress_bar p("UNCACHED", cli.no_progress, index, done, rfile.size); bad_chunks = cache->uncache(std::move(bad_chunks), [&](RChunk::Dst const& chunk, std::span data) { - if (outfile) { + if (outfile.fd()) { rlib_assert(outfile.write(chunk.uncompressed_offset, data)); } done += chunk.uncompressed_size; @@ -200,7 +200,7 @@ struct Main { progress_bar p("DOWNLOAD", cli.no_progress, index, done, rfile.size); bad_chunks = cdn->download(std::move(bad_chunks), [&](RChunk::Dst const& chunk, std::span data) { - if (outfile) { + if (outfile.fd()) { rlib_assert(outfile.write(chunk.uncompressed_offset, data)); } done += chunk.uncompressed_size; diff --git a/src/rman_ls.cpp b/src/rman_ls.cpp index 96034f3..8555b28 100644 --- a/src/rman_ls.cpp +++ b/src/rman_ls.cpp @@ -55,7 +55,7 @@ struct Main { auto run() -> void { rlib_trace("Manifest file: %s", cli.manifest.c_str()); - auto infile = IOFile(cli.manifest, false); + auto infile = IO::File(cli.manifest, IO::READ); auto manifest = RMAN::read(infile.copy(0, infile.size())); for (auto const& rfile : manifest.files) {