From b7603cae635a097a1f60b5e5acc53b81ff834a27 Mon Sep 17 00:00:00 2001 From: Fuzzbawls Date: Wed, 4 Aug 2021 02:23:56 -0700 Subject: [PATCH] Squashed 'src/leveldb/' changes from 0c40829872..f8ae182c1e f8ae182c1e Adds unicode support to Windows environment. 92ae82c78f Increase maximum read-only mmap()s used from 1000 to 4096 on 64-bit systems d42e63d49d Do not crash if filesystem can't fsync bf2c2090b7 Add filename to corruption errors git-subtree-dir: src/leveldb git-subtree-split: f8ae182c1e5176d12e816fb2217ae33a5472fdd7 --- db/db_impl.cc | 2 +- db/db_test.cc | 3 ++ db/fault_injection_test.cc | 1 + db/leveldbutil.cc | 1 + db/log_reader.cc | 2 +- db/log_test.cc | 2 + db/repair.cc | 2 +- helpers/memenv/memenv.cc | 3 ++ include/leveldb/env.h | 9 ++++ table/format.cc | 10 ++-- table/table_test.cc | 2 + util/env_posix.cc | 23 +++++++-- util/env_windows.cc | 99 ++++++++++++++++++++++++++++---------- 13 files changed, 121 insertions(+), 38 deletions(-) diff --git a/db/db_impl.cc b/db/db_impl.cc index 95e2bb4107d0..65e31724bcec 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -428,7 +428,7 @@ Status DBImpl::RecoverLogFile(uint64_t log_number, bool last_log, while (reader.ReadRecord(&record, &scratch) && status.ok()) { if (record.size() < 12) { reporter.Corruption(record.size(), - Status::Corruption("log record too small")); + Status::Corruption("log record too small", fname)); continue; } WriteBatchInternal::SetContents(&batch, record); diff --git a/db/db_test.cc b/db/db_test.cc index 9a8faf100672..beb1d3bdef61 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -158,6 +158,7 @@ class SpecialEnv : public EnvWrapper { } return base_->Sync(); } + std::string GetName() const override { return ""; } }; class ManifestFile : public WritableFile { private: @@ -183,6 +184,7 @@ class SpecialEnv : public EnvWrapper { return base_->Sync(); } } + std::string GetName() const override { return ""; } }; if (non_writable_.load(std::memory_order_acquire)) { @@ -216,6 +218,7 @@ class SpecialEnv : public EnvWrapper { counter_->Increment(); return target_->Read(offset, n, result, scratch); } + std::string GetName() const override { return ""; } }; Status s = target()->NewRandomAccessFile(f, r); diff --git a/db/fault_injection_test.cc b/db/fault_injection_test.cc index 5b31bb8053f2..bf705cb60f24 100644 --- a/db/fault_injection_test.cc +++ b/db/fault_injection_test.cc @@ -114,6 +114,7 @@ class TestWritableFile : public WritableFile { Status Close() override; Status Flush() override; Status Sync() override; + std::string GetName() const override { return ""; } private: FileState state_; diff --git a/db/leveldbutil.cc b/db/leveldbutil.cc index 55cdcc577223..9ed9667d376b 100644 --- a/db/leveldbutil.cc +++ b/db/leveldbutil.cc @@ -20,6 +20,7 @@ class StdoutPrinter : public WritableFile { Status Close() override { return Status::OK(); } Status Flush() override { return Status::OK(); } Status Sync() override { return Status::OK(); } + std::string GetName() const override { return "[stdout]"; } }; bool HandleDumpCommand(Env* env, char** files, int num) { diff --git a/db/log_reader.cc b/db/log_reader.cc index b770fee1a0de..1ccfb7b34aa8 100644 --- a/db/log_reader.cc +++ b/db/log_reader.cc @@ -176,7 +176,7 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch) { uint64_t Reader::LastRecordOffset() { return last_record_offset_; } void Reader::ReportCorruption(uint64_t bytes, const char* reason) { - ReportDrop(bytes, Status::Corruption(reason)); + ReportDrop(bytes, Status::Corruption(reason, file_->GetName())); } void Reader::ReportDrop(uint64_t bytes, const Status& reason) { diff --git a/db/log_test.cc b/db/log_test.cc index 0e31648cc8ff..41fc043068dc 100644 --- a/db/log_test.cc +++ b/db/log_test.cc @@ -168,6 +168,7 @@ class LogTest { contents_.append(slice.data(), slice.size()); return Status::OK(); } + std::string GetName() const override { return ""; } std::string contents_; }; @@ -204,6 +205,7 @@ class LogTest { return Status::OK(); } + std::string GetName() const { return ""; } Slice contents_; bool force_error_; diff --git a/db/repair.cc b/db/repair.cc index d9d12ba556f0..04847c3bbfb7 100644 --- a/db/repair.cc +++ b/db/repair.cc @@ -183,7 +183,7 @@ class Repairer { while (reader.ReadRecord(&record, &scratch)) { if (record.size() < 12) { reporter.Corruption(record.size(), - Status::Corruption("log record too small")); + Status::Corruption("log record too small", logname)); continue; } WriteBatchInternal::SetContents(&batch, record); diff --git a/helpers/memenv/memenv.cc b/helpers/memenv/memenv.cc index 31d2bc0f6f63..47e4481f7c55 100644 --- a/helpers/memenv/memenv.cc +++ b/helpers/memenv/memenv.cc @@ -178,6 +178,7 @@ class SequentialFileImpl : public SequentialFile { return Status::OK(); } + virtual std::string GetName() const override { return "[memenv]"; } private: FileState* file_; uint64_t pos_; @@ -194,6 +195,7 @@ class RandomAccessFileImpl : public RandomAccessFile { return file_->Read(offset, n, result, scratch); } + virtual std::string GetName() const override { return "[memenv]"; } private: FileState* file_; }; @@ -210,6 +212,7 @@ class WritableFileImpl : public WritableFile { Status Flush() override { return Status::OK(); } Status Sync() override { return Status::OK(); } + virtual std::string GetName() const override { return "[memenv]"; } private: FileState* file_; }; diff --git a/include/leveldb/env.h b/include/leveldb/env.h index 112fe964fb1c..96c21b3966c4 100644 --- a/include/leveldb/env.h +++ b/include/leveldb/env.h @@ -217,6 +217,9 @@ class LEVELDB_EXPORT SequentialFile { // // REQUIRES: External synchronization virtual Status Skip(uint64_t n) = 0; + + // Get a name for the file, only for error reporting + virtual std::string GetName() const = 0; }; // A file abstraction for randomly reading the contents of a file. @@ -240,6 +243,9 @@ class LEVELDB_EXPORT RandomAccessFile { // Safe for concurrent use by multiple threads. virtual Status Read(uint64_t offset, size_t n, Slice* result, char* scratch) const = 0; + + // Get a name for the file, only for error reporting + virtual std::string GetName() const = 0; }; // A file abstraction for sequential writing. The implementation @@ -258,6 +264,9 @@ class LEVELDB_EXPORT WritableFile { virtual Status Close() = 0; virtual Status Flush() = 0; virtual Status Sync() = 0; + + // Get a name for the file, only for error reporting + virtual std::string GetName() const = 0; }; // An interface for writing log messages. diff --git a/table/format.cc b/table/format.cc index e1839779e4ba..a3d67de2e41d 100644 --- a/table/format.cc +++ b/table/format.cc @@ -79,7 +79,7 @@ Status ReadBlock(RandomAccessFile* file, const ReadOptions& options, } if (contents.size() != n + kBlockTrailerSize) { delete[] buf; - return Status::Corruption("truncated block read"); + return Status::Corruption("truncated block read", file->GetName()); } // Check the crc of the type and the block contents @@ -89,7 +89,7 @@ Status ReadBlock(RandomAccessFile* file, const ReadOptions& options, const uint32_t actual = crc32c::Value(data, n + 1); if (actual != crc) { delete[] buf; - s = Status::Corruption("block checksum mismatch"); + s = Status::Corruption("block checksum mismatch", file->GetName()); return s; } } @@ -116,13 +116,13 @@ Status ReadBlock(RandomAccessFile* file, const ReadOptions& options, size_t ulength = 0; if (!port::Snappy_GetUncompressedLength(data, n, &ulength)) { delete[] buf; - return Status::Corruption("corrupted compressed block contents"); + return Status::Corruption("corrupted compressed block contents", file->GetName()); } char* ubuf = new char[ulength]; if (!port::Snappy_Uncompress(data, n, ubuf)) { delete[] buf; delete[] ubuf; - return Status::Corruption("corrupted compressed block contents"); + return Status::Corruption("corrupted compressed block contents", file->GetName()); } delete[] buf; result->data = Slice(ubuf, ulength); @@ -132,7 +132,7 @@ Status ReadBlock(RandomAccessFile* file, const ReadOptions& options, } default: delete[] buf; - return Status::Corruption("bad block type"); + return Status::Corruption("bad block type", file->GetName()); } return Status::OK(); diff --git a/table/table_test.cc b/table/table_test.cc index f689a276577e..17aaea2f9e8b 100644 --- a/table/table_test.cc +++ b/table/table_test.cc @@ -102,6 +102,7 @@ class StringSink : public WritableFile { return Status::OK(); } + std::string GetName() const override { return ""; } private: std::string contents_; }; @@ -128,6 +129,7 @@ class StringSource : public RandomAccessFile { return Status::OK(); } + std::string GetName() const { return ""; } private: std::string contents_; }; diff --git a/util/env_posix.cc b/util/env_posix.cc index 00ca9aedef58..9f5863a0f35a 100644 --- a/util/env_posix.cc +++ b/util/env_posix.cc @@ -42,8 +42,8 @@ namespace { // Set by EnvPosixTestHelper::SetReadOnlyMMapLimit() and MaxOpenFiles(). int g_open_read_only_file_limit = -1; -// Up to 1000 mmap regions for 64-bit binaries; none for 32-bit. -constexpr const int kDefaultMmapLimit = (sizeof(void*) >= 8) ? 1000 : 0; +// Up to 4096 mmap regions for 64-bit binaries; none for 32-bit. +constexpr const int kDefaultMmapLimit = (sizeof(void*) >= 8) ? 4096 : 0; // Can be set using EnvPosixTestHelper::SetReadOnlyMMapLimit(). int g_mmap_limit = kDefaultMmapLimit; @@ -135,6 +135,8 @@ class PosixSequentialFile final : public SequentialFile { return Status::OK(); } + virtual std::string GetName() const override { return filename_; } + private: const int fd_; const std::string filename_; @@ -195,6 +197,8 @@ class PosixRandomAccessFile final : public RandomAccessFile { return status; } + virtual std::string GetName() const override { return filename_; } + private: const bool has_permanent_fd_; // If false, the file is opened on every read. const int fd_; // -1 if has_permanent_fd_ is false. @@ -239,6 +243,8 @@ class PosixMmapReadableFile final : public RandomAccessFile { return Status::OK(); } + virtual std::string GetName() const override { return filename_; } + private: char* const mmap_base_; const size_t length_; @@ -319,7 +325,7 @@ class PosixWritableFile final : public WritableFile { return status; } - return SyncFd(fd_, filename_); + return SyncFd(fd_, filename_, false); } private: @@ -354,7 +360,7 @@ class PosixWritableFile final : public WritableFile { if (fd < 0) { status = PosixError(dirname_, errno); } else { - status = SyncFd(fd, dirname_); + status = SyncFd(fd, dirname_, true); ::close(fd); } return status; @@ -366,7 +372,7 @@ class PosixWritableFile final : public WritableFile { // // The path argument is only used to populate the description string in the // returned Status if an error occurs. - static Status SyncFd(int fd, const std::string& fd_path) { + static Status SyncFd(int fd, const std::string& fd_path, bool syncing_dir) { #if HAVE_FULLFSYNC // On macOS and iOS, fsync() doesn't guarantee durability past power // failures. fcntl(F_FULLFSYNC) is required for that purpose. Some @@ -386,6 +392,11 @@ class PosixWritableFile final : public WritableFile { if (sync_success) { return Status::OK(); } + // Do not crash if filesystem can't fsync directories + // (see https://github.com/bitcoin/bitcoin/pull/10000) + if (syncing_dir && errno == EINVAL) { + return Status::OK(); + } return PosixError(fd_path, errno); } @@ -426,6 +437,8 @@ class PosixWritableFile final : public WritableFile { return Basename(filename).starts_with("MANIFEST"); } + virtual std::string GetName() const override { return filename_; } + // buf_[0, pos_ - 1] contains data to be written to fd_. char buf_[kWritableFileBufferSize]; size_t pos_; diff --git a/util/env_windows.cc b/util/env_windows.cc index 2dd7794f66e4..1834206562cb 100644 --- a/util/env_windows.cc +++ b/util/env_windows.cc @@ -177,6 +177,8 @@ class WindowsSequentialFile : public SequentialFile { return Status::OK(); } + std::string GetName() const override { return filename_; } + private: const ScopedHandle handle_; const std::string filename_; @@ -209,6 +211,8 @@ class WindowsRandomAccessFile : public RandomAccessFile { return Status::OK(); } + std::string GetName() const override { return filename_; } + private: const ScopedHandle handle_; const std::string filename_; @@ -240,6 +244,8 @@ class WindowsMmapReadableFile : public RandomAccessFile { return Status::OK(); } + std::string GetName() const override { return filename_; } + private: char* const mmap_base_; const size_t length_; @@ -309,6 +315,8 @@ class WindowsWritableFile : public WritableFile { return Status::OK(); } + std::string GetName() const override { return filename_; } + private: Status FlushBuffer() { Status status = WriteUnbuffered(buf_, pos_); @@ -379,8 +387,9 @@ class WindowsEnv : public Env { *result = nullptr; DWORD desired_access = GENERIC_READ; DWORD share_mode = FILE_SHARE_READ; - ScopedHandle handle = ::CreateFileA( - filename.c_str(), desired_access, share_mode, + auto wFilename = toUtf16(filename); + ScopedHandle handle = ::CreateFileW( + wFilename.c_str(), desired_access, share_mode, /*lpSecurityAttributes=*/nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, /*hTemplateFile=*/nullptr); if (!handle.is_valid()) { @@ -396,8 +405,9 @@ class WindowsEnv : public Env { *result = nullptr; DWORD desired_access = GENERIC_READ; DWORD share_mode = FILE_SHARE_READ; + auto wFilename = toUtf16(filename); ScopedHandle handle = - ::CreateFileA(filename.c_str(), desired_access, share_mode, + ::CreateFileW(wFilename.c_str(), desired_access, share_mode, /*lpSecurityAttributes=*/nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_READONLY, /*hTemplateFile=*/nullptr); @@ -417,7 +427,7 @@ class WindowsEnv : public Env { } ScopedHandle mapping = - ::CreateFileMappingA(handle.get(), + ::CreateFileMappingW(handle.get(), /*security attributes=*/nullptr, PAGE_READONLY, /*dwMaximumSizeHigh=*/0, /*dwMaximumSizeLow=*/0, @@ -442,8 +452,9 @@ class WindowsEnv : public Env { WritableFile** result) override { DWORD desired_access = GENERIC_WRITE; DWORD share_mode = 0; // Exclusive access. - ScopedHandle handle = ::CreateFileA( - filename.c_str(), desired_access, share_mode, + auto wFilename = toUtf16(filename); + ScopedHandle handle = ::CreateFileW( + wFilename.c_str(), desired_access, share_mode, /*lpSecurityAttributes=*/nullptr, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, /*hTemplateFile=*/nullptr); if (!handle.is_valid()) { @@ -459,8 +470,9 @@ class WindowsEnv : public Env { WritableFile** result) override { DWORD desired_access = FILE_APPEND_DATA; DWORD share_mode = 0; // Exclusive access. - ScopedHandle handle = ::CreateFileA( - filename.c_str(), desired_access, share_mode, + auto wFilename = toUtf16(filename); + ScopedHandle handle = ::CreateFileW( + wFilename.c_str(), desired_access, share_mode, /*lpSecurityAttributes=*/nullptr, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, /*hTemplateFile=*/nullptr); if (!handle.is_valid()) { @@ -473,14 +485,16 @@ class WindowsEnv : public Env { } bool FileExists(const std::string& filename) override { - return GetFileAttributesA(filename.c_str()) != INVALID_FILE_ATTRIBUTES; + auto wFilename = toUtf16(filename); + return GetFileAttributesW(wFilename.c_str()) != INVALID_FILE_ATTRIBUTES; } Status GetChildren(const std::string& directory_path, std::vector* result) override { const std::string find_pattern = directory_path + "\\*"; - WIN32_FIND_DATAA find_data; - HANDLE dir_handle = ::FindFirstFileA(find_pattern.c_str(), &find_data); + WIN32_FIND_DATAW find_data; + auto wFind_pattern = toUtf16(find_pattern); + HANDLE dir_handle = ::FindFirstFileW(wFind_pattern.c_str(), &find_data); if (dir_handle == INVALID_HANDLE_VALUE) { DWORD last_error = ::GetLastError(); if (last_error == ERROR_FILE_NOT_FOUND) { @@ -492,11 +506,12 @@ class WindowsEnv : public Env { char base_name[_MAX_FNAME]; char ext[_MAX_EXT]; - if (!_splitpath_s(find_data.cFileName, nullptr, 0, nullptr, 0, base_name, - ARRAYSIZE(base_name), ext, ARRAYSIZE(ext))) { + auto find_data_filename = toUtf8(find_data.cFileName); + if (!_splitpath_s(find_data_filename.c_str(), nullptr, 0, nullptr, 0, + base_name, ARRAYSIZE(base_name), ext, ARRAYSIZE(ext))) { result->emplace_back(std::string(base_name) + ext); } - } while (::FindNextFileA(dir_handle, &find_data)); + } while (::FindNextFileW(dir_handle, &find_data)); DWORD last_error = ::GetLastError(); ::FindClose(dir_handle); if (last_error != ERROR_NO_MORE_FILES) { @@ -506,21 +521,24 @@ class WindowsEnv : public Env { } Status DeleteFile(const std::string& filename) override { - if (!::DeleteFileA(filename.c_str())) { + auto wFilename = toUtf16(filename); + if (!::DeleteFileW(wFilename.c_str())) { return WindowsError(filename, ::GetLastError()); } return Status::OK(); } Status CreateDir(const std::string& dirname) override { - if (!::CreateDirectoryA(dirname.c_str(), nullptr)) { + auto wDirname = toUtf16(dirname); + if (!::CreateDirectoryW(wDirname.c_str(), nullptr)) { return WindowsError(dirname, ::GetLastError()); } return Status::OK(); } Status DeleteDir(const std::string& dirname) override { - if (!::RemoveDirectoryA(dirname.c_str())) { + auto wDirname = toUtf16(dirname); + if (!::RemoveDirectoryW(wDirname.c_str())) { return WindowsError(dirname, ::GetLastError()); } return Status::OK(); @@ -528,7 +546,8 @@ class WindowsEnv : public Env { Status GetFileSize(const std::string& filename, uint64_t* size) override { WIN32_FILE_ATTRIBUTE_DATA file_attributes; - if (!::GetFileAttributesExA(filename.c_str(), GetFileExInfoStandard, + auto wFilename = toUtf16(filename); + if (!::GetFileAttributesExW(wFilename.c_str(), GetFileExInfoStandard, &file_attributes)) { return WindowsError(filename, ::GetLastError()); } @@ -542,7 +561,9 @@ class WindowsEnv : public Env { Status RenameFile(const std::string& from, const std::string& to) override { // Try a simple move first. It will only succeed when |to| doesn't already // exist. - if (::MoveFileA(from.c_str(), to.c_str())) { + auto wFrom = toUtf16(from); + auto wTo = toUtf16(to); + if (::MoveFileW(wFrom.c_str(), wTo.c_str())) { return Status::OK(); } DWORD move_error = ::GetLastError(); @@ -551,7 +572,7 @@ class WindowsEnv : public Env { // succeed when |to| does exist. When writing to a network share, we may not // be able to change the ACLs. Ignore ACL errors then // (REPLACEFILE_IGNORE_MERGE_ERRORS). - if (::ReplaceFileA(to.c_str(), from.c_str(), /*lpBackupFileName=*/nullptr, + if (::ReplaceFileW(wTo.c_str(), wFrom.c_str(), /*lpBackupFileName=*/nullptr, REPLACEFILE_IGNORE_MERGE_ERRORS, /*lpExclude=*/nullptr, /*lpReserved=*/nullptr)) { return Status::OK(); @@ -571,8 +592,9 @@ class WindowsEnv : public Env { Status LockFile(const std::string& filename, FileLock** lock) override { *lock = nullptr; Status result; - ScopedHandle handle = ::CreateFileA( - filename.c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ, + auto wFilename = toUtf16(filename); + ScopedHandle handle = ::CreateFileW( + wFilename.c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ, /*lpSecurityAttributes=*/nullptr, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr); if (!handle.is_valid()) { @@ -612,10 +634,11 @@ class WindowsEnv : public Env { return Status::OK(); } - char tmp_path[MAX_PATH]; - if (!GetTempPathA(ARRAYSIZE(tmp_path), tmp_path)) { + wchar_t wtmp_path[MAX_PATH]; + if (!GetTempPathW(ARRAYSIZE(wtmp_path), wtmp_path)) { return WindowsError("GetTempPath", ::GetLastError()); } + std::string tmp_path = toUtf8(std::wstring(wtmp_path)); std::stringstream ss; ss << tmp_path << "leveldbtest-" << std::this_thread::get_id(); *result = ss.str(); @@ -626,7 +649,8 @@ class WindowsEnv : public Env { } Status NewLogger(const std::string& filename, Logger** result) override { - std::FILE* fp = std::fopen(filename.c_str(), "w"); + auto wFilename = toUtf16(filename); + std::FILE* fp = _wfopen(wFilename.c_str(), L"w"); if (fp == nullptr) { *result = nullptr; return WindowsError(filename, ::GetLastError()); @@ -682,6 +706,31 @@ class WindowsEnv : public Env { GUARDED_BY(background_work_mutex_); Limiter mmap_limiter_; // Thread-safe. + + // Converts a Windows wide multi-byte UTF-16 string to a UTF-8 string. + // See http://utf8everywhere.org/#windows + std::string toUtf8(const std::wstring& wstr) { + if (wstr.empty()) return std::string(); + int size_needed = WideCharToMultiByte( + CP_UTF8, 0, &wstr[0], (int)wstr.size(), NULL, 0, NULL, NULL); + std::string strTo(size_needed, 0); + WideCharToMultiByte(CP_UTF8, 0, &wstr[0], (int)wstr.size(), &strTo[0], + size_needed, NULL, NULL); + return strTo; + } + + // Converts a UTF-8 string to a Windows UTF-16 multi-byte wide character + // string. + // See http://utf8everywhere.org/#windows + std::wstring toUtf16(const std::string& str) { + if (str.empty()) return std::wstring(); + int size_needed = + MultiByteToWideChar(CP_UTF8, 0, &str[0], (int)str.size(), NULL, 0); + std::wstring strTo(size_needed, 0); + MultiByteToWideChar(CP_UTF8, 0, &str[0], (int)str.size(), &strTo[0], + size_needed); + return strTo; + } }; // Return the maximum number of concurrent mmaps.