diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter index cccbe6a7537c7..8bbc29d280cea 100644 --- a/ci/licenses_golden/licenses_flutter +++ b/ci/licenses_golden/licenses_flutter @@ -492,6 +492,7 @@ FILE: ../../../flutter/shell/common/isolate_configuration.cc FILE: ../../../flutter/shell/common/isolate_configuration.h FILE: ../../../flutter/shell/common/persistent_cache.cc FILE: ../../../flutter/shell/common/persistent_cache.h +FILE: ../../../flutter/shell/common/persistent_cache_unittests.cc FILE: ../../../flutter/shell/common/pipeline.cc FILE: ../../../flutter/shell/common/pipeline.h FILE: ../../../flutter/shell/common/pipeline_unittests.cc diff --git a/common/settings.cc b/common/settings.cc index b4b58cb6d7714..b1ef2161fa561 100644 --- a/common/settings.cc +++ b/common/settings.cc @@ -40,6 +40,7 @@ std::string Settings::ToString() const { stream << "trace_systrace: " << trace_systrace << std::endl; stream << "dump_skp_on_shader_compilation: " << dump_skp_on_shader_compilation << std::endl; + stream << "cache_sksl: " << cache_sksl << std::endl; stream << "endless_trace_buffer: " << endless_trace_buffer << std::endl; stream << "enable_dart_profiling: " << enable_dart_profiling << std::endl; stream << "disable_dart_asserts: " << disable_dart_asserts << std::endl; diff --git a/common/settings.h b/common/settings.h index d39c2ea6ea3da..2ebcbf0cd862a 100644 --- a/common/settings.h +++ b/common/settings.h @@ -94,6 +94,7 @@ struct Settings { bool trace_startup = false; bool trace_systrace = false; bool dump_skp_on_shader_compilation = false; + bool cache_sksl = false; bool endless_trace_buffer = false; bool enable_dart_profiling = false; bool disable_dart_asserts = false; diff --git a/fml/file.cc b/fml/file.cc index 8deb76c91da0f..2f62e6c6f884b 100644 --- a/fml/file.cc +++ b/fml/file.cc @@ -5,6 +5,7 @@ #include "flutter/fml/file.h" #include "flutter/fml/logging.h" +#include "flutter/fml/unique_fd.h" namespace fml { @@ -51,6 +52,8 @@ ScopedTemporaryDirectory::ScopedTemporaryDirectory() { } ScopedTemporaryDirectory::~ScopedTemporaryDirectory() { + // Windows has to close UniqueFD first before UnlinkDirectory + dir_fd_.reset(); if (path_ != "") { if (!UnlinkDirectory(path_.c_str())) { FML_LOG(ERROR) << "Could not remove directory: " << path_; @@ -58,4 +61,35 @@ ScopedTemporaryDirectory::~ScopedTemporaryDirectory() { } } +bool VisitFilesRecursively(const fml::UniqueFD& directory, + FileVisitor visitor) { + FileVisitor recursive_visitor = [&recursive_visitor, &visitor]( + const UniqueFD& directory, + const std::string& filename) { + if (!visitor(directory, filename)) { + return false; + } + if (IsDirectory(directory, filename.c_str())) { + UniqueFD sub_dir = OpenDirectoryReadOnly(directory, filename.c_str()); + if (!sub_dir.is_valid()) { + FML_LOG(ERROR) << "Can't open sub-directory: " << filename; + return true; + } + return VisitFiles(sub_dir, recursive_visitor); + } + return true; + }; + return VisitFiles(directory, recursive_visitor); +} + +fml::UniqueFD OpenFileReadOnly(const fml::UniqueFD& base_directory, + const char* path) { + return OpenFile(base_directory, path, false, FilePermission::kRead); +} + +fml::UniqueFD OpenDirectoryReadOnly(const fml::UniqueFD& base_directory, + const char* path) { + return OpenDirectory(base_directory, path, false, FilePermission::kRead); +} + } // namespace fml diff --git a/fml/file.h b/fml/file.h index 068ddc84c999d..ee54354520695 100644 --- a/fml/file.h +++ b/fml/file.h @@ -5,6 +5,7 @@ #ifndef FLUTTER_FML_FILE_H_ #define FLUTTER_FML_FILE_H_ +#include #include #include #include @@ -28,15 +29,24 @@ enum class FilePermission { std::string CreateTemporaryDirectory(); +/// This can open a directory on POSIX, but not on Windows. fml::UniqueFD OpenFile(const char* path, bool create_if_necessary, FilePermission permission); +/// This can open a directory on POSIX, but not on Windows. fml::UniqueFD OpenFile(const fml::UniqueFD& base_directory, const char* path, bool create_if_necessary, FilePermission permission); +/// Helper method that calls `OpenFile` with create_if_necessary = false +/// and permission = kRead. +/// +/// This can open a directory on POSIX, but not on Windows. +fml::UniqueFD OpenFileReadOnly(const fml::UniqueFD& base_directory, + const char* path); + fml::UniqueFD OpenDirectory(const char* path, bool create_if_necessary, FilePermission permission); @@ -46,10 +56,17 @@ fml::UniqueFD OpenDirectory(const fml::UniqueFD& base_directory, bool create_if_necessary, FilePermission permission); +/// Helper method that calls `OpenDirectory` with create_if_necessary = false +/// and permission = kRead. +fml::UniqueFD OpenDirectoryReadOnly(const fml::UniqueFD& base_directory, + const char* path); + fml::UniqueFD Duplicate(fml::UniqueFD::element_type descriptor); bool IsDirectory(const fml::UniqueFD& directory); +bool IsDirectory(const fml::UniqueFD& base_directory, const char* path); + // Returns whether the given path is a file. bool IsFile(const std::string& path); @@ -73,12 +90,44 @@ bool WriteAtomically(const fml::UniqueFD& base_directory, const char* file_name, const Mapping& mapping); +/// Signature of a callback on a file in `directory` with `filename` (relative +/// to `directory`). The returned bool should be false if and only if further +/// traversal should be stopped. For example, a file-search visitor may return +/// false when the file is found so no more visiting is needed. +using FileVisitor = std::function; + +/// Call `visitor` on all files inside the `directory` non-recursively. The +/// trivial file "." and ".." will not be visited. +/// +/// Return false if and only if the visitor returns false during the +/// traversal. +/// +/// If recursive visiting is needed, call `VisitFiles` inside the `visitor`, or +/// use our helper method `VisitFilesRecursively`. +/// +/// @see `VisitFilesRecursively`. +bool VisitFiles(const fml::UniqueFD& directory, FileVisitor visitor); + +/// Recursively call `visitor` on all files inside the `directory`. Return false +/// if and only if the visitor returns false during the traversal. +/// +/// This is a helper method that wraps the general `VisitFiles` method. The +/// `VisitFiles` is strictly more powerful as it has the access of the recursion +/// stack to the file. For example, `VisitFiles` may be able to maintain a +/// vector of directory names that lead to a file. That could be useful to +/// compute the relative path between the root directory and the visited file. +/// +/// @see `VisitFiles`. +bool VisitFilesRecursively(const fml::UniqueFD& directory, FileVisitor visitor); + class ScopedTemporaryDirectory { public: ScopedTemporaryDirectory(); ~ScopedTemporaryDirectory(); + const std::string& path() const { return path_; } const UniqueFD& fd() { return dir_fd_; } private: diff --git a/fml/file_unittest.cc b/fml/file_unittest.cc index c492a0a36a93e..aed268b3fc7b2 100644 --- a/fml/file_unittest.cc +++ b/fml/file_unittest.cc @@ -10,6 +10,7 @@ #include "flutter/fml/build_config.h" #include "flutter/fml/file.h" #include "flutter/fml/mapping.h" +#include "flutter/fml/unique_fd.h" static bool WriteStringToFile(const fml::UniqueFD& fd, const std::string& contents) { @@ -131,6 +132,101 @@ TEST(FileTest, CreateDirectoryStructure) { ASSERT_TRUE(fml::UnlinkDirectory(dir.fd(), "a")); } +TEST(FileTest, VisitFilesCanBeCalledTwice) { + fml::ScopedTemporaryDirectory dir; + + { + auto file = fml::OpenFile(dir.fd(), "my_contents", true, + fml::FilePermission::kReadWrite); + ASSERT_TRUE(file.is_valid()); + } + + int count; + fml::FileVisitor count_visitor = [&count](const fml::UniqueFD& directory, + const std::string& filename) { + count += 1; + return true; + }; + count = 0; + fml::VisitFiles(dir.fd(), count_visitor); + ASSERT_EQ(count, 1); + + // Without `rewinddir` in `VisitFiles`, the following check would fail. + count = 0; + fml::VisitFiles(dir.fd(), count_visitor); + ASSERT_EQ(count, 1); + + ASSERT_TRUE(fml::UnlinkFile(dir.fd(), "my_contents")); +} + +TEST(FileTest, CanListFilesRecursively) { + fml::ScopedTemporaryDirectory dir; + + { + auto c = fml::CreateDirectory(dir.fd(), {"a", "b", "c"}, + fml::FilePermission::kReadWrite); + ASSERT_TRUE(c.is_valid()); + auto file1 = + fml::OpenFile(c, "file1", true, fml::FilePermission::kReadWrite); + auto file2 = + fml::OpenFile(c, "file2", true, fml::FilePermission::kReadWrite); + auto d = fml::CreateDirectory(c, {"d"}, fml::FilePermission::kReadWrite); + ASSERT_TRUE(d.is_valid()); + auto file3 = + fml::OpenFile(d, "file3", true, fml::FilePermission::kReadWrite); + ASSERT_TRUE(file1.is_valid()); + ASSERT_TRUE(file2.is_valid()); + ASSERT_TRUE(file3.is_valid()); + } + + std::set names; + fml::FileVisitor visitor = [&names](const fml::UniqueFD& directory, + const std::string& filename) { + names.insert(filename); + return true; + }; + + fml::VisitFilesRecursively(dir.fd(), visitor); + ASSERT_EQ(names, std::set( + {"a", "b", "c", "d", "file1", "file2", "file3"})); + + // Cleanup. + ASSERT_TRUE(fml::UnlinkFile(dir.fd(), "a/b/c/d/file3")); + ASSERT_TRUE(fml::UnlinkFile(dir.fd(), "a/b/c/file1")); + ASSERT_TRUE(fml::UnlinkFile(dir.fd(), "a/b/c/file2")); + ASSERT_TRUE(fml::UnlinkDirectory(dir.fd(), "a/b/c/d")); + ASSERT_TRUE(fml::UnlinkDirectory(dir.fd(), "a/b/c")); + ASSERT_TRUE(fml::UnlinkDirectory(dir.fd(), "a/b")); + ASSERT_TRUE(fml::UnlinkDirectory(dir.fd(), "a")); +} + +TEST(FileTest, CanStopVisitEarly) { + fml::ScopedTemporaryDirectory dir; + + { + auto d = fml::CreateDirectory(dir.fd(), {"a", "b", "c", "d"}, + fml::FilePermission::kReadWrite); + ASSERT_TRUE(d.is_valid()); + } + + std::set names; + fml::FileVisitor visitor = [&names](const fml::UniqueFD& directory, + const std::string& filename) { + names.insert(filename); + return filename == "c" ? false : true; // stop if c is found + }; + + // Check the d is not visited as we stop at c. + ASSERT_FALSE(fml::VisitFilesRecursively(dir.fd(), visitor)); + ASSERT_EQ(names, std::set({"a", "b", "c"})); + + // Cleanup. + ASSERT_TRUE(fml::UnlinkDirectory(dir.fd(), "a/b/c/d")); + ASSERT_TRUE(fml::UnlinkDirectory(dir.fd(), "a/b/c")); + ASSERT_TRUE(fml::UnlinkDirectory(dir.fd(), "a/b")); + ASSERT_TRUE(fml::UnlinkDirectory(dir.fd(), "a")); +} + #if OS_WIN #define AtomicWriteTest DISABLED_AtomicWriteTest #else @@ -159,13 +255,15 @@ TEST(FileTest, AtomicWriteTest) { TEST(FileTest, EmptyMappingTest) { fml::ScopedTemporaryDirectory dir; - auto file = fml::OpenFile(dir.fd(), "my_contents", true, - fml::FilePermission::kReadWrite); + { + auto file = fml::OpenFile(dir.fd(), "my_contents", true, + fml::FilePermission::kReadWrite); - fml::FileMapping mapping(file); - ASSERT_TRUE(mapping.IsValid()); - ASSERT_EQ(mapping.GetSize(), 0ul); - ASSERT_EQ(mapping.GetMapping(), nullptr); + fml::FileMapping mapping(file); + ASSERT_TRUE(mapping.IsValid()); + ASSERT_EQ(mapping.GetSize(), 0ul); + ASSERT_EQ(mapping.GetMapping(), nullptr); + } ASSERT_TRUE(fml::UnlinkFile(dir.fd(), "my_contents")); } diff --git a/fml/platform/posix/file_posix.cc b/fml/platform/posix/file_posix.cc index d30a9c2f1d2df..516565e9f7673 100644 --- a/fml/platform/posix/file_posix.cc +++ b/fml/platform/posix/file_posix.cc @@ -4,6 +4,7 @@ #include "flutter/fml/file.h" +#include #include #include #include @@ -13,7 +14,9 @@ #include #include "flutter/fml/eintr_wrapper.h" +#include "flutter/fml/logging.h" #include "flutter/fml/mapping.h" +#include "flutter/fml/unique_fd.h" namespace fml { @@ -132,6 +135,11 @@ bool IsDirectory(const fml::UniqueFD& directory) { return S_ISDIR(stat_result.st_mode); } +bool IsDirectory(const fml::UniqueFD& base_directory, const char* path) { + UniqueFD file = OpenFileReadOnly(base_directory, path); + return (file.is_valid() && IsDirectory(file)); +} + bool IsFile(const std::string& path) { struct stat buf; if (stat(path.c_str(), &buf) != 0) { @@ -162,7 +170,11 @@ bool UnlinkFile(const char* path) { } bool UnlinkFile(const fml::UniqueFD& base_directory, const char* path) { - return ::unlinkat(base_directory.get(), path, 0) == 0; + int code = ::unlinkat(base_directory.get(), path, 0); + if (code != 0) { + FML_DLOG(ERROR) << strerror(errno); + } + return code == 0; } bool FileExists(const fml::UniqueFD& base_directory, const char* path) { @@ -210,4 +222,29 @@ bool WriteAtomically(const fml::UniqueFD& base_directory, base_directory.get(), file_name) == 0; } +bool VisitFiles(const fml::UniqueFD& directory, FileVisitor visitor) { + // We cannot call closedir(dir) because it will also close the corresponding + // UniqueFD, and later reference to that UniqueFD will fail. Also, we don't + // have to call closedir because UniqueFD will call close on its destructor. + DIR* dir = ::fdopendir(directory.get()); + if (dir == nullptr) { + FML_DLOG(ERROR) << "Can't open the directory. Error: " << strerror(errno); + return true; // continue to visit other files + } + + // Without `rewinddir`, `readir` will directly return NULL (end of dir is + // reached) after a previuos `VisitFiles` call for the same `const + // fml::UniqueFd& directory`. + rewinddir(dir); + while (dirent* ent = readdir(dir)) { + std::string filename = ent->d_name; + if (filename != "." && filename != "..") { + if (!visitor(directory, filename)) { + return false; + } + } + } + return true; +} + } // namespace fml diff --git a/fml/platform/win/file_win.cc b/fml/platform/win/file_win.cc index bad9d052fb942..42a5958f4d061 100644 --- a/fml/platform/win/file_win.cc +++ b/fml/platform/win/file_win.cc @@ -252,6 +252,12 @@ bool IsDirectory(const fml::UniqueFD& directory) { return info.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY; } +bool IsDirectory(const fml::UniqueFD& base_directory, const char* path) { + std::string full_path = GetFullHandlePath(base_directory) + "\\" + path; + return ::GetFileAttributes(ConvertToWString(full_path.c_str()).c_str()) & + FILE_ATTRIBUTE_DIRECTORY; +} + bool IsFile(const std::string& path) { struct stat buf; if (stat(path.c_str(), &buf) != 0) @@ -393,4 +399,29 @@ bool WriteAtomically(const fml::UniqueFD& base_directory, return true; } +bool VisitFiles(const fml::UniqueFD& directory, FileVisitor visitor) { + std::string search_pattern = GetFullHandlePath(directory) + "\\*"; + WIN32_FIND_DATA find_file_data; + HANDLE find_handle = ::FindFirstFile( + StringToWideString(search_pattern).c_str(), &find_file_data); + + if (find_handle == INVALID_HANDLE_VALUE) { + FML_DLOG(ERROR) << "Can't open the directory. Error: " + << GetLastErrorMessage(); + return true; // continue to visit other files + } + + do { + std::string filename = WideStringToString(find_file_data.cFileName); + if (filename != "." && filename != "..") { + if (!visitor(directory, filename)) { + ::FindClose(find_handle); + return false; + } + } + } while (::FindNextFile(find_handle, &find_file_data)); + ::FindClose(find_handle); + return true; +} + } // namespace fml diff --git a/shell/common/BUILD.gn b/shell/common/BUILD.gn index b3745aef2f262..f93bca63478f0 100644 --- a/shell/common/BUILD.gn +++ b/shell/common/BUILD.gn @@ -159,6 +159,7 @@ if (current_toolchain == host_toolchain) { sources = [ "canvas_spy_unittests.cc", "input_events_unittests.cc", + "persistent_cache_unittests.cc", "pipeline_unittests.cc", "shell_test.cc", "shell_test.h", diff --git a/shell/common/persistent_cache.cc b/shell/common/persistent_cache.cc index bc19976b38263..23308fe4fa228 100644 --- a/shell/common/persistent_cache.cc +++ b/shell/common/persistent_cache.cc @@ -10,6 +10,7 @@ #include "flutter/fml/base32.h" #include "flutter/fml/file.h" +#include "flutter/fml/logging.h" #include "flutter/fml/make_copyable.h" #include "flutter/fml/mapping.h" #include "flutter/fml/paths.h" @@ -20,6 +21,9 @@ namespace flutter { std::string PersistentCache::cache_base_path_; +std::mutex PersistentCache::instance_mutex_; +std::unique_ptr PersistentCache::gPersistentCache; + static std::string SkKeyToFilePath(const SkData& data) { if (data.data() == nullptr || data.size() == 0) { return ""; @@ -39,22 +43,41 @@ static std::string SkKeyToFilePath(const SkData& data) { bool PersistentCache::gIsReadOnly = false; +std::atomic PersistentCache::cache_sksl_ = false; +std::atomic PersistentCache::strategy_set_ = false; + +void PersistentCache::SetCacheSkSL(bool value) { + if (strategy_set_) { + FML_LOG(ERROR) << "Cache SkSL can only be set before the " + "GrContextOptions::fShaderCacheStrategy is set."; + return; + } + cache_sksl_ = value; +} + PersistentCache* PersistentCache::GetCacheForProcess() { - static std::unique_ptr gPersistentCache; - static std::once_flag once = {}; - std::call_once( - once, []() { gPersistentCache.reset(new PersistentCache(gIsReadOnly)); }); + std::scoped_lock lock(instance_mutex_); + if (gPersistentCache == nullptr) { + gPersistentCache.reset(new PersistentCache(gIsReadOnly)); + } return gPersistentCache.get(); } +void PersistentCache::ResetCacheForProcess() { + std::scoped_lock lock(instance_mutex_); + gPersistentCache.reset(new PersistentCache(gIsReadOnly)); + strategy_set_ = false; +} + void PersistentCache::SetCacheDirectoryPath(std::string path) { cache_base_path_ = path; } namespace { -std::shared_ptr MakeCacheDirectory( +static std::shared_ptr MakeCacheDirectory( const std::string& global_cache_base_path, - bool read_only) { + bool read_only, + bool cache_sksl) { fml::UniqueFD cache_base_dir; if (global_cache_base_path.length()) { cache_base_dir = fml::OpenDirectory(global_cache_base_path.c_str(), false, @@ -64,20 +87,54 @@ std::shared_ptr MakeCacheDirectory( } if (cache_base_dir.is_valid()) { - return std::make_shared(CreateDirectory( - cache_base_dir, - {"flutter_engine", GetFlutterEngineVersion(), "skia", GetSkiaVersion()}, - read_only ? fml::FilePermission::kRead - : fml::FilePermission::kReadWrite)); + std::vector components = { + "flutter_engine", GetFlutterEngineVersion(), "skia", GetSkiaVersion()}; + if (cache_sksl) { + components.push_back("sksl"); + } + return std::make_shared( + CreateDirectory(cache_base_dir, components, + read_only ? fml::FilePermission::kRead + : fml::FilePermission::kReadWrite)); } else { return std::make_shared(); } } } // namespace +std::vector PersistentCache::LoadSkSLs() { + TRACE_EVENT0("flutter", "PersistentCache::LoadSkSLs"); + std::vector result; + if (!IsValid()) { + return result; + } + fml::FileVisitor visitor = [&result](const fml::UniqueFD& directory, + const std::string& filename) { + std::pair decode_result = fml::Base32Decode(filename); + if (!decode_result.first) { + FML_LOG(ERROR) << "Base32 can't decode: " << filename; + return true; // continue to visit other files + } + const std::string& data_string = decode_result.second; + sk_sp key = + SkData::MakeWithCopy(data_string.data(), data_string.length()); + sk_sp data = LoadFile(directory, filename); + if (data != nullptr) { + result.push_back({key, data}); + } else { + FML_LOG(ERROR) << "Failed to load: " << filename; + } + return true; + }; + fml::VisitFiles(*sksl_cache_directory_, visitor); + return result; +} + PersistentCache::PersistentCache(bool read_only) : is_read_only_(read_only), - cache_directory_(MakeCacheDirectory(cache_base_path_, read_only)) { + cache_directory_(MakeCacheDirectory(cache_base_path_, read_only, false)), + sksl_cache_directory_( + MakeCacheDirectory(cache_base_path_, read_only, true)) { if (!IsValid()) { FML_LOG(WARNING) << "Could not acquire the persistent cache directory. " "Caching of GPU resources on disk is disabled."; @@ -90,6 +147,19 @@ bool PersistentCache::IsValid() const { return cache_directory_ && cache_directory_->is_valid(); } +sk_sp PersistentCache::LoadFile(const fml::UniqueFD& dir, + const std::string& file_name) { + auto file = fml::OpenFileReadOnly(dir, file_name.c_str()); + if (!file.is_valid()) { + return nullptr; + } + auto mapping = std::make_unique(file); + if (mapping->GetSize() == 0) { + return nullptr; + } + return SkData::MakeWithCopy(mapping->GetMapping(), mapping->GetSize()); +} + // |GrContextOptions::PersistentCache| sk_sp PersistentCache::load(const SkData& key) { TRACE_EVENT0("flutter", "PersistentCacheLoad"); @@ -100,18 +170,13 @@ sk_sp PersistentCache::load(const SkData& key) { if (file_name.size() == 0) { return nullptr; } - auto file = fml::OpenFile(*cache_directory_, file_name.c_str(), false, - fml::FilePermission::kRead); - if (!file.is_valid()) { - return nullptr; - } - auto mapping = std::make_unique(file); - if (mapping->GetSize() == 0) { - return nullptr; + auto result = PersistentCache::LoadFile(*cache_directory_, file_name); + if (result != nullptr) { + TRACE_EVENT0("flutter", "PersistentCacheLoadHit"); + } else { + FML_LOG(INFO) << "PersistentCache::load failed: " << file_name; } - - TRACE_EVENT0("flutter", "PersistentCacheLoadHit"); - return SkData::MakeWithCopy(mapping->GetMapping(), mapping->GetSize()); + return result; } static void PersistentCacheStore(fml::RefPtr worker, @@ -169,7 +234,8 @@ void PersistentCache::store(const SkData& key, const SkData& data) { return; } - PersistentCacheStore(GetWorkerTaskRunner(), cache_directory_, + PersistentCacheStore(GetWorkerTaskRunner(), + cache_sksl_ ? sksl_cache_directory_ : cache_directory_, std::move(file_name), std::move(mapping)); } diff --git a/shell/common/persistent_cache.h b/shell/common/persistent_cache.h index 595593eba7227..a01368cb498cb 100644 --- a/shell/common/persistent_cache.h +++ b/shell/common/persistent_cache.h @@ -31,6 +31,7 @@ class PersistentCache : public GrContextOptions::PersistentCache { static bool gIsReadOnly; static PersistentCache* GetCacheForProcess(); + static void ResetCacheForProcess(); static void SetCacheDirectoryPath(std::string path); @@ -49,11 +50,38 @@ class PersistentCache : public GrContextOptions::PersistentCache { bool IsDumpingSkp() const { return is_dumping_skp_; } void SetIsDumpingSkp(bool value) { is_dumping_skp_ = value; } + // |GrContextOptions::PersistentCache| + sk_sp load(const SkData& key) override; + + using SkSLCache = std::pair, sk_sp>; + + /// Load all the SkSL shader caches in the right directory. + std::vector LoadSkSLs(); + + static bool cache_sksl() { return cache_sksl_; } + static void SetCacheSkSL(bool value); + static void MarkStrategySet() { strategy_set_ = true; } + private: static std::string cache_base_path_; + static std::mutex instance_mutex_; + static std::unique_ptr gPersistentCache + FML_GUARDED_BY(instance_mutex_); + + // Mutable static switch that can be set before GetCacheForProcess is called + // and GrContextOptions.fShaderCacheStrategy is set. If true, it means that + // we'll set `GrContextOptions::fShaderCacheStrategy` to `kSkSL`, and all the + // persistent cache should be stored and loaded from the "sksl" directory. + static std::atomic cache_sksl_; + + // Guard flag to make sure that cache_sksl_ is not modified after + // strategy_set_ becomes true. + static std::atomic strategy_set_; + const bool is_read_only_; const std::shared_ptr cache_directory_; + const std::shared_ptr sksl_cache_directory_; mutable std::mutex worker_task_runners_mutex_; std::multiset> worker_task_runners_ FML_GUARDED_BY(worker_task_runners_mutex_); @@ -61,13 +89,13 @@ class PersistentCache : public GrContextOptions::PersistentCache { bool stored_new_shaders_ = false; bool is_dumping_skp_ = false; + static sk_sp LoadFile(const fml::UniqueFD& dir, + const std::string& filen_ame); + bool IsValid() const; PersistentCache(bool read_only = false); - // |GrContextOptions::PersistentCache| - sk_sp load(const SkData& key) override; - // |GrContextOptions::PersistentCache| void store(const SkData& key, const SkData& data) override; diff --git a/shell/common/persistent_cache_unittests.cc b/shell/common/persistent_cache_unittests.cc new file mode 100644 index 0000000000000..80257cdbde01f --- /dev/null +++ b/shell/common/persistent_cache_unittests.cc @@ -0,0 +1,123 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include + +#include "flutter/flow/layers/container_layer.h" +#include "flutter/flow/layers/layer.h" +#include "flutter/flow/layers/physical_shape_layer.h" +#include "flutter/flow/layers/picture_layer.h" +#include "flutter/fml/command_line.h" +#include "flutter/fml/file.h" +#include "flutter/fml/unique_fd.h" +#include "flutter/shell/common/persistent_cache.h" +#include "flutter/shell/common/shell_test.h" +#include "flutter/shell/common/switches.h" +#include "flutter/testing/testing.h" +#include "include/core/SkPicture.h" + +namespace flutter { +namespace testing { + +static void WaitForIO(Shell* shell) { + std::promise io_task_finished; + shell->GetTaskRunners().GetIOTaskRunner()->PostTask( + [&io_task_finished]() { io_task_finished.set_value(true); }); + io_task_finished.get_future().wait(); +} + +TEST_F(ShellTest, CacheSkSLWorks) { + // Create a temp dir to store the persistent cache + fml::ScopedTemporaryDirectory dir; + PersistentCache::SetCacheDirectoryPath(dir.path()); + PersistentCache::ResetCacheForProcess(); + + auto settings = CreateSettingsForFixture(); + settings.cache_sksl = true; + settings.dump_skp_on_shader_compilation = true; + auto sksl_config = RunConfiguration::InferFromSettings(settings); + sksl_config.SetEntrypoint("emptyMain"); + std::unique_ptr shell = CreateShell(settings); + PlatformViewNotifyCreated(shell.get()); + RunEngine(shell.get(), std::move(sksl_config)); + + // Initially, we should have no SkSL cache + auto cache = PersistentCache::GetCacheForProcess()->LoadSkSLs(); + ASSERT_EQ(cache.size(), 0u); + + // Draw something to trigger shader compilations. + LayerTreeBuilder builder = [](std::shared_ptr root) { + SkPath path; + path.addCircle(50, 50, 20); + auto physical_shape_layer = std::make_shared( + SK_ColorRED, SK_ColorBLUE, 1.0f, 1.0f, 1.0f, path, Clip::antiAlias); + root->Add(physical_shape_layer); + }; + PumpOneFrame(shell.get(), 100, 100, builder); + fml::Status result = + shell->WaitForFirstFrame(fml::TimeDelta::FromMilliseconds(1000)); + ASSERT_TRUE(result.ok()); + WaitForIO(shell.get()); + + // Some skp should be dumped due to shader compilations. + int skp_count = 0; + fml::FileVisitor skp_visitor = [&skp_count](const fml::UniqueFD& directory, + const std::string& filename) { + if (filename.size() >= 4 && + filename.substr(filename.size() - 4, 4) == ".skp") { + skp_count += 1; + } + return true; + }; + fml::VisitFilesRecursively(dir.fd(), skp_visitor); + ASSERT_GT(skp_count, 0); + + // SkSL cache should be generated by the last run. + cache = PersistentCache::GetCacheForProcess()->LoadSkSLs(); + ASSERT_GT(cache.size(), 0u); + + // Run the engine again with cache_sksl = false and check that the previously + // generated SkSL cache is used for precompile. + PersistentCache::ResetCacheForProcess(); + settings.cache_sksl = false; + settings.dump_skp_on_shader_compilation = true; + auto normal_config = RunConfiguration::InferFromSettings(settings); + normal_config.SetEntrypoint("emptyMain"); + shell.reset(); + shell = CreateShell(settings); + PlatformViewNotifyCreated(shell.get()); + RunEngine(shell.get(), std::move(normal_config)); + PumpOneFrame(shell.get(), 100, 100, builder); + result = shell->WaitForFirstFrame(fml::TimeDelta::FromMilliseconds(1000)); + ASSERT_TRUE(result.ok()); + WaitForIO(shell.get()); + + // To check that all shaders are precompiled, verify that no new skp is dumped + // due to shader compilations. + int old_skp_count = skp_count; + skp_count = 0; + fml::VisitFilesRecursively(dir.fd(), skp_visitor); + ASSERT_EQ(skp_count, old_skp_count); + + // Remove all files generated + fml::FileVisitor remove_visitor = [&remove_visitor]( + const fml::UniqueFD& directory, + const std::string& filename) { + if (fml::IsDirectory(directory, filename.c_str())) { + { // To trigger fml::~UniqueFD before fml::UnlinkDirectory + fml::UniqueFD sub_dir = + fml::OpenDirectoryReadOnly(directory, filename.c_str()); + fml::VisitFiles(sub_dir, remove_visitor); + } + fml::UnlinkDirectory(directory, filename.c_str()); + } else { + fml::UnlinkFile(directory, filename.c_str()); + } + return true; + }; + fml::VisitFiles(dir.fd(), remove_visitor); +} + +} // namespace testing +} // namespace flutter diff --git a/shell/common/shell.cc b/shell/common/shell.cc index 9c97908c399f6..d31b0dcc3a8b6 100644 --- a/shell/common/shell.cc +++ b/shell/common/shell.cc @@ -208,6 +208,7 @@ std::unique_ptr Shell::Create( Shell::CreateCallback on_create_platform_view, Shell::CreateCallback on_create_rasterizer) { PerformInitializationTasks(settings); + PersistentCache::SetCacheSkSL(settings.cache_sksl); TRACE_EVENT0("flutter", "Shell::Create"); @@ -235,6 +236,7 @@ std::unique_ptr Shell::Create( Shell::CreateCallback on_create_rasterizer, DartVMRef vm) { PerformInitializationTasks(settings); + PersistentCache::SetCacheSkSL(settings.cache_sksl); TRACE_EVENT0("flutter", "Shell::CreateWithSnapshots"); diff --git a/shell/common/shell_test.cc b/shell/common/shell_test.cc index 1e7d7c4b1d8c6..fa937a86c5626 100644 --- a/shell/common/shell_test.cc +++ b/shell/common/shell_test.cc @@ -135,15 +135,18 @@ void ShellTest::VSyncFlush(Shell* shell, bool& will_draw_new_frame) { latch.Wait(); } -void ShellTest::PumpOneFrame(Shell* shell) { +void ShellTest::PumpOneFrame(Shell* shell, + double width, + double height, + LayerTreeBuilder builder) { // Set viewport to nonempty, and call Animator::BeginFrame to make the layer // tree pipeline nonempty. Without either of this, the layer tree below // won't be rasterized. fml::AutoResetWaitableEvent latch; shell->GetTaskRunners().GetUITaskRunner()->PostTask( - [&latch, engine = shell->weak_engine_]() { + [&latch, engine = shell->weak_engine_, width, height]() { engine->SetViewportMetrics( - {1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}); + {1, width, height, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}); engine->animator_->BeginFrame(fml::TimePoint::Now(), fml::TimePoint::Now()); latch.Signal(); @@ -154,12 +157,15 @@ void ShellTest::PumpOneFrame(Shell* shell) { // Call |Render| to rasterize a layer tree and trigger |OnFrameRasterized| fml::WeakPtr runtime_delegate = shell->weak_engine_; shell->GetTaskRunners().GetUITaskRunner()->PostTask( - [&latch, runtime_delegate]() { + [&latch, runtime_delegate, &builder]() { auto layer_tree = std::make_unique(); SkMatrix identity; identity.setIdentity(); auto root_layer = std::make_shared(identity); layer_tree->set_root_layer(root_layer); + if (builder) { + builder(root_layer); + } runtime_delegate->Render(std::move(layer_tree)); latch.Signal(); }); diff --git a/shell/common/shell_test.h b/shell/common/shell_test.h index ca486230a42c6..794c499a84bd0 100644 --- a/shell/common/shell_test.h +++ b/shell/common/shell_test.h @@ -8,6 +8,7 @@ #include #include "flutter/common/settings.h" +#include "flutter/flow/layers/container_layer.h" #include "flutter/fml/macros.h" #include "flutter/fml/synchronization/thread_annotations.h" #include "flutter/lib/ui/window/platform_message.h" @@ -50,7 +51,15 @@ class ShellTest : public ThreadTest { /// the `will_draw_new_frame` to true. static void VSyncFlush(Shell* shell, bool& will_draw_new_frame); - static void PumpOneFrame(Shell* shell); + /// Given the root layer, this callback builds the layer tree to be rasterized + /// in PumpOneFrame. + using LayerTreeBuilder = + std::function root)>; + static void PumpOneFrame(Shell* shell, + double width = 1, + double height = 1, + LayerTreeBuilder = {}); + static void DispatchFakePointerData(Shell* shell); // Declare |UnreportedTimingsCount|, |GetNeedsReportTimings| and diff --git a/shell/common/switches.cc b/shell/common/switches.cc index ec275b36a475c..ad3b322acad4f 100644 --- a/shell/common/switches.cc +++ b/shell/common/switches.cc @@ -355,6 +355,9 @@ Settings SettingsFromCommandLine(const fml::CommandLine& command_line) { settings.dump_skp_on_shader_compilation = command_line.HasOption(FlagForSwitch(Switch::DumpSkpOnShaderCompilation)); + settings.cache_sksl = + command_line.HasOption(FlagForSwitch(Switch::CacheSkSL)); + return settings; } diff --git a/shell/common/switches.h b/shell/common/switches.h index 60a9392eeb4a2..29e4c81ea455c 100644 --- a/shell/common/switches.h +++ b/shell/common/switches.h @@ -130,6 +130,12 @@ DEF_SWITCH(DumpSkpOnShaderCompilation, "Automatically dump the skp that triggers new shader compilations. " "This is useful for writing custom ShaderWarmUp to reduce jank. " "By default, this is not enabled to reduce the overhead. ") +DEF_SWITCH(CacheSkSL, + "cache-sksl", + "Only cache the shader in SkSL instead of binary or GLSL. This " + "should only be used during development phases. The generated SkSLs " + "can later be used in the release build for shader precompilation " + "at launch in order to eliminate the shader-compile jank.") DEF_SWITCH( TraceSystrace, "trace-systrace", diff --git a/shell/gpu/gpu_surface_gl.cc b/shell/gpu/gpu_surface_gl.cc index 3d02bb59f19c0..0aeeb3d78e65e 100644 --- a/shell/gpu/gpu_surface_gl.cc +++ b/shell/gpu/gpu_surface_gl.cc @@ -4,6 +4,7 @@ #include "gpu_surface_gl.h" +#include "flutter/fml/base32.h" #include "flutter/fml/logging.h" #include "flutter/fml/size.h" #include "flutter/fml/trace_event.h" @@ -46,6 +47,11 @@ GPUSurfaceGL::GPUSurfaceGL(GPUSurfaceGLDelegate* delegate, GrContextOptions options; + if (PersistentCache::cache_sksl()) { + FML_LOG(INFO) << "Cache SkSL"; + options.fShaderCacheStrategy = GrContextOptions::ShaderCacheStrategy::kSkSL; + PersistentCache::MarkStrategySet(); + } options.fPersistentCache = PersistentCache::GetCacheForProcess(); options.fAvoidStencilBuffers = true; @@ -69,11 +75,20 @@ GPUSurfaceGL::GPUSurfaceGL(GPUSurfaceGLDelegate* delegate, context_->setResourceCacheLimits(kGrCacheMaxCount, kGrCacheMaxByteSize); - delegate_->GLContextClearCurrent(); - context_owner_ = true; valid_ = true; + + std::vector caches = + PersistentCache::GetCacheForProcess()->LoadSkSLs(); + int compiled_count = 0; + for (const auto& cache : caches) { + compiled_count += context_->precompileShader(*cache.first, *cache.second); + } + FML_LOG(INFO) << "Found " << caches.size() << " SkSL shaders; precompiled " + << compiled_count; + + delegate_->GLContextClearCurrent(); } GPUSurfaceGL::GPUSurfaceGL(sk_sp gr_context, diff --git a/shell/platform/android/io/flutter/app/FlutterActivityDelegate.java b/shell/platform/android/io/flutter/app/FlutterActivityDelegate.java index 95d007a809895..aa57d50905ebe 100644 --- a/shell/platform/android/io/flutter/app/FlutterActivityDelegate.java +++ b/shell/platform/android/io/flutter/app/FlutterActivityDelegate.java @@ -309,6 +309,9 @@ private static String[] getArgsFromIntent(Intent intent) { if (intent.getBooleanExtra("dump-skp-on-shader-compilation", false)) { args.add("--dump-skp-on-shader-compilation"); } + if (intent.getBooleanExtra("cache-sksl", false)) { + args.add("--cache-sksl"); + } if (intent.getBooleanExtra("verbose-logging", false)) { args.add("--verbose-logging"); } diff --git a/shell/platform/android/io/flutter/embedding/engine/FlutterShellArgs.java b/shell/platform/android/io/flutter/embedding/engine/FlutterShellArgs.java index ca37866906c37..544411b275742 100644 --- a/shell/platform/android/io/flutter/embedding/engine/FlutterShellArgs.java +++ b/shell/platform/android/io/flutter/embedding/engine/FlutterShellArgs.java @@ -41,6 +41,8 @@ public class FlutterShellArgs { public static final String ARG_TRACE_SKIA = "--trace-skia"; public static final String ARG_KEY_DUMP_SHADER_SKP_ON_SHADER_COMPILATION = "dump-skp-on-shader-compilation"; public static final String ARG_DUMP_SHADER_SKP_ON_SHADER_COMPILATION = "--dump-skp-on-shader-compilation"; + public static final String ARG_KEY_CACHE_SKSL = "cache-sksl"; + public static final String ARG_CACHE_SKSL = "--cache-sksl"; public static final String ARG_KEY_VERBOSE_LOGGING = "verbose-logging"; public static final String ARG_VERBOSE_LOGGING = "--verbose-logging"; public static final String ARG_KEY_OBSERVATORY_PORT = "observatory-port"; @@ -83,7 +85,10 @@ public static FlutterShellArgs fromIntent(@NonNull Intent intent) { args.add(ARG_TRACE_SKIA); } if (intent.getBooleanExtra(ARG_KEY_DUMP_SHADER_SKP_ON_SHADER_COMPILATION, false)) { - args.add(ARG_KEY_DUMP_SHADER_SKP_ON_SHADER_COMPILATION); + args.add(ARG_DUMP_SHADER_SKP_ON_SHADER_COMPILATION); + } + if (intent.getBooleanExtra(ARG_KEY_CACHE_SKSL, false)) { + args.add(ARG_CACHE_SKSL); } if (intent.getBooleanExtra(ARG_KEY_VERBOSE_LOGGING, false)) { args.add(ARG_VERBOSE_LOGGING); diff --git a/shell/platform/embedder/embedder.h b/shell/platform/embedder/embedder.h index 7490198b24aec..f721e209d5c32 100644 --- a/shell/platform/embedder/embedder.h +++ b/shell/platform/embedder/embedder.h @@ -875,7 +875,8 @@ typedef struct { // which is used in `flutter::Settings` as `temp_directory_path`. const char* persistent_cache_path; - /// If true, we'll only read the existing cache, but not write new ones. + /// If true, the engine would only read the existing cache, but not write new + /// ones. bool is_persistent_cache_read_only; /// A callback that gets invoked by the engine when it attempts to wait for a