Skip to content

Commit afb5bbc

Browse files
joyeecheungjoyeecheung
committed
src: update compile cache storage structure
This refactors the compile cache handler in preparation for the JS API, and updates the compile cache storage structure into: - $NODE_COMPILE_CACHE_DIR - $NODE_VERION-$ARCH-$CACHE_DATA_VERSION_TAG-$UID - $FILENAME_AND_MODULE_TYPE_HASH.cache This also adds a magic number to the beginning of the cache files for verification.
1 parent 3cbeed8 commit afb5bbc

File tree

5 files changed

+132
-62
lines changed

5 files changed

+132
-62
lines changed

‎src/compile_cache.cc‎

Lines changed: 73 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
#include "compile_cache.h"
2+
#include <string>
23
#include "debug_utils-inl.h"
34
#include "env-inl.h"
45
#include "node_file.h"
@@ -27,15 +28,19 @@ uint32_t GetHash(const char* data, size_t size) {
2728
return crc32(crc, reinterpret_cast<const Bytef*>(data), size);
2829
}
2930

30-
uint32_t GetCacheVersionTag() {
31-
std::string_view node_version(NODE_VERSION);
32-
uint32_t v8_tag = v8::ScriptCompiler::CachedDataVersionTag();
33-
uLong crc = crc32(0L, Z_NULL, 0);
34-
crc = crc32(crc, reinterpret_cast<const Bytef*>(&v8_tag), sizeof(uint32_t));
35-
crc = crc32(crc,
36-
reinterpret_cast<const Bytef*>(node_version.data()),
37-
node_version.size());
38-
return crc;
31+
std::string GetCacheVersionTag() {
32+
// On platforms where uids are available, use different folders for
33+
// different users to avoid cache miss due to permission incompatibility.
34+
// On platforms where uids are not available, bare with the cache miss.
35+
// This should be fine on Windows, as there local directories tend to be
36+
// user-specific.
37+
std::string tag = std::string(NODE_VERSION) + '-' + std::string(NODE_ARCH) +
38+
'-' +
39+
Uint32ToHex(v8::ScriptCompiler::CachedDataVersionTag());
40+
#ifdef NODE_IMPLEMENTS_POSIX_CREDENTIALS
41+
tag += '-' + std::to_string(getuid());
42+
#endif
43+
return tag;
3944
}
4045

4146
uint32_t GetCacheKey(std::string_view filename, CachedCodeType type) {
@@ -63,6 +68,8 @@ v8::ScriptCompiler::CachedData* CompileCacheEntry::CopyCache() const {
6368
data, cache_size, v8::ScriptCompiler::CachedData::BufferOwned);
6469
}
6570

71+
constexpr uint32_t kCacheMagicNumber = 0x8adfdbb2;
72+
6673
void CompileCacheHandler::ReadCacheFile(CompileCacheEntry* entry) {
6774
Debug("[compile cache] reading cache from %s for %s %s...",
6875
entry->cache_filename,
@@ -100,12 +107,20 @@ void CompileCacheHandler::ReadCacheFile(CompileCacheEntry* entry) {
100107
return;
101108
}
102109

103-
Debug("[%d %d %d %d]...",
110+
Debug("[%d %d %d %d %d]...",
111+
headers[kMagicNumberOffset],
104112
headers[kCodeSizeOffset],
105113
headers[kCacheSizeOffset],
106114
headers[kCodeHashOffset],
107115
headers[kCacheHashOffset]);
108116

117+
if (headers[kMagicNumberOffset] != kCacheMagicNumber) {
118+
Debug("magic number mismatch: expected %d, actual %d\n",
119+
kCacheMagicNumber,
120+
headers[kMagicNumberOffset]);
121+
return;
122+
}
123+
109124
// Check the code size and hash which are already computed.
110125
if (headers[kCodeSizeOffset] != entry->code_size) {
111126
Debug("code size mismatch: expected %d, actual %d\n",
@@ -202,11 +217,14 @@ CompileCacheEntry* CompileCacheHandler::GetOrInsert(
202217
compiler_cache_store_.emplace(key, std::make_unique<CompileCacheEntry>());
203218
auto* result = emplaced.first->second.get();
204219

220+
std::u8string cache_filename_u8 =
221+
(compile_cache_dir_ / Uint32ToHex(key)).u8string();
205222
result->code_hash = code_hash;
206223
result->code_size = code_utf8.length();
207224
result->cache_key = key;
208225
result->cache_filename =
209-
(compile_cache_dir_ / Uint32ToHex(result->cache_key)).string();
226+
std::string(cache_filename_u8.begin(), cache_filename_u8.end()) +
227+
".cache";
210228
result->source_filename = filename_utf8.ToString();
211229
result->cache = nullptr;
212230
result->type = type;
@@ -264,6 +282,7 @@ void CompileCacheHandler::MaybeSave(CompileCacheEntry* entry,
264282
}
265283

266284
// Layout of a cache file:
285+
// [uint32_t] magic number
267286
// [uint32_t] code size
268287
// [uint32_t] code hash
269288
// [uint32_t] cache size
@@ -301,14 +320,16 @@ void CompileCacheHandler::Persist() {
301320

302321
// Generating headers.
303322
std::vector<uint32_t> headers(kHeaderCount);
323+
headers[kMagicNumberOffset] = kCacheMagicNumber;
304324
headers[kCodeSizeOffset] = entry->code_size;
305325
headers[kCacheSizeOffset] = cache_size;
306326
headers[kCodeHashOffset] = entry->code_hash;
307327
headers[kCacheHashOffset] = cache_hash;
308328

309-
Debug("[compile cache] writing cache for %s in %s [%d %d %d %d]...",
329+
Debug("[compile cache] writing cache for %s in %s [%d %d %d %d %d]...",
310330
entry->source_filename,
311331
entry->cache_filename,
332+
headers[kMagicNumberOffset],
312333
headers[kCodeSizeOffset],
313334
headers[kCacheSizeOffset],
314335
headers[kCodeHashOffset],
@@ -335,53 +356,63 @@ CompileCacheHandler::CompileCacheHandler(Environment* env)
335356

336357
// Directory structure:
337358
// - Compile cache directory (from NODE_COMPILE_CACHE)
338-
// - <cache_version_tag_1>: hash of CachedDataVersionTag + NODE_VERESION
339-
// - <cache_version_tag_2>
340-
// - <cache_version_tag_3>
341-
// - <cache_file_1>: a hash of filename + module type
342-
// - <cache_file_2>
343-
// - <cache_file_3>
344-
bool CompileCacheHandler::InitializeDirectory(Environment* env,
345-
const std::string& dir) {
346-
compiler_cache_key_ = GetCacheVersionTag();
347-
std::string compiler_cache_key_string = Uint32ToHex(compiler_cache_key_);
348-
std::vector<std::string_view> paths = {dir, compiler_cache_key_string};
349-
std::string cache_dir = PathResolve(env, paths);
350-
359+
// - $NODE_VERION-$ARCH-$CACHE_DATA_VERSION_TAG-$UID
360+
// - $FILENAME_AND_MODULE_TYPE_HASH.cache: a hash of filename + module type
361+
CompileCacheEnableResult CompileCacheHandler::Enable(Environment* env,
362+
const std::string& dir) {
363+
std::string cache_tag = GetCacheVersionTag();
364+
std::string absolute_cache_dir_base = PathResolve(env, {dir});
365+
std::filesystem::path cache_dir_with_tag =
366+
std::filesystem::path(absolute_cache_dir_base) / cache_tag;
367+
std::u8string cache_dir_with_tag_u8 = cache_dir_with_tag.u8string();
368+
std::string cache_dir_with_tag_str(cache_dir_with_tag_u8.begin(),
369+
cache_dir_with_tag_u8.end());
370+
CompileCacheEnableResult result;
351371
Debug("[compile cache] resolved path %s + %s -> %s\n",
352372
dir,
353-
compiler_cache_key_string,
354-
cache_dir);
373+
cache_tag,
374+
cache_dir_with_tag_str);
355375

356376
if (UNLIKELY(!env->permission()->is_granted(
357-
env, permission::PermissionScope::kFileSystemWrite, cache_dir))) {
358-
Debug("[compile cache] skipping cache because write permission for %s "
359-
"is not granted\n",
360-
cache_dir);
361-
return false;
377+
env,
378+
permission::PermissionScope::kFileSystemWrite,
379+
cache_dir_with_tag_str))) {
380+
result.message = "Skipping compile cache because write permission for " +
381+
cache_dir_with_tag_str + " is not granted";
382+
result.status = CompileCacheEnableStatus::kFailed;
383+
return result;
362384
}
363385

364386
if (UNLIKELY(!env->permission()->is_granted(
365-
env, permission::PermissionScope::kFileSystemRead, cache_dir))) {
366-
Debug("[compile cache] skipping cache because read permission for %s "
367-
"is not granted\n",
368-
cache_dir);
369-
return false;
387+
env,
388+
permission::PermissionScope::kFileSystemRead,
389+
cache_dir_with_tag_str))) {
390+
result.message = "Skipping compile cache because read permission for " +
391+
cache_dir_with_tag_str + " is not granted";
392+
result.status = CompileCacheEnableStatus::kFailed;
393+
return result;
370394
}
371395

372396
fs::FSReqWrapSync req_wrap;
373-
int err = fs::MKDirpSync(nullptr, &(req_wrap.req), cache_dir, 0777, nullptr);
397+
int err = fs::MKDirpSync(
398+
nullptr, &(req_wrap.req), cache_dir_with_tag_str, 0777, nullptr);
374399
if (is_debug_) {
375400
Debug("[compile cache] creating cache directory %s...%s\n",
376-
cache_dir,
401+
cache_dir_with_tag_str,
377402
err < 0 ? uv_strerror(err) : "success");
378403
}
379404
if (err != 0 && err != UV_EEXIST) {
380-
return false;
405+
result.message =
406+
"Cannot create cache directory: " + std::string(uv_strerror(err));
407+
result.status = CompileCacheEnableStatus::kFailed;
408+
return result;
381409
}
382410

383-
compile_cache_dir_ = std::filesystem::path(cache_dir);
384-
return true;
411+
compile_cache_dir_str_ = absolute_cache_dir_base;
412+
result.cache_directory = absolute_cache_dir_base;
413+
compile_cache_dir_ = cache_dir_with_tag;
414+
result.status = CompileCacheEnableStatus::kEnabled;
415+
return result;
385416
}
386417

387418
} // namespace node

‎src/compile_cache.h‎

Lines changed: 26 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,27 @@ struct CompileCacheEntry {
3434
v8::ScriptCompiler::CachedData* CopyCache() const;
3535
};
3636

37+
#define COMPILE_CACHE_STATUS(V) \
38+
V(kFailed) /* Failed to enable the cache */ \
39+
V(kEnabled) /* Was not enabled before, and now enabled. */ \
40+
V(kAlreadyEnabled) /* Was already enabled. */
41+
42+
enum class CompileCacheEnableStatus : uint8_t {
43+
#define V(status) status,
44+
COMPILE_CACHE_STATUS(V)
45+
#undef V
46+
};
47+
48+
struct CompileCacheEnableResult {
49+
CompileCacheEnableStatus status;
50+
std::string cache_directory;
51+
std::string message; // Set in case of failure.
52+
};
53+
3754
class CompileCacheHandler {
3855
public:
3956
explicit CompileCacheHandler(Environment* env);
40-
bool InitializeDirectory(Environment* env, const std::string& dir);
57+
CompileCacheEnableResult Enable(Environment* env, const std::string& dir);
4158

4259
void Persist();
4360

@@ -50,6 +67,7 @@ class CompileCacheHandler {
5067
void MaybeSave(CompileCacheEntry* entry,
5168
v8::Local<v8::Module> mod,
5269
bool rejected);
70+
const std::string& cache_dir() { return compile_cache_dir_str_; }
5371

5472
private:
5573
void ReadCacheFile(CompileCacheEntry* entry);
@@ -62,19 +80,18 @@ class CompileCacheHandler {
6280
template <typename... Args>
6381
inline void Debug(const char* format, Args&&... args) const;
6482

65-
static constexpr size_t kCodeSizeOffset = 0;
66-
static constexpr size_t kCacheSizeOffset = 1;
67-
static constexpr size_t kCodeHashOffset = 2;
68-
static constexpr size_t kCacheHashOffset = 3;
69-
static constexpr size_t kHeaderCount = 4;
83+
static constexpr size_t kMagicNumberOffset = 0;
84+
static constexpr size_t kCodeSizeOffset = 1;
85+
static constexpr size_t kCacheSizeOffset = 2;
86+
static constexpr size_t kCodeHashOffset = 3;
87+
static constexpr size_t kCacheHashOffset = 4;
88+
static constexpr size_t kHeaderCount = 5;
7089

7190
v8::Isolate* isolate_ = nullptr;
7291
bool is_debug_ = false;
7392

93+
std::string compile_cache_dir_str_;
7494
std::filesystem::path compile_cache_dir_;
75-
// The compile cache is stored in a directory whose name is the hex string of
76-
// compiler_cache_key_.
77-
uint32_t compiler_cache_key_ = 0;
7895
std::unordered_map<uint32_t, std::unique_ptr<CompileCacheEntry>>
7996
compiler_cache_store_;
8097
};

‎src/env.cc‎

Lines changed: 29 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1129,15 +1129,36 @@ void Environment::InitializeCompileCache() {
11291129
dir_from_env.empty()) {
11301130
return;
11311131
}
1132-
auto handler = std::make_unique<CompileCacheHandler>(this);
1133-
if (handler->InitializeDirectory(this, dir_from_env)) {
1134-
compile_cache_handler_ = std::move(handler);
1135-
AtExit(
1136-
[](void* env) {
1137-
static_cast<Environment*>(env)->compile_cache_handler()->Persist();
1138-
},
1139-
this);
1132+
EnableCompileCache(dir_from_env);
1133+
}
1134+
1135+
CompileCacheEnableResult Environment::EnableCompileCache(
1136+
const std::string& cache_dir) {
1137+
CompileCacheEnableResult result;
1138+
1139+
if (!compile_cache_handler_) {
1140+
std::unique_ptr<CompileCacheHandler> handler =
1141+
std::make_unique<CompileCacheHandler>(this);
1142+
result = handler->Enable(this, cache_dir);
1143+
if (result.status == CompileCacheEnableStatus::kEnabled) {
1144+
compile_cache_handler_ = std::move(handler);
1145+
AtExit(
1146+
[](void* env) {
1147+
static_cast<Environment*>(env)->compile_cache_handler()->Persist();
1148+
},
1149+
this);
1150+
}
1151+
if (!result.message.empty()) {
1152+
Debug(this,
1153+
DebugCategory::COMPILE_CACHE,
1154+
"[compile cache] %s\n",
1155+
result.message);
1156+
}
1157+
} else {
1158+
result.status = CompileCacheEnableStatus::kAlreadyEnabled;
1159+
result.cache_directory = compile_cache_handler_->cache_dir();
11401160
}
1161+
return result;
11411162
}
11421163

11431164
void Environment::ExitEnv(StopFlags::Flags flags) {

‎src/env.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1028,6 +1028,7 @@ class Environment final : public MemoryRetainer {
10281028
inline CompileCacheHandler* compile_cache_handler();
10291029
inline bool use_compile_cache() const;
10301030
void InitializeCompileCache();
1031+
CompileCacheEnableResult EnableCompileCache(const std::string& cache_dir);
10311032

10321033
void RunAndClearNativeImmediates(bool only_refed = false);
10331034
void RunAndClearInterrupts();

‎test/parallel/test-compile-cache-permission-disallowed.js‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ function testDisallowed(dummyDir, cacheDirInPermission, cacheDirInEnv) {
3939
},
4040
{
4141
stderr(output) {
42-
assert.match(output, /skipping cache because write permission for .* is not granted/);
42+
assert.match(output, /Skipping compile cache because write permission for .* is not granted/);
4343
return true;
4444
}
4545
});
@@ -63,7 +63,7 @@ function testDisallowed(dummyDir, cacheDirInPermission, cacheDirInEnv) {
6363
},
6464
{
6565
stderr(output) {
66-
assert.match(output, /skipping cache because write permission for .* is not granted/);
66+
assert.match(output, /Skipping compile cache because write permission for .* is not granted/);
6767
return true;
6868
}
6969
});
@@ -86,7 +86,7 @@ function testDisallowed(dummyDir, cacheDirInPermission, cacheDirInEnv) {
8686
},
8787
{
8888
stderr(output) {
89-
assert.match(output, /skipping cache because read permission for .* is not granted/);
89+
assert.match(output, /Skipping compile cache because read permission for .* is not granted/);
9090
return true;
9191
}
9292
});

0 commit comments

Comments
 (0)