Skip to content

[libstd]: Enable CacheHash in WASI #5636

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
kubkon wants to merge 5 commits into from
Closed

[libstd]: Enable CacheHash in WASI #5636

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
536fd4b
Add realpathat fn
kubkon Jun 19, 2020
5635a40
Add unit test
kubkon Jun 19, 2020
93b1a8f
Add std.fs.path.resolveat fn and use it in CacheHash
kubkon Jun 19, 2020
043da82
Add std.fs.path.resolveRelative helper fn
kubkon Jun 19, 2020
fd5fc00
Tweak realpathat tests
kubkon Jun 19, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
111 changes: 60 additions & 51 deletions lib/std/cache_hash.zig
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
const std = @import("std.zig");
const builtin = @import("builtin");
const std = @import("std");
const Blake3 = std.crypto.Blake3;
const fs = std.fs;
const base64 = std.base64;
const ArrayList = std.ArrayList;
const assert = std.debug.assert;
const testing = std.testing;
const tmpDir = testing.tmpDir;
const mem = std.mem;
const fmt = std.fmt;
const Allocator = std.mem.Allocator;
Expand Down Expand Up @@ -41,18 +43,20 @@ pub const File = struct {
pub const CacheHash = struct {
allocator: *Allocator,
blake3: Blake3,
work_dir: fs.Dir,
manifest_dir: fs.Dir,
manifest_file: ?fs.File,
manifest_dirty: bool,
files: ArrayList(File),
b64_digest: [BASE64_DIGEST_LEN]u8,

/// Be sure to call release after successful initialization.
pub fn init(allocator: *Allocator, dir: fs.Dir, manifest_dir_path: []const u8) !CacheHash {
pub fn init(allocator: *Allocator, work_dir: fs.Dir, manifest_dir_path: []const u8) !CacheHash {
return CacheHash{
.allocator = allocator,
.blake3 = Blake3.init(),
.manifest_dir = try dir.makeOpenPath(manifest_dir_path, .{}),
.work_dir = work_dir,
.manifest_dir = try work_dir.makeOpenPath(manifest_dir_path, .{}),
.manifest_file = null,
.manifest_dirty = false,
.files = ArrayList(File).init(allocator),
Expand Down Expand Up @@ -100,7 +104,13 @@ pub const CacheHash = struct {
assert(self.manifest_file == null);

try self.files.ensureCapacity(self.files.items.len + 1);
const resolved_path = try fs.path.resolve(self.allocator, &[_][]const u8{file_path});

var resolved_path: []u8 = undefined;
if (builtin.os.tag == .wasi) {
resolved_path = (try fs.path.resolveRelative(self.allocator, &[_][]const u8{file_path})) orelse unreachable;
} else {
resolved_path = try fs.path.resolveat(self.allocator, self.work_dir, &[_][]const u8{file_path});
}

const idx = self.files.items.len;
self.files.addOneAssumeCapacity().* = .{
Expand Down Expand Up @@ -210,7 +220,7 @@ pub const CacheHash = struct {
cache_hash_file.path = try mem.dupe(self.allocator, u8, file_path);
}

const this_file = fs.cwd().openFile(cache_hash_file.path.?, .{ .read = true }) catch {
const this_file = self.work_dir.openFile(cache_hash_file.path.?, .{ .read = true }) catch {
return error.CacheUnavailable;
};
defer this_file.close();
Expand Down Expand Up @@ -276,7 +286,7 @@ pub const CacheHash = struct {
}

fn populateFileHash(self: *CacheHash, ch_file: *File) !void {
const file = try fs.cwd().openFile(ch_file.path.?, .{});
const file = try self.work_dir.openFile(ch_file.path.?, .{});
defer file.close();

ch_file.stat = try file.stat();
Expand Down Expand Up @@ -322,7 +332,12 @@ pub const CacheHash = struct {
pub fn addFilePostFetch(self: *CacheHash, file_path: []const u8, max_file_size: usize) ![]u8 {
assert(self.manifest_file != null);

const resolved_path = try fs.path.resolve(self.allocator, &[_][]const u8{file_path});
var resolved_path: []u8 = undefined;
if (builtin.os.tag == .wasi) {
resolved_path = (try fs.path.resolveRelative(self.allocator, &[_][]const u8{file_path})) orelse unreachable;
} else {
resolved_path = try fs.path.resolveat(self.allocator, self.work_dir, &[_][]const u8{file_path});
}
errdefer self.allocator.free(resolved_path);

const new_ch_file = try self.files.addOne();
Expand All @@ -347,7 +362,12 @@ pub const CacheHash = struct {
pub fn addFilePost(self: *CacheHash, file_path: []const u8) !void {
assert(self.manifest_file != null);

const resolved_path = try fs.path.resolve(self.allocator, &[_][]const u8{file_path});
var resolved_path: []u8 = undefined;
if (builtin.os.tag == .wasi) {
resolved_path = (try fs.path.resolveRelative(self.allocator, &[_][]const u8{file_path})) orelse unreachable;
} else {
resolved_path = try fs.path.resolveat(self.allocator, self.work_dir, &[_][]const u8{file_path});
}
errdefer self.allocator.free(resolved_path);

const new_ch_file = try self.files.addOne();
Expand Down Expand Up @@ -470,16 +490,13 @@ fn isProblematicTimestamp(fs_clock: i128) bool {
}

test "cache file and then recall it" {
if (std.Target.current.os.tag == .wasi) {
// https://github.com/ziglang/zig/issues/5437
return error.SkipZigTest;
}
const cwd = fs.cwd();
var tmp = tmpDir(.{});
defer tmp.cleanup();

const temp_file = "test.txt";
const temp_manifest_dir = "temp_manifest_dir";

try cwd.writeFile(temp_file, "Hello, world!\n");
try tmp.dir.writeFile(temp_file, "Hello, world!\n");

while (isProblematicTimestamp(std.time.nanoTimestamp())) {
std.time.sleep(1);
Expand All @@ -489,7 +506,7 @@ test "cache file and then recall it" {
var digest2: [BASE64_DIGEST_LEN]u8 = undefined;

{
var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
var ch = try CacheHash.init(testing.allocator, tmp.dir, temp_manifest_dir);
defer ch.release();

ch.add(true);
Expand All @@ -503,7 +520,7 @@ test "cache file and then recall it" {
digest1 = ch.final();
}
{
var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
var ch = try CacheHash.init(testing.allocator, tmp.dir, temp_manifest_dir);
defer ch.release();

ch.add(true);
Expand All @@ -517,8 +534,8 @@ test "cache file and then recall it" {

testing.expectEqual(digest1, digest2);

try cwd.deleteTree(temp_manifest_dir);
try cwd.deleteFile(temp_file);
try tmp.dir.deleteTree(temp_manifest_dir);
try tmp.dir.deleteFile(temp_file);
}

test "give problematic timestamp" {
Expand All @@ -534,18 +551,15 @@ test "give nonproblematic timestamp" {
}

test "check that changing a file makes cache fail" {
if (std.Target.current.os.tag == .wasi) {
// https://github.com/ziglang/zig/issues/5437
return error.SkipZigTest;
}
const cwd = fs.cwd();
var tmp = tmpDir(.{});
defer tmp.cleanup();

const temp_file = "cache_hash_change_file_test.txt";
const temp_manifest_dir = "cache_hash_change_file_manifest_dir";
const original_temp_file_contents = "Hello, world!\n";
const updated_temp_file_contents = "Hello, world; but updated!\n";

try cwd.writeFile(temp_file, original_temp_file_contents);
try tmp.dir.writeFile(temp_file, original_temp_file_contents);

while (isProblematicTimestamp(std.time.nanoTimestamp())) {
std.time.sleep(1);
Expand All @@ -555,7 +569,7 @@ test "check that changing a file makes cache fail" {
var digest2: [BASE64_DIGEST_LEN]u8 = undefined;

{
var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
var ch = try CacheHash.init(testing.allocator, tmp.dir, temp_manifest_dir);
defer ch.release();

ch.add("1234");
Expand All @@ -569,14 +583,14 @@ test "check that changing a file makes cache fail" {
digest1 = ch.final();
}

try cwd.writeFile(temp_file, updated_temp_file_contents);
try tmp.dir.writeFile(temp_file, updated_temp_file_contents);

while (isProblematicTimestamp(std.time.nanoTimestamp())) {
std.time.sleep(1);
}

{
var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
var ch = try CacheHash.init(testing.allocator, tmp.dir, temp_manifest_dir);
defer ch.release();

ch.add("1234");
Expand All @@ -593,24 +607,22 @@ test "check that changing a file makes cache fail" {

testing.expect(!mem.eql(u8, digest1[0..], digest2[0..]));

try cwd.deleteTree(temp_manifest_dir);
try cwd.deleteFile(temp_file);
try tmp.dir.deleteTree(temp_manifest_dir);
try tmp.dir.deleteFile(temp_file);
}

test "no file inputs" {
if (std.Target.current.os.tag == .wasi) {
// https://github.com/ziglang/zig/issues/5437
return error.SkipZigTest;
}
const cwd = fs.cwd();
var tmp = tmpDir(.{});
defer tmp.cleanup();

const temp_manifest_dir = "no_file_inputs_manifest_dir";
defer cwd.deleteTree(temp_manifest_dir) catch unreachable;
defer tmp.dir.deleteTree(temp_manifest_dir) catch unreachable;

var digest1: [BASE64_DIGEST_LEN]u8 = undefined;
var digest2: [BASE64_DIGEST_LEN]u8 = undefined;

{
var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
var ch = try CacheHash.init(testing.allocator, tmp.dir, temp_manifest_dir);
defer ch.release();

ch.add("1234");
Expand All @@ -621,7 +633,7 @@ test "no file inputs" {
digest1 = ch.final();
}
{
var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
var ch = try CacheHash.init(testing.allocator, tmp.dir, temp_manifest_dir);
defer ch.release();

ch.add("1234");
Expand All @@ -633,18 +645,15 @@ test "no file inputs" {
}

test "CacheHashes with files added after initial hash work" {
if (std.Target.current.os.tag == .wasi) {
// https://github.com/ziglang/zig/issues/5437
return error.SkipZigTest;
}
const cwd = fs.cwd();
var tmp = tmpDir(.{});
defer tmp.cleanup();

const temp_file1 = "cache_hash_post_file_test1.txt";
const temp_file2 = "cache_hash_post_file_test2.txt";
const temp_manifest_dir = "cache_hash_post_file_manifest_dir";

try cwd.writeFile(temp_file1, "Hello, world!\n");
try cwd.writeFile(temp_file2, "Hello world the second!\n");
try tmp.dir.writeFile(temp_file1, "Hello, world!\n");
try tmp.dir.writeFile(temp_file2, "Hello world the second!\n");

while (isProblematicTimestamp(std.time.nanoTimestamp())) {
std.time.sleep(1);
Expand All @@ -655,7 +664,7 @@ test "CacheHashes with files added after initial hash work" {
var digest3: [BASE64_DIGEST_LEN]u8 = undefined;

{
var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
var ch = try CacheHash.init(testing.allocator, tmp.dir, temp_manifest_dir);
defer ch.release();

ch.add("1234");
Expand All @@ -669,7 +678,7 @@ test "CacheHashes with files added after initial hash work" {
digest1 = ch.final();
}
{
var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
var ch = try CacheHash.init(testing.allocator, tmp.dir, temp_manifest_dir);
defer ch.release();

ch.add("1234");
Expand All @@ -680,14 +689,14 @@ test "CacheHashes with files added after initial hash work" {
testing.expect(mem.eql(u8, &digest1, &digest2));

// Modify the file added after initial hash
try cwd.writeFile(temp_file2, "Hello world the second, updated\n");
try tmp.dir.writeFile(temp_file2, "Hello world the second, updated\n");

while (isProblematicTimestamp(std.time.nanoTimestamp())) {
std.time.sleep(1);
}

{
var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
var ch = try CacheHash.init(testing.allocator, tmp.dir, temp_manifest_dir);
defer ch.release();

ch.add("1234");
Expand All @@ -703,7 +712,7 @@ test "CacheHashes with files added after initial hash work" {

testing.expect(!mem.eql(u8, &digest1, &digest3));

try cwd.deleteTree(temp_manifest_dir);
try cwd.deleteFile(temp_file1);
try cwd.deleteFile(temp_file2);
try tmp.dir.deleteTree(temp_manifest_dir);
try tmp.dir.deleteFile(temp_file1);
try tmp.dir.deleteFile(temp_file2);
}
Loading