Skip to content

[libstd]: reworked CacheHash means its WASI compatible now #5784

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
kubkon wants to merge 14 commits into from
Closed

[libstd]: reworked CacheHash means its WASI compatible now #5784

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
333a699
Add std.os.realpathat and std.fs.Dir.realpath
kubkon Jun 24, 2020
c833cec
Create file with sharing on on Windows
kubkon Jun 24, 2020
a3857cb
Close the handle after creating the file
kubkon Jun 25, 2020
48a767c
Use Dir.realpath in 'directories operations on file' testcase
kubkon Jun 26, 2020
71b3e04
Add null-terminated and WTF16 versions
kubkon Jun 27, 2020
4f2f19a
Fix compilation error in realpathatW
kubkon Jun 27, 2020
5b53746
Update lib/std/fs.zig
kubkon Jul 2, 2020
1988108
Use Dir.realpathAlloc instead of fs.path.resolve
kubkon Jul 2, 2020
76d3b83
Implement Dir.realpathWasi and os.realpathatWasi
kubkon Jul 2, 2020
381bba2
Handle symlinks in os.realpathatWasi and add docs
kubkon Jul 2, 2020
171a755
Handle too long a path in link resolution
kubkon Jul 2, 2020
9f1a930
Fix handling symlinks in realpathatWasi
kubkon Jul 2, 2020
9c9062f
Tweak std.os and std.fs tests
kubkon Jul 3, 2020
7cc613c
Fix std.os.realpathatW to properly canonicalize relative paths
kubkon Jul 4, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 34 additions & 53 deletions lib/std/cache_hash.zig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ const testing = std.testing;
const mem = std.mem;
const fmt = std.fmt;
const Allocator = std.mem.Allocator;
const tmpDir = testing.tmpDir;

const base64_encoder = fs.base64_encoder;
const base64_decoder = fs.base64_decoder;
Expand Down Expand Up @@ -41,18 +42,20 @@ pub const File = struct {
pub const CacheHash = struct {
allocator: *Allocator,
blake3: Blake3,
work_dir: fs.Dir,
manifest_dir: fs.Dir,
manifest_file: ?fs.File,
manifest_dirty: bool,
files: ArrayList(File),
b64_digest: [BASE64_DIGEST_LEN]u8,

/// Be sure to call release after successful initialization.
pub fn init(allocator: *Allocator, dir: fs.Dir, manifest_dir_path: []const u8) !CacheHash {
pub fn init(allocator: *Allocator, work_dir: fs.Dir, manifest_dir_path: []const u8) !CacheHash {
return CacheHash{
.allocator = allocator,
.blake3 = Blake3.init(),
.manifest_dir = try dir.makeOpenPath(manifest_dir_path, .{}),
.work_dir = work_dir,
.manifest_dir = try work_dir.makeOpenPath(manifest_dir_path, .{}),
.manifest_file = null,
.manifest_dirty = false,
.files = ArrayList(File).init(allocator),
Expand Down Expand Up @@ -100,7 +103,7 @@ pub const CacheHash = struct {
assert(self.manifest_file == null);

try self.files.ensureCapacity(self.files.items.len + 1);
const resolved_path = try fs.path.resolve(self.allocator, &[_][]const u8{file_path});
const resolved_path = try self.work_dir.realpathAlloc(self.allocator, file_path);

const idx = self.files.items.len;
self.files.addOneAssumeCapacity().* = .{
Expand Down Expand Up @@ -210,7 +213,7 @@ pub const CacheHash = struct {
cache_hash_file.path = try self.allocator.dupe(u8, file_path);
}

const this_file = fs.cwd().openFile(cache_hash_file.path.?, .{ .read = true }) catch {
const this_file = self.work_dir.openFile(cache_hash_file.path.?, .{ .read = true }) catch {
return error.CacheUnavailable;
};
defer this_file.close();
Expand Down Expand Up @@ -276,7 +279,7 @@ pub const CacheHash = struct {
}

fn populateFileHash(self: *CacheHash, ch_file: *File) !void {
const file = try fs.cwd().openFile(ch_file.path.?, .{});
const file = try self.work_dir.openFile(ch_file.path.?, .{});
defer file.close();

ch_file.stat = try file.stat();
Expand Down Expand Up @@ -322,7 +325,7 @@ pub const CacheHash = struct {
pub fn addFilePostFetch(self: *CacheHash, file_path: []const u8, max_file_size: usize) ![]u8 {
assert(self.manifest_file != null);

const resolved_path = try fs.path.resolve(self.allocator, &[_][]const u8{file_path});
const resolved_path = try self.work_dir.realpathAlloc(self.allocator, file_path);
errdefer self.allocator.free(resolved_path);

const new_ch_file = try self.files.addOne();
Expand All @@ -347,7 +350,7 @@ pub const CacheHash = struct {
pub fn addFilePost(self: *CacheHash, file_path: []const u8) !void {
assert(self.manifest_file != null);

const resolved_path = try fs.path.resolve(self.allocator, &[_][]const u8{file_path});
const resolved_path = try self.work_dir.realpathAlloc(self.allocator, file_path);
errdefer self.allocator.free(resolved_path);

const new_ch_file = try self.files.addOne();
Expand Down Expand Up @@ -470,16 +473,13 @@ fn isProblematicTimestamp(fs_clock: i128) bool {
}

test "cache file and then recall it" {
if (std.Target.current.os.tag == .wasi) {
// https://github.com/ziglang/zig/issues/5437
return error.SkipZigTest;
}
const cwd = fs.cwd();
var tmp = tmpDir(.{});
defer tmp.cleanup();

const temp_file = "test.txt";
const temp_manifest_dir = "temp_manifest_dir";

try cwd.writeFile(temp_file, "Hello, world!\n");
try tmp.dir.writeFile(temp_file, "Hello, world!\n");

while (isProblematicTimestamp(std.time.nanoTimestamp())) {
std.time.sleep(1);
Expand All @@ -489,7 +489,7 @@ test "cache file and then recall it" {
var digest2: [BASE64_DIGEST_LEN]u8 = undefined;

{
var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
var ch = try CacheHash.init(testing.allocator, tmp.dir, temp_manifest_dir);
defer ch.release();

ch.add(true);
Expand All @@ -503,7 +503,7 @@ test "cache file and then recall it" {
digest1 = ch.final();
}
{
var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
var ch = try CacheHash.init(testing.allocator, tmp.dir, temp_manifest_dir);
defer ch.release();

ch.add(true);
Expand All @@ -516,9 +516,6 @@ test "cache file and then recall it" {
}

testing.expectEqual(digest1, digest2);

try cwd.deleteTree(temp_manifest_dir);
try cwd.deleteFile(temp_file);
}

test "give problematic timestamp" {
Expand All @@ -534,18 +531,15 @@ test "give nonproblematic timestamp" {
}

test "check that changing a file makes cache fail" {
if (std.Target.current.os.tag == .wasi) {
// https://github.com/ziglang/zig/issues/5437
return error.SkipZigTest;
}
const cwd = fs.cwd();
var tmp = tmpDir(.{});
defer tmp.cleanup();

const temp_file = "cache_hash_change_file_test.txt";
const temp_manifest_dir = "cache_hash_change_file_manifest_dir";
const original_temp_file_contents = "Hello, world!\n";
const updated_temp_file_contents = "Hello, world; but updated!\n";

try cwd.writeFile(temp_file, original_temp_file_contents);
try tmp.dir.writeFile(temp_file, original_temp_file_contents);

while (isProblematicTimestamp(std.time.nanoTimestamp())) {
std.time.sleep(1);
Expand All @@ -555,7 +549,7 @@ test "check that changing a file makes cache fail" {
var digest2: [BASE64_DIGEST_LEN]u8 = undefined;

{
var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
var ch = try CacheHash.init(testing.allocator, tmp.dir, temp_manifest_dir);
defer ch.release();

ch.add("1234");
Expand All @@ -569,14 +563,14 @@ test "check that changing a file makes cache fail" {
digest1 = ch.final();
}

try cwd.writeFile(temp_file, updated_temp_file_contents);
try tmp.dir.writeFile(temp_file, updated_temp_file_contents);

while (isProblematicTimestamp(std.time.nanoTimestamp())) {
std.time.sleep(1);
}

{
var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
var ch = try CacheHash.init(testing.allocator, tmp.dir, temp_manifest_dir);
defer ch.release();

ch.add("1234");
Expand All @@ -592,25 +586,19 @@ test "check that changing a file makes cache fail" {
}

testing.expect(!mem.eql(u8, digest1[0..], digest2[0..]));

try cwd.deleteTree(temp_manifest_dir);
try cwd.deleteFile(temp_file);
}

test "no file inputs" {
if (std.Target.current.os.tag == .wasi) {
// https://github.com/ziglang/zig/issues/5437
return error.SkipZigTest;
}
const cwd = fs.cwd();
var tmp = tmpDir(.{});
defer tmp.cleanup();

const temp_manifest_dir = "no_file_inputs_manifest_dir";
defer cwd.deleteTree(temp_manifest_dir) catch unreachable;

var digest1: [BASE64_DIGEST_LEN]u8 = undefined;
var digest2: [BASE64_DIGEST_LEN]u8 = undefined;

{
var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
var ch = try CacheHash.init(testing.allocator, tmp.dir, temp_manifest_dir);
defer ch.release();

ch.add("1234");
Expand All @@ -621,7 +609,7 @@ test "no file inputs" {
digest1 = ch.final();
}
{
var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
var ch = try CacheHash.init(testing.allocator, tmp.dir, temp_manifest_dir);
defer ch.release();

ch.add("1234");
Expand All @@ -633,18 +621,15 @@ test "no file inputs" {
}

test "CacheHashes with files added after initial hash work" {
if (std.Target.current.os.tag == .wasi) {
// https://github.com/ziglang/zig/issues/5437
return error.SkipZigTest;
}
const cwd = fs.cwd();
var tmp = tmpDir(.{});
defer tmp.cleanup();

const temp_file1 = "cache_hash_post_file_test1.txt";
const temp_file2 = "cache_hash_post_file_test2.txt";
const temp_manifest_dir = "cache_hash_post_file_manifest_dir";

try cwd.writeFile(temp_file1, "Hello, world!\n");
try cwd.writeFile(temp_file2, "Hello world the second!\n");
try tmp.dir.writeFile(temp_file1, "Hello, world!\n");
try tmp.dir.writeFile(temp_file2, "Hello world the second!\n");

while (isProblematicTimestamp(std.time.nanoTimestamp())) {
std.time.sleep(1);
Expand All @@ -655,7 +640,7 @@ test "CacheHashes with files added after initial hash work" {
var digest3: [BASE64_DIGEST_LEN]u8 = undefined;

{
var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
var ch = try CacheHash.init(testing.allocator, tmp.dir, temp_manifest_dir);
defer ch.release();

ch.add("1234");
Expand All @@ -669,7 +654,7 @@ test "CacheHashes with files added after initial hash work" {
digest1 = ch.final();
}
{
var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
var ch = try CacheHash.init(testing.allocator, tmp.dir, temp_manifest_dir);
defer ch.release();

ch.add("1234");
Expand All @@ -680,14 +665,14 @@ test "CacheHashes with files added after initial hash work" {
testing.expect(mem.eql(u8, &digest1, &digest2));

// Modify the file added after initial hash
try cwd.writeFile(temp_file2, "Hello world the second, updated\n");
try tmp.dir.writeFile(temp_file2, "Hello world the second, updated\n");

while (isProblematicTimestamp(std.time.nanoTimestamp())) {
std.time.sleep(1);
}

{
var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
var ch = try CacheHash.init(testing.allocator, tmp.dir, temp_manifest_dir);
defer ch.release();

ch.add("1234");
Expand All @@ -702,8 +687,4 @@ test "CacheHashes with files added after initial hash work" {
}

testing.expect(!mem.eql(u8, &digest1, &digest3));

try cwd.deleteTree(temp_manifest_dir);
try cwd.deleteFile(temp_file1);
try cwd.deleteFile(temp_file2);
}
99 changes: 98 additions & 1 deletion lib/std/fs.zig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -931,6 +931,103 @@ pub const Dir = struct {
return self.openDir(sub_path, open_dir_options);
}

/// This function behaves differently on different hosts.
/// In WASI, returns the canonicalized relative pathname of `pathname` relative
/// to this `Dir`. If `pathname` is absolute, or an attempt is made at
/// escaping beyond this `Dir`, return `error.AccessDenied`.
/// On other hosts, this function returns the canonicalized absolute pathname of
/// `pathname` relative to this `Dir`. If `pathname` is absolute, ignores this
/// `Dir` handle and returns the canonicalized absolute pathname of `pathname`
/// argument.
/// See also `Dir.realpathZ`, `Dir.realpathW`, `Dir.realpathWasi`,
/// and `Dir.realpathAlloc`.
pub fn realpath(self: Dir, pathname: []const u8, out_buffer: []u8) ![]u8 {
if (builtin.os.tag == .wasi) {
return self.realpathWasi(pathname, out_buffer);
}
if (builtin.os.tag == .windows) {
const pathname_w = try os.windows.sliceToPrefixedFileW(pathname);
return self.realpathW(pathname_w.span().ptr, out_buffer);
}
const pathname_c = try os.toPosixPath(pathname);
return self.realpathZ(&pathname_c, out_buffer);
}

/// Same as `Dir.realpath` except `pathname` is null-terminated.
/// See also `Dir.realpath`, `realpathZ`.
pub fn realpathZ(self: Dir, pathname: [*:0]const u8, out_buffer: []u8) ![]u8 {
// Use of MAX_PATH_BYTES here is valid as the realpath function does not
// have a variant that takes an arbitrary-size buffer.
// TODO(#4812): Consider reimplementing realpath or using the POSIX.1-2008
// NULL out parameter (GNU's canonicalize_file_name) to handle overelong
// paths. musl supports passing NULL but restricts the output to PATH_MAX
// anyway.
var buf: [MAX_PATH_BYTES]u8 = undefined;
const out_path = try os.realpathatZ(self.fd, pathname, &buf);

if (out_path.len > out_buffer.len) {
return error.NameTooLong;
}

mem.copy(u8, out_buffer, out_path);

return out_buffer[0..out_path.len];
}

/// Windows-only. Same as `Dir.realpath` except `pathname` is null-terminated,
/// WTF16 encoded.
/// See also `Dir.realpath`, `realpathW`.
pub fn realpathW(self: Dir, pathname: [*:0]const u16, out_buffer: []u8) ![]u8 {
// Use of MAX_PATH_BYTES here is valid as the realpath function does not
// have a variant that takes an arbitrary-size buffer.
// TODO(#4812): Consider reimplementing realpath or using the POSIX.1-2008
// NULL out parameter (GNU's canonicalize_file_name) to handle overelong
// paths. musl supports passing NULL but restricts the output to PATH_MAX
// anyway.
var buf: [MAX_PATH_BYTES]u8 = undefined;
const out_path = try os.realpathatW(self.fd, pathname, &buf);

if (out_path.len > out_buffer.len) {
return error.NameTooLong;
}

mem.copy(u8, out_buffer, out_path);

return out_buffer[0..out_path.len];
}

/// WASI-only. Returns the canonicalized relative pathname of `pathname` relative
/// to this `Dir`. If `pathname` is absolute, or an attempt is made at
/// escaping beyond this `Dir`, return `error.AccessDenied`.
/// See also `Dir.realpath`, and `Dir.realpathAlloc`.
pub fn realpathWasi(self: Dir, pathname: []const u8, out_buffer: []u8) ![]u8 {
// Use of MAX_PATH_BYTES here is valid as the realpath function does not
// have a variant that takes an arbitrary-size buffer.
var buf: [MAX_PATH_BYTES]u8 = undefined;
const out_path = try os.realpathatWasi(self.fd, pathname, &buf);

if (out_path.len > out_buffer.len) {
return error.NameTooLong;
}

mem.copy(u8, out_buffer, out_path);

return out_buffer[0..out_path.len];
}

/// Same as `Dir.realpath` except caller must free the returned memory.
/// See also `Dir.realpath`.
pub fn realpathAlloc(self: Dir, allocator: *Allocator, pathname: []const u8) ![]u8 {
// Use of MAX_PATH_BYTES here is valid as the realpath function does not
// have a variant that takes an arbitrary-size buffer.
// TODO(#4812): Consider reimplementing realpath or using the POSIX.1-2008
// NULL out parameter (GNU's canonicalize_file_name) to handle overelong
// paths. musl supports passing NULL but restricts the output to PATH_MAX
// anyway.
var buf: [MAX_PATH_BYTES]u8 = undefined;
return allocator.dupe(u8, try os.realpathat(self.fd, pathname, &buf));
}

/// Changes the current working directory to the open directory handle.
/// This modifies global state and can have surprising effects in multi-
/// threaded applications. Most applications and especially libraries should
Expand Down Expand Up @@ -1903,7 +2000,7 @@ pub fn selfExeDirPath(out_buffer: []u8) SelfExePathError![]const u8 {
}

/// `realpath`, except caller must free the returned memory.
/// TODO integrate with `Dir`
/// See also `Dir.realpath`.
pub fn realpathAlloc(allocator: *Allocator, pathname: []const u8) ![]u8 {
// Use of MAX_PATH_BYTES here is valid as the realpath function does not
// have a variant that takes an arbitrary-size buffer.
Expand Down
Loading