From 20cc0b39c4cfad5785e9d65a5db5f13fb51a4438 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20Anic=CC=81?= Date: Mon, 26 Feb 2024 00:26:38 +0100 Subject: [PATCH 1/2] fetch: fix failing test Prior to [this](https://github.com/ziglang/zig/commit/ef9966c9855dd855afda767f212abec6e5a36307#diff-08c935ef8c633bb630641d44230597f1cff5afb0e736d451e2ba5569fa53d915R805) commit tar was not a valid extension. After that this one is valid case. --- src/Package/Fetch.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index e4e944d18692..3adb61333348 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -867,9 +867,9 @@ const FileType = enum { try std.testing.expectEqual(@as(?FileType, .@"tar.xz"), fromContentDisposition("ATTACHMENT; filename=\"stuff.tar.xz\"")); try std.testing.expectEqual(@as(?FileType, .@"tar.xz"), fromContentDisposition("attachment; FileName=\"stuff.tar.xz\"")); try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attachment; FileName*=UTF-8\'\'xyz%2Fstuff.tar.gz")); + try std.testing.expectEqual(@as(?FileType, .tar), fromContentDisposition("attachment; FileName=\"stuff.tar\"")); try std.testing.expect(fromContentDisposition("attachment FileName=\"stuff.tar.gz\"") == null); - try std.testing.expect(fromContentDisposition("attachment; FileName=\"stuff.tar\"") == null); try std.testing.expect(fromContentDisposition("attachment; FileName\"stuff.gz\"") == null); try std.testing.expect(fromContentDisposition("attachment; size=42") == null); try std.testing.expect(fromContentDisposition("inline; size=42") == null); From 299f889dc4a914073d6b1a6f5ea255dcc467b522 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20Anic=CC=81?= Date: Mon, 26 Feb 2024 12:15:35 +0100 Subject: [PATCH 2/2] package: allow setting tar_strip_components in build.zig.zon. That allows us to unpack tars which don't have leading root folder. Like examples from #17779. Trying to fetch `https://chromium.googlesource.com/chromium/tools/depot_tools.git/+archive/refs/heads/main.tar.gz` with default settings results in error. That archive has files in the root: ``` $ tar tvf main.tar | head -rw-r--r-- 0/0 3848 2024-02-26 12:21 .cipd_impl.ps1 -rw-r--r-- 0/0 31 2024-02-26 12:21 .flake8 -rw-r--r-- 0/0 1682 2024-02-26 12:21 .gitattributes -rw-r--r-- 0/0 2040 2024-02-26 12:21 .gitignore -rw-r--r-- 0/0 857 2024-02-26 12:21 .isort.cfg -rw-r--r-- 0/0 48 2024-02-26 12:21 .style.yapf -rw-r--r-- 0/0 1978 2024-02-26 12:21 .vpython -rw-r--r-- 0/0 2429 2024-02-26 12:21 .vpython3 -rw-r--r-- 0/0 128 2024-02-26 12:21 BUILD_OWNERS -rw-r--r-- 0/0 156 2024-02-26 12:21 CROS_OWNERS ``` stripping one folder (default) results in empty files names. Setting tar_strip_components = 0 in build.zig.zon enables fetching such archives. ``` .dependencies = .{ .chromium = .{ .url = "https://chromium.googlesource.com/chromium/tools/depot_tools.git/+archive/refs/heads/main.tar.gz", .hash = "122089fd900653f199701726fe43bf956d9613f2c7ef4e216d363700f5c7df0cb4d8", .tar_strip_components = 0, }, ``` --- src/Package/Fetch.zig | 5 +++- src/Package/Manifest.zig | 63 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index 3adb61333348..0c3c0abe6122 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -44,6 +44,7 @@ omit_missing_hash_error: bool, /// which specifies inclusion rules. This is intended to be true for the first /// fetch task and false for the recursive dependencies. allow_missing_paths_field: bool, +tar_strip_components: u32 = 1, // Above this are fields provided as inputs to `run`. // Below this are fields populated by `run`. @@ -674,6 +675,7 @@ fn queueJobsForDeps(f: *Fetch) RunError!void { prog_names[new_fetch_index] = dep_name; new_fetch_index += 1; f.job_queue.all_fetches.appendAssumeCapacity(new_fetch); + new_fetch.* = .{ .arena = std.heap.ArenaAllocator.init(gpa), .location = location, @@ -687,6 +689,7 @@ fn queueJobsForDeps(f: *Fetch) RunError!void { .job_queue = f.job_queue, .omit_missing_hash_error = false, .allow_missing_paths_field = true, + .tar_strip_components = dep.tar_strip_components, .package_root = undefined, .error_bundle = undefined, @@ -1152,7 +1155,7 @@ fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!void { std.tar.pipeToFileSystem(out_dir, reader, .{ .diagnostics = &diagnostics, - .strip_components = 1, + .strip_components = f.tar_strip_components, // TODO: we would like to set this to executable_bit_only, but two // things need to happen before that: // 1. the tar implementation needs to support it diff --git a/src/Package/Manifest.zig b/src/Package/Manifest.zig index 589be91357fa..85561495e08c 100644 --- a/src/Package/Manifest.zig +++ b/src/Package/Manifest.zig @@ -14,6 +14,7 @@ pub const Dependency = struct { node: Ast.Node.Index, name_tok: Ast.TokenIndex, lazy: bool, + tar_strip_components: u32 = 1, pub const Location = union(enum) { url: []const u8, @@ -307,6 +308,7 @@ const Parse = struct { .node = node, .name_tok = 0, .lazy = false, + .tar_strip_components = 1, }; var has_location = false; @@ -352,6 +354,11 @@ const Parse = struct { error.ParseFailure => continue, else => |e| return e, }; + } else if (mem.eql(u8, field_name, "tar_strip_components")) { + dep.tar_strip_components = parseInt(u32, p, field_init) catch |err| switch (err) { + error.ParseFailure => continue, + else => |e| return e, + }; } else { // Ignore unknown fields so that we can add fields in future zig // versions without breaking older zig versions. @@ -402,6 +409,22 @@ const Parse = struct { } } + fn parseInt(comptime T: type, p: *Parse, node: Ast.Node.Index) !T { + const ast = p.ast; + const node_tags = ast.nodes.items(.tag); + const main_tokens = ast.nodes.items(.main_token); + if (node_tags[node] != .number_literal) { + return fail(p, main_tokens[node], "expected identifier", .{}); + } + const ident_token = main_tokens[node]; + const token_bytes = ast.tokenSlice(ident_token); + + const value: T = std.fmt.parseInt(T, token_bytes, 10) catch { + return fail(p, ident_token, "expected integer", .{}); + }; + return value; + } + fn parseString(p: *Parse, node: Ast.Node.Index) ![]const u8 { const ast = p.ast; const node_tags = ast.nodes.items(.tag); @@ -703,3 +726,43 @@ test "minimum_zig_version - invalid version" { try testing.expect(manifest.minimum_zig_version == null); } + +test "tar_strip_components" { + const gpa = testing.allocator; + + const example = + \\.{ + \\ .name = "foo", + \\ .version = "3.2.1", + \\ .dependencies = .{ + \\ .bar = .{ + \\ .url = "https://example.com/baz.tar.gz", + \\ .tar_strip_components = 123, + \\ }, + \\ }, + \\ .paths = .{""}, + \\} + ; + + var ast = try Ast.parse(gpa, example, .zon); + defer ast.deinit(gpa); + + try testing.expect(ast.errors.len == 0); + + var manifest = try Manifest.parse(gpa, ast, .{}); + defer manifest.deinit(gpa); + + try testing.expect(manifest.errors.len == 0); + try testing.expectEqualStrings("foo", manifest.name); + + try testing.expect(manifest.dependencies.count() == 1); + try testing.expectEqualStrings("bar", manifest.dependencies.keys()[0]); + try testing.expectEqualStrings( + "https://example.com/baz.tar.gz", + manifest.dependencies.values()[0].location.url, + ); + try testing.expectEqual( + 123, + manifest.dependencies.values()[0].tar_strip_components, + ); +}