From 15f8bf325f63b8b62fbcd28089097c23a4fb9205 Mon Sep 17 00:00:00 2001 From: anonymous Date: Tue, 26 Apr 2016 22:08:29 +0200 Subject: [PATCH 1/4] new function: copyRecurse copyRecurse differs from copy in that it accepts directories and copies them recursively. --- changelog.dd | 9 +++ std/file.d | 213 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 222 insertions(+) diff --git a/changelog.dd b/changelog.dd index 10eed4b3b8d..71ba9b78146 100644 --- a/changelog.dd +++ b/changelog.dd @@ -25,6 +25,7 @@ $(BUGSTITLE Library Changes, equal values was added.) $(LI $(REF readLink, std,file) and $(REF symlink, std,file) have been rangified.) + $(LI $(RELATIVE_LINK2 copyRecurse, New Function: `std.file.copyRecurse`.)) ) $(BUGSTITLE Library Changes, @@ -78,6 +79,14 @@ $(LI $(P $(XREF range, iota)'s `.length` property is now always returned as `size_t.max` in your call to `iota`. ) +$(LI $(LNAME2 copyRecurse, New Function: `std.file.copyRecurse`.) + + $(P A new function has been added: $(XREF file, copyRecurse). It differs + from $(XREF file, copy) in that `copyRecurse` accepts directories and + copies them recursively. + ) +) + Macros: TITLE=Change Log diff --git a/std/file.d b/std/file.d index 3c7b38ee7e3..f16af2723db 100644 --- a/std/file.d +++ b/std/file.d @@ -3372,6 +3372,219 @@ unittest // issue 15865 assert(readText(t) == "a"); } +/*************************************************** +Copy file or directory $(D from) to $(D to). When a directory is copied, its +contents are copied as well, recursively. + +File timestamps are preserved, except for symbolic links. +File attributes are preserved, if $(D preserve) equals $(D PreserveAttributes.yes). +On Windows only $(D PreserveAttributes.yes) (the default on Windows) is supported. + +Existing target files are overwritten. Existing target directories are written +into. + +With symbolic links, the links themselves are copied. That is, for every +symbolic link in $(D from), a new link with the same target path is created in +$(D to). Relative target paths are not changed. They may be invalidated by the +copying, when they point outside of the copied directory tree. + +Special files (e.g. device files, FIFOs) cannot be copied with $(D copyRecurse). + +Params: + from = string or range of characters representing the existing + file/directory name + to = string or range of characters representing the target file/directory + name + preserve = whether to preserve the file attributes + +Throws: $(D FileException) on error. + */ +void copyRecurse(string from, string to, + PreserveAttributes preserve = preserveAttributesDefault) +{ + // Remove any trailing directory separators. + from = buildNormalizedPath(from); + to = buildNormalizedPath(to); + + immutable attrs = getLinkAttributes(from); + + if (attrIsFile(attrs)) copy(from, to, preserve); + else if (attrIsDir(attrs)) + { + mkdirRecurse(std.conv.to!string(to)); + + if (preserve == PreserveAttributes.yes) setAttributes(to, attrs); + + foreach (entry; dirEntries(from, SpanMode.shallow, false)) + { + immutable path = entry.name[from.length + 1 .. $]; + // + 1 for the directory separator + copyRecurse(entry.name, buildPath(to, path), preserve); + } + + SysTime accessTime, modificationTime; + getTimes(from, accessTime, modificationTime); + setTimes(to, accessTime, modificationTime); + } + else if (attrIsSymlink(attrs)) + { + version (Posix) + { + if (exists(to)) remove(to); + symlink(readLink(from), to); + if (preserve == PreserveAttributes.yes) + { + /*NOTE: Would copy attributes of the symlink here, but there + is no setLinkAttributes here, and no fchmodat in the C headers. + Just enforcing that the new attributes are the same as the old + ones, for now. */ + if (getLinkAttributes(to) != attrs) + { + throw new FileException(from, "Cannot preserve " ~ + "non-default attributes of symbolic link."); + } + } + /*NOTE: Would copy timestamps of the symlink here, but there is no + setLinkTimes here, and no lutimes in the C headers. Just ignoring + for now. */ + } + else version (Windows) assert(false); + else static assert(false); + } + else throw new FileException(from, "Cannot copy special file."); +} + +unittest // copyRecurse can still copy mere files +{ + auto t1 = deleteme, t2 = deleteme~"2"; + scope(exit) foreach (t; [t1, t2]) if (t.exists) t.remove(); + write(t1, "1"); + copyRecurse(t1, t2); + assert(readText(t2) == "1"); + write(t1, "2"); + copyRecurse(t1, t2); + assert(readText(t2) == "2"); +} + +unittest // copying directories +{ + auto t1 = deleteme ~ ".srcdir", t2 = deleteme~".dstdir"; + scope(exit) foreach (t; [t1, t2]) if (t.exists) rmdirRecurse(t); + mkdirRecurse(t1 ~ "/a/b1"); + mkdirRecurse(t1 ~ "/a/b2"); + write(t1 ~ "/a/b1/f", "1"); + write(t1 ~ "/f", "1"); + copyRecurse(t1, t2); + assert(readText(t2 ~ "/a/b1/f") == "1"); + assert(readText(t2 ~ "/f") == "1"); + assert((t2 ~ "/a/b2").exists); + write(t2 ~ "/g", ""); + write(t1 ~ "/f", "2"); + copyRecurse(t1, t2); + assert((t2 ~ "/g").exists); + assert(readText(t2 ~ "/f") == "2"); +} + +unittest // 'from' ending in a directory separator +{ + auto t1 = deleteme ~ ".srcdir", t2 = deleteme~".dstdir"; + scope(exit) foreach (t; [t1, t2]) if (t.exists) rmdirRecurse(t); + mkdirRecurse(t1 ~ "/a/b1"); + copyRecurse(t1 ~ "/", t2); + assert((t2 ~ "/a/b1").exists); +} + +unittest // 'to' ending in a directory separator +{ + auto t1 = deleteme ~ ".srcdir", t2 = deleteme~".dstdir"; + scope(exit) foreach (t; [t1, t2]) if (t.exists) rmdirRecurse(t); + mkdirRecurse(t1 ~ "/a/b1"); + copyRecurse(t1, t2 ~ "/"); + assert((t2 ~ "/a/b1").exists); +} + +version (Posix) unittest // symlinks +{ + auto t1 = deleteme ~ ".srcdir", t2 = deleteme~".dstdir"; + scope(exit) foreach (t; [t1, t2]) if (t.exists) rmdirRecurse(t); + mkdirRecurse(t1 ~ "/a"); + write(t1 ~ "/f", "1"); + symlink("f", t1 ~ "/l"); + symlink("../f", t1 ~ "/a/l"); + copyRecurse(t1, t2); + assert(readText(t2 ~ "/f") == "1"); + assert(readText(t2 ~ "/l") == "1"); + assert(readText(t2 ~ "/a/l") == "1"); + write(t2 ~ "/f", "2"); + assert(readText(t2 ~ "/l") == "2"); + assert(readText(t2 ~ "/a/l") == "2"); +} + +version (Posix) unittest // special files are rejected +{ + import core.sys.posix.sys.stat: mkfifo; + import std.string: toStringz; + auto t1 = deleteme ~ ".srcdir", t2 = deleteme~".dstdir"; + scope(exit) foreach (t; [t1, t2]) if (t.exists) rmdirRecurse(t); + mkdirRecurse(t1); + mkfifo(toStringz(t1 ~ "/fifo"), octal!"600"); + assertThrown!FileException(copyRecurse(t1, t2)); +} + +unittest // overwriting existing files +{ + auto t1 = deleteme ~ ".srcdir", t2 = deleteme~".dstdir"; + scope(exit) foreach (t; [t1, t2]) if (t.exists) rmdirRecurse(t); + mkdirRecurse(t1); + write(t1 ~ "/f", "1"); + version (Posix) symlink("l1", t1 ~ "/l"); + mkdirRecurse(t2); + write(t2 ~ "/f", "2"); + version (Posix) symlink("l2", t2 ~ "/l"); + copyRecurse(t1, t2); + assert(readText(t2 ~ "/f") == "1"); + version (Posix) assert(readLink(t2 ~ "/l") == "l1"); +} + +version (Posix) unittest // PreserveAttributes +{ + auto t1 = deleteme ~ ".srcdir", t2 = deleteme~".dstdir"; + scope(exit) foreach (t; [t1, t2]) if (t.exists) rmdirRecurse(t); + mkdirRecurse(t1 ~ "/d"); + write(t1 ~ "/f", "1"); + enum rw = octal!"600"; + enum rwx = octal!"700"; + setAttributes(t1 ~ "/d", rw); + setAttributes(t1 ~ "/f", rw); + setAttributes(t1, rwx); + copyRecurse(t1, t2, PreserveAttributes.yes); + assert((getAttributes(t2 ~ "/d") & rwx) == rw); + assert((getAttributes(t2 ~ "/f") & rwx) == rw); + assert((getAttributes(t2) & rwx) == rwx); +} + +unittest // timestamps are preserved +{ + auto t1 = deleteme ~ ".srcdir", t2 = deleteme~".dstdir"; + scope(exit) foreach (t; [t1, t2]) if (t.exists) rmdirRecurse(t); + mkdirRecurse(t1 ~ "/d"); + write(t1 ~ "/f", "1"); + setTimes(t1, SysTime(DateTime(1877, 08, 17)), + SysTime(DateTime(1877, 08, 17))); + copyRecurse(t1, t2); + void assertSameTimes(in char[] f1, in char[] f2) + { + SysTime a1, m1, a2, m2; + getTimes(f1, a1, m1); + getTimes(f2, a2, m2); + assert(a1 == a2, text(f1, " ", a1, " != ", a2)); + assert(m1 == m2, f1); + } + assertSameTimes(t1, t2); + assertSameTimes(t1 ~ "/d", t2 ~ "/d"); + assertSameTimes(t1 ~ "/f", t2 ~ "/f"); +} + /++ Remove directory and all of its content and subdirectories, recursively. From ae0eb80e5cdf4b6367667aa9e58473d2b4ffd85f Mon Sep 17 00:00:00 2001 From: anonymous Date: Mon, 25 Jan 2016 18:18:24 +0100 Subject: [PATCH 2/4] remove left-over std.conv.to!string --- std/file.d | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/std/file.d b/std/file.d index f16af2723db..463702fe930 100644 --- a/std/file.d +++ b/std/file.d @@ -3411,7 +3411,7 @@ void copyRecurse(string from, string to, if (attrIsFile(attrs)) copy(from, to, preserve); else if (attrIsDir(attrs)) { - mkdirRecurse(std.conv.to!string(to)); + mkdirRecurse(to); if (preserve == PreserveAttributes.yes) setAttributes(to, attrs); From 265ca230fcc6202bb34a071a9fdbc6943e7bae38 Mon Sep 17 00:00:00 2001 From: anonymous Date: Mon, 25 Jan 2016 18:18:42 +0100 Subject: [PATCH 3/4] use enforce --- std/file.d | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/std/file.d b/std/file.d index 463702fe930..1f2f9e800cf 100644 --- a/std/file.d +++ b/std/file.d @@ -3438,11 +3438,9 @@ void copyRecurse(string from, string to, is no setLinkAttributes here, and no fchmodat in the C headers. Just enforcing that the new attributes are the same as the old ones, for now. */ - if (getLinkAttributes(to) != attrs) - { - throw new FileException(from, "Cannot preserve " ~ - "non-default attributes of symbolic link."); - } + enforce(getLinkAttributes(to) == attrs, + new FileException(from, "Cannot preserve non-default " ~ + "attributes of symbolic link.")); } /*NOTE: Would copy timestamps of the symlink here, but there is no setLinkTimes here, and no lutimes in the C headers. Just ignoring From c3ab587662dc45b16d0f645da830b223b387a0d9 Mon Sep 17 00:00:00 2001 From: anonymous Date: Mon, 1 Feb 2016 00:41:10 +0100 Subject: [PATCH 4/4] rangify copyRecurse --- std/file.d | 194 +++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 151 insertions(+), 43 deletions(-) diff --git a/std/file.d b/std/file.d index 1f2f9e800cf..58452fa7c16 100644 --- a/std/file.d +++ b/std/file.d @@ -3399,57 +3399,137 @@ Params: Throws: $(D FileException) on error. */ -void copyRecurse(string from, string to, +void copyRecurse(RF, RT)(RF from, RT to, PreserveAttributes preserve = preserveAttributesDefault) + if ((isForwardRange!RF && isSomeChar!(ElementType!RF) || + isConvertibleToString!RF) && + (isForwardRange!RT && isSomeChar!(ElementType!RT) || + isConvertibleToString!RT)) { - // Remove any trailing directory separators. - from = buildNormalizedPath(from); - to = buildNormalizedPath(to); + static if (isConvertibleToString!RF || isConvertibleToString!RT) + { + import std.meta : staticMap; + alias Types = staticMap!(convertToString, RF, RT); + copyRecurse!Types(from, to, preserve); + } + else + { + // Remove any trailing directory separators. + auto fromNoSep = NoTrailingDirSep!RF(from); + auto toNoSep = NoTrailingDirSep!RT(to); - immutable attrs = getLinkAttributes(from); + copyRecurseImpl!true(fromNoSep, toNoSep, "", preserve); + } +} - if (attrIsFile(attrs)) copy(from, to, preserve); +private void copyRecurseImpl(bool isRootCall, RF, RT) + (NoTrailingDirSep!RF baseFrom, NoTrailingDirSep!RT baseTo, string path, + PreserveAttributes preserve) +{ + import std.algorithm : endsWith; + import std.range : chain, walkLength; + + static if (isRootCall) + { + assert(path == ""); + alias src = baseFrom; + alias dst = baseTo; + } + else + { + assert(path != ""); + /* Not using std.path.chainPath, because that demands a random access + range for some reason. */ + auto src = chain(baseFrom, dirSeparator, path); + auto dst = chain(baseTo, dirSeparator, path); + } + + immutable attrs = getLinkAttributes(src.save); + + if (attrIsFile(attrs)) copy(src.save, dst.save, preserve); else if (attrIsDir(attrs)) { - mkdirRecurse(to); + static if (isRootCall) mkdirRecurse(text(dst.save)); + else if (!exists(dst.save)) mkdir(dst.save); - if (preserve == PreserveAttributes.yes) setAttributes(to, attrs); + if (preserve) setAttributes(dst.save, attrs); - foreach (entry; dirEntries(from, SpanMode.shallow, false)) + foreach (entry; dirEntries(text(src.save), SpanMode.shallow, false)) { - immutable path = entry.name[from.length + 1 .. $]; - // + 1 for the directory separator - copyRecurse(entry.name, buildPath(to, path), preserve); + immutable baseFromLen = walkLength(baseFrom.save); + string subPath = entry.name[baseFromLen + dirSeparator.length .. $]; + copyRecurseImpl!false(baseFrom, baseTo, subPath, preserve); } SysTime accessTime, modificationTime; - getTimes(from, accessTime, modificationTime); - setTimes(to, accessTime, modificationTime); + getTimes(src.save, accessTime, modificationTime); + setTimes(dst.save, accessTime, modificationTime); } else if (attrIsSymlink(attrs)) { version (Posix) { - if (exists(to)) remove(to); - symlink(readLink(from), to); - if (preserve == PreserveAttributes.yes) - { - /*NOTE: Would copy attributes of the symlink here, but there - is no setLinkAttributes here, and no fchmodat in the C headers. - Just enforcing that the new attributes are the same as the old - ones, for now. */ - enforce(getLinkAttributes(to) == attrs, - new FileException(from, "Cannot preserve non-default " ~ - "attributes of symbolic link.")); - } + if (exists(dst.save)) remove(dst.save); + symlink(readLink(src.save), dst.save); + + /*NOTE: Would copy attributes of the symlink here if preserve is + set, but there is no setLinkAttributes here, and no fchmodat in the + C headers. Just enforcing that the new attributes are the same as + the old ones, for now. */ + enforce(!preserve || getLinkAttributes(dst.save) == attrs, + new FileException(text(src.save), "Cannot preserve " ~ + "non-default attributes of symbolic link.")); + /*NOTE: Would copy timestamps of the symlink here, but there is no setLinkTimes here, and no lutimes in the C headers. Just ignoring for now. */ } - else version (Windows) assert(false); + else version (Windows) + { + throw new FileException(text(src.save), "Copying symbolic links " ~ + "is currently not supported on Windows."); + } else static assert(false); } - else throw new FileException(from, "Cannot copy special file."); + else throw new FileException(text(src.save), "Cannot copy special file."); +} + +// helper for copyRecurse +private struct NoTrailingDirSep(R) + if (isForwardRange!R && isSomeChar!(ElementType!R)) +{ + static assert(dirSeparator.length == 1); + + private R path; + private dchar front_; + private bool empty_ = false; + + this(R path) + { + this.path = path; + popFront(); + } + + @property bool empty() const {return empty_;} + @property dchar front() const {return front_;} + + void popFront() + { + if (path.empty) empty_ = true; + else + { + front_ = path.front; + path.popFront(); + if (path.empty && isDirSeparator(front_)) empty_ = true; + } + } + + @property typeof(this) save() + { + auto saved = this; + saved.path = this.path.save; + return saved; + } } unittest // copyRecurse can still copy mere files @@ -3466,21 +3546,49 @@ unittest // copyRecurse can still copy mere files unittest // copying directories { - auto t1 = deleteme ~ ".srcdir", t2 = deleteme~".dstdir"; - scope(exit) foreach (t; [t1, t2]) if (t.exists) rmdirRecurse(t); - mkdirRecurse(t1 ~ "/a/b1"); - mkdirRecurse(t1 ~ "/a/b2"); - write(t1 ~ "/a/b1/f", "1"); - write(t1 ~ "/f", "1"); - copyRecurse(t1, t2); - assert(readText(t2 ~ "/a/b1/f") == "1"); - assert(readText(t2 ~ "/f") == "1"); - assert((t2 ~ "/a/b2").exists); - write(t2 ~ "/g", ""); - write(t1 ~ "/f", "2"); - copyRecurse(t1, t2); - assert((t2 ~ "/g").exists); - assert(readText(t2 ~ "/f") == "2"); + static void test(R)(R f, R t) + { + auto fstr = text(f.save); + auto tstr = text(t.save); + scope(exit) foreach (d; [fstr, tstr]) if (d.exists) + rmdirRecurse(text(d)); + mkdirRecurse(fstr ~ "/a/b1"); + mkdirRecurse(fstr ~ "/a/b2"); + write(fstr ~ "/a/b1/f", "1"); + write(fstr ~ "/f", "1"); + copyRecurse(f.save, t.save); + assert(readText(tstr ~ "/a/b1/f") == "1"); + assert(readText(tstr ~ "/f") == "1"); + assert((tstr ~ "/a/b2").exists); + write(tstr ~ "/g", ""); + write(fstr ~ "/f", "2"); + copyRecurse(f.save, t.save); + assert((tstr ~ "/g").exists); + assert(readText(tstr ~ "/f") == "2"); + } + + string f = deleteme ~ ".srcdir"; + string t = deleteme ~ ".dstdir"; + test(f, t); + + { + import std.range.interfaces : ForwardRange, inputRangeObject; + alias R = ForwardRange!dchar; + static assert(!isBidirectionalRange!R && isForwardRange!R && + isSomeChar!(ElementType!R)); + R fobj = inputRangeObject(f); + R tobj = inputRangeObject(t); + test(fobj, tobj); + } + { + static struct R + { + string s; + alias s this; + } + static assert(!isForwardRange!R && isConvertibleToString!R); + test(R(f), R(t)); + } } unittest // 'from' ending in a directory separator