Skip to content

Make std.utf.toUTF8 DRY by using byUTF internally #4640

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
andralex merged 6 commits into from
Sep 17, 2016
Merged

Make std.utf.toUTF8 DRY by using byUTF internally #4640

Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
0c550b7
Make std.utf.toUTF8 DRY by using byChar internally
JackStouffer Jul 22, 2016
65c4648
Remove over applied pure attribute from std.utf
JackStouffer Jul 22, 2016
2f24a0a
use appender instead of array
JackStouffer Jul 22, 2016
3cc0a7c
Added string specific logic
JackStouffer Jul 23, 2016
ed92b3d
Range-ified
JackStouffer Jul 23, 2016
e096f29
Improved docs and added tests
JackStouffer Jul 24, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
114 changes: 50 additions & 64 deletions std/utf.d
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2418,11 +2418,7 @@ void validate(S)(in S str) @safe pure
}

/* =================== Conversion to UTF8 ======================= */

pure
{

char[] toUTF8(return out char[4] buf, dchar c) nothrow @nogc @safe
char[] toUTF8(return out char[4] buf, dchar c) nothrow @nogc @safe pure
{
if (c <= 0x7F)
{
Expand Down Expand Up @@ -2462,73 +2458,66 @@ char[] toUTF8(return out char[4] buf, dchar c) nothrow @nogc @safe
}
}

/*******************
* Encodes string $(D_PARAM s) into UTF-8 and returns the encoded string.
/**
* Encodes the elements of `s` to UTF-8 and returns a newly allocated
* string of the elements.
*
* Params:
* s = the string to encode
* Returns:
* A UTF-8 string
* See_Also:
* For a lazy, non-allocating version of these functions, see $(LREF byUTF).
*/
string toUTF8(scope const char[] s) @safe
string toUTF8(S)(S s) if (isInputRange!S && isSomeChar!(ElementEncodingType!S))
{
validate(s);
return s.idup;
}
static if (is(S : string))
{
return s.idup;
}
else
{
import std.array : appender;
auto app = appender!string();

/// ditto
string toUTF8(scope const wchar[] s) @safe
{
char[] r;
size_t i;
immutable slen = s.length;
static if (hasLength!S || isSomeString!S)
app.reserve(s.length);

r.length = slen;
for (i = 0; i < slen; i++)
{
immutable c = s[i];
foreach (c; s.byUTF!char)
app.put(c);

if (c <= 0x7F)
r[i] = cast(char)c; // fast path for ascii
else
{
r.length = i;
while (i < slen)
encode(r, decode(s, i));
break;
}
return app.data;
}

return r;
}

/// ditto
string toUTF8(scope const dchar[] s) @safe
///
@safe pure unittest
{
char[] r;
size_t i;
immutable slen = s.length;
import std.algorithm.comparison : equal;

r.length = slen;
for (i = 0; i < slen; i++)
{
immutable c = s[i];
// The ö is represented by two UTF-8 code units
assert("Hellø"w.toUTF8.equal(['H', 'e', 'l', 'l', 0xC3, 0xB8]));

if (c <= 0x7F)
r[i] = cast(char)c; // fast path for ascii
else
{
r.length = i;
foreach (dchar d; s[i .. slen])
{
encode(r, d);
}
break;
}
}
// 𐐷 is four code units in UTF-8
assert("𐐷"d.toUTF8.equal([0xF0, 0x90, 0x90, 0xB7]));
}

return r;
@system pure unittest
{
import std.internal.test.dummyrange : ReferenceInputRange;
import std.algorithm.comparison : equal;

auto r1 = new ReferenceInputRange!dchar("Hellø");
auto r2 = new ReferenceInputRange!dchar("𐐷");

assert(r1.toUTF8.equal(['H', 'e', 'l', 'l', 0xC3, 0xB8]));
assert(r2.toUTF8.equal([0xF0, 0x90, 0x90, 0xB7]));
}


/* =================== Conversion to UTF16 ======================= */

wchar[] toUTF16(return ref wchar[2] buf, dchar c) nothrow @nogc @safe
wchar[] toUTF16(return ref wchar[2] buf, dchar c) nothrow @nogc @safe pure
in
{
assert(isValidDchar(c));
Expand All @@ -2551,7 +2540,7 @@ body
/****************
* Encodes string $(D s) into UTF-16 and returns the encoded string.
*/
wstring toUTF16(scope const char[] s) @safe
wstring toUTF16(scope const char[] s) @safe pure
{
wchar[] r;
immutable slen = s.length;
Expand All @@ -2577,14 +2566,14 @@ wstring toUTF16(scope const char[] s) @safe
}

/// ditto
wstring toUTF16(scope const wchar[] s) @safe
wstring toUTF16(scope const wchar[] s) @safe pure
{
validate(s);
return s.idup;
}

/// ditto
wstring toUTF16(scope const dchar[] s) @safe
wstring toUTF16(scope const dchar[] s) @safe pure
{
wchar[] r;
immutable slen = s.length;
Expand All @@ -2605,7 +2594,7 @@ wstring toUTF16(scope const dchar[] s) @safe
/*****
* Encodes string $(D_PARAM s) into UTF-32 and returns the encoded string.
*/
dstring toUTF32(scope const char[] s) @safe
dstring toUTF32(scope const char[] s) @safe pure
{
dchar[] r;
immutable slen = s.length;
Expand All @@ -2626,7 +2615,7 @@ dstring toUTF32(scope const char[] s) @safe
}

/// ditto
dstring toUTF32(scope const wchar[] s) @safe
dstring toUTF32(scope const wchar[] s) @safe pure
{
dchar[] r;
immutable slen = s.length;
Expand All @@ -2647,15 +2636,12 @@ dstring toUTF32(scope const wchar[] s) @safe
}

/// ditto
dstring toUTF32(scope const dchar[] s) @safe
dstring toUTF32(scope const dchar[] s) @safe pure
{
validate(s);
return s.idup;
}

} // Convert functions are @safe


/* =================== toUTFz ======================= */

/++
Expand Down