diff --git a/src/dmd/cli.d b/src/dmd/cli.d index 7cb4da968d20..43526acc160d 100644 --- a/src/dmd/cli.d +++ b/src/dmd/cli.d @@ -101,6 +101,9 @@ struct Usage Option("c", "do not link" ), + Option("cache=mmap", + "use persistent cache for CTFE evaluations" + ), Option("color", "turn colored console output on" ), diff --git a/src/dmd/dinterpret.d b/src/dmd/dinterpret.d index 73b4a71ce088..4ff378daf673 100644 --- a/src/dmd/dinterpret.d +++ b/src/dmd/dinterpret.d @@ -12,16 +12,23 @@ module dmd.dinterpret; +import core.stdc.stdlib; import core.stdc.stdio; import core.stdc.string; +import core.stdc.time; import dmd.apply; import dmd.arraytypes; +import dmd.astcodegen; import dmd.attrib; import dmd.builtin; import dmd.constfold; import dmd.ctfeexpr; +import dmd.dcache; import dmd.dclass; import dmd.declaration; +import dmd.dimport; +import dmd.dmodule; +import dmd.dscope; import dmd.dstruct; import dmd.dsymbol; import dmd.dsymbolsem; @@ -30,14 +37,18 @@ import dmd.errors; import dmd.expression; import dmd.expressionsem; import dmd.func; +import dmd.hdrgen; import dmd.globals; import dmd.id; import dmd.identifier; import dmd.init; import dmd.initsem; import dmd.mtype; +import dmd.parse; import dmd.root.array; +import dmd.root.outbuffer; import dmd.root.rootobject; +import dmd.root.sha; import dmd.statement; import dmd.tokens; import dmd.utf; @@ -720,6 +731,161 @@ extern (C++) Expression ctfeInterpretForPragmaMsg(Expression e) return e; } +ubyte[20] ctfeHashingSeed() +{ + static ubyte[20] hash; + static bool inited; + + if (inited) return hash; + else + { + SHA1 hasher; + hasher.start(); + // predefined versions indicate target, bitness and asserts + if (global.versionids) + { + foreach (id; *global.versionids) + { + const(char)* s = id.toChars(); + hasher.put(cast(ubyte[])s[0..strlen(s)]); + } + } + if (global.debugids) + { + foreach (id; *global.debugids) + { + const(char)* s = id.toChars(); + hasher.put(cast(ubyte[])s[0 .. strlen(s)]); + } + } + hash = hasher.finish(); + inited = true; + return hash; + } +} + +ubyte[20] generateCtfeCacheKey(FuncDeclaration fd, ref Expressions eargs) +{ + SHA1 hasher; + hasher.start(); + auto signature = fd.toFullSignature(); + // Add in a hash of compilation parameters + hasher.put(ctfeHashingSeed); + debug(DCACHE) printf("Func's parent %s \n", fd.parent.toChars()); + // Include modules of type arguments + if (TemplateInstance ti = fd.parent.isTemplateInstance()) + { + if (ti.tiargs) + { + foreach (arg; *ti.tiargs) + { + Dsymbol sym = getDsymbol(arg); + if (sym) + { + /*if (ScopeDsymbol sd = sym.isScopeDsymbol()) + { + foreach(sc; *sd.importedScopes) + { + if (Module md = sc.isModule()) + { + debug(DCACHE) auto str = toHexString(md.signature[]); + debug(DCACHE) printf("\tmod %s sig: %.*s\n", + md.toChars(), str.length, str.ptr); + hasher.put(md.signature[]); + } + } + }*/ + Dsymbol last; + while (sym) + { + last = sym; + sym = sym.parent; + } + if (Module md = last.isModule()) + { + foreach (imp; fd._scope._module.aimports) + { + hasher.put(imp.signature[]); + } + debug(DCACHE) auto str = toHexString(md.signature[]); + debug(DCACHE) printf("\tmod %s sig: %.*s\n", + md.toChars(), str.length, str.ptr); + hasher.put(md.signature[]); + } + } + } + } + } + debug(DCACHE) printf("Imports from function's module:\n"); + foreach (imp; fd._scope._module.aimports) + { + hasher.put(imp.signature[]); + auto str = toHexString(imp.signature[]); + debug(DCACHE) printf("\tmod %s sig: %.*s\n", imp.toChars(), str.length, str.ptr); + } + hasher.put(fd._scope._module.signature); + hasher.put(0x0); // separator + hasher.put(cast(ubyte[])signature[0 .. strlen(signature)]); + foreach(i, earg; eargs) + { + hasher.put(0x0); // separator + auto argument = earg.toChars(); + hasher.put(cast(ubyte[])argument[0 .. strlen(argument)]); + } + return hasher.finish(); +} + +extern(C++) class ExtractSymbols : StoppableVisitor { + Scope* _sc; + Dsymbol[] symbols; + + this(Scope* sc) + { + _sc = sc; + } + + override void visit(Expression e) + { + if (!e.type.isscalar()) + { + Dsymbol sym = e.type.toDsymbol(_sc); + if (sym) + symbols ~= sym; + } + } + alias visit = Visitor.visit; +} + +void writeModuleName(Dsymbol sym, ref OutBuffer dest) +{ + const(char)*[] parts; + sym = sym.getModule(); + while (sym) + { + parts ~= sym.toChars(); + sym = sym.parent; + } + for (size_t i = 0; i < parts.length; i++) + { + if (i != 0) + dest.writestring(cast(char*)"."); + dest.writestring(parts[$ - i - 1]); + } +} + +void uniqueInPlace(T)(ref T[] args) +{ + size_t i = 0; + for (size_t j = 0; j < args.length; j++) + { + if (args[i] != args[j]) + { + args[i++] = args[j]; + } + } + args.length = i; +} + /************************************* * Attempt to interpret a function given the arguments. * Input: @@ -836,6 +1002,51 @@ private Expression interpret(FuncDeclaration fd, InterState* istate, Expressions } eargs[i] = earg; } + + // Skip generated symbols and only permit caching of top-level CTFE calls + bool cacheable = (istate == null || istate.caller == null) + && strncmp(fd.ident.toChars(), "__", 2) != 0; + clock_t ctfeStart = clock(); + ubyte[20] cacheKey = void; + if (global.params.cache && cacheable) + { + cacheKey = generateCtfeCacheKey(fd, eargs); + auto value = dcache.get(cacheKey[]); + if (value != null) + { + clock_t startMixin = clock(); + debug (DCACHE) + { + printf("interpreting global %s: %s\n", fd.loc.toChars(), fd.toFullSignature()); + foreach (i, earg; eargs[]) + { + printf("\targ %d : %s\n", i, earg.toChars()); + } + printf("\tFound in cache\n"); + printf("BODY %*s\n", value.length, value.ptr); + } + scope p = new Parser!ASTCodegen(fd.loc, fd._scope._module, cast(string)value ~ "\0", false); + p.nextToken(); + Expression e = p.parseExpression(); + if (p.errors) + { + return new ErrorExp(); + } + if (p.token.value != TOK.endOfFile) + { + e.error("incomplete mixin expression (%s)", e.toChars()); + return new ErrorExp(); + } + cachedSemantics = true; + e = expressionSemantic(e, fd._scope); + cachedSemantics = false; + clock_t endMixin = clock(); + double delta = 1.0*(endMixin - startMixin) / CLOCKS_PER_SEC; + debug(DCACHE) printf("\tSuccessfully mixed in!\n\n"); + version(DCACHE_STATS) printf("Mixed in %s from cache in %lf\n", fd.toFullSignature(), delta); + return e; + } + } // Now that we've evaluated all the arguments, we can start the frame // (this is the moment when the 'call' actually takes place). @@ -978,7 +1189,48 @@ private Expression interpret(FuncDeclaration fd, InterState* istate, Expressions (cast(ThrownExceptionExp)e).generateUncaughtError(); e = CTFEExp.cantexp; } - + if (global.params.cache && cacheable) + { + clock_t ctfeEnd = clock(); + double delta = 1.0*(ctfeEnd - ctfeStart) / CLOCKS_PER_SEC; + if (delta > 0.0) + { + debug(DCACHE) + { + printf("interpreting global %s: %s\n", fd.loc.toChars(), fd.toFullSignature()); + foreach (i, earg; eargs[]) + { + printf("\targ %d : %s\n", i, earg.toChars()); + } + } + OutBuffer buf; + scope extractor = new ExtractSymbols(fd._scope); + walkPostorder(e, extractor); + extern(C) static int comp(scope const(void*) *a , scope const(void*)* b) + { + return a > b ? 1 : (a == b ? 0 : -1); + } + qsort(extractor.symbols.ptr, extractor.symbols.length, Dsymbol.sizeof, cast(_compare_fp_t)&comp); + uniqueInPlace(extractor.symbols); + buf.writestring("(){\n"); + foreach (t; extractor.symbols) + { + buf.writestring("import "); + writeModuleName(t, buf); + buf.writestring(" : "); + buf.writestring(t.toChars()); + buf.writestring(";\n"); + } + buf.writestring("return "); + buf.writestring(e.toChars()); + buf.writestring(";\n"); + buf.writestring("}()\n"); + auto value = buf.extractString(); + dcache.put(cacheKey[], cast(ubyte[])value[0 .. strlen(value)]); + debug(DCACHE) printf("\treturns (%lf) CACHED %s\n", delta, value); + version(DCACHE_STATS) printf("Recomputed %s in %lf\n", fd.toFullSignature(), delta); + } + } return e; } diff --git a/src/dmd/dmodule.d b/src/dmd/dmodule.d index 579e13e3b370..00bbadb62c6e 100644 --- a/src/dmd/dmodule.d +++ b/src/dmd/dmodule.d @@ -407,6 +407,8 @@ extern (C++) final class Module : Package size_t nameoffset; // offset of module name from start of ModuleInfo size_t namelen; // length of module name in characters + + ubyte[20] signature; // A signature of contents, currently SHA1 extern (D) this(const(char)* filename, Identifier ident, int doDocComment, int doHdrGen) { diff --git a/src/dmd/dscope.d b/src/dmd/dscope.d index b35482d5e2d2..efd9dddae8ae 100644 --- a/src/dmd/dscope.d +++ b/src/dmd/dscope.d @@ -16,10 +16,12 @@ import core.stdc.stdio; import core.stdc.string; import dmd.aggregate; import dmd.attrib; +import dmd.dcache; import dmd.dclass; import dmd.declaration; import dmd.dmodule; import dmd.doc; +import dmd.dimport; import dmd.dsymbol; import dmd.dsymbolsem; import dmd.dtemplate; @@ -554,7 +556,7 @@ struct Scope * checked by the compiler remain usable. Once the deprecation is over, * this should be moved to search_correct instead. */ - if (!s && !(flags & IgnoreSymbolVisibility)) + if (!cachedSemantics && !s && !(flags & IgnoreSymbolVisibility)) { s = searchScopes(flags | SearchLocalsOnly | IgnoreSymbolVisibility); if (!s) diff --git a/src/dmd/dsymbol.d b/src/dmd/dsymbol.d index 0f5dc5c50ff1..3146ee4d87a7 100644 --- a/src/dmd/dsymbol.d +++ b/src/dmd/dsymbol.d @@ -22,6 +22,7 @@ import dmd.aliasthis; import dmd.arraytypes; import dmd.attrib; import dmd.gluelayer; +import dmd.dcache; import dmd.dclass; import dmd.declaration; import dmd.denum; @@ -1380,7 +1381,7 @@ public: AliasDeclaration ad = void; // accessing private selective and renamed imports is // deprecated by restricting the symbol visibility - if (s.isImport() || (ad = s.isAliasDeclaration()) !is null && ad._import !is null) + if (cachedSemantics || s.isImport() || (ad = s.isAliasDeclaration()) !is null && ad._import !is null) {} else error(loc, "%s `%s` is `private`", s.kind(), s.toPrettyChars()); diff --git a/src/dmd/expressionsem.d b/src/dmd/expressionsem.d index 345f64343f97..eba00476f882 100644 --- a/src/dmd/expressionsem.d +++ b/src/dmd/expressionsem.d @@ -23,6 +23,7 @@ import dmd.arraytypes; import dmd.attrib; import dmd.astcodegen; import dmd.canthrow; +import dmd.dcache; import dmd.dscope; import dmd.dsymbol; import dmd.declaration; @@ -2962,6 +2963,8 @@ private extern (C++) final class ExpressionSemanticVisitor : Visitor if (!sd.ctor) sd.ctor = sd.searchCtor(); + // A cached value is a literal + if (cachedSemantics) goto Lx; // First look for constructor if (exp.e1.op == TOK.type && sd.ctor) { diff --git a/src/dmd/globals.d b/src/dmd/globals.d index 14fcc8514c6d..5ee9c87a18d2 100644 --- a/src/dmd/globals.d +++ b/src/dmd/globals.d @@ -146,6 +146,7 @@ struct Param * used to hide a feature that will have to go through deprecate-then-error * before becoming default. */ + bool cache; // use persistent caching of CTFE (DCache) bool showGaggedErrors; // print gagged errors anyway bool manual; // open browser on compiler manual diff --git a/src/dmd/hdrgen.d b/src/dmd/hdrgen.d index dbb5dc4fd86c..17dd06006077 100644 --- a/src/dmd/hdrgen.d +++ b/src/dmd/hdrgen.d @@ -1530,7 +1530,10 @@ public: override void visit(TemplateInstance ti) { - buf.writestring(ti.name.toChars()); + if (ti.aliasdecl) + buf.writestring(ti.aliasdecl.toChars()); + else + buf.writestring(ti.name.toChars()); tiargsToBuffer(ti); if (hgs.fullDump) @@ -2497,7 +2500,7 @@ public: override void visit(StructLiteralExp e) { - buf.writestring(e.sd.toChars()); + buf.writestring(e.sd.type.toChars()); buf.writeByte('('); // CTFE can generate struct literals that contain an AddrExp pointing // to themselves, need to avoid infinite recursion: diff --git a/src/dmd/mars.d b/src/dmd/mars.d index 1a5ff3b219ae..d80a7a3fa340 100644 --- a/src/dmd/mars.d +++ b/src/dmd/mars.d @@ -32,6 +32,7 @@ import dmd.dinifile; import dmd.dinterpret; import dmd.dmodule; import dmd.doc; +import dmd.dcache; import dmd.dscope; import dmd.dsymbol; import dmd.dsymbolsem; @@ -478,6 +479,8 @@ private int tryMain(size_t argc, const(char)** argv) Expression._init(); Objc._init(); builtin_init(); + if (global.params.cache) + dcache.initialize(global._version[0..strlen(global._version)]); if (global.params.verbose) { @@ -1538,6 +1541,10 @@ private bool parseCommandLine(const ref Strings arguments, const size_t argc, re else if (p[4]) goto Lerror; } + else if (strcmp(p + 1, cast(char*)"cache=mmap") == 0) + { + params.cache = true; + } else if (arg == "-shared") params.dll = true; else if (arg == "-dylib") diff --git a/src/dmd/parse.d b/src/dmd/parse.d index 081bed9dc659..605da3950213 100644 --- a/src/dmd/parse.d +++ b/src/dmd/parse.d @@ -237,7 +237,12 @@ final class Parser(AST) : Lexer Loc endloc; // set to location of last right curly int inBrackets; // inside [] of array index or slice Loc lookingForElse; // location of lonely if looking for an else - + enum withSignature = __traits(hasMember, typeof(mod), "signature"); + static if (withSignature) + { + import dmd.root.sha; + SHA1 hasher; + } /********************* * Use this constructor for string mixins. * Input: @@ -246,8 +251,14 @@ final class Parser(AST) : Lexer extern (D) this(Loc loc, AST.Module _module, const(char)[] input, bool doDocComment) { super(_module ? _module.srcfile.toChars() : null, input.ptr, 0, input.length, doDocComment, false); - //printf("Parser::Parser()\n"); + static if (withSignature) + { + hasher.start(); + hasher.put(_module.signature[]); + hasher.put(cast(ubyte[])input); + _module.signature = hasher.finish(); + } scanloc = loc; if (loc.filename) @@ -268,7 +279,12 @@ final class Parser(AST) : Lexer extern (D) this(AST.Module _module, const(char)[] input, bool doDocComment) { super(_module ? _module.srcfile.toChars() : null, input.ptr, 0, input.length, doDocComment, false); - + static if (withSignature) + { + hasher.start(); + hasher.put(cast(ubyte[])input); + _module.signature = hasher.finish(); + } //printf("Parser::Parser()\n"); mod = _module; linkage = LINK.d; diff --git a/src/posix.mak b/src/posix.mak index 276ac94ac0de..669a0704f517 100644 --- a/src/posix.mak +++ b/src/posix.mak @@ -287,12 +287,16 @@ D_OBJC := 1 endif endif +# Enable stats for DCache +#DFLAGS += -version=DCACHE_STATS +# Enable debug for DCache +#DFLAGS += -debug=DCACHE ######## DMD frontend source files FRONT_SRCS=$(addsuffix .d, $(addprefix $D/,access aggregate aliasthis apply argtypes arrayop \ arraytypes astcodegen attrib builtin canthrow cli clone compiler complex cond constfold \ - cppmangle cppmanglewin ctfeexpr dcast dclass declaration delegatize denum dimport \ + cppmangle cppmanglewin ctfeexpr dcache dcast dclass declaration delegatize denum dimport \ dinifile dinterpret dmacro dmangle dmodule doc dscope dstruct dsymbol dsymbolsem \ dtemplate dversion escape expression expressionsem func \ hdrgen id impcnvtab imphint init initsem inline inlinecost intrange \ @@ -304,7 +308,7 @@ FRONT_SRCS=$(addsuffix .d, $(addprefix $D/,access aggregate aliasthis apply argt LEXER_SRCS=$(addsuffix .d, $(addprefix $D/, console entity errors globals id identifier lexer tokens utf)) LEXER_ROOT=$(addsuffix .d, $(addprefix $(ROOT)/, array ctfloat file filename outbuffer port rmem \ - rootobject stringtable hash)) + rootobject stringtable hash sha sha_SSSE3)) ROOT_SRCS = $(addsuffix .d,$(addprefix $(ROOT)/,aav array ctfloat file \ filename man outbuffer port response rmem rootobject speller \ diff --git a/src/win32.mak b/src/win32.mak index f1e98b8e3790..a259c19c0a5b 100644 --- a/src/win32.mak +++ b/src/win32.mak @@ -150,7 +150,7 @@ DMDMAKE=$(MAKE) -fwin32.mak C=$C TK=$(TK) ROOT=$(ROOT) MAKE="$(MAKE)" HOST_DC="$ # D front end FRONT_SRCS=$D/access.d $D/aggregate.d $D/aliasthis.d $D/apply.d $D/argtypes.d $D/arrayop.d \ $D/arraytypes.d $D/astcodegen.d $D/attrib.d $D/builtin.d $D/canthrow.d $D/cli.d $D/clone.d $D/compiler.d $D/complex.d \ - $D/cond.d $D/constfold.d $D/cppmangle.d $D/cppmanglewin.d $D/ctfeexpr.d $D/dcast.d $D/dclass.d \ + $D/cond.d $D/constfold.d $D/cppmangle.d $D/cppmanglewin.d $D/ctfeexpr.d $D/cache.d $D/dcast.d $D/dclass.d \ $D/declaration.d $D/delegatize.d $D/denum.d $D/dimport.d $D/dinifile.d $D/dinterpret.d \ $D/dmacro.d $D/dmangle.d $D/dmodule.d $D/doc.d $D/dscope.d $D/dstruct.d $D/dsymbol.d $D/dsymbolsem.d \ $D/dtemplate.d $D/dversion.d $D/escape.d \ @@ -204,7 +204,7 @@ GBACKOBJ= $G/go.obj $G/gdag.obj $G/gother.obj $G/gflow.obj $G/gloop.obj $G/var.o ROOT_SRCS=$(ROOT)/aav.d $(ROOT)/array.d $(ROOT)/ctfloat.d $(ROOT)/file.d \ $(ROOT)/filename.d $(ROOT)/man.d $(ROOT)/outbuffer.d $(ROOT)/port.d \ $(ROOT)/response.d $(ROOT)/rmem.d $(ROOT)/rootobject.d \ - $(ROOT)/speller.d $(ROOT)/stringtable.d $(ROOT)/hash.d + $(ROOT)/speller.d $(ROOT)/stringtable.d $(ROOT)/hash.d $(ROOT)/sha.d $(ROOT)/sha_SSSE3.d # D front end SRCS = $D/aggregate.h $D/aliasthis.h $D/arraytypes.h \