diff --git a/src/org/rascalmpl/exceptions/RuntimeExceptionFactory.java b/src/org/rascalmpl/exceptions/RuntimeExceptionFactory.java index 335e46da2e6..fbb88d3653f 100644 --- a/src/org/rascalmpl/exceptions/RuntimeExceptionFactory.java +++ b/src/org/rascalmpl/exceptions/RuntimeExceptionFactory.java @@ -99,6 +99,9 @@ public class RuntimeExceptionFactory { // NotImplemented public static final Type ParseError = TF.constructor(TS, Exception, "ParseError", TF.sourceLocationType(), "location"); + // this comes from lang::json::IO + public static final Type NoOffsetParseError = TF.constructor(TS, Exception, "NoOffsetParseError", TF.sourceLocationType(), "location", TF.integerType(), "line", TF.integerType(), "column"); + public static final Type PathNotFound = TF.constructor(TS,Exception,"PathNotFound",TF.sourceLocationType(), "location"); public static final Type PermissionDenied = TF.constructor(TS,Exception,"PermissionDenied",TF.stringType(), "message"); @@ -684,7 +687,12 @@ public static Throw jsonParseError(ISourceLocation loc, String cause, String pat .asWithKeywordParameters().setParameter("path", VF.string(path))); } - + public static Throw jsonParseError(ISourceLocation file, int line, int col, String cause, String path) { + return new Throw(VF.constructor(NoOffsetParseError, file, VF.integer(line), VF.integer(col)) + .asWithKeywordParameters().setParameter("reason", VF.string(cause)) + .asWithKeywordParameters().setParameter("path", VF.string(path))); + } + public static Throw parseError(ISourceLocation loc, AbstractAST ast, StackTrace trace) { return new Throw(VF.constructor(ParseError, loc), ast != null ? ast.getLocation() : null, trace); } @@ -793,4 +801,6 @@ public static Throw parseErrorRecovery(IValue trigger, ISourceLocation loc) { public static Throw parseErrorRecoveryNoSuchField(String name, ISourceLocation loc) { return new Throw(VF.constructor(ParseErrorRecovery, VF.constructor(NoSuchField, VF.string(name)), loc)); } + + } diff --git a/src/org/rascalmpl/library/lang/json/IO.java b/src/org/rascalmpl/library/lang/json/IO.java index f359b3a92d8..e5b91d89fda 100644 --- a/src/org/rascalmpl/library/lang/json/IO.java +++ b/src/org/rascalmpl/library/lang/json/IO.java @@ -5,13 +5,16 @@ * * Contributors: * - * * Jurgen J. Vinju - Jurgen.Vinju@cwi.nl - CWI * Mark Hills - Mark.Hills@cwi.nl (CWI) * Arnold - * Lankamp - Arnold.Lankamp@cwi.nl * Bert Lisser - Bert.Lisser@cwi.nl + * * Jurgen J. Vinju - Jurgen.Vinju@cwi.nl - CWI + * * Mark Hills - Mark.Hills@cwi.nl (CWI) + * * Arnold - Lankamp - Arnold.Lankamp@cwi.nl + * * Bert Lisser - Bert.Lisser@cwi.nl *******************************************************************************/ package org.rascalmpl.library.lang.json; import java.io.IOException; import java.io.OutputStreamWriter; +import java.io.Reader; import java.io.StringReader; import java.io.StringWriter; import java.nio.charset.Charset; @@ -38,7 +41,6 @@ import io.usethesource.vallang.type.Type; import io.usethesource.vallang.type.TypeStore; -import com.google.gson.stream.JsonReader; import com.google.gson.stream.JsonWriter; public class IO { @@ -51,63 +53,55 @@ public IO(IRascalValueFactory values, IRascalMonitor monitor) { this.monitor = monitor; } - public IValue readJSON(IValue type, ISourceLocation loc, IString dateTimeFormat, IBool lenient, IBool trackOrigins, - IFunction parsers, IMap nulls, IBool explicitConstructorNames, IBool explicitDataTypes) { + private IValue doReadJSON(Reader in, + IValue type, ISourceLocation loc, IString dateTimeFormat, IBool lenient, IBool trackOrigins, + IFunction parsers, IMap nulls, IBool explicitConstructorNames, IBool explicitDataTypes) throws IOException { + TypeStore store = new TypeStore(); Type start = new TypeReifier(values).valueToType((IConstructor) type, store); - + if (parsers.getType() instanceof ReifiedType && parsers.getType().getTypeParameters().getFieldType(0).isTop()) { // ignore the default parser parsers = null; } - try (JsonReader in = new JsonReader(URIResolverRegistry.getInstance().getCharacterReader(loc))) { - in.setLenient(lenient.getValue()); + try { return new JsonValueReader(values, store, monitor, loc) - .setCalendarFormat(dateTimeFormat.getValue()) - .setParsers(parsers) - .setNulls(unreify(nulls)) - .setExplicitConstructorNames(explicitConstructorNames.getValue()) - .setExplicitDataTypes(explicitDataTypes.getValue()) - .setTrackOrigins(trackOrigins.getValue()) - .read(in, start); - } - catch (IOException e) { - throw RuntimeExceptionFactory.io(e); + .setCalendarFormat(dateTimeFormat.getValue()) + .setLenient(lenient.getValue()) + .setParsers(parsers) + .setNulls(unreify(nulls)) + .setExplicitConstructorNames(explicitConstructorNames.getValue()) + .setExplicitDataTypes(explicitDataTypes.getValue()) + .setTrackOrigins(trackOrigins.getValue()) + .read(in, start); } catch (NullPointerException e) { throw RuntimeExceptionFactory.io("NPE in error handling code"); } } + + public IValue readJSON( + IValue type, ISourceLocation loc, IString dateTimeFormat, IBool lenient, IBool trackOrigins, + IFunction parsers, IMap nulls, IBool explicitConstructorNames, IBool explicitDataTypes) { - private Map unreify(IMap nulls) { - var tr = new TypeReifier(values); - return nulls.stream().map(t -> (ITuple) t) - .collect(Collectors.toMap(t -> tr.valueToType((IConstructor) t.get(0)), t -> t.get(1))); + try (Reader in = URIResolverRegistry.getInstance().getCharacterReader(loc)) { + return doReadJSON(in, type, loc, dateTimeFormat, lenient, trackOrigins, parsers, nulls, explicitConstructorNames, explicitDataTypes); + } + catch (IOException e) { + throw RuntimeExceptionFactory.io(e); + } } public IValue parseJSON(IValue type, IString src, IString dateTimeFormat, IBool lenient, IBool trackOrigins, IFunction parsers, IMap nulls, IBool explicitConstructorNames, IBool explicitDataTypes) { - TypeStore store = new TypeStore(); - Type start = new TypeReifier(values).valueToType((IConstructor) type, store); - - try (JsonReader in = new JsonReader(new StringReader(src.getValue()))) { - in.setLenient(lenient.getValue()); - return new JsonValueReader(values, store, monitor,null) - .setCalendarFormat(dateTimeFormat.getValue()) - .setParsers(parsers) - .setNulls(unreify(nulls)) - .setTrackOrigins(trackOrigins.getValue()) - .setExplicitConstructorNames(explicitConstructorNames.getValue()) - .setExplicitDataTypes(explicitDataTypes.getValue()) - .read(in, start); + + try (Reader in = new StringReader(src.getValue())) { + return doReadJSON(in, type, null, dateTimeFormat, lenient, trackOrigins, parsers, nulls, explicitConstructorNames, explicitDataTypes); } catch (IOException e) { throw RuntimeExceptionFactory.io(e); } - catch (NullPointerException e) { - throw RuntimeExceptionFactory.io("NPE"); - } } public void writeJSON(ISourceLocation loc, IValue value, IBool unpackedLocations, IString dateTimeFormat, @@ -162,4 +156,10 @@ public IString asJSON(IValue value, IBool unpackedLocations, IString dateTimeFor throw RuntimeExceptionFactory.io(e); } } + + private Map unreify(IMap nulls) { + var tr = new TypeReifier(values); + return nulls.stream().map(t -> (ITuple) t) + .collect(Collectors.toMap(t -> tr.valueToType((IConstructor) t.get(0)), t -> t.get(1))); + } } diff --git a/src/org/rascalmpl/library/lang/json/IO.rsc b/src/org/rascalmpl/library/lang/json/IO.rsc index 8c71c94ed72..2198c8e28d6 100644 --- a/src/org/rascalmpl/library/lang/json/IO.rsc +++ b/src/org/rascalmpl/library/lang/json/IO.rsc @@ -74,7 +74,14 @@ import Exception; * `cause` is a factual diagnosis of what was expected at that position, versus what was found. * `path` is a path query string into the JSON value from the root down to the leaf where the error was detected. } -data RuntimeException(str cause="", str path=""); +@benefits{ +* ((NoOffsetParseError)) is for when accurate offset tracking is turned off. Typically this is _on_ +even if `trackOrigins=false`, when we call the json parsers from Rascal. +} +data RuntimeException(str cause="", str path="") + = ParseError(loc location) + | NoOffsetParseError(loc location, int line, int column) + ; private str DEFAULT_DATETIME_FORMAT = "yyyy-MM-dd\'T\'HH:mm:ssZ"; diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index 5af18d864dd..02e2af38579 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -23,7 +23,9 @@ package org.rascalmpl.library.lang.json.internal; import java.io.EOFException; +import java.io.FilterReader; import java.io.IOException; +import java.io.Reader; import java.io.StringReader; import java.lang.invoke.MethodHandles; import java.lang.invoke.VarHandle; @@ -38,6 +40,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; + import org.rascalmpl.debug.IRascalMonitor; import org.rascalmpl.exceptions.RuntimeExceptionFactory; import org.rascalmpl.exceptions.Throw; @@ -63,6 +66,7 @@ import io.usethesource.vallang.type.TypeStore; import com.google.gson.JsonParseException; +import com.google.gson.Strictness; import com.google.gson.stream.JsonReader; import com.google.gson.stream.JsonToken; import com.google.gson.stream.MalformedJsonException; @@ -75,35 +79,50 @@ public class JsonValueReader { private static final TypeFactory TF = TypeFactory.getInstance(); private final TypeStore store; - private final IValueFactory vf; - private ThreadLocal format; + private final IValueFactory vf; private final IRascalMonitor monitor; - private ISourceLocation src; - private boolean trackOrigins = false; - private boolean disableTracking = false; + private final ISourceLocation src; private VarHandle posHandler; private VarHandle lineHandler; private VarHandle lineStartHandler; + + /* options */ + private ThreadLocal format; + private boolean trackOrigins = false; + private boolean stopTracking = false; private boolean explicitConstructorNames; private boolean explicitDataTypes; + private boolean lenient; private IFunction parsers; private Map nulls = Collections.emptyMap(); + private final class ExpectedTypeDispatcher implements ITypeVisitor { private final JsonReader in; - private int offset = 0; - private int lastPos = 0; - private int lastLimit = 0; - private boolean stopTracking = true; /* origin tracking is turned off while we work on debugging the offsets on another PR */ + private final OriginTrackingReader tracker; - private ExpectedTypeDispatcher(JsonReader in, boolean noTracking) { + /** + * In this mode we read directly from a given JsonReader, under which we can not + * encapsulate its Reader for counting offsets. This is used by the JSON-RPC bridge. + * @param in + */ + private ExpectedTypeDispatcher(JsonReader in) { + this(in, null); + } + + /** + * In this mode we have created an OriginTrackingReader which feeds the JsonReader from below. + * Accurate offsets can be tracked like this, which enables accurate error locations. When + * trackOrigins=true we get accurate origin src fields for objects. + */ + public ExpectedTypeDispatcher(JsonReader in, OriginTrackingReader tracker) { this.in = in; - this.stopTracking = true; // noTracking; NB origin tracking is turned off while we work on debugging the offsets on another PR + this.tracker = tracker; } @Override public IValue visitInteger(Type type) throws IOException { - lastPos = getPos(); + try { switch (in.peek()) { case NUMBER: @@ -124,7 +143,7 @@ public IValue visitInteger(Type type) throws IOException { @Override public IValue visitReal(Type type) throws IOException { - lastPos = getPos(); + try { switch (in.peek()) { case NUMBER: @@ -176,7 +195,7 @@ public IValue visitTuple(Type type) throws IOException { } List l = new ArrayList<>(); - lastPos = getPos(); + in.beginArray(); if (type.hasFieldNames()) { @@ -458,30 +477,15 @@ public IValue visitBool(Type type) throws IOException { * because it has not been called in between from JsonValueReader to JsonReader and the condition * `internalPos < lastPos` will not have had the opportunity to evaluate to `true`. */ - private int getPos() { + private int getOffset() { if (stopTracking) { return 0; } - var internalPos = (int) posHandler.get(in); - var internalLimit = getLimit(); - - if (internalPos < lastPos) { - // so we detected we are in trouble, but we do not have enough information for a solution here. - // TODO: fix this code in another PR by wrapping the CharacterReader. - offset = offset + (lastLimit - lastPos) + internalPos /* gson copies the tail of the buffer to the front */; - } - else { - // the offset advances by the number of parsed characters - offset += (internalPos - lastPos); - } - - // save the previous state - lastPos = internalPos; - lastLimit = internalLimit; - try { - return Math.max(0, offset - 1); + assert posHandler != null; + var internalPos = (int) posHandler.get(in); + return tracker.getOffsetAtBufferStart() + internalPos; } catch (IllegalArgumentException | SecurityException e) { // we stop trying to track positions if it fails so hard, @@ -491,21 +495,6 @@ private int getPos() { } } - private int getLimit() { - if (stopTracking) { - return 0; - } - - try { - return (int) lineHandler.get(in) + 1; - } - catch (IllegalArgumentException | SecurityException e) { - // stop trying to recover the positions - stopTracking = true; - return 0; - } - } - private int getLine() { if (stopTracking) { return 0; @@ -544,13 +533,22 @@ private int getCol() { } protected Throw parseErrorHere(String cause) { - var location = src == null ? URIUtil.rootLocation("unknown") : src; - int offset = getPos(); + var location = getRootLoc(); + int offset = getOffset(); int line = getLine(); int col = getCol(); - return RuntimeExceptionFactory - .jsonParseError(vf.sourceLocation(location, offset, 1, line, line, col, col + 1), cause, in.getPath()); + if (!stopTracking) { + return RuntimeExceptionFactory + .jsonParseError(vf.sourceLocation(location, offset, 1, line, line, col, col + 1), cause, in.getPath()); + } + else { + // if we didn't track the offset, we can at least produce line and column information, but not as a + // default Rascal ParseError with '0' or '-1' for offset, because that can trigger assertions and + // break other assumptions clients make about the source location values. + return RuntimeExceptionFactory + .jsonParseError(location, line, col, cause, in.getPath()); + } } /** @@ -616,11 +614,12 @@ private IValue visitStringAsAbstractData(Type type) throws IOException { private IValue visitObjectAsAbstractData(Type type) throws IOException { Set alternatives = null; - in.beginObject(); - int startPos = getPos(); + int startPos = getOffset() - 1; int startLine = getLine(); - int startCol = getCol(); + int startCol = getCol() - 1; + in.beginObject(); + // use explicit information in the JSON to select and filter constructors from the TypeStore // we expect always to have the field _constructor before _type. if (explicitConstructorNames || explicitDataTypes) { @@ -735,12 +734,14 @@ else if (!explicitDataTypes && "_type".equals(label)) { } } - in.endObject(); - int endPos = getPos(); - assert endPos >= startPos : "as assumpion on the internals of gson"; + int endPos = getOffset() - 1; + assert endPos > startPos : "offset tracking messed up while stopTracking is " + stopTracking + " and trackOrigins is " + trackOrigins; + int endLine = getLine(); - int endCol = getCol(); + int endCol = getCol() - 1; + in.endObject(); + for (int i = 0; i < args.length; i++) { if (args[i] == null) { throw parseErrorHere( @@ -750,12 +751,21 @@ else if (!explicitDataTypes && "_type".equals(label)) { if (trackOrigins && !stopTracking) { kwParams.put(kwParams.containsKey("src") ? "rascal-src" : "src", - vf.sourceLocation(src, startPos, endPos - startPos + 1, startLine, endLine, startCol, endCol + 1)); + vf.sourceLocation(getRootLoc(), startPos, endPos - startPos + 1, startLine, endLine, startCol, endCol + 1)); } return vf.constructor(cons, args, kwParams); } + private ISourceLocation getRootLoc() { + if (src == null) { + return URIUtil.rootLocation("unknown"); + } + else { + return src; + } + } + @Override public IValue visitAbstractData(Type type) throws IOException { if (UtilMaybe.isMaybe(type)) { @@ -792,9 +802,9 @@ public IValue visitNode(Type type) throws IOException { return inferNullValue(nulls, type); } - int startPos = getPos(); + int startPos = getOffset() - 1; int startLine = getLine(); - int startCol = getCol(); + int startCol = getCol() - 1; in.beginObject(); @@ -829,14 +839,15 @@ public IValue visitNode(Type type) throws IOException { } } - in.endObject(); - int endPos = getPos(); + int endPos = getOffset() - 1; int endLine = getLine(); - int endCol = getCol(); + int endCol = getCol() - 1; + in.endObject(); + if (trackOrigins && !stopTracking) { kws.put(kws.containsKey("src") ? "rascal-src" : "src", - vf.sourceLocation(src, startPos, endPos - startPos + 1, startLine, endLine, startCol, endCol + 1)); + vf.sourceLocation(getRootLoc(), startPos, endPos - startPos + 1, startLine, endLine, startCol, endCol + 1)); } IValue[] argArray = args.entrySet().stream().sorted((e, f) -> e.getKey().compareTo(f.getKey())) @@ -898,6 +909,7 @@ public IValue visitList(Type type) throws IOException { } IListWriter w = vf.listWriter(); + getOffset(); in.beginArray(); while (in.hasNext()) { // here we pass label from the higher context @@ -910,6 +922,7 @@ public IValue visitList(Type type) throws IOException { } in.endArray(); + getOffset(); return w.done(); } @@ -959,24 +972,26 @@ public JsonValueReader(IValueFactory vf, TypeStore store, IRascalMonitor monitor this.store = store; this.monitor = monitor; this.src = src; - this.disableTracking = false; + this.stopTracking = false; setCalendarFormat("yyyy-MM-dd'T'HH:mm:ssZ"); - if (src != null) { - try { - var lookup = MethodHandles.lookup(); - var privateLookup = MethodHandles.privateLookupIn(JsonReader.class, lookup); - this.posHandler = privateLookup.findVarHandle(JsonReader.class, "pos", int.class); - this.lineHandler = privateLookup.findVarHandle(JsonReader.class, "lineNumber", int.class); - this.lineStartHandler = privateLookup.findVarHandle(JsonReader.class, "lineStart", int.class); - } - catch (NoSuchFieldException | SecurityException | IllegalAccessException e) { - // we disable the origin tracking if we can not get to the fields - disableTracking = true; - monitor.warning("Unable to retrieve origin information due to: " + e.getMessage(), src); + try { + var lookup = MethodHandles.lookup(); + var privateLookup = MethodHandles.privateLookupIn(JsonReader.class, lookup); + this.posHandler = privateLookup.findVarHandle(JsonReader.class, "pos", int.class); + this.lineHandler = privateLookup.findVarHandle(JsonReader.class, "lineNumber", int.class); + this.lineStartHandler = privateLookup.findVarHandle(JsonReader.class, "lineStart", int.class); + + if (posHandler == null) { + stopTracking = true; } } + catch (NoSuchFieldException | SecurityException | IllegalAccessException e) { + // we disable the origin tracking if we can not get to the fields + stopTracking = true; + monitor.warning("Unable to retrieve origin information due to: " + e.getMessage(), src); + } } public JsonValueReader(IValueFactory vf, IRascalMonitor monitor, ISourceLocation src) { @@ -1019,7 +1034,7 @@ public JsonValueReader(IRascalValueFactory vf, TypeStore store, IRascalMonitor m } catch (NoSuchFieldException | SecurityException | IllegalAccessException e) { // we disable the origin tracking if we can not get to the fields - disableTracking = true; + stopTracking = true; monitor.warning("Unable to retrieve origin information due to: " + e.getMessage(), src); } } @@ -1035,10 +1050,6 @@ public JsonValueReader setNulls(Map nulls) { public JsonValueReader setTrackOrigins(boolean trackOrigins) { this.trackOrigins = trackOrigins; - if (trackOrigins) { - monitor.warning("The origin tracking feature of the JSON parser is temporarily disabled.", src); - this.trackOrigins = false; - } return this; } @@ -1066,8 +1077,14 @@ public JsonValueReader setParsers(IFunction parsers) { return this; } + public JsonValueReader setLenient(boolean value) { + this.lenient = true; + return this; + } + /** - * Read and validate a Json stream as an IValue + * Read and validate a Json stream as an IValue. This version does not support accurate error messages + * or origin tracking. * * @param in json stream * @param expected type to validate against (recursively) @@ -1075,7 +1092,11 @@ public JsonValueReader setParsers(IFunction parsers) { * @throws IOException when either a parse error or a validation error occurs */ public IValue read(JsonReader in, Type expected) throws IOException { - var dispatch = new ExpectedTypeDispatcher(in, disableTracking); + in.setStrictness(lenient ? Strictness.LENIENT : Strictness.LEGACY_STRICT); + + // we can't track accurately because we don't have a handle to the raw buffer under `in` + this.stopTracking = true; + var dispatch = new ExpectedTypeDispatcher(in); try { var result = expected.accept(dispatch); @@ -1090,4 +1111,112 @@ public IValue read(JsonReader in, Type expected) throws IOException { throw dispatch.parseErrorHere(e.getMessage()); } } + + /** + * Read and validate a Json stream as an IValue. This version supports accurate error messages + * and origin tracking. + * + * @param in json stream + * @param expected type to validate against (recursively) + * @return an IValue of the expected type + * @throws IOException when either a parse error or a validation error occurs + */ + public IValue read(Reader in, Type expected) throws IOException { + try (OriginTrackingReader wrappedIn = new OriginTrackingReader(in); JsonReader jsonIn = new JsonReader(wrappedIn)) { + jsonIn.setStrictness(lenient ? Strictness.LENIENT : Strictness.LEGACY_STRICT); + + var dispatch = new ExpectedTypeDispatcher(jsonIn, wrappedIn); + + try { + var result = expected.accept(dispatch); + if (result == null) { + throw new JsonParseException("null occurred outside an optionality context and without a registered representation."); + } + return result; + } + catch (NullPointerException e) { + throw dispatch.parseErrorHere("Unexpected internal NullPointerException"); + } + catch (EOFException | JsonParseException | NumberFormatException | MalformedJsonException | IllegalStateException e) { + throw dispatch.parseErrorHere(e.getMessage()); + } + } + } + + /** + * This wraps a normal reader to make it possible for a client to detect accurate + * character offsets (> buffersize) in a large file, even if the underlying stream is buffered. + * + * This implementation is tightly coupled (semantically) with the internals of JsonReader. It provides + * just enough information, together with internal private fields of JsonReader, to compute Rascal-required + * offsets. We get only the character offset in the file, at the start of each streamed buffer contents. + * That should be just enough information to recompute the actual offset of every Json element, using the + * current position in the buffer (the private field `pos` of JsonReader). + */ + public static class OriginTrackingReader extends FilterReader { + // offset is always pointing at the point in the file where JsonReader.pos == 0 + private int offset = 0; + // limit is always pointing to the amount of no-junk characters in the underlying buffer below buffer.length + private int limit = 0; + + protected OriginTrackingReader(Reader in) { + super(in); + } + + /* This private method from JsonReader must be mirrored by `read` + private boolean fillBuffer(int minimum) throws IOException { + char[] buffer = this.buffer; + lineStart -= pos; + if (limit != pos) { + limit -= pos; + System.arraycopy(buffer, pos, buffer, 0, limit); + } else { + limit = 0; + } + + pos = 0; + int total; + while ((total = in.read(buffer, limit, buffer.length - limit)) != -1) { + limit += total; + + // if this is the first read, consume an optional byte order mark (BOM) if it exists + if (lineNumber == 0 && lineStart == 0 && limit > 0 && buffer[0] == '\ufeff') { + pos++; + lineStart++; + minimum++; + } + + if (limit >= minimum) { + return true; + } + } + return false; + } */ + @Override + public int read(char[] cbuf, int off, int len) throws IOException { + // Note that `fillBuffer.limit != fillBuffer.pos <==> reader.off != 0`. + // Moreover, `fillBuffer.limit == reader.off` at the start of this method. + + // we know take the previous limit and add it to the + // offset, to arrive at the new `pos=0` of `buffer[0]`, + // rewinding `off` characters which were reused from the previous buffer + // with System.arraycopy. + offset += limit - off; + + // make sure we are only a facade for the real reader. + // parameters are mapped one-to-one without mutations. + var charsRead = in.read(cbuf, off, len); + + // the next buffer[0] offset will be after this increment. + // Note that `fillBuffer.limit == read.limit` + limit = off + charsRead; + + // and return only the number of characters read. + return charsRead; + } + + public int getOffsetAtBufferStart() { + return offset; + } + } } diff --git a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/Issue2633.rsc b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/Issue2633.rsc index 3de22d49e20..09e94442a4b 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/Issue2633.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/Issue2633.rsc @@ -1,59 +1,59 @@ -module lang::rascal::tests::library::lang::json::Issue2633 - -import lang::json::IO; -import IO; - -data OuterData = outer(str name, str \type, list[Nested] nested_abc); -data Nested = nested(str \type, map[str, value] property__1, map[value, value] property__2); - -test bool failsOnCertainJSON() { - str input = "{ - ' \"name\": \"TESTING_ Adding one more character to this data causes the test to fail\", - ' \"type\": \"type_abcd\", - ' \"nested_abc\": [ - ' { - ' \"type\": \"line\", - ' \"property__1\": { \"y\": { \"value\": 2, \"absolute\": false } }, - ' \"property__2\": {} - ' }, - ' { - ' \"type\": \"line\", - ' \"property__1\": { \"y\": { \"value\": 2, \"absolute\": false } }, - ' \"property__2\": {} - ' }, - ' { - ' \"type\": \"line\", - ' \"property__1\": { \"y\": { \"value\": 2, \"absolute\": false } }, - ' \"property__2\": {} - ' }, - ' { - ' \"type\": \"line\", - ' \"property__1\": { \"y\": { \"value\": 2, \"absolute\": false } }, - ' \"property__2\": {} - ' }, - ' { - ' \"type\": \"line\", - ' \"property__1\": { \"y\": { \"value\": 2, \"absolute\": false } }, - ' \"property__2\": {} - ' }, - ' { - ' \"type\": \"line\", - ' \"property__1\": { \"y\": { \"value\": 2, \"absolute\": false } }, - ' \"property__2\": {} - ' }, - ' { - ' \"type\": \"line\", - ' \"property__1\": { \"y\": { \"value\": 2, \"absolute\": false } }, - ' \"property__2\": {} - ' } - ' ] - '} - '"; - - // Fails when the original data above is increased by even one character (e.g. the name, or changing a 2 to 20, or even adding a meaningless space anywhere) - parsedVFD = parseJSON(#OuterData, input); - println("Parsed VisualFormData (After)"); - iprintln(parsedVFD); - - return true; +module lang::rascal::tests::library::lang::json::Issue2633 + +import lang::json::IO; +import IO; + +data OuterData = outer(str name, str \type, list[Nested] nested_abc); +data Nested = nested(str \type, map[str, value] property__1, map[value, value] property__2); + +test bool failsOnCertainJSON() { + str input = "{ + ' \"name\": \"TESTING_ Adding one more character to this data causes the test to fail\", + ' \"type\": \"type_abcd\", + ' \"nested_abc\": [ + ' { + ' \"type\": \"line\", + ' \"property__1\": { \"y\": { \"value\": 2, \"absolute\": false } }, + ' \"property__2\": {} + ' }, + ' { + ' \"type\": \"line\", + ' \"property__1\": { \"y\": { \"value\": 2, \"absolute\": false } }, + ' \"property__2\": {} + ' }, + ' { + ' \"type\": \"line\", + ' \"property__1\": { \"y\": { \"value\": 2, \"absolute\": false } }, + ' \"property__2\": {} + ' }, + ' { + ' \"type\": \"line\", + ' \"property__1\": { \"y\": { \"value\": 2, \"absolute\": false } }, + ' \"property__2\": {} + ' }, + ' { + ' \"type\": \"line\", + ' \"property__1\": { \"y\": { \"value\": 2, \"absolute\": false } }, + ' \"property__2\": {} + ' }, + ' { + ' \"type\": \"line\", + ' \"property__1\": { \"y\": { \"value\": 2, \"absolute\": false } }, + ' \"property__2\": {} + ' }, + ' { + ' \"type\": \"line\", + ' \"property__1\": { \"y\": { \"value\": 2, \"absolute\": false } }, + ' \"property__2\": {} + ' } + ' ] + '} + '"; + + // Fails when the original data above is increased by even one character (e.g. the name, or changing a 2 to 20, or even adding a meaningless space anywhere) + parsedVFD = parseJSON(#OuterData, input, trackOrigins=true, lenient=true); + println("Parsed VisualFormData (After)"); + iprintln(parsedVFD); + + return true; } \ No newline at end of file diff --git a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc index 4ea773c85a8..65602a9e73b 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc @@ -94,22 +94,24 @@ test bool json4(Enum e) = writeRead(#DATA4, data4(e=e)); bool originTest(loc example) { ex2 = readJSON(#node, example, trackOrigins=true); content = readFile(example); - + lines = split("\n", content); poss = [ | /node x := ex2, x.line?]; // every node has a .src field, otherwise this fails with an exception for ( <- poss) { assert content[p.offset] == "{"; // all nodes start with a { assert content[p.offset + p.length - 1] == "}"; // all nodes end with a } assert p.begin.line == line; + assert lines[p.begin.line - 1][p.begin.column] == "{"; + assert lines[p.end.line - 1][p.end.column - 1] == "}"; } return true; } -@ignore{awaiting fix of 2633} test bool originTracking() { - return originTest(|std:///lang/rascal/tests/library/lang/json/glossary.json|) - && originTest(|std:///lang/rascal/tests/library/lang/json/testing.json|); + files = [ l | loc l <- |std:///lang/rascal/tests/library/lang/json|.ls, l.extension == "json"]; + + return (true | it && originTest(example) | loc example <- files); } value numNormalizer(int i) = i % maxLong when abs(i) > maxLong; @@ -181,7 +183,6 @@ value toDefaultValue(real r) = r - round(r) == 0 : fitDouble(r); default value toDefaultValue(value x) = x; -@ignore{awaiting fix of 2633} test bool accurateParseErrors() { ex = readFile(|std:///lang/rascal/tests/library/lang/json/glossary.json|); broken = ex[..size(ex)/2] + ex[size(ex)/2+10..]; @@ -312,5 +313,48 @@ test bool explicitDataTypes() { // here we can't be sure to get z() back, but we will get some Enum assert data4(e=Enum _) := parseJSON(#DATA4, json, explicitDataTypes=false); + return true; +} + +data X(loc src=|unkown:///|) = v1(int x=0, str s = ""); + +test bool jsonVerifyOriginCorrect() { + ref = v1(x=123456789); + refExpected = asJSON(ref); + t1 = [v1(s="hoi"), ref]; + writeJSON(|memory:///test.json|, t1); + v = readJSON(#list[X],|memory:///test.json|, trackOrigins=true); + return refExpected == readFile(v[1].src); +} + +test bool triggerIssue2633() { + return jsonVerifyOriginCorrectAcrossBufferBoundaries(1023); +} + +test bool jsonVerifyOriginCorrectAcrossBufferBoundaries() { + /* twice just before and after the 1024 buffer size of JsonReader */ + for (int sSize <- [1000..1025] + [2000..2050]) { + jsonVerifyOriginCorrectAcrossBufferBoundaries(sSize); + } + return true; +} + +bool jsonVerifyOriginCorrectAcrossBufferBoundaries(int sSize) { + ref = v1(x=123456789); + refExpected = asJSON(ref); + + t1 = [v1(s="a<}>"), ref]; + writeJSON(|memory:///test.json|, t1); + + //s this throws exceptions and asserts if there are bugs with the + // origin tracker. In particular it triggers #2633 + v = readJSON(#list[X],|memory:///test.json|, trackOrigins=true); + + // checking the last element + if (refExpected != readFile(v[1].src)) { + println("Failed for : != "); + return false; + } + return true; } \ No newline at end of file diff --git a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/longcomment.json b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/longcomment.json new file mode 100644 index 00000000000..001e102dca2 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/longcomment.json @@ -0,0 +1,8 @@ +// 66XU0Cylu7Sur4Iiae/KOiEr7HnJOWgaHqwPTYusrEsX1Imws/Qxpc9EZISLCq+G60OVedHbctmJF1KY2Vx962khn5xcKZ7IpdavqI1Xi5FVz2PB4nNb2SD4zwxFK1Fz6NhS2B80vftdwtik9ya3CGgMV5d8dllRivihC+hhdY4sLAjdUXiFGSgEqhPIZYbRtFHi4kWNBJd0BJHx1gWEuimd6WDbBS0FyVsRqjqLutmKjy6pOiF0s5XHYYHkm+LjFN+eTT1xV6AKvuqslRt8q7cz2Cy2HRGTmfSunlLCei1Q/Bff/jtGEG+sYsfRZy7FQwa1vNxAGXOGUqOaYrQOrg68Y/TbVnQ2Wq/KFjQIxEEZJWq5z5MKRGZfV+EfQ2gLSjE+owPr+y/AAH4QjSehItW6qYGo0k8Vt7ptD95xNg4K/b2ebXOuJh4vOpOdIXUpRuwZfndVacYT61ozLOGzlukdqofQF5zaE1XuegssUUJNPxvT6gpVF1d3P0MoIVSJpCi4boSgM0xpAYYnC+Trl9B+zz74HIvS3oePc+Bm2CrL3Ap2rC9BreJKRljLwzJdWw/Snnb5ssZJtKI1S4EqelMEOg0vxoUQDYiGGysRHxZBsvTU0sHzKEe85m7ovR3iDnd5mnAhx58BS/YVXemd3zefcisnyAnjreW1g+fiDXG66le90C0lGvBPEg15xRPt298AjGulnWylJ3Q4pKZ367x41L134GvAGtnihuYMnPoSuWo2bakvHHDP2iexVobSK64QzlfXFLHydqNyJ/kSPztvsX5QV1kWIIHiqxGZEttOpbP4OvZDP82/5kvnI4hNv92ZcCdu1eKfgCNrd+cSYBeRXvz2W7KH+Ts2FbZDrwHNQWhvkfgBEpDvk+YwQTZFvMwenj4VA6VmYZvw5pzOiS2t3kdOJz60CZsgCqKaF7slaJV/SeG6zfi5rDkioQJZyB97Tjmed8S8etMexEjrq7q1S2Wkn7uAdDY/cEjhwuH1+YvNlECS6naT+oFu3thk66XU0Cylu7Sur4Iiae/KOiEr7HnJOWgaHqwPTYusrEsX1Imws/Qxpc9EZISLCq+G60OVedHbctmJF1KY2Vx962khn5xcKZ7IpdavqI1Xi5FVz2PB4nNb2SD4zwxFK1Fz6NhS2B80vftdwtik9ya3CGgMV5d8dllRivihC+hhdY4sLAjdUXiFGSgEqhPIZYbRtFHi4kWNBJd0BJHx1gWEuimd6WDbBS0FyVsRqjqLutmKjy6pOiF0s5XHYYHkm+LjFN+eTT1xV6AKvuqslRt8q7cz2Cy2HRGTmfSunlLCei1Q/Bff/jtGEG+sYsfRZy7FQwa1vNxAGXOGUqOaYrQOrg68Y/TbVnQ2Wq/KFjQIxEEZJWq5z5MKRGZfV+EfQ2gLSjE+owPr+y/AAH4QjSehItW6qYGo0k8Vt7ptD95xNg4K/b2ebXOuJh4vOpOdIXUpRuwZfndVacYT61ozLOGzlukdqofQF5zaE1XuegssUUJNPxvT6gpVF1d3P0MoIVSJpCi4boSgM0xpAYYnC+Trl9B+zz74HIvS3oePc+Bm2CrL3Ap2rC9BreJKRljLwzJdWw/Snnb5ssZJtKI1S4EqelMEOg0vxoUQDYiGGysRHxZBsvTU0sHzKEe85m7ovR3iDnd5mnAhx58BS/YVXemd3zefcisnyAnjreW1g+fiDXG66le90C0lGvBPEg15xRPt298AjGulnWylJ3Q4pKZ367x41L134GvAGtnihuYMnPoSuWo2bakvHHDP2iexVobSK64QzlfXFLHydqNyJ/kSPztvsX5QV1kWIIHiqxGZEttOpbP4OvZDP82/5kvnI4hNv92ZcCdu1eKfgCNrd+cSYBeRXvz2W7KH+Ts2FbZDrwHNQWhvkfgBEpDvk+YwQTZFvMwenj4VA6VmYZvw5pzOiS2t3kdOJz60CZsgCqKaF7slaJV/SeG6zfi5rDkioQJZyB97Tjmed8S8etMexEjrq7q1S2Wkn7uAdDY/cEjhwuH1+YvNlECS6naT+oFu3thk +{ + "name": "this object has a very long comment of >1024 characters", + + "nested" : { + "type": "" + } +} \ No newline at end of file diff --git a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/longstring.json b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/longstring.json new file mode 100644 index 00000000000..a7830c82ab8 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/longstring.json @@ -0,0 +1,6 @@ +{ + "name": "this object has a very long string of 1024 characters", + "nested" : { + "type": "66XU0Cylu7Sur4Iiae/KOiEr7HnJOWgaHqwPTYusrEsX1Imws/Qxpc9EZISLCq+G60OVedHbctmJF1KY2Vx962khn5xcKZ7IpdavqI1Xi5FVz2PB4nNb2SD4zwxFK1Fz6NhS2B80vftdwtik9ya3CGgMV5d8dllRivihC+hhdY4sLAjdUXiFGSgEqhPIZYbRtFHi4kWNBJd0BJHx1gWEuimd6WDbBS0FyVsRqjqLutmKjy6pOiF0s5XHYYHkm+LjFN+eTT1xV6AKvuqslRt8q7cz2Cy2HRGTmfSunlLCei1Q/Bff/jtGEG+sYsfRZy7FQwa1vNxAGXOGUqOaYrQOrg68Y/TbVnQ2Wq/KFjQIxEEZJWq5z5MKRGZfV+EfQ2gLSjE+owPr+y/AAH4QjSehItW6qYGo0k8Vt7ptD95xNg4K/b2ebXOuJh4vOpOdIXUpRuwZfndVacYT61ozLOGzlukdqofQF5zaE1XuegssUUJNPxvT6gpVF1d3P0MoIVSJpCi4boSgM0xpAYYnC+Trl9B+zz74HIvS3oePc+Bm2CrL3Ap2rC9BreJKRljLwzJdWw/Snnb5ssZJtKI1S4EqelMEOg0vxoUQDYiGGysRHxZBsvTU0sHzKEe85m7ovR3iDnd5mnAhx58BS/YVXemd3zefcisnyAnjreW1g+fiDXG66le90C0lGvBPEg15xRPt298AjGulnWylJ3Q4pKZ367x41L134GvAGtnihuYMnPoSuWo2bakvHHDP2iexVobSK64QzlfXFLHydqNyJ/kSPztvsX5QV1kWIIHiqxGZEttOpbP4OvZDP82/5kvnI4hNv92ZcCdu1eKfgCNrd+cSYBeRXvz2W7KH+Ts2FbZDrwHNQWhvkfgBEpDvk+YwQTZFvMwenj4VA6VmYZvw5pzOiS2t3kdOJz60CZsgCqKaF7slaJV/SeG6zfi5rDkioQJZyB97Tjmed8S8etMexEjrq7q1S2Wkn7uAdDY/cEjhwuH1+YvNlECS6naT+oFu3thk66XU0Cylu7Sur4Iiae/KOiEr7HnJOWgaHqwPTYusrEsX1Imws/Qxpc9EZISLCq+G60OVedHbctmJF1KY2Vx962khn5xcKZ7IpdavqI1Xi5FVz2PB4nNb2SD4zwxFK1Fz6NhS2B80vftdwtik9ya3CGgMV5d8dllRivihC+hhdY4sLAjdUXiFGSgEqhPIZYbRtFHi4kWNBJd0BJHx1gWEuimd6WDbBS0FyVsRqjqLutmKjy6pOiF0s5XHYYHkm+LjFN+eTT1xV6AKvuqslRt8q7cz2Cy2HRGTmfSunlLCei1Q/Bff/jtGEG+sYsfRZy7FQwa1vNxAGXOGUqOaYrQOrg68Y/TbVnQ2Wq/KFjQIxEEZJWq5z5MKRGZfV+EfQ2gLSjE+owPr+y/AAH4QjSehItW6qYGo0k8Vt7ptD95xNg4K/b2ebXOuJh4vOpOdIXUpRuwZfndVacYT61ozLOGzlukdqofQF5zaE1XuegssUUJNPxvT6gpVF1d3P0MoIVSJpCi4boSgM0xpAYYnC+Trl9B+zz74HIvS3oePc+Bm2CrL3Ap2rC9BreJKRljLwzJdWw/Snnb5ssZJtKI1S4EqelMEOg0vxoUQDYiGGysRHxZBsvTU0sHzKEe85m7ovR3iDnd5mnAhx58BS/YVXemd3zefcisnyAnjreW1g+fiDXG66le90C0lGvBPEg15xRPt298AjGulnWylJ3Q4pKZ367x41L134GvAGtnihuYMnPoSuWo2bakvHHDP2iexVobSK64QzlfXFLHydqNyJ/kSPztvsX5QV1kWIIHiqxGZEttOpbP4OvZDP82/5kvnI4hNv92ZcCdu1eKfgCNrd+cSYBeRXvz2W7KH+Ts2FbZDrwHNQWhvkfgBEpDvk+YwQTZFvMwenj4VA6VmYZvw5pzOiS2t3kdOJz60CZsgCqKaF7slaJV/SeG6zfi5rDkioQJZyB97Tjmed8S8etMexEjrq7q1S2Wkn7uAdDY/cEjhwuH1+YvNlECS6naT+oFu3thk" + } +} \ No newline at end of file diff --git a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/testing.json b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/testing.json index 4211534ee3b..bb778c26d15 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/testing.json +++ b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/testing.json @@ -1,41 +1,42 @@ -{ - "name": "TESTING Adding one more character to this data causes the test to fail", - "type": "type_abcd", - "nested_abc": [ - { - "type": "line", - "property__1": { "y": { "value": 2, "absolute": false } }, - "property__2": {} - }, - { - "type": "line", - "property__1": { "y": { "value": 2, "absolute": false } }, - "property__2": {} - }, - { - "type": "line", - "property__1": { "y": { "value": 2, "absolute": false } }, - "property__2": {} - }, - { - "type": "line", - "property__1": { "y": { "value": 2, "absolute": false } }, - "property__2": {} - }, - { - "type": "line", - "property__1": { "y": { "value": 2, "absolute": false } }, - "property__2": {} - }, - { - "type": "line", - "property__1": { "y": { "value": 2, "absolute": false } }, - "property__2": {} - }, - { - "type": "line", - "property__1": { "y": { "value": 2, "absolute": false } }, - "property__2": {} - } - ] -} +// This comment pushes the length of the file beyong the 1024 limit. +{ + "name": "TESTING Adding one more character to this data caused the test to fail", + "type": "type_abcd", + "nested_abc": [ + { + "type": "line", + "property__1": { "y": { "value": 2, "absolute": false } }, + "property__2": {} + }, + { + "type": "line", + "property__1": { "y": { "value": 2, "absolute": false } }, + "property__2": {} + }, + { + "type": "line", + "property__1": { "y": { "value": 2, "absolute": false } }, + "property__2": {} + }, + { + "type": "line", + "property__1": { "y": { "value": 2, "absolute": false } }, + "property__2": {} + }, + { + "type": "line", + "property__1": { "y": { "value": 2, "absolute": false } }, + "property__2": {} + }, + { + "type": "line", + "property__1": { "y": { "value": 2, "absolute": false } }, + "property__2": {} + }, + { + "type": "line", + "property__1": { "y": { "value": 2, "absolute": false } }, + "property__2": {} + } + ] +}