From 429ee1314fd5ed1e14fb598b0b1f6cc004605280 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Wed, 11 Feb 2026 13:21:11 +0100 Subject: [PATCH 01/23] basic setup for fixing #2633 by wrapping the character buffer under the JsonReader. idea via @DavyLandman --- src/org/rascalmpl/library/lang/json/IO.java | 4 + .../lang/json/internal/JsonValueReader.java | 91 +++++++++++++++++-- 2 files changed, 86 insertions(+), 9 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/IO.java b/src/org/rascalmpl/library/lang/json/IO.java index f359b3a92d8..b1ea8a9bad0 100644 --- a/src/org/rascalmpl/library/lang/json/IO.java +++ b/src/org/rascalmpl/library/lang/json/IO.java @@ -10,8 +10,10 @@ *******************************************************************************/ package org.rascalmpl.library.lang.json; +import java.io.FilterReader; import java.io.IOException; import java.io.OutputStreamWriter; +import java.io.Reader; import java.io.StringReader; import java.io.StringWriter; import java.nio.charset.Charset; @@ -162,4 +164,6 @@ public IString asJSON(IValue value, IBool unpackedLocations, IString dateTimeFor throw RuntimeExceptionFactory.io(e); } } + + } diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index 5af18d864dd..dc7f8f727f3 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -23,7 +23,9 @@ package org.rascalmpl.library.lang.json.internal; import java.io.EOFException; +import java.io.FilterReader; import java.io.IOException; +import java.io.Reader; import java.io.StringReader; import java.lang.invoke.MethodHandles; import java.lang.invoke.VarHandle; @@ -91,14 +93,23 @@ public class JsonValueReader { private final class ExpectedTypeDispatcher implements ITypeVisitor { private final JsonReader in; + private final OriginTrackingReader tracker; + private int offset = 0; private int lastPos = 0; private int lastLimit = 0; - private boolean stopTracking = true; /* origin tracking is turned off while we work on debugging the offsets on another PR */ + private boolean stopTracking = false; + + private ExpectedTypeDispatcher(JsonReader in) { + this.in = in; + this.stopTracking = true; + this.tracker = null; + } - private ExpectedTypeDispatcher(JsonReader in, boolean noTracking) { + public ExpectedTypeDispatcher(JsonReader in, OriginTrackingReader tracker, boolean disableTracking) { this.in = in; - this.stopTracking = true; // noTracking; NB origin tracking is turned off while we work on debugging the offsets on another PR + this.tracker = tracker; + this.stopTracking = disableTracking; } @Override @@ -465,11 +476,10 @@ private int getPos() { var internalPos = (int) posHandler.get(in); var internalLimit = getLimit(); + var baseOffset = tracker.getLimitOffset(); if (internalPos < lastPos) { - // so we detected we are in trouble, but we do not have enough information for a solution here. - // TODO: fix this code in another PR by wrapping the CharacterReader. - offset = offset + (lastLimit - lastPos) + internalPos /* gson copies the tail of the buffer to the front */; + offset = baseOffset + internalPos; } else { // the offset advances by the number of parsed characters @@ -860,7 +870,7 @@ public IValue visitNumber(Type type) throws IOException { if (numberString.contains("r")) { return vf.rational(numberString); } - if (numberString.matches(".*[\\.eE].*")) { + if (numberString.matches(".*[\.eE].*")) { return vf.real(numberString); } else { @@ -1067,7 +1077,8 @@ public JsonValueReader setParsers(IFunction parsers) { } /** - * Read and validate a Json stream as an IValue + * Read and validate a Json stream as an IValue. This version does not support accurate error messages + * or origin tracking. * * @param in json stream * @param expected type to validate against (recursively) @@ -1075,7 +1086,8 @@ public JsonValueReader setParsers(IFunction parsers) { * @throws IOException when either a parse error or a validation error occurs */ public IValue read(JsonReader in, Type expected) throws IOException { - var dispatch = new ExpectedTypeDispatcher(in, disableTracking); + assert !trackOrigins : "use the other read method to track Json origins"; + var dispatch = new ExpectedTypeDispatcher(in); try { var result = expected.accept(dispatch); @@ -1090,4 +1102,65 @@ public IValue read(JsonReader in, Type expected) throws IOException { throw dispatch.parseErrorHere(e.getMessage()); } } + + /** + * Read and validate a Json stream as an IValue. This version supports accurate error messages + * and origin tracking. + * + * @param in json stream + * @param expected type to validate against (recursively) + * @return an IValue of the expected type + * @throws IOException when either a parse error or a validation error occurs + */ + public IValue read(Reader in, Type expected) throws IOException { + try (OriginTrackingReader wrappedIn = new OriginTrackingReader(in); JsonReader jsonIn = new JsonReader(wrappedIn)) { + var dispatch = new ExpectedTypeDispatcher(jsonIn, wrappedIn, disableTracking); + + try { + var result = expected.accept(dispatch); + if (result == null) { + throw new JsonParseException( + "null occurred outside an optionality context and without a registered representation."); + } + return result; + } catch (EOFException | JsonParseException | NumberFormatException | MalformedJsonException | IllegalStateException | NullPointerException e) { + throw dispatch.parseErrorHere(e.getMessage()); + } + } + } + + /** + * This wraps a normal reader to make it possible for a client to detect accurate + * character offsets (> buffersize) in a large file, even if the underlying stream is buffered. + * + * This implementation is tightly coupled (semantically) with the internals of JsonReader. It provides + * just enough information, together with internal private fields of JsonReader, to compute Rascal-required + * offsets. We get only the character offset in the file, at the start of each streamed buffer contents. + * That should be just enough information to recompute the actual offset of every Json element, using the + * current position in the buffer. + */ + public static class OriginTrackingReader extends FilterReader { + private int offset = 0; + private int limit = 0; + + protected OriginTrackingReader(Reader in) { + super(in); + } + + @Override + public int read(char[] cbuf, int off, int len) throws IOException { + // we've read until here before we were reset to 0. + offset += limit; + + // get the new limit + limit = in.read(cbuf, off, len); + + // and return it. + return limit; + } + + public int getLimitOffset() { + return offset; + } + } } From fcac23c2120d073fd53626f7270fa53b35cd5eda Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Wed, 11 Feb 2026 13:25:02 +0100 Subject: [PATCH 02/23] cleaning up a bit --- .../lang/json/internal/JsonValueReader.java | 20 +------------------ 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index dc7f8f727f3..e44dc549fbe 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -97,7 +97,6 @@ private final class ExpectedTypeDispatcher implements ITypeVisitor Date: Wed, 11 Feb 2026 14:07:03 +0100 Subject: [PATCH 03/23] this works. but now a better way to detect a read --- src/org/rascalmpl/library/lang/json/IO.java | 49 +++++++++++++------ .../lang/json/internal/JsonValueReader.java | 4 -- 2 files changed, 34 insertions(+), 19 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/IO.java b/src/org/rascalmpl/library/lang/json/IO.java index b1ea8a9bad0..b83b54a61e5 100644 --- a/src/org/rascalmpl/library/lang/json/IO.java +++ b/src/org/rascalmpl/library/lang/json/IO.java @@ -63,22 +63,41 @@ public IValue readJSON(IValue type, ISourceLocation loc, IString dateTimeFormat, parsers = null; } - try (JsonReader in = new JsonReader(URIResolverRegistry.getInstance().getCharacterReader(loc))) { - in.setLenient(lenient.getValue()); - return new JsonValueReader(values, store, monitor, loc) - .setCalendarFormat(dateTimeFormat.getValue()) - .setParsers(parsers) - .setNulls(unreify(nulls)) - .setExplicitConstructorNames(explicitConstructorNames.getValue()) - .setExplicitDataTypes(explicitDataTypes.getValue()) - .setTrackOrigins(trackOrigins.getValue()) - .read(in, start); - } - catch (IOException e) { - throw RuntimeExceptionFactory.io(e); + if (trackOrigins.getValue()) { + try (Reader in = URIResolverRegistry.getInstance().getCharacterReader(loc)) { + return new JsonValueReader(values, store, monitor, loc) + .setCalendarFormat(dateTimeFormat.getValue()) + .setParsers(parsers) + .setNulls(unreify(nulls)) + .setExplicitConstructorNames(explicitConstructorNames.getValue()) + .setExplicitDataTypes(explicitDataTypes.getValue()) + .setTrackOrigins(trackOrigins.getValue()) + .read(in, start); + } + catch (IOException e) { + throw RuntimeExceptionFactory.io(e); + } + catch (NullPointerException e) { + throw RuntimeExceptionFactory.io("NPE in error handling code"); + } } - catch (NullPointerException e) { - throw RuntimeExceptionFactory.io("NPE in error handling code"); + else { + try (JsonReader in = new JsonReader(URIResolverRegistry.getInstance().getCharacterReader(loc))) { + in.setLenient(lenient.getValue()); + return new JsonValueReader(values, store, monitor, loc) + .setCalendarFormat(dateTimeFormat.getValue()) + .setParsers(parsers) + .setNulls(unreify(nulls)) + .setExplicitConstructorNames(explicitConstructorNames.getValue()) + .setExplicitDataTypes(explicitDataTypes.getValue()) + .read(in, start); + } + catch (IOException e) { + throw RuntimeExceptionFactory.io(e); + } + catch (NullPointerException e) { + throw RuntimeExceptionFactory.io("NPE in error handling code"); + } } } diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index e44dc549fbe..a90fea924b3 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -1027,10 +1027,6 @@ public JsonValueReader setNulls(Map nulls) { public JsonValueReader setTrackOrigins(boolean trackOrigins) { this.trackOrigins = trackOrigins; - if (trackOrigins) { - monitor.warning("The origin tracking feature of the JSON parser is temporarily disabled.", src); - this.trackOrigins = false; - } return this; } From 6b0804ee52c7e8ccbd0a968a227a025d2b201082 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Wed, 11 Feb 2026 14:15:26 +0100 Subject: [PATCH 04/23] knowing when to start from the new tracker position is important --- .../lang/json/internal/JsonValueReader.java | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index a90fea924b3..f91bd8b2add 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -96,6 +96,7 @@ private final class ExpectedTypeDispatcher implements ITypeVisitor Date: Thu, 12 Feb 2026 09:44:05 +0100 Subject: [PATCH 05/23] cleaning up parameters and error reporting --- .../exceptions/RuntimeExceptionFactory.java | 8 ++ src/org/rascalmpl/library/lang/json/IO.java | 87 +++++++------------ src/org/rascalmpl/library/lang/json/IO.rsc | 9 +- .../lang/json/internal/JsonValueReader.java | 52 ++++++++--- 4 files changed, 88 insertions(+), 68 deletions(-) diff --git a/src/org/rascalmpl/exceptions/RuntimeExceptionFactory.java b/src/org/rascalmpl/exceptions/RuntimeExceptionFactory.java index 335e46da2e6..0d150a558f2 100644 --- a/src/org/rascalmpl/exceptions/RuntimeExceptionFactory.java +++ b/src/org/rascalmpl/exceptions/RuntimeExceptionFactory.java @@ -98,6 +98,7 @@ public class RuntimeExceptionFactory { // NotImplemented public static final Type ParseError = TF.constructor(TS, Exception, "ParseError", TF.sourceLocationType(), "location"); + public static final Type NoOffsetParseError = TF.constructor(TS, Exception, "NoOffsetParseError", TF.sourceLocationType(), "location", TF.integerType(), "line", TF.integerType(), "column"); public static final Type PathNotFound = TF.constructor(TS,Exception,"PathNotFound",TF.sourceLocationType(), "location"); @@ -684,6 +685,11 @@ public static Throw jsonParseError(ISourceLocation loc, String cause, String pat .asWithKeywordParameters().setParameter("path", VF.string(path))); } + public static Throw jsonParseError(ISourceLocation file, int line, int col, String cause, String path) { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'jsonParseError'"); + } + public static Throw parseError(ISourceLocation loc, AbstractAST ast, StackTrace trace) { return new Throw(VF.constructor(ParseError, loc), ast != null ? ast.getLocation() : null, trace); @@ -793,4 +799,6 @@ public static Throw parseErrorRecovery(IValue trigger, ISourceLocation loc) { public static Throw parseErrorRecoveryNoSuchField(String name, ISourceLocation loc) { return new Throw(VF.constructor(ParseErrorRecovery, VF.constructor(NoSuchField, VF.string(name)), loc)); } + + } diff --git a/src/org/rascalmpl/library/lang/json/IO.java b/src/org/rascalmpl/library/lang/json/IO.java index b83b54a61e5..c9f98152ff6 100644 --- a/src/org/rascalmpl/library/lang/json/IO.java +++ b/src/org/rascalmpl/library/lang/json/IO.java @@ -10,7 +10,6 @@ *******************************************************************************/ package org.rascalmpl.library.lang.json; -import java.io.FilterReader; import java.io.IOException; import java.io.OutputStreamWriter; import java.io.Reader; @@ -40,7 +39,6 @@ import io.usethesource.vallang.type.Type; import io.usethesource.vallang.type.TypeStore; -import com.google.gson.stream.JsonReader; import com.google.gson.stream.JsonWriter; public class IO { @@ -53,82 +51,55 @@ public IO(IRascalValueFactory values, IRascalMonitor monitor) { this.monitor = monitor; } - public IValue readJSON(IValue type, ISourceLocation loc, IString dateTimeFormat, IBool lenient, IBool trackOrigins, - IFunction parsers, IMap nulls, IBool explicitConstructorNames, IBool explicitDataTypes) { + private IValue doReadJSON(Reader in, + IValue type, ISourceLocation loc, IString dateTimeFormat, IBool lenient, IBool trackOrigins, + IFunction parsers, IMap nulls, IBool explicitConstructorNames, IBool explicitDataTypes) throws IOException { + TypeStore store = new TypeStore(); Type start = new TypeReifier(values).valueToType((IConstructor) type, store); - + if (parsers.getType() instanceof ReifiedType && parsers.getType().getTypeParameters().getFieldType(0).isTop()) { // ignore the default parser parsers = null; } - if (trackOrigins.getValue()) { - try (Reader in = URIResolverRegistry.getInstance().getCharacterReader(loc)) { - return new JsonValueReader(values, store, monitor, loc) - .setCalendarFormat(dateTimeFormat.getValue()) - .setParsers(parsers) - .setNulls(unreify(nulls)) - .setExplicitConstructorNames(explicitConstructorNames.getValue()) - .setExplicitDataTypes(explicitDataTypes.getValue()) - .setTrackOrigins(trackOrigins.getValue()) - .read(in, start); - } - catch (IOException e) { - throw RuntimeExceptionFactory.io(e); - } - catch (NullPointerException e) { - throw RuntimeExceptionFactory.io("NPE in error handling code"); - } - } - else { - try (JsonReader in = new JsonReader(URIResolverRegistry.getInstance().getCharacterReader(loc))) { - in.setLenient(lenient.getValue()); - return new JsonValueReader(values, store, monitor, loc) + try { + return new JsonValueReader(values, store, monitor, loc) .setCalendarFormat(dateTimeFormat.getValue()) - .setParsers(parsers) + .setLenient(lenient.getValue()) + .setParsers(parsers) .setNulls(unreify(nulls)) .setExplicitConstructorNames(explicitConstructorNames.getValue()) .setExplicitDataTypes(explicitDataTypes.getValue()) + .setTrackOrigins(trackOrigins.getValue()) .read(in, start); - } - catch (IOException e) { - throw RuntimeExceptionFactory.io(e); - } - catch (NullPointerException e) { - throw RuntimeExceptionFactory.io("NPE in error handling code"); - } + } + catch (NullPointerException e) { + throw RuntimeExceptionFactory.io("NPE in error handling code"); } } + + public IValue readJSON( + IValue type, ISourceLocation loc, IString dateTimeFormat, IBool lenient, IBool trackOrigins, + IFunction parsers, IMap nulls, IBool explicitConstructorNames, IBool explicitDataTypes) { - private Map unreify(IMap nulls) { - var tr = new TypeReifier(values); - return nulls.stream().map(t -> (ITuple) t) - .collect(Collectors.toMap(t -> tr.valueToType((IConstructor) t.get(0)), t -> t.get(1))); + try (Reader in = URIResolverRegistry.getInstance().getCharacterReader(loc)) { + return doReadJSON(in, type, loc, dateTimeFormat, lenient, trackOrigins, parsers, nulls, explicitConstructorNames, explicitDataTypes); + } + catch (IOException e) { + throw RuntimeExceptionFactory.io(e); + } } public IValue parseJSON(IValue type, IString src, IString dateTimeFormat, IBool lenient, IBool trackOrigins, IFunction parsers, IMap nulls, IBool explicitConstructorNames, IBool explicitDataTypes) { - TypeStore store = new TypeStore(); - Type start = new TypeReifier(values).valueToType((IConstructor) type, store); - - try (JsonReader in = new JsonReader(new StringReader(src.getValue()))) { - in.setLenient(lenient.getValue()); - return new JsonValueReader(values, store, monitor,null) - .setCalendarFormat(dateTimeFormat.getValue()) - .setParsers(parsers) - .setNulls(unreify(nulls)) - .setTrackOrigins(trackOrigins.getValue()) - .setExplicitConstructorNames(explicitConstructorNames.getValue()) - .setExplicitDataTypes(explicitDataTypes.getValue()) - .read(in, start); + + try (Reader in = new StringReader(src.getValue())) { + return doReadJSON(in, type, null, dateTimeFormat, lenient, trackOrigins, parsers, nulls, explicitConstructorNames, explicitDataTypes); } catch (IOException e) { throw RuntimeExceptionFactory.io(e); } - catch (NullPointerException e) { - throw RuntimeExceptionFactory.io("NPE"); - } } public void writeJSON(ISourceLocation loc, IValue value, IBool unpackedLocations, IString dateTimeFormat, @@ -184,5 +155,9 @@ public IString asJSON(IValue value, IBool unpackedLocations, IString dateTimeFor } } - + private Map unreify(IMap nulls) { + var tr = new TypeReifier(values); + return nulls.stream().map(t -> (ITuple) t) + .collect(Collectors.toMap(t -> tr.valueToType((IConstructor) t.get(0)), t -> t.get(1))); + } } diff --git a/src/org/rascalmpl/library/lang/json/IO.rsc b/src/org/rascalmpl/library/lang/json/IO.rsc index 8c71c94ed72..2198c8e28d6 100644 --- a/src/org/rascalmpl/library/lang/json/IO.rsc +++ b/src/org/rascalmpl/library/lang/json/IO.rsc @@ -74,7 +74,14 @@ import Exception; * `cause` is a factual diagnosis of what was expected at that position, versus what was found. * `path` is a path query string into the JSON value from the root down to the leaf where the error was detected. } -data RuntimeException(str cause="", str path=""); +@benefits{ +* ((NoOffsetParseError)) is for when accurate offset tracking is turned off. Typically this is _on_ +even if `trackOrigins=false`, when we call the json parsers from Rascal. +} +data RuntimeException(str cause="", str path="") + = ParseError(loc location) + | NoOffsetParseError(loc location, int line, int column) + ; private str DEFAULT_DATETIME_FORMAT = "yyyy-MM-dd\'T\'HH:mm:ssZ"; diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index f91bd8b2add..65829723cf6 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -65,6 +65,7 @@ import io.usethesource.vallang.type.TypeStore; import com.google.gson.JsonParseException; +import com.google.gson.Strictness; import com.google.gson.stream.JsonReader; import com.google.gson.stream.JsonToken; import com.google.gson.stream.MalformedJsonException; @@ -82,14 +83,16 @@ public class JsonValueReader { private final IRascalMonitor monitor; private ISourceLocation src; private boolean trackOrigins = false; - private boolean disableTracking = false; + private boolean stopTracking = false; private VarHandle posHandler; private VarHandle lineHandler; private VarHandle lineStartHandler; private boolean explicitConstructorNames; private boolean explicitDataTypes; + private boolean lenient; private IFunction parsers; private Map nulls = Collections.emptyMap(); + private final class ExpectedTypeDispatcher implements ITypeVisitor { private final JsonReader in; @@ -98,18 +101,27 @@ private final class ExpectedTypeDispatcher implements ITypeVisitor Date: Thu, 12 Feb 2026 09:44:59 +0100 Subject: [PATCH 06/23] fixed broken header --- src/org/rascalmpl/library/lang/json/IO.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/IO.java b/src/org/rascalmpl/library/lang/json/IO.java index c9f98152ff6..e5b91d89fda 100644 --- a/src/org/rascalmpl/library/lang/json/IO.java +++ b/src/org/rascalmpl/library/lang/json/IO.java @@ -5,8 +5,10 @@ * * Contributors: * - * * Jurgen J. Vinju - Jurgen.Vinju@cwi.nl - CWI * Mark Hills - Mark.Hills@cwi.nl (CWI) * Arnold - * Lankamp - Arnold.Lankamp@cwi.nl * Bert Lisser - Bert.Lisser@cwi.nl + * * Jurgen J. Vinju - Jurgen.Vinju@cwi.nl - CWI + * * Mark Hills - Mark.Hills@cwi.nl (CWI) + * * Arnold - Lankamp - Arnold.Lankamp@cwi.nl + * * Bert Lisser - Bert.Lisser@cwi.nl *******************************************************************************/ package org.rascalmpl.library.lang.json; @@ -54,7 +56,7 @@ public IO(IRascalValueFactory values, IRascalMonitor monitor) { private IValue doReadJSON(Reader in, IValue type, ISourceLocation loc, IString dateTimeFormat, IBool lenient, IBool trackOrigins, IFunction parsers, IMap nulls, IBool explicitConstructorNames, IBool explicitDataTypes) throws IOException { - + TypeStore store = new TypeStore(); Type start = new TypeReifier(values).valueToType((IConstructor) type, store); From db3001ccafa730200f2810a94f4aafbc027cc412 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 12 Feb 2026 10:13:26 +0100 Subject: [PATCH 07/23] testing.json is now longer to no longer need the carriage returns to trigger #2633 --- .../lang/json/internal/JsonValueReader.java | 3 +- .../tests/library/lang/json/longcomment.json | 8 ++ .../tests/library/lang/json/longstring.json | 6 ++ .../tests/library/lang/json/testing.json | 83 ++++++++++--------- 4 files changed, 58 insertions(+), 42 deletions(-) create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/longcomment.json create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/longstring.json diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index 65829723cf6..b310a0f35bb 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -752,7 +752,8 @@ else if (!explicitDataTypes && "_type".equals(label)) { in.endObject(); int endPos = getPos(); - assert endPos >= startPos : "as assumpion on the internals of gson"; + assert endPos >= startPos : "offset tracking messed up while stopTracking is " + stopTracking + " and trackOrigins is " + trackOrigins; + int endLine = getLine(); int endCol = getCol(); diff --git a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/longcomment.json b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/longcomment.json new file mode 100644 index 00000000000..001e102dca2 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/longcomment.json @@ -0,0 +1,8 @@ +// 66XU0Cylu7Sur4Iiae/KOiEr7HnJOWgaHqwPTYusrEsX1Imws/Qxpc9EZISLCq+G60OVedHbctmJF1KY2Vx962khn5xcKZ7IpdavqI1Xi5FVz2PB4nNb2SD4zwxFK1Fz6NhS2B80vftdwtik9ya3CGgMV5d8dllRivihC+hhdY4sLAjdUXiFGSgEqhPIZYbRtFHi4kWNBJd0BJHx1gWEuimd6WDbBS0FyVsRqjqLutmKjy6pOiF0s5XHYYHkm+LjFN+eTT1xV6AKvuqslRt8q7cz2Cy2HRGTmfSunlLCei1Q/Bff/jtGEG+sYsfRZy7FQwa1vNxAGXOGUqOaYrQOrg68Y/TbVnQ2Wq/KFjQIxEEZJWq5z5MKRGZfV+EfQ2gLSjE+owPr+y/AAH4QjSehItW6qYGo0k8Vt7ptD95xNg4K/b2ebXOuJh4vOpOdIXUpRuwZfndVacYT61ozLOGzlukdqofQF5zaE1XuegssUUJNPxvT6gpVF1d3P0MoIVSJpCi4boSgM0xpAYYnC+Trl9B+zz74HIvS3oePc+Bm2CrL3Ap2rC9BreJKRljLwzJdWw/Snnb5ssZJtKI1S4EqelMEOg0vxoUQDYiGGysRHxZBsvTU0sHzKEe85m7ovR3iDnd5mnAhx58BS/YVXemd3zefcisnyAnjreW1g+fiDXG66le90C0lGvBPEg15xRPt298AjGulnWylJ3Q4pKZ367x41L134GvAGtnihuYMnPoSuWo2bakvHHDP2iexVobSK64QzlfXFLHydqNyJ/kSPztvsX5QV1kWIIHiqxGZEttOpbP4OvZDP82/5kvnI4hNv92ZcCdu1eKfgCNrd+cSYBeRXvz2W7KH+Ts2FbZDrwHNQWhvkfgBEpDvk+YwQTZFvMwenj4VA6VmYZvw5pzOiS2t3kdOJz60CZsgCqKaF7slaJV/SeG6zfi5rDkioQJZyB97Tjmed8S8etMexEjrq7q1S2Wkn7uAdDY/cEjhwuH1+YvNlECS6naT+oFu3thk66XU0Cylu7Sur4Iiae/KOiEr7HnJOWgaHqwPTYusrEsX1Imws/Qxpc9EZISLCq+G60OVedHbctmJF1KY2Vx962khn5xcKZ7IpdavqI1Xi5FVz2PB4nNb2SD4zwxFK1Fz6NhS2B80vftdwtik9ya3CGgMV5d8dllRivihC+hhdY4sLAjdUXiFGSgEqhPIZYbRtFHi4kWNBJd0BJHx1gWEuimd6WDbBS0FyVsRqjqLutmKjy6pOiF0s5XHYYHkm+LjFN+eTT1xV6AKvuqslRt8q7cz2Cy2HRGTmfSunlLCei1Q/Bff/jtGEG+sYsfRZy7FQwa1vNxAGXOGUqOaYrQOrg68Y/TbVnQ2Wq/KFjQIxEEZJWq5z5MKRGZfV+EfQ2gLSjE+owPr+y/AAH4QjSehItW6qYGo0k8Vt7ptD95xNg4K/b2ebXOuJh4vOpOdIXUpRuwZfndVacYT61ozLOGzlukdqofQF5zaE1XuegssUUJNPxvT6gpVF1d3P0MoIVSJpCi4boSgM0xpAYYnC+Trl9B+zz74HIvS3oePc+Bm2CrL3Ap2rC9BreJKRljLwzJdWw/Snnb5ssZJtKI1S4EqelMEOg0vxoUQDYiGGysRHxZBsvTU0sHzKEe85m7ovR3iDnd5mnAhx58BS/YVXemd3zefcisnyAnjreW1g+fiDXG66le90C0lGvBPEg15xRPt298AjGulnWylJ3Q4pKZ367x41L134GvAGtnihuYMnPoSuWo2bakvHHDP2iexVobSK64QzlfXFLHydqNyJ/kSPztvsX5QV1kWIIHiqxGZEttOpbP4OvZDP82/5kvnI4hNv92ZcCdu1eKfgCNrd+cSYBeRXvz2W7KH+Ts2FbZDrwHNQWhvkfgBEpDvk+YwQTZFvMwenj4VA6VmYZvw5pzOiS2t3kdOJz60CZsgCqKaF7slaJV/SeG6zfi5rDkioQJZyB97Tjmed8S8etMexEjrq7q1S2Wkn7uAdDY/cEjhwuH1+YvNlECS6naT+oFu3thk +{ + "name": "this object has a very long comment of >1024 characters", + + "nested" : { + "type": "" + } +} \ No newline at end of file diff --git a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/longstring.json b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/longstring.json new file mode 100644 index 00000000000..a7830c82ab8 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/longstring.json @@ -0,0 +1,6 @@ +{ + "name": "this object has a very long string of 1024 characters", + "nested" : { + "type": "66XU0Cylu7Sur4Iiae/KOiEr7HnJOWgaHqwPTYusrEsX1Imws/Qxpc9EZISLCq+G60OVedHbctmJF1KY2Vx962khn5xcKZ7IpdavqI1Xi5FVz2PB4nNb2SD4zwxFK1Fz6NhS2B80vftdwtik9ya3CGgMV5d8dllRivihC+hhdY4sLAjdUXiFGSgEqhPIZYbRtFHi4kWNBJd0BJHx1gWEuimd6WDbBS0FyVsRqjqLutmKjy6pOiF0s5XHYYHkm+LjFN+eTT1xV6AKvuqslRt8q7cz2Cy2HRGTmfSunlLCei1Q/Bff/jtGEG+sYsfRZy7FQwa1vNxAGXOGUqOaYrQOrg68Y/TbVnQ2Wq/KFjQIxEEZJWq5z5MKRGZfV+EfQ2gLSjE+owPr+y/AAH4QjSehItW6qYGo0k8Vt7ptD95xNg4K/b2ebXOuJh4vOpOdIXUpRuwZfndVacYT61ozLOGzlukdqofQF5zaE1XuegssUUJNPxvT6gpVF1d3P0MoIVSJpCi4boSgM0xpAYYnC+Trl9B+zz74HIvS3oePc+Bm2CrL3Ap2rC9BreJKRljLwzJdWw/Snnb5ssZJtKI1S4EqelMEOg0vxoUQDYiGGysRHxZBsvTU0sHzKEe85m7ovR3iDnd5mnAhx58BS/YVXemd3zefcisnyAnjreW1g+fiDXG66le90C0lGvBPEg15xRPt298AjGulnWylJ3Q4pKZ367x41L134GvAGtnihuYMnPoSuWo2bakvHHDP2iexVobSK64QzlfXFLHydqNyJ/kSPztvsX5QV1kWIIHiqxGZEttOpbP4OvZDP82/5kvnI4hNv92ZcCdu1eKfgCNrd+cSYBeRXvz2W7KH+Ts2FbZDrwHNQWhvkfgBEpDvk+YwQTZFvMwenj4VA6VmYZvw5pzOiS2t3kdOJz60CZsgCqKaF7slaJV/SeG6zfi5rDkioQJZyB97Tjmed8S8etMexEjrq7q1S2Wkn7uAdDY/cEjhwuH1+YvNlECS6naT+oFu3thk66XU0Cylu7Sur4Iiae/KOiEr7HnJOWgaHqwPTYusrEsX1Imws/Qxpc9EZISLCq+G60OVedHbctmJF1KY2Vx962khn5xcKZ7IpdavqI1Xi5FVz2PB4nNb2SD4zwxFK1Fz6NhS2B80vftdwtik9ya3CGgMV5d8dllRivihC+hhdY4sLAjdUXiFGSgEqhPIZYbRtFHi4kWNBJd0BJHx1gWEuimd6WDbBS0FyVsRqjqLutmKjy6pOiF0s5XHYYHkm+LjFN+eTT1xV6AKvuqslRt8q7cz2Cy2HRGTmfSunlLCei1Q/Bff/jtGEG+sYsfRZy7FQwa1vNxAGXOGUqOaYrQOrg68Y/TbVnQ2Wq/KFjQIxEEZJWq5z5MKRGZfV+EfQ2gLSjE+owPr+y/AAH4QjSehItW6qYGo0k8Vt7ptD95xNg4K/b2ebXOuJh4vOpOdIXUpRuwZfndVacYT61ozLOGzlukdqofQF5zaE1XuegssUUJNPxvT6gpVF1d3P0MoIVSJpCi4boSgM0xpAYYnC+Trl9B+zz74HIvS3oePc+Bm2CrL3Ap2rC9BreJKRljLwzJdWw/Snnb5ssZJtKI1S4EqelMEOg0vxoUQDYiGGysRHxZBsvTU0sHzKEe85m7ovR3iDnd5mnAhx58BS/YVXemd3zefcisnyAnjreW1g+fiDXG66le90C0lGvBPEg15xRPt298AjGulnWylJ3Q4pKZ367x41L134GvAGtnihuYMnPoSuWo2bakvHHDP2iexVobSK64QzlfXFLHydqNyJ/kSPztvsX5QV1kWIIHiqxGZEttOpbP4OvZDP82/5kvnI4hNv92ZcCdu1eKfgCNrd+cSYBeRXvz2W7KH+Ts2FbZDrwHNQWhvkfgBEpDvk+YwQTZFvMwenj4VA6VmYZvw5pzOiS2t3kdOJz60CZsgCqKaF7slaJV/SeG6zfi5rDkioQJZyB97Tjmed8S8etMexEjrq7q1S2Wkn7uAdDY/cEjhwuH1+YvNlECS6naT+oFu3thk" + } +} \ No newline at end of file diff --git a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/testing.json b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/testing.json index 4211534ee3b..bb778c26d15 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/testing.json +++ b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/testing.json @@ -1,41 +1,42 @@ -{ - "name": "TESTING Adding one more character to this data causes the test to fail", - "type": "type_abcd", - "nested_abc": [ - { - "type": "line", - "property__1": { "y": { "value": 2, "absolute": false } }, - "property__2": {} - }, - { - "type": "line", - "property__1": { "y": { "value": 2, "absolute": false } }, - "property__2": {} - }, - { - "type": "line", - "property__1": { "y": { "value": 2, "absolute": false } }, - "property__2": {} - }, - { - "type": "line", - "property__1": { "y": { "value": 2, "absolute": false } }, - "property__2": {} - }, - { - "type": "line", - "property__1": { "y": { "value": 2, "absolute": false } }, - "property__2": {} - }, - { - "type": "line", - "property__1": { "y": { "value": 2, "absolute": false } }, - "property__2": {} - }, - { - "type": "line", - "property__1": { "y": { "value": 2, "absolute": false } }, - "property__2": {} - } - ] -} +// This comment pushes the length of the file beyong the 1024 limit. +{ + "name": "TESTING Adding one more character to this data caused the test to fail", + "type": "type_abcd", + "nested_abc": [ + { + "type": "line", + "property__1": { "y": { "value": 2, "absolute": false } }, + "property__2": {} + }, + { + "type": "line", + "property__1": { "y": { "value": 2, "absolute": false } }, + "property__2": {} + }, + { + "type": "line", + "property__1": { "y": { "value": 2, "absolute": false } }, + "property__2": {} + }, + { + "type": "line", + "property__1": { "y": { "value": 2, "absolute": false } }, + "property__2": {} + }, + { + "type": "line", + "property__1": { "y": { "value": 2, "absolute": false } }, + "property__2": {} + }, + { + "type": "line", + "property__1": { "y": { "value": 2, "absolute": false } }, + "property__2": {} + }, + { + "type": "line", + "property__1": { "y": { "value": 2, "absolute": false } }, + "property__2": {} + } + ] +} From a191a8258b85ccc9e3f9278b6535ad792f06f34f Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 12 Feb 2026 10:47:16 +0100 Subject: [PATCH 08/23] testing and working on asserts --- .../library/lang/json/internal/JsonValueReader.java | 7 ++++++- .../lang/rascal/tests/library/lang/json/JSONIOTests.rsc | 7 +++---- .../lang/rascal/tests/library/lang/json/testing.json | 2 +- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index b310a0f35bb..b2451ba5fbf 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -482,6 +482,8 @@ public IValue visitBool(Type type) throws IOException { * `internalPos < lastPos` will not have had the opportunity to evaluate to `true`. */ private int getPos() { + assert !(!stopTracking && posHandler == null) : "if we don't have the posHandler stopTracking should be true"; + if (stopTracking) { return 0; } @@ -986,6 +988,10 @@ public JsonValueReader(IValueFactory vf, TypeStore store, IRascalMonitor monitor this.posHandler = privateLookup.findVarHandle(JsonReader.class, "pos", int.class); this.lineHandler = privateLookup.findVarHandle(JsonReader.class, "lineNumber", int.class); this.lineStartHandler = privateLookup.findVarHandle(JsonReader.class, "lineStart", int.class); + + if (posHandler == null || lineHandler == null || lineStartHandler == null) { + stopTracking = true; + } } catch (NoSuchFieldException | SecurityException | IllegalAccessException e) { // we disable the origin tracking if we can not get to the fields @@ -1093,7 +1099,6 @@ public JsonValueReader setLenient(boolean value) { * @throws IOException when either a parse error or a validation error occurs */ public IValue read(JsonReader in, Type expected) throws IOException { - assert !trackOrigins : "use the other read method to track Json origins"; in.setStrictness(lenient ? Strictness.LENIENT : Strictness.LEGACY_STRICT); var dispatch = new ExpectedTypeDispatcher(in); diff --git a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc index 4ea773c85a8..0d23ce9f86c 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc @@ -106,10 +106,10 @@ bool originTest(loc example) { return true; } -@ignore{awaiting fix of 2633} test bool originTracking() { - return originTest(|std:///lang/rascal/tests/library/lang/json/glossary.json|) - && originTest(|std:///lang/rascal/tests/library/lang/json/testing.json|); + files = [ l | loc l <- |std:///lang/rascal/tests/library/lang/json|.ls, l.extension == "json"]; + + return (true | it && originTest(example) | example <- files, bprintln("testing origins of ")); } value numNormalizer(int i) = i % maxLong when abs(i) > maxLong; @@ -181,7 +181,6 @@ value toDefaultValue(real r) = r - round(r) == 0 : fitDouble(r); default value toDefaultValue(value x) = x; -@ignore{awaiting fix of 2633} test bool accurateParseErrors() { ex = readFile(|std:///lang/rascal/tests/library/lang/json/glossary.json|); broken = ex[..size(ex)/2] + ex[size(ex)/2+10..]; diff --git a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/testing.json b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/testing.json index bb778c26d15..6d758bfccfa 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/testing.json +++ b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/testing.json @@ -1,7 +1,7 @@ // This comment pushes the length of the file beyong the 1024 limit. { "name": "TESTING Adding one more character to this data caused the test to fail", - "type": "type_abcd", + "type": "66XU0Cylu7Sur4Iiae/KOiEr7HnJOWgaHqwPTYusrEsX1Imws/Qxpc9EZISLCq+G60OVedHbctmJF1KY2Vx962khn5xcKZ7IpdavqI1Xi5FVz2PB4nNb2SD4zwxFK1Fz6NhS2B80vftdwtik9ya3CGgMV5d8dllRivihC+hhdY4sLAjdUXiFGSgEqhPIZYbRtFHi4kWNBJd0BJHx1gWEuimd6WDbBS0FyVsRqjqLutmKjy6pOiF0s5XHYYHkm+LjFN+eTT1xV6AKvuqslRt8q7cz2Cy2HRGTmfSunlLCei1Q/Bff/jtGEG+sYsfRZy7FQwa1vNxAGXOGUqOaYrQOrg68Y/TbVnQ2Wq/KFjQIxEEZJWq5z5MKRGZfV+EfQ2gLSjE+owPr+y/AAH4QjSehItW6qYGo0k8Vt7ptD95xNg4K/b2ebXOuJh4vOpOdIXUpRuwZfndVacYT61ozLOGzlukdqofQF5zaE1XuegssUUJNPxvT6gpVF1d3P0MoIVSJpCi4boSgM0xpAYYnC+Trl9B+zz74HIvS3oePc+Bm2CrL3Ap2rC9BreJKRljLwzJdWw/Snnb5ssZJtKI1S4EqelMEOg0vxoUQDYiGGysRHxZBsvTU0sHzKEe85m7ovR3iDnd5mnAhx58BS/YVXemd3zefcisnyAnjreW1g+fiDXG66le90C0lGvBPEg15xRPt298AjGulnWylJ3Q4pKZ367x41L134GvAGtnihuYMnPoSuWo2bakvHHDP2iexVobSK64QzlfXFLHydqNyJ/kSPztvsX5QV1kWIIHiqxGZEttOpbP4OvZDP82/5kvnI4hNv92ZcCdu1eKfgCNrd+cSYBeRXvz2W7KH+Ts2FbZDrwHNQWhvkfgBEpDvk+YwQTZFvMwenj4VA6VmYZvw5pzOiS2t3kdOJz60CZsgCqKaF7slaJV/SeG6zfi5rDkioQJZyB97Tjmed8S8etMexEjrq7q1S2Wkn7uAdDY/cEjhwuH1+YvNlECS6naT+oFu3thk", "nested_abc": [ { "type": "line", From 974101335ac4a40c0fe7b46a1d31d2cea8e8718e Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 12 Feb 2026 12:12:23 +0100 Subject: [PATCH 09/23] make sure tracking is really of when a JsonReader is passed in, to avoid NPEs and other crashes --- .../lang/json/internal/JsonValueReader.java | 34 ++++++++++--------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index b2451ba5fbf..d75a04ef092 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -981,24 +981,23 @@ public JsonValueReader(IValueFactory vf, TypeStore store, IRascalMonitor monitor setCalendarFormat("yyyy-MM-dd'T'HH:mm:ssZ"); - if (src != null) { - try { - var lookup = MethodHandles.lookup(); - var privateLookup = MethodHandles.privateLookupIn(JsonReader.class, lookup); - this.posHandler = privateLookup.findVarHandle(JsonReader.class, "pos", int.class); - this.lineHandler = privateLookup.findVarHandle(JsonReader.class, "lineNumber", int.class); - this.lineStartHandler = privateLookup.findVarHandle(JsonReader.class, "lineStart", int.class); - - if (posHandler == null || lineHandler == null || lineStartHandler == null) { - stopTracking = true; - } - } - catch (NoSuchFieldException | SecurityException | IllegalAccessException e) { - // we disable the origin tracking if we can not get to the fields + + try { + var lookup = MethodHandles.lookup(); + var privateLookup = MethodHandles.privateLookupIn(JsonReader.class, lookup); + this.posHandler = privateLookup.findVarHandle(JsonReader.class, "pos", int.class); + this.lineHandler = privateLookup.findVarHandle(JsonReader.class, "lineNumber", int.class); + this.lineStartHandler = privateLookup.findVarHandle(JsonReader.class, "lineStart", int.class); + + if (posHandler == null) { stopTracking = true; - monitor.warning("Unable to retrieve origin information due to: " + e.getMessage(), src); } } + catch (NoSuchFieldException | SecurityException | IllegalAccessException e) { + // we disable the origin tracking if we can not get to the fields + stopTracking = true; + monitor.warning("Unable to retrieve origin information due to: " + e.getMessage(), src); + } } public JsonValueReader(IValueFactory vf, IRascalMonitor monitor, ISourceLocation src) { @@ -1100,7 +1099,10 @@ public JsonValueReader setLenient(boolean value) { */ public IValue read(JsonReader in, Type expected) throws IOException { in.setStrictness(lenient ? Strictness.LENIENT : Strictness.LEGACY_STRICT); - + + // we can't track accurately because we don't have a handle to the raw buffer under `in` + this.stopTracking = true; + var dispatch = new ExpectedTypeDispatcher(in); try { From 1a328240685a00a143188b0e2d22822ec450898e Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 12 Feb 2026 12:18:03 +0100 Subject: [PATCH 10/23] forgot to implement the degenerate parse error --- src/org/rascalmpl/exceptions/RuntimeExceptionFactory.java | 8 +++++--- .../library/lang/json/internal/JsonValueReader.java | 1 - 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/org/rascalmpl/exceptions/RuntimeExceptionFactory.java b/src/org/rascalmpl/exceptions/RuntimeExceptionFactory.java index 0d150a558f2..fbb88d3653f 100644 --- a/src/org/rascalmpl/exceptions/RuntimeExceptionFactory.java +++ b/src/org/rascalmpl/exceptions/RuntimeExceptionFactory.java @@ -98,6 +98,8 @@ public class RuntimeExceptionFactory { // NotImplemented public static final Type ParseError = TF.constructor(TS, Exception, "ParseError", TF.sourceLocationType(), "location"); + + // this comes from lang::json::IO public static final Type NoOffsetParseError = TF.constructor(TS, Exception, "NoOffsetParseError", TF.sourceLocationType(), "location", TF.integerType(), "line", TF.integerType(), "column"); public static final Type PathNotFound = TF.constructor(TS,Exception,"PathNotFound",TF.sourceLocationType(), "location"); @@ -686,11 +688,11 @@ public static Throw jsonParseError(ISourceLocation loc, String cause, String pat } public static Throw jsonParseError(ISourceLocation file, int line, int col, String cause, String path) { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'jsonParseError'"); + return new Throw(VF.constructor(NoOffsetParseError, file, VF.integer(line), VF.integer(col)) + .asWithKeywordParameters().setParameter("reason", VF.string(cause)) + .asWithKeywordParameters().setParameter("path", VF.string(path))); } - public static Throw parseError(ISourceLocation loc, AbstractAST ast, StackTrace trace) { return new Throw(VF.constructor(ParseError, loc), ast != null ? ast.getLocation() : null, trace); } diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index d75a04ef092..0d1fb636d38 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -1102,7 +1102,6 @@ public IValue read(JsonReader in, Type expected) throws IOException { // we can't track accurately because we don't have a handle to the raw buffer under `in` this.stopTracking = true; - var dispatch = new ExpectedTypeDispatcher(in); try { From f0356b89648602fc9845eec4e25f648fc3fb72ff Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 12 Feb 2026 12:22:14 +0100 Subject: [PATCH 11/23] cleaning up --- .../lang/json/internal/JsonValueReader.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index 0d1fb636d38..636ee7eec3e 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -78,15 +78,17 @@ public class JsonValueReader { private static final TypeFactory TF = TypeFactory.getInstance(); private final TypeStore store; - private final IValueFactory vf; - private ThreadLocal format; + private final IValueFactory vf; private final IRascalMonitor monitor; - private ISourceLocation src; - private boolean trackOrigins = false; - private boolean stopTracking = false; + private final ISourceLocation src; private VarHandle posHandler; private VarHandle lineHandler; private VarHandle lineStartHandler; + + /* options */ + private ThreadLocal format; + private boolean trackOrigins = false; + private boolean stopTracking = false; private boolean explicitConstructorNames; private boolean explicitDataTypes; private boolean lenient; @@ -101,7 +103,6 @@ private final class ExpectedTypeDispatcher implements ITypeVisitor Date: Thu, 12 Feb 2026 12:25:58 +0100 Subject: [PATCH 12/23] added some comments --- .../rascalmpl/library/lang/json/internal/JsonValueReader.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index 636ee7eec3e..326617bc7ff 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -492,7 +492,10 @@ private int getPos() { var trackerCount = tracker.getReadCount(); if (readCount < trackerCount) { + // the tracker indicates that `read` has happened and so the buffer has been rewound. readCount = trackerCount; + // we learn from the tracker how far the offset is until the new first character in the buffer + // and we add the current offset since that reset to found our new current offset. offset = tracker.getLimitOffset() + internalPos; } else { @@ -504,6 +507,7 @@ private int getPos() { lastPos = internalPos; try { + // never go below 0. might happen with a parse error at the first character. return Math.max(0, offset - 1); } catch (IllegalArgumentException | SecurityException e) { From 44c9e7714f3dc1802bb878c5ac9f0587e34e13e3 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 12 Feb 2026 12:31:08 +0100 Subject: [PATCH 13/23] reset testing.json --- .../library/lang/rascal/tests/library/lang/json/testing.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/testing.json b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/testing.json index 6d758bfccfa..bb778c26d15 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/testing.json +++ b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/testing.json @@ -1,7 +1,7 @@ // This comment pushes the length of the file beyong the 1024 limit. { "name": "TESTING Adding one more character to this data caused the test to fail", - "type": "66XU0Cylu7Sur4Iiae/KOiEr7HnJOWgaHqwPTYusrEsX1Imws/Qxpc9EZISLCq+G60OVedHbctmJF1KY2Vx962khn5xcKZ7IpdavqI1Xi5FVz2PB4nNb2SD4zwxFK1Fz6NhS2B80vftdwtik9ya3CGgMV5d8dllRivihC+hhdY4sLAjdUXiFGSgEqhPIZYbRtFHi4kWNBJd0BJHx1gWEuimd6WDbBS0FyVsRqjqLutmKjy6pOiF0s5XHYYHkm+LjFN+eTT1xV6AKvuqslRt8q7cz2Cy2HRGTmfSunlLCei1Q/Bff/jtGEG+sYsfRZy7FQwa1vNxAGXOGUqOaYrQOrg68Y/TbVnQ2Wq/KFjQIxEEZJWq5z5MKRGZfV+EfQ2gLSjE+owPr+y/AAH4QjSehItW6qYGo0k8Vt7ptD95xNg4K/b2ebXOuJh4vOpOdIXUpRuwZfndVacYT61ozLOGzlukdqofQF5zaE1XuegssUUJNPxvT6gpVF1d3P0MoIVSJpCi4boSgM0xpAYYnC+Trl9B+zz74HIvS3oePc+Bm2CrL3Ap2rC9BreJKRljLwzJdWw/Snnb5ssZJtKI1S4EqelMEOg0vxoUQDYiGGysRHxZBsvTU0sHzKEe85m7ovR3iDnd5mnAhx58BS/YVXemd3zefcisnyAnjreW1g+fiDXG66le90C0lGvBPEg15xRPt298AjGulnWylJ3Q4pKZ367x41L134GvAGtnihuYMnPoSuWo2bakvHHDP2iexVobSK64QzlfXFLHydqNyJ/kSPztvsX5QV1kWIIHiqxGZEttOpbP4OvZDP82/5kvnI4hNv92ZcCdu1eKfgCNrd+cSYBeRXvz2W7KH+Ts2FbZDrwHNQWhvkfgBEpDvk+YwQTZFvMwenj4VA6VmYZvw5pzOiS2t3kdOJz60CZsgCqKaF7slaJV/SeG6zfi5rDkioQJZyB97Tjmed8S8etMexEjrq7q1S2Wkn7uAdDY/cEjhwuH1+YvNlECS6naT+oFu3thk", + "type": "type_abcd", "nested_abc": [ { "type": "line", From d8dec4d1218012d6a4622adcb33a4bc5e739f8d4 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 12 Feb 2026 13:18:57 +0100 Subject: [PATCH 14/23] commented read better and added fix by @DavyLandman for when off!=0 --- .../lang/json/internal/JsonValueReader.java | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index 326617bc7ff..cf327696418 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -1160,8 +1160,11 @@ public IValue read(Reader in, Type expected) throws IOException { * current position in the buffer. */ public static class OriginTrackingReader extends FilterReader { + // offset is always pointing at the point in the file where JsonReader.pos == 0 private int offset = 0; + // limit is always pointing to the amount of no-junk characters in the underlying buffer below buffer.length private int limit = 0; + // readCount increases by 1 with every call to `read` private int readCount = 0; protected OriginTrackingReader(Reader in) { @@ -1170,16 +1173,19 @@ protected OriginTrackingReader(Reader in) { @Override public int read(char[] cbuf, int off, int len) throws IOException { - // we've read until here before we were reset to 0. - offset += limit; + // we've read until here before we were reset to starting point `off`. + // the offset of the new current 0-based buffer starts here: + offset += limit - off; - // get the new limit - limit = in.read(cbuf, off, len); + var charsRead = in.read(cbuf, off, len); + + // get the new limit (to where we've filled the buffer) + limit = charsRead + off; readCount++; - // and return it. - return limit; + // and return the number of characters read. + return charsRead; } public int getLimitOffset() { From 1f6c294a50a080a8cad358fe43bcb4538ceaa540 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Fri, 13 Feb 2026 10:17:15 +0100 Subject: [PATCH 15/23] added tests by @davylandman and fixed off-by-one with length of origin positions on objects --- .../lang/json/internal/JsonValueReader.java | 6 ++-- .../tests/library/lang/json/JSONIOTests.rsc | 35 +++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index cf327696418..a8870880cb9 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -772,7 +772,7 @@ else if (!explicitDataTypes && "_type".equals(label)) { if (trackOrigins && !stopTracking) { kwParams.put(kwParams.containsKey("src") ? "rascal-src" : "src", - vf.sourceLocation(src, startPos, endPos - startPos + 1, startLine, endLine, startCol, endCol + 1)); + vf.sourceLocation(src, startPos, endPos - startPos, startLine, endLine, startCol, endCol + 1)); } return vf.constructor(cons, args, kwParams); @@ -858,7 +858,7 @@ public IValue visitNode(Type type) throws IOException { if (trackOrigins && !stopTracking) { kws.put(kws.containsKey("src") ? "rascal-src" : "src", - vf.sourceLocation(src, startPos, endPos - startPos + 1, startLine, endLine, startCol, endCol + 1)); + vf.sourceLocation(src, startPos, endPos - startPos, startLine, endLine, startCol, endCol + 1)); } IValue[] argArray = args.entrySet().stream().sorted((e, f) -> e.getKey().compareTo(f.getKey())) @@ -920,6 +920,7 @@ public IValue visitList(Type type) throws IOException { } IListWriter w = vf.listWriter(); + getPos(); in.beginArray(); while (in.hasNext()) { // here we pass label from the higher context @@ -932,6 +933,7 @@ public IValue visitList(Type type) throws IOException { } in.endArray(); + getPos(); return w.done(); } diff --git a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc index 0d23ce9f86c..f9b3dfbd0dd 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc @@ -311,5 +311,40 @@ test bool explicitDataTypes() { // here we can't be sure to get z() back, but we will get some Enum assert data4(e=Enum _) := parseJSON(#DATA4, json, explicitDataTypes=false); + return true; +} + +data X(loc src=|unkown:///|) = v1(int x=0, str s = ""); + +// BUG: off-by-one in current implementation +loc origin(X x) = x.src[length = x.src.length -1]; + +test bool jsonVerifyOriginCorrect() { + ref = v1(x=123456789); + refExpected = asJSON(ref); + t1 = [v1(s="hoi"), ref]; + writeJSON(|memory:///test.json|, t1); + v = readJSON(#list[X],|memory:///test.json|, trackOrigins=true); + return refExpected == readFile(origin(v[1])); +} + +test bool jsonVerifyOriginCorrectAcrossBufferBoundaries() { + ref = v1(x=123456789); + refExpected = asJSON(ref); + for (sSize <- [900..1024]) { + println(sSize); + t1 = [v1(s="a<}>"), ref]; + writeJSON(|memory:///test.json|, t1); + + // this throws exceptions and asserts if there are bugs with the + // origin tracker. In particular it triggers #2633 + v = readJSON(#list[X],|memory:///test.json|, trackOrigins=true); + + // checking the last element + if (refExpected != readFile(origin(v[1]))) { + println("Failed for : "); + return false; + } + } return true; } \ No newline at end of file From 37077dcb1051c2b5c49ecfeadd5a87c1b34b1acb Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Fri, 13 Feb 2026 12:01:20 +0100 Subject: [PATCH 16/23] fixing off-by-ones --- .../lang/json/internal/JsonValueReader.java | 12 ++++---- .../tests/library/lang/json/JSONIOTests.rsc | 9 ++---- .../tests/library/lang/json/glossary.json | 28 +------------------ 3 files changed, 11 insertions(+), 38 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index a8870880cb9..19397d1d28b 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -756,13 +756,14 @@ else if (!explicitDataTypes && "_type".equals(label)) { } } - in.endObject(); int endPos = getPos(); - assert endPos >= startPos : "offset tracking messed up while stopTracking is " + stopTracking + " and trackOrigins is " + trackOrigins; + assert endPos > startPos : "offset tracking messed up while stopTracking is " + stopTracking + " and trackOrigins is " + trackOrigins; int endLine = getLine(); int endCol = getCol(); + in.endObject(); + for (int i = 0; i < args.length; i++) { if (args[i] == null) { throw parseErrorHere( @@ -772,7 +773,7 @@ else if (!explicitDataTypes && "_type".equals(label)) { if (trackOrigins && !stopTracking) { kwParams.put(kwParams.containsKey("src") ? "rascal-src" : "src", - vf.sourceLocation(src, startPos, endPos - startPos, startLine, endLine, startCol, endCol + 1)); + vf.sourceLocation(src, startPos, endPos - startPos + 1, startLine, endLine, startCol, endCol + 1)); } return vf.constructor(cons, args, kwParams); @@ -851,14 +852,15 @@ public IValue visitNode(Type type) throws IOException { } } - in.endObject(); int endPos = getPos(); int endLine = getLine(); int endCol = getCol(); + in.endObject(); + if (trackOrigins && !stopTracking) { kws.put(kws.containsKey("src") ? "rascal-src" : "src", - vf.sourceLocation(src, startPos, endPos - startPos, startLine, endLine, startCol, endCol + 1)); + vf.sourceLocation(src, startPos, endPos - startPos + 1, startLine, endLine, startCol, endCol + 1)); } IValue[] argArray = args.entrySet().stream().sorted((e, f) -> e.getKey().compareTo(f.getKey())) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc index f9b3dfbd0dd..ecf73e70eb2 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc @@ -316,16 +316,13 @@ test bool explicitDataTypes() { data X(loc src=|unkown:///|) = v1(int x=0, str s = ""); -// BUG: off-by-one in current implementation -loc origin(X x) = x.src[length = x.src.length -1]; - test bool jsonVerifyOriginCorrect() { ref = v1(x=123456789); refExpected = asJSON(ref); t1 = [v1(s="hoi"), ref]; writeJSON(|memory:///test.json|, t1); v = readJSON(#list[X],|memory:///test.json|, trackOrigins=true); - return refExpected == readFile(origin(v[1])); + return refExpected == readFile(v[1].src); } test bool jsonVerifyOriginCorrectAcrossBufferBoundaries() { @@ -341,8 +338,8 @@ test bool jsonVerifyOriginCorrectAcrossBufferBoundaries() { v = readJSON(#list[X],|memory:///test.json|, trackOrigins=true); // checking the last element - if (refExpected != readFile(origin(v[1]))) { - println("Failed for : "); + if (refExpected != readFile(v[1].src)) { + println("Failed for : "); return false; } } diff --git a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/glossary.json b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/glossary.json index 804e8e54c89..9e26dfeeb6e 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/glossary.json +++ b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/glossary.json @@ -1,27 +1 @@ -{ - "glossary": { - "title": "example glossary", - "line" : 2, - "GlossDiv": { - "line" : 5, - "title": "S", - "GlossList": { - "line" : 8, - "GlossEntry": { - "line" : 10, - "ID": "SGML", - "SortAs": "SGML", - "GlossTerm": "Standard Generalized Markup Language", - "Acronym": "SGML", - "Abbrev": "ISO 8879:1986", - "GlossDef": { - "line" : 17, - "para": "A meta-markup language, used to create markup languages such as DocBook.", - "GlossSeeAlso": ["GML", "XML"] - }, - "GlossSee": "markup" - } - } - } - } -} \ No newline at end of file +{} \ No newline at end of file From b5e1a04d9db327b6e1be89716cdbd2a82baf071b Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Fri, 13 Feb 2026 12:02:03 +0100 Subject: [PATCH 17/23] reset --- .../tests/library/lang/json/glossary.json | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/glossary.json b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/glossary.json index 9e26dfeeb6e..804e8e54c89 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/glossary.json +++ b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/glossary.json @@ -1 +1,27 @@ -{} \ No newline at end of file +{ + "glossary": { + "title": "example glossary", + "line" : 2, + "GlossDiv": { + "line" : 5, + "title": "S", + "GlossList": { + "line" : 8, + "GlossEntry": { + "line" : 10, + "ID": "SGML", + "SortAs": "SGML", + "GlossTerm": "Standard Generalized Markup Language", + "Acronym": "SGML", + "Abbrev": "ISO 8879:1986", + "GlossDef": { + "line" : 17, + "para": "A meta-markup language, used to create markup languages such as DocBook.", + "GlossSeeAlso": ["GML", "XML"] + }, + "GlossSee": "markup" + } + } + } + } +} \ No newline at end of file From 4b8ea994fea4b9b177f8536c5ad8a8ef5d0ba5fd Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Tue, 17 Feb 2026 10:48:01 +0100 Subject: [PATCH 18/23] refactored and simplified use of the wrapped reader. Also improved tests to include line/column accuracy testing. Fixed some off-by-one errors --- .../lang/json/internal/JsonValueReader.java | 140 +++++++++++------- .../tests/library/lang/json/Issue2633.rsc | 116 +++++++-------- .../tests/library/lang/json/JSONIOTests.rsc | 47 ++++-- 3 files changed, 174 insertions(+), 129 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index 19397d1d28b..1040800c9cc 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -40,6 +40,8 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; + +import org.apache.commons.lang3.ObjectUtils.Null; import org.rascalmpl.debug.IRascalMonitor; import org.rascalmpl.exceptions.RuntimeExceptionFactory; import org.rascalmpl.exceptions.Throw; @@ -100,10 +102,6 @@ private final class ExpectedTypeDispatcher implements ITypeVisitor l = new ArrayList<>(); - lastPos = getPos(); + in.beginArray(); if (type.hasFieldNames()) { @@ -481,34 +479,15 @@ public IValue visitBool(Type type) throws IOException { * because it has not been called in between from JsonValueReader to JsonReader and the condition * `internalPos < lastPos` will not have had the opportunity to evaluate to `true`. */ - private int getPos() { - assert !(!stopTracking && posHandler == null) : "if we don't have the posHandler stopTracking should be true"; - + private int getOffset() { if (stopTracking) { return 0; } - var internalPos = (int) posHandler.get(in); - var trackerCount = tracker.getReadCount(); - - if (readCount < trackerCount) { - // the tracker indicates that `read` has happened and so the buffer has been rewound. - readCount = trackerCount; - // we learn from the tracker how far the offset is until the new first character in the buffer - // and we add the current offset since that reset to found our new current offset. - offset = tracker.getLimitOffset() + internalPos; - } - else { - // the offset advances by the number of parsed characters - offset += (internalPos - lastPos); - } - - // save the previous state - lastPos = internalPos; - try { - // never go below 0. might happen with a parse error at the first character. - return Math.max(0, offset - 1); + assert posHandler != null; + var internalPos = (int) posHandler.get(in); + return tracker.getOffsetAtBufferStart() + internalPos; } catch (IllegalArgumentException | SecurityException e) { // we stop trying to track positions if it fails so hard, @@ -556,8 +535,8 @@ private int getCol() { } protected Throw parseErrorHere(String cause) { - var location = src == null ? URIUtil.rootLocation("unknown") : src; - int offset = getPos(); + var location = getRootLoc(); + int offset = getOffset(); int line = getLine(); int col = getCol(); @@ -637,11 +616,12 @@ private IValue visitStringAsAbstractData(Type type) throws IOException { private IValue visitObjectAsAbstractData(Type type) throws IOException { Set alternatives = null; - in.beginObject(); - int startPos = getPos(); + int startPos = getOffset() - 1; int startLine = getLine(); - int startCol = getCol(); + int startCol = getCol() - 1; + in.beginObject(); + // use explicit information in the JSON to select and filter constructors from the TypeStore // we expect always to have the field _constructor before _type. if (explicitConstructorNames || explicitDataTypes) { @@ -756,11 +736,11 @@ else if (!explicitDataTypes && "_type".equals(label)) { } } - int endPos = getPos(); + int endPos = getOffset() - 1; assert endPos > startPos : "offset tracking messed up while stopTracking is " + stopTracking + " and trackOrigins is " + trackOrigins; int endLine = getLine(); - int endCol = getCol(); + int endCol = getCol() - 1; in.endObject(); @@ -773,12 +753,21 @@ else if (!explicitDataTypes && "_type".equals(label)) { if (trackOrigins && !stopTracking) { kwParams.put(kwParams.containsKey("src") ? "rascal-src" : "src", - vf.sourceLocation(src, startPos, endPos - startPos + 1, startLine, endLine, startCol, endCol + 1)); + vf.sourceLocation(getRootLoc(), startPos, endPos - startPos + 1, startLine, endLine, startCol, endCol + 1)); } return vf.constructor(cons, args, kwParams); } + private ISourceLocation getRootLoc() { + if (src == null) { + return URIUtil.rootLocation("unknown"); + } + else { + return src; + } + } + @Override public IValue visitAbstractData(Type type) throws IOException { if (UtilMaybe.isMaybe(type)) { @@ -815,9 +804,9 @@ public IValue visitNode(Type type) throws IOException { return inferNullValue(nulls, type); } - int startPos = getPos(); + int startPos = getOffset() - 1; int startLine = getLine(); - int startCol = getCol(); + int startCol = getCol() - 1; in.beginObject(); @@ -852,15 +841,15 @@ public IValue visitNode(Type type) throws IOException { } } - int endPos = getPos(); + int endPos = getOffset() - 1; int endLine = getLine(); - int endCol = getCol(); + int endCol = getCol() - 1; in.endObject(); if (trackOrigins && !stopTracking) { kws.put(kws.containsKey("src") ? "rascal-src" : "src", - vf.sourceLocation(src, startPos, endPos - startPos + 1, startLine, endLine, startCol, endCol + 1)); + vf.sourceLocation(getRootLoc(), startPos, endPos - startPos + 1, startLine, endLine, startCol, endCol + 1)); } IValue[] argArray = args.entrySet().stream().sorted((e, f) -> e.getKey().compareTo(f.getKey())) @@ -922,7 +911,7 @@ public IValue visitList(Type type) throws IOException { } IListWriter w = vf.listWriter(); - getPos(); + getOffset(); in.beginArray(); while (in.hasNext()) { // here we pass label from the higher context @@ -935,7 +924,7 @@ public IValue visitList(Type type) throws IOException { } in.endArray(); - getPos(); + getOffset(); return w.done(); } @@ -1143,11 +1132,14 @@ public IValue read(Reader in, Type expected) throws IOException { try { var result = expected.accept(dispatch); if (result == null) { - throw new JsonParseException( - "null occurred outside an optionality context and without a registered representation."); + throw new JsonParseException("null occurred outside an optionality context and without a registered representation."); } return result; - } catch (EOFException | JsonParseException | NumberFormatException | MalformedJsonException | IllegalStateException | NullPointerException e) { + } + catch (NullPointerException e) { + throw dispatch.parseErrorHere("Unexpected internal NullPointerException"); + } + catch (EOFException | JsonParseException | NumberFormatException | MalformedJsonException | IllegalStateException e) { throw dispatch.parseErrorHere(e.getMessage()); } } @@ -1161,7 +1153,7 @@ public IValue read(Reader in, Type expected) throws IOException { * just enough information, together with internal private fields of JsonReader, to compute Rascal-required * offsets. We get only the character offset in the file, at the start of each streamed buffer contents. * That should be just enough information to recompute the actual offset of every Json element, using the - * current position in the buffer. + * current position in the buffer (the private field `pos` of JsonReader). */ public static class OriginTrackingReader extends FilterReader { // offset is always pointing at the point in the file where JsonReader.pos == 0 @@ -1175,24 +1167,62 @@ protected OriginTrackingReader(Reader in) { super(in); } + /* This private method from JsonReader must be mirrored by `read` + private boolean fillBuffer(int minimum) throws IOException { + char[] buffer = this.buffer; + lineStart -= pos; + if (limit != pos) { + limit -= pos; + System.arraycopy(buffer, pos, buffer, 0, limit); + } else { + limit = 0; + } + + pos = 0; + int total; + while ((total = in.read(buffer, limit, buffer.length - limit)) != -1) { + limit += total; + + // if this is the first read, consume an optional byte order mark (BOM) if it exists + if (lineNumber == 0 && lineStart == 0 && limit > 0 && buffer[0] == '\ufeff') { + pos++; + lineStart++; + minimum++; + } + + if (limit >= minimum) { + return true; + } + } + return false; + } */ @Override public int read(char[] cbuf, int off, int len) throws IOException { - // we've read until here before we were reset to starting point `off`. - // the offset of the new current 0-based buffer starts here: + // Note that `fillBuffer.limit != fillBuffer.pos <==> reader.off != 0`. + // Moreover, `fillBuffer.limit == reader.off` at the start of this method. + + // we know take the previous limit and add it to the + // offset, to arrive at the new `pos=0` of `buffer[0]`, + // rewinding `off` characters which were reused from the previous buffer + // with System.arraycopy. offset += limit - off; + // make sure we are only a facade for the real reader. + // parameters are mapped one-to-one without mutations. var charsRead = in.read(cbuf, off, len); - // get the new limit (to where we've filled the buffer) - limit = charsRead + off; + // the next buffer[0] offset will be after this increment. + // Note that `fillBuffer.limit == read.limit` + limit = off + charsRead; + // to help decide if a read happened readCount++; - // and return the number of characters read. + // and return only the number of characters read. return charsRead; } - public int getLimitOffset() { + public int getOffsetAtBufferStart() { return offset; } diff --git a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/Issue2633.rsc b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/Issue2633.rsc index 3de22d49e20..09e94442a4b 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/Issue2633.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/Issue2633.rsc @@ -1,59 +1,59 @@ -module lang::rascal::tests::library::lang::json::Issue2633 - -import lang::json::IO; -import IO; - -data OuterData = outer(str name, str \type, list[Nested] nested_abc); -data Nested = nested(str \type, map[str, value] property__1, map[value, value] property__2); - -test bool failsOnCertainJSON() { - str input = "{ - ' \"name\": \"TESTING_ Adding one more character to this data causes the test to fail\", - ' \"type\": \"type_abcd\", - ' \"nested_abc\": [ - ' { - ' \"type\": \"line\", - ' \"property__1\": { \"y\": { \"value\": 2, \"absolute\": false } }, - ' \"property__2\": {} - ' }, - ' { - ' \"type\": \"line\", - ' \"property__1\": { \"y\": { \"value\": 2, \"absolute\": false } }, - ' \"property__2\": {} - ' }, - ' { - ' \"type\": \"line\", - ' \"property__1\": { \"y\": { \"value\": 2, \"absolute\": false } }, - ' \"property__2\": {} - ' }, - ' { - ' \"type\": \"line\", - ' \"property__1\": { \"y\": { \"value\": 2, \"absolute\": false } }, - ' \"property__2\": {} - ' }, - ' { - ' \"type\": \"line\", - ' \"property__1\": { \"y\": { \"value\": 2, \"absolute\": false } }, - ' \"property__2\": {} - ' }, - ' { - ' \"type\": \"line\", - ' \"property__1\": { \"y\": { \"value\": 2, \"absolute\": false } }, - ' \"property__2\": {} - ' }, - ' { - ' \"type\": \"line\", - ' \"property__1\": { \"y\": { \"value\": 2, \"absolute\": false } }, - ' \"property__2\": {} - ' } - ' ] - '} - '"; - - // Fails when the original data above is increased by even one character (e.g. the name, or changing a 2 to 20, or even adding a meaningless space anywhere) - parsedVFD = parseJSON(#OuterData, input); - println("Parsed VisualFormData (After)"); - iprintln(parsedVFD); - - return true; +module lang::rascal::tests::library::lang::json::Issue2633 + +import lang::json::IO; +import IO; + +data OuterData = outer(str name, str \type, list[Nested] nested_abc); +data Nested = nested(str \type, map[str, value] property__1, map[value, value] property__2); + +test bool failsOnCertainJSON() { + str input = "{ + ' \"name\": \"TESTING_ Adding one more character to this data causes the test to fail\", + ' \"type\": \"type_abcd\", + ' \"nested_abc\": [ + ' { + ' \"type\": \"line\", + ' \"property__1\": { \"y\": { \"value\": 2, \"absolute\": false } }, + ' \"property__2\": {} + ' }, + ' { + ' \"type\": \"line\", + ' \"property__1\": { \"y\": { \"value\": 2, \"absolute\": false } }, + ' \"property__2\": {} + ' }, + ' { + ' \"type\": \"line\", + ' \"property__1\": { \"y\": { \"value\": 2, \"absolute\": false } }, + ' \"property__2\": {} + ' }, + ' { + ' \"type\": \"line\", + ' \"property__1\": { \"y\": { \"value\": 2, \"absolute\": false } }, + ' \"property__2\": {} + ' }, + ' { + ' \"type\": \"line\", + ' \"property__1\": { \"y\": { \"value\": 2, \"absolute\": false } }, + ' \"property__2\": {} + ' }, + ' { + ' \"type\": \"line\", + ' \"property__1\": { \"y\": { \"value\": 2, \"absolute\": false } }, + ' \"property__2\": {} + ' }, + ' { + ' \"type\": \"line\", + ' \"property__1\": { \"y\": { \"value\": 2, \"absolute\": false } }, + ' \"property__2\": {} + ' } + ' ] + '} + '"; + + // Fails when the original data above is increased by even one character (e.g. the name, or changing a 2 to 20, or even adding a meaningless space anywhere) + parsedVFD = parseJSON(#OuterData, input, trackOrigins=true, lenient=true); + println("Parsed VisualFormData (After)"); + iprintln(parsedVFD); + + return true; } \ No newline at end of file diff --git a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc index ecf73e70eb2..9d6355bff2f 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc @@ -94,13 +94,15 @@ test bool json4(Enum e) = writeRead(#DATA4, data4(e=e)); bool originTest(loc example) { ex2 = readJSON(#node, example, trackOrigins=true); content = readFile(example); - + lines = split("\n", content); poss = [ | /node x := ex2, x.line?]; // every node has a .src field, otherwise this fails with an exception for ( <- poss) { assert content[p.offset] == "{"; // all nodes start with a { assert content[p.offset + p.length - 1] == "}"; // all nodes end with a } assert p.begin.line == line; + assert lines[p.begin.line - 1][p.begin.column] == "{"; + assert lines[p.end.line - 1][p.end.column - 1] == "}"; } return true; @@ -201,7 +203,7 @@ test bool accurateParseErrors() { return true; } -@ignore{until #2133 is fixed} +// @ignore{until #2133 is fixed} test bool regression1() = jsonRandom1(("a":12,[]:{})); data Cons = cons(str bla = "null"); @@ -322,26 +324,39 @@ test bool jsonVerifyOriginCorrect() { t1 = [v1(s="hoi"), ref]; writeJSON(|memory:///test.json|, t1); v = readJSON(#list[X],|memory:///test.json|, trackOrigins=true); + iprintln(refExpected); + iprintln(readFile(v[1].src)); return refExpected == readFile(v[1].src); } +test bool triggerIssue2633() { + return jsonVerifyOriginCorrectAcrossBufferBoundaries(1023); +} + test bool jsonVerifyOriginCorrectAcrossBufferBoundaries() { + /* twice just before and after the 1024 buffer size of JsonReader */ + for (int sSize <- [1000..1025] + [2000..2050]) { + jsonVerifyOriginCorrectAcrossBufferBoundaries(sSize); + } + return true; +} + +bool jsonVerifyOriginCorrectAcrossBufferBoundaries(int sSize) { ref = v1(x=123456789); refExpected = asJSON(ref); - for (sSize <- [900..1024]) { - println(sSize); - t1 = [v1(s="a<}>"), ref]; - writeJSON(|memory:///test.json|, t1); - - // this throws exceptions and asserts if there are bugs with the - // origin tracker. In particular it triggers #2633 - v = readJSON(#list[X],|memory:///test.json|, trackOrigins=true); - - // checking the last element - if (refExpected != readFile(v[1].src)) { - println("Failed for : "); - return false; - } + + t1 = [v1(s="a<}>"), ref]; + writeJSON(|memory:///test.json|, t1); + + //s this throws exceptions and asserts if there are bugs with the + // origin tracker. In particular it triggers #2633 + v = readJSON(#list[X],|memory:///test.json|, trackOrigins=true); + + // checking the last element + if (refExpected != readFile(v[1].src)) { + println("Failed for : != "); + return false; } + return true; } \ No newline at end of file From dd3f686820483daa324d13760220b53ed1b6df7a Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Tue, 17 Feb 2026 10:48:31 +0100 Subject: [PATCH 19/23] removed unused readCount --- .../library/lang/json/internal/JsonValueReader.java | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index 1040800c9cc..20c0283c349 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -1160,9 +1160,7 @@ public static class OriginTrackingReader extends FilterReader { private int offset = 0; // limit is always pointing to the amount of no-junk characters in the underlying buffer below buffer.length private int limit = 0; - // readCount increases by 1 with every call to `read` - private int readCount = 0; - + protected OriginTrackingReader(Reader in) { super(in); } @@ -1215,9 +1213,6 @@ public int read(char[] cbuf, int off, int len) throws IOException { // Note that `fillBuffer.limit == read.limit` limit = off + charsRead; - // to help decide if a read happened - readCount++; - // and return only the number of characters read. return charsRead; } @@ -1225,9 +1220,5 @@ public int read(char[] cbuf, int off, int len) throws IOException { public int getOffsetAtBufferStart() { return offset; } - - public int getReadCount() { - return readCount; - } } } From f4ca19cb14297799075ad843da6618446243566e Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Tue, 17 Feb 2026 10:55:07 +0100 Subject: [PATCH 20/23] ignored a test again --- .../library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc index 9d6355bff2f..22350468e55 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc @@ -203,7 +203,7 @@ test bool accurateParseErrors() { return true; } -// @ignore{until #2133 is fixed} +@ignore{until #2133 is fixed} test bool regression1() = jsonRandom1(("a":12,[]:{})); data Cons = cons(str bla = "null"); From 38de5d593dbbf5c61cea4bcdcfd86bdea61620e7 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Tue, 17 Feb 2026 12:30:09 +0100 Subject: [PATCH 21/23] removed test printlns --- .../lang/rascal/tests/library/lang/json/JSONIOTests.rsc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc index 22350468e55..65602a9e73b 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc @@ -111,7 +111,7 @@ bool originTest(loc example) { test bool originTracking() { files = [ l | loc l <- |std:///lang/rascal/tests/library/lang/json|.ls, l.extension == "json"]; - return (true | it && originTest(example) | example <- files, bprintln("testing origins of ")); + return (true | it && originTest(example) | loc example <- files); } value numNormalizer(int i) = i % maxLong when abs(i) > maxLong; @@ -324,8 +324,6 @@ test bool jsonVerifyOriginCorrect() { t1 = [v1(s="hoi"), ref]; writeJSON(|memory:///test.json|, t1); v = readJSON(#list[X],|memory:///test.json|, trackOrigins=true); - iprintln(refExpected); - iprintln(readFile(v[1].src)); return refExpected == readFile(v[1].src); } From e7a4679ed3d97a983c96a651eaf2f966629caf5e Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Wed, 18 Feb 2026 12:47:55 +0100 Subject: [PATCH 22/23] removed dead import --- .../rascalmpl/library/lang/json/internal/JsonValueReader.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index 20c0283c349..d9593b1102c 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -41,7 +41,6 @@ import java.util.Map.Entry; import java.util.Set; -import org.apache.commons.lang3.ObjectUtils.Null; import org.rascalmpl.debug.IRascalMonitor; import org.rascalmpl.exceptions.RuntimeExceptionFactory; import org.rascalmpl.exceptions.Throw; From 199423e915a51cbf82d398d688ffbe10cf42f050 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Wed, 18 Feb 2026 12:51:20 +0100 Subject: [PATCH 23/23] fixed comment by @davylandman --- .../rascalmpl/library/lang/json/internal/JsonValueReader.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index d9593b1102c..02e2af38579 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -107,8 +107,7 @@ private final class ExpectedTypeDispatcher implements ITypeVisitor