From 9573836853e2a91889972d44ccb9df2dd5ea9587 Mon Sep 17 00:00:00 2001 From: Gustavo Lopes Date: Fri, 3 Mar 2023 08:43:53 +0000 Subject: [PATCH] Multiple optimizations --- LICENSE-3rdparty.csv | 1 + build.gradle | 1 + dl_dbgsyms | 4 +- src/main/c/jni/io_sqreen_powerwaf_Additive.h | 4 +- src/main/c/jni/io_sqreen_powerwaf_Powerwaf.h | 4 +- src/main/c/powerwaf_jni.c | 57 +++-- .../java/io/sqreen/powerwaf/Additive.java | 25 +- .../sqreen/powerwaf/ByteBufferSerializer.java | 236 ++++++++++++------ .../sqreen/powerwaf/MapIterableWithSize.java | 7 + .../powerwaf/NativeStringAddressable.java | 7 + .../java/io/sqreen/powerwaf/Powerwaf.java | 2 +- .../io/sqreen/powerwaf/PowerwafContext.java | 16 +- .../io/sqreen/powerwaf/AdditiveTest.groovy | 24 +- .../io/sqreen/powerwaf/BasicTests.groovy | 39 +++ .../powerwaf/ByteBufferSerializerTests.groovy | 57 ++++- .../CharSequenceSerializationTests.groovy | 32 +++ .../powerwaf/InvalidInvocationTests.groovy | 3 +- .../io/sqreen/powerwaf/LimitsTests.groovy | 19 ++ 18 files changed, 432 insertions(+), 106 deletions(-) create mode 100644 src/main/java/io/sqreen/powerwaf/MapIterableWithSize.java create mode 100644 src/main/java/io/sqreen/powerwaf/NativeStringAddressable.java diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 90f47ad8..71d12519 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -1,6 +1,7 @@ Component,Origin,License,Copyright main,slf4j,mit,QOS.ch main,libddwaf,apache-2.0,DataDog +main,weak-lock-free,apache-2.0,Rafael Winterhalter tests,junit,epl-1.0,various tests,ant,apache-2.0,Apache Software Foundation tests,groovy,apache-2.0,various diff --git a/build.gradle b/build.gradle index 97ce7b3d..8b656620 100644 --- a/build.gradle +++ b/build.gradle @@ -143,6 +143,7 @@ configurations { def SLF4J_VERSION = '1.7.30' dependencies { implementation group: 'org.slf4j', name: 'slf4j-api', version: SLF4J_VERSION + implementation group: 'com.blogspot.mydailyjava', name: 'weak-lock-free', version: '0.17' testRuntimeOnly group: 'org.slf4j', name: 'slf4j-simple', version: SLF4J_VERSION testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: '2.2' diff --git a/dl_dbgsyms b/dl_dbgsyms index 61f32f41..eae93b10 100755 --- a/dl_dbgsyms +++ b/dl_dbgsyms @@ -39,8 +39,8 @@ function extract { function main { local readonly version=$1 dl_zip $version - extract $version linux_64_glibc/libsqreen_jni - extract $version linux_64/libddwaf + extract $version linux/x86_64/glibc/libsqreen_jni + extract $version linux/x86_64/libddwaf } main $1 diff --git a/src/main/c/jni/io_sqreen_powerwaf_Additive.h b/src/main/c/jni/io_sqreen_powerwaf_Additive.h index 78bc5580..3a949eb9 100644 --- a/src/main/c/jni/io_sqreen_powerwaf_Additive.h +++ b/src/main/c/jni/io_sqreen_powerwaf_Additive.h @@ -18,9 +18,9 @@ JNIEXPORT jlong JNICALL Java_io_sqreen_powerwaf_Additive_initAdditive /* * Class: io_sqreen_powerwaf_Additive * Method: runAdditive - * Signature: (Ljava/util/Map;Lio/sqreen/powerwaf/Powerwaf$Limits;Lio/sqreen/powerwaf/PowerwafMetrics;)Lio/sqreen/powerwaf/Powerwaf$ResultWithData; + * Signature: (Ljava/lang/Object;Lio/sqreen/powerwaf/Powerwaf$Limits;Lio/sqreen/powerwaf/PowerwafMetrics;)Lio/sqreen/powerwaf/Powerwaf$ResultWithData; */ -JNIEXPORT jobject JNICALL Java_io_sqreen_powerwaf_Additive_runAdditive__Ljava_util_Map_2Lio_sqreen_powerwaf_Powerwaf_00024Limits_2Lio_sqreen_powerwaf_PowerwafMetrics_2 +JNIEXPORT jobject JNICALL Java_io_sqreen_powerwaf_Additive_runAdditive__Ljava_lang_Object_2Lio_sqreen_powerwaf_Powerwaf_00024Limits_2Lio_sqreen_powerwaf_PowerwafMetrics_2 (JNIEnv *, jobject, jobject, jobject, jobject); /* diff --git a/src/main/c/jni/io_sqreen_powerwaf_Powerwaf.h b/src/main/c/jni/io_sqreen_powerwaf_Powerwaf.h index 98989665..37c319dc 100644 --- a/src/main/c/jni/io_sqreen_powerwaf_Powerwaf.h +++ b/src/main/c/jni/io_sqreen_powerwaf_Powerwaf.h @@ -42,9 +42,9 @@ JNIEXPORT jobject JNICALL Java_io_sqreen_powerwaf_Powerwaf_runRules__Lio_sqreen_ /* * Class: io_sqreen_powerwaf_Powerwaf * Method: runRules - * Signature: (Lio/sqreen/powerwaf/PowerwafHandle;Ljava/util/Map;Lio/sqreen/powerwaf/Powerwaf$Limits;Lio/sqreen/powerwaf/PowerwafMetrics;)Lio/sqreen/powerwaf/Powerwaf$ResultWithData; + * Signature: (Lio/sqreen/powerwaf/PowerwafHandle;Ljava/lang/Object;Lio/sqreen/powerwaf/Powerwaf$Limits;Lio/sqreen/powerwaf/PowerwafMetrics;)Lio/sqreen/powerwaf/Powerwaf$ResultWithData; */ -JNIEXPORT jobject JNICALL Java_io_sqreen_powerwaf_Powerwaf_runRules__Lio_sqreen_powerwaf_PowerwafHandle_2Ljava_util_Map_2Lio_sqreen_powerwaf_Powerwaf_00024Limits_2Lio_sqreen_powerwaf_PowerwafMetrics_2 +JNIEXPORT jobject JNICALL Java_io_sqreen_powerwaf_Powerwaf_runRules__Lio_sqreen_powerwaf_PowerwafHandle_2Ljava_lang_Object_2Lio_sqreen_powerwaf_Powerwaf_00024Limits_2Lio_sqreen_powerwaf_PowerwafMetrics_2 (JNIEnv *, jclass, jobject, jobject, jobject, jobject); /* diff --git a/src/main/c/powerwaf_jni.c b/src/main/c/powerwaf_jni.c index 70910656..5c9da806 100644 --- a/src/main/c/powerwaf_jni.c +++ b/src/main/c/powerwaf_jni.c @@ -142,6 +142,9 @@ jclass *number_cls = &number_longValue.class_glob; static struct j_method _boolean_booleanValue; static jclass *_boolean_cls = &_boolean_booleanValue.class_glob; +// MapIterableWithSize +static jclass _miws_cls; + static struct j_method _hashmap_init; static struct j_method _map_put; struct j_method map_entryset; @@ -596,10 +599,10 @@ static jobject _run_rule_common(bool is_byte_buffer, JNIEnv *env, jclass clazz, /* * Class: io_sqreen_powerwaf_Powerwaf * Method: runRules - * Signature: (Lio/sqreen/powerwaf/PowerwafHandle;Ljava/util/Map;Lio/sqreen/powerwaf/Powerwaf$Limits;)Lio/sqreen/powerwaf/Powerwaf$ResultWithData; + * Signature: (Lio/sqreen/powerwaf/PowerwafHandle;Ljava/util/Object;Lio/sqreen/powerwaf/Powerwaf$Limits;)Lio/sqreen/powerwaf/Powerwaf$ResultWithData; */ JNIEXPORT jobject JNICALL -Java_io_sqreen_powerwaf_Powerwaf_runRules__Lio_sqreen_powerwaf_PowerwafHandle_2Ljava_util_Map_2Lio_sqreen_powerwaf_Powerwaf_00024Limits_2Lio_sqreen_powerwaf_PowerwafMetrics_2( +Java_io_sqreen_powerwaf_Powerwaf_runRules__Lio_sqreen_powerwaf_PowerwafHandle_2Ljava_lang_Object_2Lio_sqreen_powerwaf_Powerwaf_00024Limits_2Lio_sqreen_powerwaf_PowerwafMetrics_2( JNIEnv *env, jclass clazz, jobject handle_obj, jobject parameters, jobject limits_obj, jobject metrics_obj) { @@ -811,10 +814,10 @@ static jobject _run_additive_common(JNIEnv *env, jobject this, * Class: io_sqreen_powerwaf_Additive * Method: runAdditive * Signature: - * (Ljava/util/Map;Lio/sqreen/powerwaf/Powerwaf$Limits;)Lio/sqreen/powerwaf/Powerwaf$ResultWithData; + * (Ljava/lang/Object;Lio/sqreen/powerwaf/Powerwaf$Limits;)Lio/sqreen/powerwaf/Powerwaf$ResultWithData; */ JNIEXPORT jobject JNICALL -Java_io_sqreen_powerwaf_Additive_runAdditive__Ljava_util_Map_2Lio_sqreen_powerwaf_Powerwaf_00024Limits_2Lio_sqreen_powerwaf_PowerwafMetrics_2( +Java_io_sqreen_powerwaf_Additive_runAdditive__Ljava_lang_Object_2Lio_sqreen_powerwaf_Powerwaf_00024Limits_2Lio_sqreen_powerwaf_PowerwafMetrics_2( JNIEnv *env, jobject this, jobject parameters, jobject limits_obj, jobject metrics_obj) { @@ -1128,17 +1131,18 @@ static bool _cache_single_class_weak(JNIEnv *env, static bool _cache_classes(JNIEnv *env) { - return _cache_single_class_weak(env, "java/lang/RuntimeException", - &jcls_rte) && - _cache_single_class_weak(env, "java/lang/IllegalArgumentException", - &jcls_iae) && - _cache_single_class_weak(env, "java/lang/CharSequence", - &charSequence_cls) && - _cache_single_class_weak(env, "java/nio/Buffer", - &buffer_cls) && - _cache_single_class_weak(env, "java/nio/CharBuffer", - &charBuffer_cls) && - _cache_single_class_weak(env, "java/lang/String", &string_cls); + return _cache_single_class_weak(env, "java/lang/RuntimeException", + &jcls_rte) && + _cache_single_class_weak(env, "java/lang/IllegalArgumentException", + &jcls_iae) && + _cache_single_class_weak(env, "java/lang/CharSequence", + &charSequence_cls) && + _cache_single_class_weak(env, "java/nio/Buffer", &buffer_cls) && + _cache_single_class_weak(env, "java/nio/CharBuffer", + &charBuffer_cls) && + _cache_single_class_weak(env, "java/lang/String", &string_cls) && + _cache_single_class_weak( + env, "io/sqreen/powerwaf/MapIterableWithSize", &_miws_cls); } static void _dispose_of_weak_classes(JNIEnv *env) @@ -1154,6 +1158,7 @@ static void _dispose_of_weak_classes(JNIEnv *env) DESTROY_CLASS_REF(charSequence_cls) DESTROY_CLASS_REF(buffer_cls) DESTROY_CLASS_REF(charBuffer_cls) + DESTROY_CLASS_REF(_miws_cls) // leave jcls_rte for last in OnUnload; we might still need it } @@ -1487,6 +1492,10 @@ static ddwaf_object _convert_checked_ex(JNIEnv *env, bool use_bools, JNI(DeleteLocalRef, clazz); } + // shared between two branches + jobject entry_set = NULL; + jobject entry_set_it; + if (JNI(IsSameObject, obj, NULL)) { // replace NULLs with empty maps. // DDWAF_OBJ_NULL is actually invalid; it can't be added to containers @@ -1526,6 +1535,18 @@ static ddwaf_object _convert_checked_ex(JNIEnv *env, bool use_bools, goto error; } } + } else if (JNI(IsInstanceOf, obj, _miws_cls)) { + ddwaf_object_map(&result); + if (rec_level >= lims->max_depth) { + JAVA_LOG(DDWAF_LOG_DEBUG, + "Leaving map empty because max depth of %d " + "has been reached", + lims->max_depth); + goto early_return; + } + JAVA_CALL(entry_set_it, iterable_iterator, obj); + goto iterator_map_entry; + } else if (JNI(IsInstanceOf, obj, *map_cls)) { ddwaf_object_map(&result); // can't fail if (rec_level >= lims->max_depth) { @@ -1536,10 +1557,10 @@ static ddwaf_object _convert_checked_ex(JNIEnv *env, bool use_bools, goto early_return; } - jobject entry_set, entry_set_it; JAVA_CALL(entry_set, map_entryset, obj); JAVA_CALL(entry_set_it, iterable_iterator, entry_set); +iterator_map_entry: while (JNI(CallBooleanMethod, entry_set_it, iterator_hasNext.meth_id)) { if (JNI(ExceptionCheck)) { goto error; @@ -1592,7 +1613,9 @@ static ddwaf_object _convert_checked_ex(JNIEnv *env, bool use_bools, } JNI(DeleteLocalRef, entry_set_it); - JNI(DeleteLocalRef, entry_set); + if (entry_set) { + JNI(DeleteLocalRef, entry_set); + } } else if (JNI(IsInstanceOf, obj, *iterable_cls)) { ddwaf_object_array(&result); diff --git a/src/main/java/io/sqreen/powerwaf/Additive.java b/src/main/java/io/sqreen/powerwaf/Additive.java index c60bf825..ae004a28 100644 --- a/src/main/java/io/sqreen/powerwaf/Additive.java +++ b/src/main/java/io/sqreen/powerwaf/Additive.java @@ -48,7 +48,7 @@ public final class Additive implements Closeable { private static native long initAdditive(PowerwafHandle handle); private native Powerwaf.ResultWithData runAdditive( - Map parameters, Powerwaf.Limits limits, PowerwafMetrics metrics) throws AbstractPowerwafException; + Object parameters, Powerwaf.Limits limits, PowerwafMetrics metrics) throws AbstractPowerwafException; private native Powerwaf.ResultWithData runAdditive( ByteBuffer firstPWArgsBuffer, Powerwaf.Limits limits, PowerwafMetrics metrics) throws AbstractPowerwafException; @@ -70,9 +70,30 @@ private native Powerwaf.ResultWithData runAdditive( * @return execution results * @throws AbstractPowerwafException rethrow from native code, timeout or param serialization failure */ - public Powerwaf.ResultWithData run(Map parameters, + public Powerwaf.ResultWithData run(Map parameters, Powerwaf.Limits limits, PowerwafMetrics metrics) throws AbstractPowerwafException { + return run((Object) parameters, limits, metrics); + } + + /** + * Push params to PowerWAF with given limits + * + * @param parameters data to push to PowerWAF + * @param limits request execution limits + * @param metrics a metrics collector, or null + * @return execution results + * @throws AbstractPowerwafException rethrow from native code, timeout or param serialization failure + */ + public Powerwaf.ResultWithData run(MapIterableWithSize parameters, + Powerwaf.Limits limits, + PowerwafMetrics metrics) throws AbstractPowerwafException { + return run((Object) parameters, limits, metrics); + } + + private Powerwaf.ResultWithData run(Object parameters, + Powerwaf.Limits limits, + PowerwafMetrics metrics) throws AbstractPowerwafException { if (limits == null) { throw new IllegalArgumentException("limits must be provided"); } diff --git a/src/main/java/io/sqreen/powerwaf/ByteBufferSerializer.java b/src/main/java/io/sqreen/powerwaf/ByteBufferSerializer.java index a31103fc..2b1a5c42 100644 --- a/src/main/java/io/sqreen/powerwaf/ByteBufferSerializer.java +++ b/src/main/java/io/sqreen/powerwaf/ByteBufferSerializer.java @@ -8,10 +8,9 @@ package io.sqreen.powerwaf; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - +import com.blogspot.mydailyjava.weaklockfree.WeakConcurrentMap; import java.io.Closeable; +import java.io.PrintWriter; import java.lang.reflect.Array; import java.lang.reflect.UndeclaredThrowableException; import java.nio.ByteBuffer; @@ -26,10 +25,13 @@ import java.util.Collection; import java.util.ConcurrentModificationException; import java.util.Deque; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentLinkedDeque; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class ByteBufferSerializer { private static final long NULLPTR = 0; @@ -45,7 +47,11 @@ public ByteBufferSerializer(Powerwaf.Limits limits) { this.limits = limits; } - public ArenaLease serialize(Map map) { + static void release() { + ArenaPool.INSTANCE.arenas.clear(); + } + + public ArenaLease serialize(Object map) { if (map == null) { throw new NullPointerException("map can't be null"); } @@ -65,17 +71,17 @@ public static ArenaLease getBlankLease() { return ArenaPool.INSTANCE.getLease(); } - private static ByteBuffer serializeMore(ArenaLease lease, Powerwaf.Limits limits, Map map) { + public static void debugDump(PrintWriter pw) { + ArenaPool.INSTANCE.debugDump(pw); + } + + private static ByteBuffer serializeMore(ArenaLease lease, Powerwaf.Limits limits, Object map) { Arena arena = lease.getArena(); // limits apply per-serialization run int[] remainingElements = new int[]{limits.maxElements}; // The address of this ByteBuffer will be accessed from native code via GetDirectBufferAddress - PWArgsArrayBuffer pwArgsArrayBuffer = arena.allocateGetAddressCompatiblePWArgsBuffer(1); - if (pwArgsArrayBuffer == null) { - throw new OutOfMemoryError(); - } - PWArgsBuffer initialValue = pwArgsArrayBuffer.get(0); + PWArgsBuffer initialValue = arena.getRoot(); doSerialize(arena, limits, initialValue, null, map, remainingElements, limits.maxDepth); return initialValue.buffer; @@ -84,12 +90,12 @@ private static ByteBuffer serializeMore(ArenaLease lease, Powerwaf.Limits limits } private static void doSerialize(Arena arena, Powerwaf.Limits limits, PWArgsBuffer pwargsSlot, - String parameterName, Object value, int[] remainingElements, + CharSequence parameterName, Object value, int[] remainingElements, int depthRemaining) { if (parameterName != null && parameterName.length() > limits.maxStringSize) { LOGGER.debug("Truncating parameter string from size {} to size {}", parameterName.length(), limits.maxStringSize); - parameterName = parameterName.substring(0, limits.maxStringSize); + parameterName = parameterName.subSequence(0, limits.maxStringSize); } remainingElements[0]--; @@ -131,6 +137,29 @@ private static void doSerialize(Arena arena, Powerwaf.Limits limits, PWArgsBuffe if (!pwargsSlot.writeLong(arena, parameterName, ((Number) value).longValue())) { throw new RuntimeException("Could not write number"); } + } else if (value instanceof MapIterableWithSize) { + MapIterableWithSize mpis = (MapIterableWithSize) value; + int size = Math.min(mpis.size(), remainingElements[0]); + PWArgsArrayBuffer pwArgsArrayBuffer = pwargsSlot.writeMap(arena, parameterName, size); + + Iterator> iterator = mpis.iterator(); + PWArgsBuffer pwArgsBuffer = new PWArgsBuffer(); + int i; + for (i = 0; iterator.hasNext() && i < size; i++) { + Map.Entry entry = iterator.next(); + PWArgsBuffer newSlot = pwArgsArrayBuffer.get(pwArgsBuffer, i); + Object key = entry.getKey(); + if (key == null) { + key = ""; + } else if (!(key instanceof CharSequence)) { + key = key.toString(); + } + doSerialize(arena, limits, newSlot, (CharSequence) key, entry.getValue(), + remainingElements, depthRemaining - 1); + } + if (i != size) { + throw new ConcurrentModificationException("i=" + i + ", size=" + size); + } } else if (value instanceof Collection) { int size = Math.min(((Collection) value).size(), remainingElements[0]); @@ -167,14 +196,17 @@ private static void doSerialize(Arena arena, Powerwaf.Limits limits, PWArgsBuffe } int i = 0; Iterator> iterator = ((Map) value).entrySet().iterator(); + PWArgsBuffer pwArgsBuffer = new PWArgsBuffer(); for (; iterator.hasNext() && i < size; i++) { Map.Entry entry = iterator.next(); - PWArgsBuffer newSlot = pwArgsArrayBuffer.get(i); + PWArgsBuffer newSlot = pwArgsArrayBuffer.get(pwArgsBuffer, i); Object key = entry.getKey(); if (key == null) { key = ""; + } else if (!(key instanceof CharSequence)) { + key = key.toString(); } - doSerialize(arena, limits, newSlot, key.toString(), entry.getValue(), + doSerialize(arena, limits, newSlot, (CharSequence) key, entry.getValue(), remainingElements, depthRemaining - 1); } if (i != size) { @@ -196,7 +228,7 @@ private static void doSerialize(Arena arena, Powerwaf.Limits limits, PWArgsBuffe private static void serializeIterable(Arena arena, Powerwaf.Limits limits, PWArgsBuffer pwArgsSlot, - String parameterName, + CharSequence parameterName, int[] remainingElements, int depthRemaining, Iterator iterator, @@ -207,9 +239,10 @@ private static void serializeIterable(Arena arena, } int i; + PWArgsBuffer pwArgsBuffer = new PWArgsBuffer(); for (i = 0; iterator.hasNext() && i < size; i++) { Object newObj = iterator.next(); - PWArgsBuffer newSlot = pwArgsArrayBuffer.get(i); + PWArgsBuffer newSlot = pwArgsArrayBuffer.get(pwArgsBuffer, i); doSerialize(arena, limits, newSlot, null, newObj, remainingElements, depthRemaining - 1); } if (i != size) { @@ -227,8 +260,9 @@ private static class Arena { .replaceWith(new byte[]{(byte) 0xEF, (byte) 0xBF, (byte) 0xBD}); List pwargsSegments = new ArrayList<>(); + PWArgsBuffer root; int curPWArgsSegment; - int idxOfFirstUsedPWArgsSegment = -1; + boolean gotRoot; List stringsSegments = new ArrayList<>(); int curStringsSegment; CharBuffer currentWrapper = null; @@ -243,6 +277,13 @@ public final CharsetEncoder getCharsetEncoder() { Arena() { pwargsSegments.add(new PWArgsSegment(PWARGS_MIN_SEGMENTS_SIZE)); stringsSegments.add(new StringsSegment(STRINGS_MIN_SEGMENTS_SIZE)); + root = new PWArgsBuffer(); + } + + PWArgsBuffer getRoot() { + pwargsSegments.get(0).allocateRoot(this.root); + gotRoot = true; + return this.root; } void reset() { @@ -256,14 +297,14 @@ void reset() { } curPWArgsSegment = 0; curStringsSegment = 0; - idxOfFirstUsedPWArgsSegment = -1; + gotRoot = false; } ByteBuffer getFirstUsedPWArgsBuffer() { - if (idxOfFirstUsedPWArgsSegment == -1) { + if (!gotRoot) { throw new IllegalStateException("No PWArgs written"); } - return pwargsSegments.get(idxOfFirstUsedPWArgsSegment).buffer; + return root.buffer; } static class WrittenString { @@ -276,7 +317,9 @@ static class WrittenString { } public void release() { - arena.cachedWS = this; + if (arena != null) { + arena.cachedWS = this; + } } public WrittenString update(long ptr, int utf8len) { @@ -286,12 +329,35 @@ public WrittenString update(long ptr, int utf8len) { } } + private static final WeakConcurrentMap CACHED_BB_ADDRESSES = + new WeakConcurrentMap.WithInlinedExpunction() { + @Override + protected Arena.WrittenString defaultValue(ByteBuffer bb) { + if (!bb.isDirect()) { + throw new IllegalArgumentException(); + } + int limit = bb.limit(); + if (limit == 0 || bb.get(limit - 1) != 0x00) { + throw new IllegalArgumentException(); + } + Arena.WrittenString writtenString = new Arena.WrittenString(null); + writtenString.ptr = getByteBufferAddress(bb); + writtenString.utf8len = limit - 1; + return writtenString; + } + }; + /** * @param s the string to serialize * @return the native pointer to the string and its size in bytes, * or null if the string is too large */ WrittenString writeStringUnlimited(CharSequence s) { + if (s instanceof NativeStringAddressable) { + ByteBuffer bb = ((NativeStringAddressable) s).getNativeStringBuffer(); + return CACHED_BB_ADDRESSES.get(bb); + } + CharBuffer cb; if (s instanceof CharBuffer) { cb = ((CharBuffer) s).duplicate(); @@ -321,24 +387,13 @@ WrittenString writeStringUnlimited(CharSequence s) { return str; } - PWArgsArrayBuffer allocateGetAddressCompatiblePWArgsBuffer(int num) { - return allocatePWArgsBuffer(num, true); - } - - PWArgsArrayBuffer allocatePWArgsBuffer(int num) { - return allocatePWArgsBuffer(num, false); - } - - private PWArgsArrayBuffer allocatePWArgsBuffer(int num, boolean getAddressCompatible) { + private PWArgsArrayBuffer allocatePWArgsBuffer(int num) { PWArgsSegment segment; segment = pwargsSegments.get(curPWArgsSegment); PWArgsArrayBuffer array; - while ((array = segment.allocate(num, getAddressCompatible)) == null) { + while ((array = segment.allocate(num)) == null) { segment = changePWArgsSegment(Math.max(PWARGS_MIN_SEGMENTS_SIZE, num)); } - if (idxOfFirstUsedPWArgsSegment == -1) { - idxOfFirstUsedPWArgsSegment = curPWArgsSegment; - } return array; } @@ -365,6 +420,23 @@ private StringsSegment changeStringsSegment(int capacity) { curStringsSegment++; return s; } + + private Map debugStats() { + Map result = new HashMap<>(); + result.put("num_pwargs_seg", pwargsSegments.size()); + result.put("num_str_seg", stringsSegments.size()); + + int totPWArgsNatMem = pwargsSegments.stream() + .map(s -> s.buffer.capacity()).reduce(0, Integer::sum); + result.put("total_pwargs_mem", totPWArgsNatMem); + int totStrNatMem = stringsSegments.stream() + .map(s -> s.buffer.capacity()).reduce(0, Integer::sum); + result.put("total_str_mem", totStrNatMem); + int totPWArgsBufPooled = pwargsSegments.stream() + .map(s -> s.pwargsArrays.size()).reduce(0, Integer::sum); + result.put("total_pwargs_buf_pooled", totPWArgsBufPooled); + return result; + } } /* we want to reuse our ByteBuffers because they live off heap */ @@ -382,6 +454,21 @@ ArenaLease getLease() { } } + void debugDump(PrintWriter w) { + w.format("Number of parked arenas: %d\n", arenas.size()); + long totalNatMemory = 0; + long totalPooled = 0; + int i = 0; + for (Arena arena : arenas) { + Map d = arena.debugStats(); + totalNatMemory += d.get("total_pwargs_mem") + d.get("total_str_mem"); + totalPooled += d.get("total_pwargs_buf_pooled"); + w.format("Arena %d: %s\n", ++i, d); + } + w.format("Total native memory: %d\n", totalNatMemory); + w.format("Total pooled PWArgsArrayBuffer objects: %d\n", totalPooled); + w.flush(); + } } public static class ArenaLease implements AutoCloseable, Closeable { @@ -400,8 +487,8 @@ public ByteBuffer getFirstPWArgsByteBuffer() { return this.arena.getFirstUsedPWArgsBuffer(); } - public ByteBuffer serializeMore(Powerwaf.Limits limits, Map map) { - return ByteBufferSerializer.serializeMore(this, limits, map); + public ByteBuffer serializeMore(Powerwaf.Limits limits, Object mapOrMpwz) { + return ByteBufferSerializer.serializeMore(this, limits, mapOrMpwz); } @Override @@ -427,25 +514,39 @@ static class PWArgsSegment { this.buffer.order(ByteOrder.nativeOrder()); } - PWArgsArrayBuffer allocate(int num, boolean getAddressCompatible) { + // Replace the first element at the start of the segment. This should + // ONLY be done for the first segment. + // libddwaf expects the data from previous runs to be available. The + // exception is the very first element, the root, which we can replace + // freely. + void allocateRoot(PWArgsBuffer pwArgs) { + pwArgs.reset(this.buffer, 0); + int curPosition = this.buffer.position(); + // we do not need to advance the position if we have already arrays + // in the segment, because we are inserting/replacing the 1st + // element + if (curPosition == 0) { + this.buffer.position(curPosition + SIZEOF_PWARGS); + } + } + + PWArgsArrayBuffer allocate(int num) { if (left() < num) { return null; } int position = this.buffer.position(); PWArgsArrayBuffer arrayBuffer; - if (getAddressCompatible) { - ByteBuffer slice = this.buffer.slice().order(ByteOrder.nativeOrder()); - arrayBuffer = new PWArgsArrayBuffer(slice, 0, num); - } else if (idxOfNextUnusedPWArgsArrayBuffer >= pwargsArrays.size()) { + if (idxOfNextUnusedPWArgsArrayBuffer >= pwargsArrays.size()) { + // do not slice() so we can reuse the object later without + // replacing the buffer ByteBuffer duplicate = this.buffer.duplicate().order(ByteOrder.nativeOrder()); arrayBuffer = new PWArgsArrayBuffer(duplicate, position, num); pwargsArrays.add(arrayBuffer); - idxOfNextUnusedPWArgsArrayBuffer++; } else { arrayBuffer = pwargsArrays.get(idxOfNextUnusedPWArgsArrayBuffer); arrayBuffer.reset(position, num); - idxOfNextUnusedPWArgsArrayBuffer++; } + idxOfNextUnusedPWArgsArrayBuffer++; this.buffer.position(position + num * SIZEOF_PWARGS); return arrayBuffer; } @@ -462,9 +563,9 @@ private int left() { static class PWArgsArrayBuffer { private final ByteBuffer buffer; + private long base; private int start; private int num; - private final List pwArgsBuffers; static final PWArgsArrayBuffer EMPTY_BUFFER = new PWArgsArrayBuffer(); @@ -473,18 +574,17 @@ static class PWArgsArrayBuffer { throw new IllegalArgumentException(); } this.buffer = buffer; + this.base = getByteBufferAddress(buffer); this.start = start; this.num = num; buffer.limit(start + num * SIZEOF_PWARGS); buffer.position(start); - this.pwArgsBuffers = new ArrayList<>(num); } private PWArgsArrayBuffer() { this.buffer = null; this.start = 0; this.num = 0; - this.pwArgsBuffers = null; } void reset(int start, int num) { @@ -494,17 +594,11 @@ void reset(int start, int num) { this.buffer.position(start); } - PWArgsBuffer get(int i) { + PWArgsBuffer get(PWArgsBuffer pwArgsBuffer, int i) { if (i < 0 || i >= num) { throw new ArrayIndexOutOfBoundsException(); } - assert this.buffer != null; - while (i >= pwArgsBuffers.size()) { - ByteBuffer duplicate = this.buffer.duplicate().order(ByteOrder.nativeOrder()); - pwArgsBuffers.add(new PWArgsBuffer(duplicate, start + i * SIZEOF_PWARGS)); - } - PWArgsBuffer pwArgsBuffer = pwArgsBuffers.get(i); - pwArgsBuffer.reset(start + i * SIZEOF_PWARGS); + pwArgsBuffer.reset(this.buffer, start + i * SIZEOF_PWARGS); return pwArgsBuffer; } @@ -512,8 +606,7 @@ long getAddress() { if (buffer == null) { return NULLPTR; } - long address = getByteBufferAddress(buffer); - return address + start; + return this.base + start; } } @@ -539,20 +632,21 @@ long getAddress() { * }; */ static class PWArgsBuffer { - private final ByteBuffer buffer; + private ByteBuffer origBuffer; + private ByteBuffer buffer; - PWArgsBuffer(ByteBuffer buffer, int start) { - this.buffer = buffer; - this.buffer.limit(start + SIZEOF_PWARGS); - this.buffer.position(start); - } + PWArgsBuffer() {} - void reset(int start) { - this.buffer.limit(start + SIZEOF_PWARGS); - this.buffer.position(start); + void reset(ByteBuffer buffer, int offset) { + if (this.buffer == null || buffer != this.origBuffer) { + this.origBuffer = buffer; + this.buffer = buffer.duplicate().order(ByteOrder.nativeOrder()); + } + this.buffer.position(offset); + this.buffer.limit(offset + SIZEOF_PWARGS); } - boolean writeString(Arena arena, String parameterName, CharSequence value) { + boolean writeString(Arena arena, CharSequence parameterName, CharSequence value) { if (!putParameterName(arena, parameterName)) { // string too large return false; } @@ -566,7 +660,7 @@ boolean writeString(Arena arena, String parameterName, CharSequence value) { return true; } - boolean writeLong(Arena arena, String parameterName, long value) { + boolean writeLong(Arena arena, CharSequence parameterName, long value) { if (!putParameterName(arena, parameterName)) { // string too large return false; } @@ -575,15 +669,15 @@ boolean writeLong(Arena arena, String parameterName, long value) { return true; } - PWArgsArrayBuffer writeArray(Arena arena, String parameterName, int numElements) { + PWArgsArrayBuffer writeArray(Arena arena, CharSequence parameterName, int numElements) { return writeArrayOrMap(arena, parameterName, numElements, PWInputType.PWI_ARRAY); } - PWArgsArrayBuffer writeMap(Arena arena, String parameterName, int numElements) { + PWArgsArrayBuffer writeMap(Arena arena, CharSequence parameterName, int numElements) { return writeArrayOrMap(arena, parameterName, numElements, PWInputType.PWI_MAP); } - private PWArgsArrayBuffer writeArrayOrMap(Arena arena, String parameterName, int numElements, + private PWArgsArrayBuffer writeArrayOrMap(Arena arena, CharSequence parameterName, int numElements, PWInputType type) { if (!putParameterName(arena, parameterName)) { // string too large return null; @@ -610,7 +704,7 @@ private PWArgsArrayBuffer writeArrayOrMap(Arena arena, String parameterName, int } - private boolean putParameterName(Arena arena, String parameterName) { + private boolean putParameterName(Arena arena, CharSequence parameterName) { if (parameterName == null) { this.buffer.putLong(0L).putLong(0L); } else { diff --git a/src/main/java/io/sqreen/powerwaf/MapIterableWithSize.java b/src/main/java/io/sqreen/powerwaf/MapIterableWithSize.java new file mode 100644 index 00000000..9cce1234 --- /dev/null +++ b/src/main/java/io/sqreen/powerwaf/MapIterableWithSize.java @@ -0,0 +1,7 @@ +package io.sqreen.powerwaf; + +import java.util.Map; + +public interface MapIterableWithSize extends Iterable> { + int size(); +} diff --git a/src/main/java/io/sqreen/powerwaf/NativeStringAddressable.java b/src/main/java/io/sqreen/powerwaf/NativeStringAddressable.java new file mode 100644 index 00000000..e0a48ea9 --- /dev/null +++ b/src/main/java/io/sqreen/powerwaf/NativeStringAddressable.java @@ -0,0 +1,7 @@ +package io.sqreen.powerwaf; + +import java.nio.ByteBuffer; + +public interface NativeStringAddressable extends CharSequence { + ByteBuffer getNativeStringBuffer(); +} diff --git a/src/main/java/io/sqreen/powerwaf/Powerwaf.java b/src/main/java/io/sqreen/powerwaf/Powerwaf.java index 51735416..132602b4 100644 --- a/src/main/java/io/sqreen/powerwaf/Powerwaf.java +++ b/src/main/java/io/sqreen/powerwaf/Powerwaf.java @@ -131,7 +131,7 @@ static native ResultWithData runRules(PowerwafHandle handle, PowerwafMetrics metrics) throws AbstractPowerwafException; static native ResultWithData runRules(PowerwafHandle handle, - Map parameters, + Object parameters, Limits limits, PowerwafMetrics metrics) throws AbstractPowerwafException; diff --git a/src/main/java/io/sqreen/powerwaf/PowerwafContext.java b/src/main/java/io/sqreen/powerwaf/PowerwafContext.java index 709dc82f..fd47ae5f 100644 --- a/src/main/java/io/sqreen/powerwaf/PowerwafContext.java +++ b/src/main/java/io/sqreen/powerwaf/PowerwafContext.java @@ -110,8 +110,20 @@ public String[] getUsedAddresses() { } public Powerwaf.ResultWithData runRules(Map parameters, - Powerwaf.Limits limits, - PowerwafMetrics metrics) throws AbstractPowerwafException { + Powerwaf.Limits limits, + PowerwafMetrics metrics) throws AbstractPowerwafException { + return runRules((Object) parameters, limits, metrics); + } + + public Powerwaf.ResultWithData runRules(MapIterableWithSize parameters, + Powerwaf.Limits limits, + PowerwafMetrics metrics) throws AbstractPowerwafException { + return runRules((Object) parameters, limits, metrics); + } + + private Powerwaf.ResultWithData runRules(Object parameters, + Powerwaf.Limits limits, + PowerwafMetrics metrics) throws AbstractPowerwafException { this.readLock.lock(); try { checkIfOnline(); diff --git a/src/test/groovy/io/sqreen/powerwaf/AdditiveTest.groovy b/src/test/groovy/io/sqreen/powerwaf/AdditiveTest.groovy index 274c26d5..305c51fc 100644 --- a/src/test/groovy/io/sqreen/powerwaf/AdditiveTest.groovy +++ b/src/test/groovy/io/sqreen/powerwaf/AdditiveTest.groovy @@ -15,6 +15,7 @@ import org.slf4j.LoggerFactory import static groovy.test.GroovyAssert.shouldFail import static org.hamcrest.MatcherAssert.assertThat +import static org.hamcrest.Matchers.containsString import static org.hamcrest.Matchers.is class AdditiveTest implements ReactiveTrait { @@ -72,6 +73,20 @@ class AdditiveTest implements ReactiveTrait { assert metrics.totalRunTimeNs >= metrics.totalDdwafRunTimeNs } + @Test + void 'variant with MapIterableWithSize'() { + ctx = new PowerwafContext('test', null, ARACHNI_ATOM_V2_1) + additive = ctx.openAdditive() + Map map = ['server.request.headers.no_cookies': ['user-agent': 'Arachni/v1']] + def miws = [ + size: { -> map.size() }, + iterator: { -> map.entrySet().iterator() } + ] as MapIterableWithSize + + Powerwaf.ResultWithData awd = additive.run(miws, limits, metrics) + assertThat awd.result, is(Powerwaf.Result.MATCH) + } + @Test void 'constructor throws if given a null context'() { shouldFail(NullPointerException) { @@ -79,13 +94,16 @@ class AdditiveTest implements ReactiveTrait { } } - @Test(expected = RuntimeException) + @Test void 'Should throw RuntimeException if double free'() { - ctx = new PowerwafContext('test', ARACHNI_ATOM) + ctx = new PowerwafContext('test', null, ARACHNI_ATOM_V2_1) additive = ctx.openAdditive() additive.close() try { - additive.close() + def exc = shouldFail(IllegalStateException) { + additive.close() + } + assertThat exc.message, containsString('is no longer online') } finally { additive = null } diff --git a/src/test/groovy/io/sqreen/powerwaf/BasicTests.groovy b/src/test/groovy/io/sqreen/powerwaf/BasicTests.groovy index ea390774..f1e8a013 100644 --- a/src/test/groovy/io/sqreen/powerwaf/BasicTests.groovy +++ b/src/test/groovy/io/sqreen/powerwaf/BasicTests.groovy @@ -90,6 +90,45 @@ class BasicTests implements PowerwafTrait { assert metrics.totalRunTimeNs >= metrics.totalDdwafRunTimeNs } + @Test + void 'test running basic rule v2_1 — MapIterableWithSizeVariant'() { + def ruleSet = ARACHNI_ATOM_V2_1 + + ctx = Powerwaf.createContext('test', ruleSet) + metrics = ctx.createMetrics() + + Map map = ['server.request.headers.no_cookies': ['user-agent': 'Arachni/v1']] + def miws = [ + size: { -> map.size() }, + iterator: { -> map.entrySet().iterator() } + ] as MapIterableWithSize + + ResultWithData awd = ctx.runRules(miws, limits, metrics) + assertThat awd.result, is(Powerwaf.Result.MATCH) + + def json = slurper.parseText(awd.data) + + assert json[0].rule.id == 'arachni_rule' + assert json[0].rule.name == 'Arachni' + assert json[0].rule.tags == [category: 'attack_attempt', type: 'security_scanner'] + assert json[0].rule_matches[0]['operator'] == 'match_regex' + assert json[0].rule_matches[0]['operator_value'] == '^Arachni\\/v' + assert json[0].rule_matches[0]['parameters'][0].address == 'server.request.headers.no_cookies' + assert json[0].rule_matches[0]['parameters'][0].key_path == ['user-agent'] + assert json[0].rule_matches[0]['parameters'][0].value == 'Arachni/v1' + assert json[0].rule_matches[0]['parameters'][0].highlight == ['Arachni/v'] + + def rsi = ctx.ruleSetInfo + assert rsi.numRulesOK == 1 + assert rsi.numRulesError == 0 + assert rsi.errors == [:] + assert rsi.fileVersion == '1.2.6' + + assert metrics.totalRunTimeNs > 0 + assert metrics.totalDdwafRunTimeNs > 0 + assert metrics.totalRunTimeNs >= metrics.totalDdwafRunTimeNs + } + @Test void 'test blocking action'() { def ruleSet = ARACHNI_ATOM_BLOCK diff --git a/src/test/groovy/io/sqreen/powerwaf/ByteBufferSerializerTests.groovy b/src/test/groovy/io/sqreen/powerwaf/ByteBufferSerializerTests.groovy index dec88003..62f4bd76 100644 --- a/src/test/groovy/io/sqreen/powerwaf/ByteBufferSerializerTests.groovy +++ b/src/test/groovy/io/sqreen/powerwaf/ByteBufferSerializerTests.groovy @@ -154,6 +154,24 @@ class ByteBufferSerializerTests implements PowerwafTrait { assertThat res, is(exp) } + @Test + void 'can serialize MapIterableWithSize'() { + def sdata = [foo: 1, bar: '2', (null): '3'] + def data = [ + size: { -> sdata.size() }, + iterator: { -> sdata.entrySet().iterator() } + ] as MapIterableWithSize + lease = serializer.serialize(data) + String res = Powerwaf.pwArgsBufferToString(lease.firstPWArgsByteBuffer) + def exp = p ''' + + foo: 1 + bar: 2 + : 3 + ''' + assertThat res, is(exp) + } + @Test void 'unknown values are serialized as empty maps'() { lease = serializer.serialize([my_key: new Object()]) @@ -308,24 +326,57 @@ class ByteBufferSerializerTests implements PowerwafTrait { } @Test + @SuppressWarnings('LineLength') void 'additive basic usage'() { + ByteBufferSerializer.release() + lease = ByteBufferSerializer.blankLease - ByteBuffer bb1 = lease.serializeMore(limits, [a: 'b']) - ByteBuffer bb2 = lease.serializeMore(limits, [c: 'd']) + ByteBuffer bb1 = lease.serializeMore(limits, [a: ['b', 'c']]) String res = Powerwaf.pwArgsBufferToString(bb1) def exp = p ''' - a: b + a: + b + c ''' assertThat res, is(exp) + ByteBuffer bb2 = lease.serializeMore(limits, [c: 'd']) res = Powerwaf.pwArgsBufferToString(bb2) exp = p ''' c: d ''' assertThat res, is(exp) + + // the original root map has been written over + assertThat ByteBufferSerializer.getByteBufferAddress(bb1), + is(ByteBufferSerializer.getByteBufferAddress(bb2)) + + // though not the contents of the root map + bb1.position(ByteBufferSerializer.SIZEOF_PWARGS) + res = Powerwaf.pwArgsBufferToString(bb1.slice()) + exp = p ''' + a: + b + c + ''' + assertThat res, is(exp) + + lease.close() + lease = null + + def sw = new StringWriter() + def pw = new PrintWriter(sw) + ByteBufferSerializer.debugDump pw + exp = p ''' + Number of parked arenas: 1 + Arena 1: {num_str_seg=1, total_str_mem=81920, total_pwargs_buf_pooled=3, total_pwargs_mem=20480, num_pwargs_seg=1} + Total native memory: 102400 + Total pooled PWArgsArrayBuffer objects: 3 + ''' + assertThat sw.toString(), is(exp) } @Test diff --git a/src/test/groovy/io/sqreen/powerwaf/CharSequenceSerializationTests.groovy b/src/test/groovy/io/sqreen/powerwaf/CharSequenceSerializationTests.groovy index 436b0b6f..a9efd3a1 100644 --- a/src/test/groovy/io/sqreen/powerwaf/CharSequenceSerializationTests.groovy +++ b/src/test/groovy/io/sqreen/powerwaf/CharSequenceSerializationTests.groovy @@ -13,6 +13,7 @@ import org.junit.Test import java.nio.ByteBuffer import java.nio.ByteOrder import java.nio.CharBuffer +import java.nio.charset.StandardCharsets import static org.hamcrest.MatcherAssert.assertThat import static org.hamcrest.Matchers.is @@ -26,6 +27,13 @@ class CharSequenceSerializationTests implements ReqBodyTrait { assertThat awd.result, is(Powerwaf.Result.MATCH) } + @Test + void 'Should MATCH with data passed as NativeStringAddressable'() { + NativeStringAddressable nsa = new NSAImpl('my string') + Powerwaf.ResultWithData awd = testWithData(nsa) + assertThat awd.result, is(Powerwaf.Result.MATCH) + } + @Test void 'Should MATCH with data passed as CharBuffer'() { char[] storedBody = 'my string' as char[] @@ -100,4 +108,28 @@ class CharSequenceSerializationTests implements ReqBodyTrait { Powerwaf.ResultWithData awd = testWithData(cs) assertThat awd.result, is(Powerwaf.Result.OK) } + + static class NSAImpl implements NativeStringAddressable { + @Delegate + CharSequence s + ByteBuffer bb + + NSAImpl(String s) { + this.s = s + def bytes = s.getBytes(StandardCharsets.UTF_8) + this.bb = ByteBuffer.allocateDirect(bytes.length + 1) + this.bb.put(bytes) + this.bb.put((byte)0) + } + + @Override + ByteBuffer getNativeStringBuffer() { + bb + } + + @Override + String toString() { + s.toString() + } + } } diff --git a/src/test/groovy/io/sqreen/powerwaf/InvalidInvocationTests.groovy b/src/test/groovy/io/sqreen/powerwaf/InvalidInvocationTests.groovy index 463e3585..e9da7653 100644 --- a/src/test/groovy/io/sqreen/powerwaf/InvalidInvocationTests.groovy +++ b/src/test/groovy/io/sqreen/powerwaf/InvalidInvocationTests.groovy @@ -101,8 +101,9 @@ class InvalidInvocationTests implements ReactiveTrait { ByteBufferSerializer serializer = new ByteBufferSerializer(limits) serializer.serialize([a: 'b']).withCloseable { lease -> ByteBuffer buffer = lease.firstPWArgsByteBuffer + buffer.limit(buffer.capacity()) + buffer.position(ByteBufferSerializer.SIZEOF_PWARGS) def slice = buffer.slice() - slice.position(ByteBufferSerializer.SIZEOF_PWARGS) shouldFail(InvalidObjectPowerwafException) { additive.runAdditive(slice, limits, metrics) } diff --git a/src/test/groovy/io/sqreen/powerwaf/LimitsTests.groovy b/src/test/groovy/io/sqreen/powerwaf/LimitsTests.groovy index 648c9218..5e45da32 100644 --- a/src/test/groovy/io/sqreen/powerwaf/LimitsTests.groovy +++ b/src/test/groovy/io/sqreen/powerwaf/LimitsTests.groovy @@ -61,6 +61,18 @@ class LimitsTests implements PowerwafTrait { assertThat awd.result, is(Powerwaf.Result.OK) } + @Test + void 'maxDepth is respected — MapIterableWithSize variant'() { + ctx = ctxWithArachniAtom + maxDepth = 3 + + Powerwaf.ResultWithData awd = runRules(asMiws(a: 'Arachni')) + assertThat awd.result, is(Powerwaf.Result.MATCH) + + awd = runRules([a: asMiws(a: 'Arachni')]) + assertThat awd.result, is(Powerwaf.Result.OK) + } + @Test void 'maxElements is respected'() { ctx = ctxWithArachniAtom @@ -195,4 +207,11 @@ class LimitsTests implements PowerwafTrait { def json = slurper.parseText(res.data) assertThat json.ret_code, hasItem(is(new TimeoutPowerwafException().code)) } + + private MapIterableWithSize asMiws(Map m) { + [ + size: { -> m.size() }, + iterator: { -> m.entrySet().iterator() } + ] as MapIterableWithSize + } }