From a4cf82d10ec87367ab9fa1611c69fb6b46a9971b Mon Sep 17 00:00:00 2001 From: Michael Kaufmann Date: Mon, 26 Oct 2015 17:25:23 +0100 Subject: [PATCH] Fix the handling of supplementary characters (characters > U+FFFF). JNI uses a modified UTF-8 encoding. For supplementary characters, an invalid UTF-8 sequence was written to the database, which resulted in interoperability problems. The solution is to avoid UTF-8 in the native code and use the UTF-16 functions of SQLite (where possible). SQLite will then convert the UTF-16 to standards-compliant, unmodified UTF-8. This also fixes related bugs in JDBC3PreparedStatement and improves the "out of memory" handling in the native code. Fixed Issues: - https://bitbucket.org/xerial/sqlite-jdbc/issues/200/wrong-utf-8-decoding-of-unicode-code , same as https://github.com/xerial/sqlite-jdbc/issues/61 - https://bitbucket.org/xerial/sqlite-jdbc/issues/144/nativedbexec-throws-an-exception-without - https://bitbucket.org/xerial/sqlite-jdbc/issues/84/bug-in-nativedbc-bind_1blob - https://bitbucket.org/xerial/sqlite-jdbc/issues/70/setting-a-blob-in-prepstmt --- src/main/java/org/sqlite/core/NativeDB.c | 205 ++++++++++++------ .../sqlite/jdbc3/JDBC3PreparedStatement.java | 25 ++- src/test/java/org/sqlite/PrepStmtTest.java | 34 ++- 3 files changed, 189 insertions(+), 75 deletions(-) diff --git a/src/main/java/org/sqlite/core/NativeDB.c b/src/main/java/org/sqlite/core/NativeDB.c index 3ef12707f..41847bcd8 100644 --- a/src/main/java/org/sqlite/core/NativeDB.c +++ b/src/main/java/org/sqlite/core/NativeDB.c @@ -49,7 +49,7 @@ static void throwex(JNIEnv *env, jobject this) (*env)->CallVoidMethod(env, this, mth_throwex); } -static void throw_errorcode(JNIEnv *env, jobject this, int errorCode) +static void throwex_errorcode(JNIEnv *env, jobject this, int errorCode) { static jmethodID mth_throwex = 0; @@ -59,7 +59,7 @@ static void throw_errorcode(JNIEnv *env, jobject this, int errorCode) (*env)->CallVoidMethod(env, this, mth_throwex, (jint) errorCode); } -static void throwexmsg(JNIEnv *env, const char *str) +static void throwex_msg(JNIEnv *env, const char *str) { static jmethodID mth_throwexmsg = 0; @@ -70,7 +70,7 @@ static void throwexmsg(JNIEnv *env, const char *str) (*env)->NewStringUTF(env, str)); } -static void throw_errorcod_and_msg(JNIEnv *env, int errorCode, const char *str) +static void throwex_errorcode_and_msg(JNIEnv *env, int errorCode, const char *str) { static jmethodID mth_throwexmsg = 0; @@ -81,6 +81,12 @@ static void throw_errorcod_and_msg(JNIEnv *env, int errorCode, const char *str) (*env)->NewStringUTF(env, str)); } +static void throwex_outofmemory(JNIEnv *env) +{ + throwex_msg(env, "Out of memory"); +} + + static sqlite3 * gethandle(JNIEnv *env, jobject this) { @@ -130,14 +136,14 @@ static sqlite3_value * tovalue(JNIEnv *env, jobject function, jint arg) } // check we have any business being here - if (arg < 0) { throwexmsg(env, "negative arg out of range"); return 0; } - if (!function) { throwexmsg(env, "inconstent function"); return 0; } + if (arg < 0) { throwex_msg(env, "negative arg out of range"); return 0; } + if (!function) { throwex_msg(env, "inconstent function"); return 0; } value_pntr = (*env)->GetLongField(env, function, func_value); numArgs = (*env)->GetIntField(env, function, func_args); - if (value_pntr == 0) { throwexmsg(env, "no current value"); return 0; } - if (arg >= numArgs) { throwexmsg(env, "arg out of range"); return 0; } + if (value_pntr == 0) { throwex_msg(env, "no current value"); return 0; } + if (arg >= numArgs) { throwex_msg(env, "arg out of range"); return 0; } return ((sqlite3_value**)toref(value_pntr))[arg]; } @@ -145,9 +151,9 @@ static sqlite3_value * tovalue(JNIEnv *env, jobject function, jint arg) /* called if an exception occured processing xFunc */ static void xFunc_error(sqlite3_context *context, JNIEnv *env) { - const char *strmsg = 0; + const jchar *msgstr = 0; jstring msg = 0; - jint msgsize = 0; + jint msglength = 0; jclass exclass = 0; static jmethodID exp_msg = 0; @@ -164,13 +170,13 @@ static void xFunc_error(sqlite3_context *context, JNIEnv *env) msg = (jstring)(*env)->CallObjectMethod(env, ex, exp_msg); if (!msg) { sqlite3_result_error(context, "unknown error", 13); return; } - msgsize = (*env)->GetStringUTFLength(env, msg); - strmsg = (*env)->GetStringUTFChars(env, msg, 0); - assert(strmsg); // out-of-memory + msglength = (*env)->GetStringLength(env, msg); + msgstr = (*env)->GetStringCritical(env, msg, 0); + if (!msgstr) { sqlite3_result_error_nomem(context); return; } - sqlite3_result_error(context, strmsg, msgsize); + sqlite3_result_error16(context, msgstr, msglength * sizeof(jchar)); - (*env)->ReleaseStringUTFChars(env, msg, strmsg); + (*env)->ReleaseStringCritical(env, msg, msgstr); } /* used to call xFunc, xStep and xFinal */ @@ -333,19 +339,20 @@ JNIEXPORT void JNICALL Java_org_sqlite_core_NativeDB__1open( const char *str; if (db) { - throwexmsg(env, "DB already open"); + throwex_msg(env, "DB already open"); sqlite3_close(db); return; } str = (*env)->GetStringUTFChars(env, file, 0); ret = sqlite3_open_v2(str, &db, flags, NULL); + (*env)->ReleaseStringUTFChars(env, file, str); + if (ret) { - throw_errorcode(env, this, ret); + throwex_errorcode(env, this, ret); sqlite3_close(db); return; } - (*env)->ReleaseStringUTFChars(env, file, str); sethandle(env, this, db); } @@ -375,12 +382,15 @@ JNIEXPORT jlong JNICALL Java_org_sqlite_core_NativeDB_prepare( sqlite3* db = gethandle(env, this); sqlite3_stmt* stmt; - const char *strsql = (*env)->GetStringUTFChars(env, sql, 0); - int status = sqlite3_prepare_v2(db, strsql, -1, &stmt, 0); - (*env)->ReleaseStringUTFChars(env, sql, strsql); + jsize sqllength = (*env)->GetStringLength(env, sql); + const jchar *sqlstr = (*env)->GetStringCritical(env, sql, 0); + if (!sqlstr) { throwex_outofmemory(env); return fromref(0); } + + int status = sqlite3_prepare16_v2(db, sqlstr, sqllength * sizeof(jchar), &stmt, 0); + (*env)->ReleaseStringCritical(env, sql, sqlstr); if (status != SQLITE_OK) { - throw_errorcode(env, this, status); + throwex_errorcode(env, this, status); return fromref(0); } return fromref(stmt); @@ -390,25 +400,79 @@ JNIEXPORT jint JNICALL Java_org_sqlite_core_NativeDB__1exec( JNIEnv *env, jobject this, jstring sql) { sqlite3* db = gethandle(env, this); - const char *strsql; - char* errorMsg; - int status; - - if(!db) - { - throw_errorcode(env, this, 21); - return 21; - } - - strsql = (*env)->GetStringUTFChars(env, sql, 0); - status = sqlite3_exec(db, strsql, 0, 0, &errorMsg); - - (*env)->ReleaseStringUTFChars(env, sql, strsql); + sqlite3_stmt* stmt = 0; + jsize sqllength; + const jchar *sqlstr; + const jchar *sqlstrend; + const jchar *sqlstrstmt; + const jchar *leftover; // Tail of unprocessed SQL + int status = SQLITE_OK; + + if (!db) + { + throwex_errorcode(env, this, SQLITE_MISUSE); + return SQLITE_MISUSE; + } - if (status != SQLITE_OK) { - throwexmsg(env, errorMsg); - sqlite3_free(errorMsg); + sqllength = (*env)->GetStringLength(env, sql); + + // Do not use GetStringCritical() here, because SQLite may call + // Java methods while evaluating the SQL query + sqlstr = (*env)->GetStringChars(env, sql, 0); + if (!sqlstr) { throwex_outofmemory(env); return 0; } + + sqlstrstmt = sqlstr; + sqlstrend = sqlstr + sqllength; + + while (status == SQLITE_OK && sqlstrstmt && sqlstrstmt < sqlstrend) + { + status = sqlite3_prepare16_v2(db, sqlstrstmt, (sqlstrend - sqlstrstmt) * sizeof(jchar), + &stmt, (const void**)&leftover); + if (status != SQLITE_OK) + { + continue; + } + + if (!stmt) + { + // this happens for a comment or white-space + sqlstrstmt = leftover; + continue; + } + + while (1) + { + status = sqlite3_step(stmt); + + if (status != SQLITE_ROW) + { + status = sqlite3_finalize(stmt); + stmt = 0; + sqlstrstmt = leftover; + + while ( sqlstrstmt && sqlstrstmt < sqlstrend + && (*sqlstrstmt == ' ' || *sqlstrstmt == '\t' || *sqlstrstmt == '\n' + || *sqlstrstmt == '\v' || *sqlstrstmt == '\f' || *sqlstrstmt == '\r' )) + { + sqlstrstmt++; + } + break; + } + } + } + + (*env)->ReleaseStringChars(env, sql, sqlstr); + + if (stmt) + { + sqlite3_finalize(stmt); + } + + if (status != SQLITE_OK) + { + throwex_errorcode(env, this, status); } + return status; } @@ -416,7 +480,8 @@ JNIEXPORT jint JNICALL Java_org_sqlite_core_NativeDB__1exec( JNIEXPORT jstring JNICALL Java_org_sqlite_core_NativeDB_errmsg(JNIEnv *env, jobject this) { - return (*env)->NewStringUTF(env, sqlite3_errmsg(gethandle(env, this))); + const jchar *str = (const jchar*) sqlite3_errmsg16(gethandle(env, this)); + return str ? (*env)->NewString(env, str, jstrlen(str)) : NULL; } JNIEXPORT jstring JNICALL Java_org_sqlite_core_NativeDB_libversion( @@ -482,8 +547,8 @@ JNIEXPORT jint JNICALL Java_org_sqlite_core_NativeDB_column_1type( JNIEXPORT jstring JNICALL Java_org_sqlite_core_NativeDB_column_1decltype( JNIEnv *env, jobject this, jlong stmt, jint col) { - const char *str = sqlite3_column_decltype(toref(stmt), col); - return (*env)->NewStringUTF(env, str); + const jchar *str = (const jchar*) sqlite3_column_decltype16(toref(stmt), col); + return str ? (*env)->NewString(env, str, jstrlen(str)) : NULL; } JNIEXPORT jstring JNICALL Java_org_sqlite_core_NativeDB_column_1table_1name( @@ -503,8 +568,12 @@ JNIEXPORT jstring JNICALL Java_org_sqlite_core_NativeDB_column_1name( JNIEXPORT jstring JNICALL Java_org_sqlite_core_NativeDB_column_1text( JNIEnv *env, jobject this, jlong stmt, jint col) { - return (*env)->NewStringUTF( - env, (const char*)sqlite3_column_text(toref(stmt), col)); + const jchar *str = 0; + jint strlength = 0; + + str = (const jchar*) sqlite3_column_text16(toref(stmt), col); + strlength = sqlite3_column_bytes16(toref(stmt), col) / sizeof(jchar); + return str ? (*env)->NewString(env, str, strlength) : NULL; } JNIEXPORT jbyteArray JNICALL Java_org_sqlite_core_NativeDB_column_1blob( @@ -518,7 +587,7 @@ JNIEXPORT jbyteArray JNICALL Java_org_sqlite_core_NativeDB_column_1blob( length = sqlite3_column_bytes(toref(stmt), col); jBlob = (*env)->NewByteArray(env, length); - assert(jBlob); // out-of-memory + if (!jBlob) { throwex_outofmemory(env); return 0; } a = (*env)->GetPrimitiveArrayCritical(env, jBlob, 0); memcpy(a, blob, length); @@ -572,9 +641,11 @@ JNIEXPORT jint JNICALL Java_org_sqlite_core_NativeDB_bind_1double( JNIEXPORT jint JNICALL Java_org_sqlite_core_NativeDB_bind_1text( JNIEnv *env, jobject this, jlong stmt, jint pos, jstring v) { - const char *chars = (*env)->GetStringUTFChars(env, v, 0); - int rc = sqlite3_bind_text(toref(stmt), pos, chars, -1, SQLITE_TRANSIENT); - (*env)->ReleaseStringUTFChars(env, v, chars); + jsize vlength = (*env)->GetStringLength(env, v); + const jchar *vstr = (*env)->GetStringCritical(env, v, 0); + if (!vstr) { throwex_outofmemory(env); return 0; } + int rc = sqlite3_bind_text16(toref(stmt), pos, vstr, vlength * sizeof(jchar), SQLITE_TRANSIENT); + (*env)->ReleaseStringCritical(env, v, vstr); return rc; } @@ -584,7 +655,8 @@ JNIEXPORT jint JNICALL Java_org_sqlite_core_NativeDB_bind_1blob( jint rc; void *a; jsize size = (*env)->GetArrayLength(env, v); - assert(a = (*env)->GetPrimitiveArrayCritical(env, v, 0)); + a = (*env)->GetPrimitiveArrayCritical(env, v, 0); + if (!a) { throwex_outofmemory(env); return 0; } rc = sqlite3_bind_blob(toref(stmt), pos, a, size, SQLITE_TRANSIENT); (*env)->ReleasePrimitiveArrayCritical(env, v, a, JNI_ABORT); return rc; @@ -599,16 +671,16 @@ JNIEXPORT void JNICALL Java_org_sqlite_core_NativeDB_result_1null( JNIEXPORT void JNICALL Java_org_sqlite_core_NativeDB_result_1text( JNIEnv *env, jobject this, jlong context, jstring value) { - const jchar *str; - jsize size; + const jchar *valuestr; + jsize valuelength; if (value == NULL) { sqlite3_result_null(toref(context)); return; } - size = (*env)->GetStringLength(env, value) * 2; - str = (*env)->GetStringCritical(env, value, 0); - assert(str); // out-of-memory - sqlite3_result_text16(toref(context), str, size, SQLITE_TRANSIENT); - (*env)->ReleaseStringCritical(env, value, str); + valuelength = (*env)->GetStringLength(env, value); + valuestr = (*env)->GetStringCritical(env, value, 0); + if (!valuestr) { throwex_outofmemory(env); return; } + sqlite3_result_text16(toref(context), valuestr, valuelength * sizeof(jchar), SQLITE_TRANSIENT); + (*env)->ReleaseStringCritical(env, value, valuestr); } JNIEXPORT void JNICALL Java_org_sqlite_core_NativeDB_result_1blob( @@ -618,11 +690,10 @@ JNIEXPORT void JNICALL Java_org_sqlite_core_NativeDB_result_1blob( jsize size; if (value == NULL) { sqlite3_result_null(toref(context)); return; } - size = (*env)->GetArrayLength(env, value); - // be careful with *Critical + size = (*env)->GetArrayLength(env, value); bytes = (*env)->GetPrimitiveArrayCritical(env, value, 0); - assert(bytes); // out-of-memory + if (!bytes) { throwex_outofmemory(env); return; } sqlite3_result_blob(toref(context), bytes, size, SQLITE_TRANSIENT); (*env)->ReleasePrimitiveArrayCritical(env, value, bytes, JNI_ABORT); } @@ -651,14 +722,14 @@ JNIEXPORT void JNICALL Java_org_sqlite_core_NativeDB_result_1int( JNIEXPORT jstring JNICALL Java_org_sqlite_core_NativeDB_value_1text( JNIEnv *env, jobject this, jobject f, jint arg) { - jint length = 0; const void *str = 0; + jint strlength = 0; sqlite3_value *value = tovalue(env, f, arg); if (!value) return NULL; - length = sqlite3_value_bytes16(value) / 2; // in jchars str = sqlite3_value_text16(value); - return str ? (*env)->NewString(env, str, length) : NULL; + strlength = sqlite3_value_bytes16(value) / sizeof(jchar); + return str ? (*env)->NewString(env, str, strlength) : NULL; } JNIEXPORT jbyteArray JNICALL Java_org_sqlite_core_NativeDB_value_1blob( @@ -676,7 +747,7 @@ JNIEXPORT jbyteArray JNICALL Java_org_sqlite_core_NativeDB_value_1blob( length = sqlite3_value_bytes(value); jBlob = (*env)->NewByteArray(env, length); - assert(jBlob); // out-of-memory + if (!jBlob) { throwex_outofmemory(env); return 0; } a = (*env)->GetPrimitiveArrayCritical(env, jBlob, 0); memcpy(a, blob, length); @@ -723,7 +794,7 @@ JNIEXPORT jint JNICALL Java_org_sqlite_core_NativeDB_create_1function( static jfieldID udfdatalist = 0; struct UDFData *udf = malloc(sizeof(struct UDFData)); - assert(udf); // out-of-memory + if (!udf) { throwex_outofmemory(env); return 0; } if (!udfdatalist) udfdatalist = (*env)->GetFieldID(env, dbclass, "udfdatalist", "J"); @@ -737,7 +808,7 @@ JNIEXPORT jint JNICALL Java_org_sqlite_core_NativeDB_create_1function( (*env)->SetLongField(env, this, udfdatalist, fromref(udf)); strname = (*env)->GetStringUTFChars(env, name, 0); - assert(strname); // out-of-memory + if (!strname) { throwex_outofmemory(env); return 0; } ret = sqlite3_create_function( gethandle(env, this), @@ -809,10 +880,10 @@ JNIEXPORT jobjectArray JNICALL Java_org_sqlite_core_NativeDB_column_1metadata( colCount = sqlite3_column_count(dbstmt); array = (*env)->NewObjectArray( env, colCount, (*env)->FindClass(env, "[Z"), NULL) ; - assert(array); // out-of-memory + if (!array) { throwex_outofmemory(env); return 0; } colDataRaw = (jboolean*)malloc(3 * sizeof(jboolean)); - assert(colDataRaw); // out-of-memory + if (!colDataRaw) { throwex_outofmemory(env); return 0; } for (i = 0; i < colCount; i++) { // load passed column name and table name @@ -837,7 +908,7 @@ JNIEXPORT jobjectArray JNICALL Java_org_sqlite_core_NativeDB_column_1metadata( colDataRaw[2] = pAutoinc; colData = (*env)->NewBooleanArray(env, 3); - assert(colData); // out-of-memory + if (!colData) { throwex_outofmemory(env); return 0; } (*env)->SetBooleanArrayRegion(env, colData, 0, 3, colDataRaw); (*env)->SetObjectArrayElement(env, array, i, colData); diff --git a/src/main/java/org/sqlite/jdbc3/JDBC3PreparedStatement.java b/src/main/java/org/sqlite/jdbc3/JDBC3PreparedStatement.java index 06eeeeef3..8c3a05996 100644 --- a/src/main/java/org/sqlite/jdbc3/JDBC3PreparedStatement.java +++ b/src/main/java/org/sqlite/jdbc3/JDBC3PreparedStatement.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.Reader; +import java.io.UnsupportedEncodingException; import java.math.BigDecimal; import java.net.URL; import java.sql.Array; @@ -202,12 +203,21 @@ private byte[] readBytes(InputStream istream, int length) throws SQLException { throw exception; } - + byte[] bytes = new byte[length]; try { - istream.read(bytes); + int bytesRead; + int totalBytesRead = 0; + + while (totalBytesRead < length) { + bytesRead = istream.read(bytes, totalBytesRead, length - totalBytesRead); + if (bytesRead == -1) { + throw new IOException("End of stream has been reached"); + } + totalBytesRead += bytesRead; + } return bytes; } @@ -216,7 +226,7 @@ private byte[] readBytes(InputStream istream, int length) throws SQLException { SQLException exception = new SQLException("Error reading stream"); exception.initCause(cause); - throw(exception); + throw exception; } } @@ -246,7 +256,14 @@ public void setUnicodeStream(int pos, InputStream istream, int length) throws SQ setString(pos, null); } - setString(pos, new String(readBytes(istream, length))); + try { + setString(pos, new String(readBytes(istream, length), "UTF-8")); + } catch (UnsupportedEncodingException e) { + SQLException exception = new SQLException("UTF-8 is not supported"); + + exception.initCause(e); + throw exception; + } } /** diff --git a/src/test/java/org/sqlite/PrepStmtTest.java b/src/test/java/org/sqlite/PrepStmtTest.java index 36ba8e35f..ab651843f 100644 --- a/src/test/java/org/sqlite/PrepStmtTest.java +++ b/src/test/java/org/sqlite/PrepStmtTest.java @@ -23,8 +23,8 @@ public class PrepStmtTest { static byte[] b1 = new byte[] { 1, 2, 7, 4, 2, 6, 2, 8, 5, 2, 3, 1, 5, 3, 6, 3, 3, 6, 2, 5 }; - static byte[] b2 = "To be or not to be.".getBytes(); - static byte[] b3 = "Question!#$%".getBytes(); + static byte[] b2 = getUtf8Bytes("To be or not to be."); + static byte[] b3 = getUtf8Bytes("Question!#$%"); static String utf01 = "\uD840\uDC40"; static String utf02 = "\uD840\uDC47 "; static String utf03 = " \uD840\uDC43"; @@ -37,6 +37,16 @@ public class PrepStmtTest private Connection conn; private Statement stat; + private static byte[] getUtf8Bytes(String str) { + try { + return str.getBytes("UTF-8"); + } + catch (UnsupportedEncodingException e) { + fail(e.getMessage()); + return null; + } + } + @Before public void connect() throws Exception { conn = DriverManager.getConnection("jdbc:sqlite:"); @@ -203,8 +213,8 @@ public void set() throws SQLException, UnsupportedEncodingException { rs = prep.executeQuery(); assertTrue(rs.next()); assertArrayEq(b1, rs.getBytes(1)); - assertEquals(new String(b2), rs.getString(2)); - assertEquals(new String(b3), rs.getString(3)); + assertEquals(new String(b2, "UTF-8"), rs.getString(2)); + assertEquals(new String(b3, "UTF-8"), rs.getString(3)); assertFalse(rs.next()); rs.close(); } @@ -306,6 +316,14 @@ public void tokens() throws SQLException { public void utf() throws SQLException { ResultSet rs = stat.executeQuery("select '" + utf01 + "','" + utf02 + "','" + utf03 + "','" + utf04 + "','" + utf05 + "','" + utf06 + "','" + utf07 + "','" + utf08 + "';"); + assertArrayEq(rs.getBytes(1), getUtf8Bytes(utf01)); + assertArrayEq(rs.getBytes(2), getUtf8Bytes(utf02)); + assertArrayEq(rs.getBytes(3), getUtf8Bytes(utf03)); + assertArrayEq(rs.getBytes(4), getUtf8Bytes(utf04)); + assertArrayEq(rs.getBytes(5), getUtf8Bytes(utf05)); + assertArrayEq(rs.getBytes(6), getUtf8Bytes(utf06)); + assertArrayEq(rs.getBytes(7), getUtf8Bytes(utf07)); + assertArrayEq(rs.getBytes(8), getUtf8Bytes(utf08)); assertEquals(rs.getString(1), utf01); assertEquals(rs.getString(2), utf02); assertEquals(rs.getString(3), utf03); @@ -327,6 +345,14 @@ public void utf() throws SQLException { prep.setString(8, utf08); rs = prep.executeQuery(); assertTrue(rs.next()); + assertArrayEq(rs.getBytes(1), getUtf8Bytes(utf01)); + assertArrayEq(rs.getBytes(2), getUtf8Bytes(utf02)); + assertArrayEq(rs.getBytes(3), getUtf8Bytes(utf03)); + assertArrayEq(rs.getBytes(4), getUtf8Bytes(utf04)); + assertArrayEq(rs.getBytes(5), getUtf8Bytes(utf05)); + assertArrayEq(rs.getBytes(6), getUtf8Bytes(utf06)); + assertArrayEq(rs.getBytes(7), getUtf8Bytes(utf07)); + assertArrayEq(rs.getBytes(8), getUtf8Bytes(utf08)); assertEquals(rs.getString(1), utf01); assertEquals(rs.getString(2), utf02); assertEquals(rs.getString(3), utf03);