From b5ad3a21c5b0764270c3c8e2df751b0acb10d12b Mon Sep 17 00:00:00 2001 From: Vladimir Krivosheev Date: Wed, 21 Jun 2023 13:28:02 +0200 Subject: [PATCH] sqlite - introduce executeBatch to reduce JNI calls overhead GitOrigin-RevId: ef7d95f263e89c0bf843b86379f1fce4acbdb5fd --- .idea/libraries/sqlite_native.xml | 4 +- platform/sqlite/Makefile | 2 +- platform/sqlite/build.sh | 16 ++-- platform/sqlite/sqlite/NativeDB.c | 43 +++++++++- platform/sqlite/sqlite/NativeDB.h | 2 + .../src/org/jetbrains/sqlite/IntBinder.kt | 23 +++-- .../src/org/jetbrains/sqlite/NativeDB.kt | 2 + .../jetbrains/sqlite/SafeStatementPointer.kt | 8 +- .../sqlite/SqliteIntPreparedStatement.kt | 71 +++------------- .../test/org/jetbrains/sqlite/SqliteTest.kt | 84 ++++++++++++++++++- .../data/index/SqliteVcsLogStorageBackend.kt | 16 ++-- 11 files changed, 165 insertions(+), 106 deletions(-) diff --git a/.idea/libraries/sqlite_native.xml b/.idea/libraries/sqlite_native.xml index bfb0fff1ffa6..bb9b972a2450 100644 --- a/.idea/libraries/sqlite_native.xml +++ b/.idea/libraries/sqlite_native.xml @@ -1,8 +1,8 @@ - + - + diff --git a/platform/sqlite/Makefile b/platform/sqlite/Makefile index 165b4093a270..ff2829504f15 100644 --- a/platform/sqlite/Makefile +++ b/platform/sqlite/Makefile @@ -1,4 +1,4 @@ -VERSION := 3.42.0-jb.0 +VERSION := 3.42.0-jb.1 .phony: archive-native install-native diff --git a/platform/sqlite/build.sh b/platform/sqlite/build.sh index 6cf3597d36a2..dce3b455a52b 100755 --- a/platform/sqlite/build.sh +++ b/platform/sqlite/build.sh @@ -31,16 +31,18 @@ createSysRoot() { createSysRoot x86_64 #createSysRoot aarch64 -OS=mac ARCH=aarch64 ./make.sh -OS=mac ARCH=x86_64 ./make.sh -OS=linux ARCH=aarch64 ./make.sh -OS=linux ARCH=x86_64 ./make.sh +OS=mac ARCH=aarch64 ./make.sh & +OS=mac ARCH=x86_64 ./make.sh & +OS=linux ARCH=aarch64 ./make.sh & +OS=linux ARCH=x86_64 ./make.sh & nerdctl run --rm --platform linux/amd64 -v "$SCRIPT_DIR":/work --workdir=/work \ - dockcross/windows-static-x64@sha256:8a53628099d9ce085303aa962120cd45b8f7a2b58b86fefc9797e9cbf43ce906 bash -c 'OS=win ARCH=x86_64 CC=gcc CROSS_PREFIX=x86_64-w64-mingw32.static- ./make.sh' + dockcross/windows-static-x64@sha256:8a53628099d9ce085303aa962120cd45b8f7a2b58b86fefc9797e9cbf43ce906 bash -c 'OS=win ARCH=x86_64 CC=gcc CROSS_PREFIX=x86_64-w64-mingw32.static- ./make.sh' & nerdctl run --rm --platform linux/amd64 -v "$SCRIPT_DIR":/work --workdir=/work \ - dockcross/windows-arm64@sha256:345e3c190fbdf44384ce256dd09f5ca9ace831a5344308c8dccb36b78c382d95 bash -c 'OS=win ARCH=aarch64 CC=clang CROSS_PREFIX=aarch64-w64-mingw32- ./make.sh' + dockcross/windows-arm64@sha256:345e3c190fbdf44384ce256dd09f5ca9ace831a5344308c8dccb36b78c382d95 bash -c 'OS=win ARCH=aarch64 CC=clang CROSS_PREFIX=aarch64-w64-mingw32- ./make.sh' & nerdctl run --rm --platform linux/amd64 -v "$SCRIPT_DIR":/work --workdir=/work \ - dockcross/linux-arm64-lts@sha256:3bbb880b002f6cc1b5332719bbb0c2ba5c646260140c2ff15bff8d63a16187ba bash -c 'OS=linux ARCH=aarch64 CC=gcc CROSS_PREFIX=aarch64-unknown-linux-gnu- ./make.sh' \ No newline at end of file + dockcross/linux-arm64-lts@sha256:3bbb880b002f6cc1b5332719bbb0c2ba5c646260140c2ff15bff8d63a16187ba bash -c 'OS=linux ARCH=aarch64 CC=gcc CROSS_PREFIX=aarch64-unknown-linux-gnu- ./make.sh' & + +wait \ No newline at end of file diff --git a/platform/sqlite/sqlite/NativeDB.c b/platform/sqlite/sqlite/NativeDB.c index f4187998b49b..fb4bbaa06d9e 100644 --- a/platform/sqlite/sqlite/NativeDB.c +++ b/platform/sqlite/sqlite/NativeDB.c @@ -435,7 +435,6 @@ JNIEXPORT jlong JNICALL Java_org_jetbrains_sqlite_NativeDB_prepare_1utf8( return fromref(stmt); } - JNIEXPORT jint JNICALL Java_org_jetbrains_sqlite_NativeDB__1exec_1utf8( JNIEnv *env, jobject this, jbyteArray sql) { @@ -777,6 +776,48 @@ JNIEXPORT jint JNICALL Java_org_jetbrains_sqlite_NativeDB_bind_1int( return sqlite3_bind_int(toref(stmt), pos, v); } +JNIEXPORT jint JNICALL Java_org_jetbrains_sqlite_NativeDB_executeBatch +( + JNIEnv *env, jobject this, jlong statement, jint queryCount, jint paramCount, jintArray data +) +{ + if (!statement) + { + throwex_stmt_finalized(env); + return SQLITE_MISUSE; + } + + int batchIndex; + int position; + int status; + jint *body = (*env)->GetIntArrayElements(env, data, NULL); + for (batchIndex = 0; batchIndex < queryCount; batchIndex++) { + sqlite3_reset(toref(statement)); + + for (position = 0; position < paramCount; position++) { + status = sqlite3_bind_int(toref(statement), position + 1, body[(batchIndex * paramCount) + position]); + if (status != SQLITE_OK) { + (*env)->ReleaseIntArrayElements(env, data, body, JNI_ABORT); + throwex(env, this); + return SQLITE_MISUSE; + } + } + + status = sqlite3_step(toref(statement)); + if (status != SQLITE_DONE) { + (*env)->ReleaseIntArrayElements(env, data, body, JNI_ABORT); + sqlite3_reset(toref(statement)); + throwex(env, this); + return SQLITE_MISUSE; + } + } + (*env)->ReleaseIntArrayElements(env, data, body, JNI_ABORT); + sqlite3_reset(toref(statement)); + sqlite3_clear_bindings(toref(statement)); + + return SQLITE_OK; +} + JNIEXPORT jint JNICALL Java_org_jetbrains_sqlite_NativeDB_bind_1long( JNIEnv *env, jobject this, jlong stmt, jint pos, jlong v) { diff --git a/platform/sqlite/sqlite/NativeDB.h b/platform/sqlite/sqlite/NativeDB.h index 093bf6c4a9d7..6a70e7e4a671 100644 --- a/platform/sqlite/sqlite/NativeDB.h +++ b/platform/sqlite/sqlite/NativeDB.h @@ -244,6 +244,8 @@ JNIEXPORT jint JNICALL Java_org_jetbrains_sqlite_NativeDB_bind_1null JNIEXPORT jint JNICALL Java_org_jetbrains_sqlite_NativeDB_bind_1int (JNIEnv *, jobject, jlong, jint, jint); +JNIEXPORT jint JNICALL Java_org_jetbrains_sqlite_NativeDB_executeBatch(JNIEnv *, jobject, jlong, jint, jint, jintArray); + /* * Class: org_jetbrains_sqlite_NativeDB * Method: bind_long diff --git a/platform/sqlite/src/org/jetbrains/sqlite/IntBinder.kt b/platform/sqlite/src/org/jetbrains/sqlite/IntBinder.kt index ab5c3cac6e77..c9c3c5a9bd3b 100644 --- a/platform/sqlite/src/org/jetbrains/sqlite/IntBinder.kt +++ b/platform/sqlite/src/org/jetbrains/sqlite/IntBinder.kt @@ -24,6 +24,15 @@ class IntBinder(paramCount: Int, batchCountHint: Int = 1) : BaseBinder(paramCoun batch[batchPosition + 2] = v3 } + internal fun ensureCapacity(count: Int) { + val expectedSize = count * paramCount + if (expectedSize > batch.size) { + val newBatch = IntArray(expectedSize) + batch.copyInto(newBatch) + this.batch = newBatch + } + } + override fun bindParams(pointer: Long, db: SqliteDb) { assert(batchQueryCount == 0) for ((index, value) in batch.withIndex()) { @@ -47,22 +56,12 @@ class IntBinder(paramCount: Int, batchCountHint: Int = 1) : BaseBinder(paramCoun } override fun executeBatch(pointer: Long, db: NativeDB) { - for (batchIndex in 0 until batchQueryCount) { - db.reset(pointer) - for (index in 0 until paramCount) { - val status = db.bind_int(pointer, index + 1, batch[batchIndex * paramCount + index]) and 0xFF - if (status != SqliteCodes.SQLITE_OK) { - throw db.newException(status) - } - } - - stepInBatch(statementPointer = pointer, db = db, batchIndex = batchIndex) - } + db.executeBatch(statementPointer = pointer, queryCount = batchQueryCount, paramCount = paramCount, data = batch) } } class LongBinder(paramCount: Int, batchCountHint: Int = 1) : BaseBinder(paramCount) { - private var batch: LongArray = LongArray(paramCount * batchCountHint) + private var batch = LongArray(paramCount * batchCountHint) fun bind(v1: Long) { assert(paramCount == 1) diff --git a/platform/sqlite/src/org/jetbrains/sqlite/NativeDB.kt b/platform/sqlite/src/org/jetbrains/sqlite/NativeDB.kt index 8a9558cf6f8c..bd9478df5bc0 100644 --- a/platform/sqlite/src/org/jetbrains/sqlite/NativeDB.kt +++ b/platform/sqlite/src/org/jetbrains/sqlite/NativeDB.kt @@ -177,6 +177,8 @@ internal class NativeDB : SqliteDb() { @Synchronized external override fun bind_int(stmt: Long, oneBasedColumnIndex: Int, v: Int): Int + external fun executeBatch(statementPointer: Long, queryCount: Int, paramCount: Int, data: IntArray) + @Synchronized external override fun bind_long(stmt: Long, oneBasedColumnIndex: Int, v: Long): Int diff --git a/platform/sqlite/src/org/jetbrains/sqlite/SafeStatementPointer.kt b/platform/sqlite/src/org/jetbrains/sqlite/SafeStatementPointer.kt index 1759eeb29dd2..f829aa4dab89 100644 --- a/platform/sqlite/src/org/jetbrains/sqlite/SafeStatementPointer.kt +++ b/platform/sqlite/src/org/jetbrains/sqlite/SafeStatementPointer.kt @@ -5,11 +5,7 @@ package org.jetbrains.sqlite * A class for safely wrapping calls to a native pointer to a statement, ensuring no other thread * has access to the pointer while it is run */ -internal class SafeStatementPointer( - private val connection: SqliteConnection, - @JvmField - internal val pointer: Long, -) { +internal class SafeStatementPointer(private val connection: SqliteConnection, @JvmField internal val pointer: Long) { /** * Check whether this pointer has been closed */ @@ -18,7 +14,6 @@ internal class SafeStatementPointer( private set internal fun close(db: SqliteDb) { - // if this is already closed, return or throw the previous result if (isClosed) { return } @@ -28,7 +23,6 @@ internal class SafeStatementPointer( if (status != SqliteCodes.SQLITE_OK && status != SqliteCodes.SQLITE_MISUSE) { throw db.newException(status) } - return } finally { isClosed = true diff --git a/platform/sqlite/src/org/jetbrains/sqlite/SqliteIntPreparedStatement.kt b/platform/sqlite/src/org/jetbrains/sqlite/SqliteIntPreparedStatement.kt index 81c92104bb76..356026012929 100644 --- a/platform/sqlite/src/org/jetbrains/sqlite/SqliteIntPreparedStatement.kt +++ b/platform/sqlite/src/org/jetbrains/sqlite/SqliteIntPreparedStatement.kt @@ -1,44 +1,26 @@ // Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package org.jetbrains.sqlite -class SqliteIntPreparedStatement internal constructor(private val connection: SqliteConnection, - private val sql: String) : SqliteStatement { - private var pointer: SafeStatementPointer? - - private var batchPosition = 0 - private var batch: IntArray - - private val columnCount: Int - private val paramCount: Int - private var batchQueryCount: Int +class SqliteIntPreparedStatement internal constructor(private val connection: SqliteConnection, private val sql: String) : SqliteStatement { + private val pointer: SafeStatementPointer + val binder: IntBinder init { lateinit var pointer: SafeStatementPointer - var columnCount = 0 var paramCount = 0 connection.useDb { db -> pointer = db.addStatement(SafeStatementPointer(connection = connection, pointer = db.prepare_utf8(sql.encodeToByteArray()))) - columnCount = db.column_count(pointer.pointer) paramCount = db.bind_parameter_count(pointer.pointer) } - - this.pointer = pointer - this.columnCount = columnCount - this.paramCount = paramCount - require(paramCount > 0) - batchQueryCount = 0 + this.pointer = pointer // pre-allocate twice more than needed to reduce overall allocations - batch = IntArray(paramCount * 2) - batchPosition = 0 + binder = IntBinder(paramCount, paramCount * 2) } override fun close() { connection.useDb { db -> - val pointer = pointer ?: return - this.pointer = null - batchPosition = 0 pointer.close(db) } } @@ -46,59 +28,26 @@ class SqliteIntPreparedStatement internal constructor(private val connection: Sq override fun toString(): String = sql fun ensureCapacity(count: Int) { - val expectedSize = count * paramCount - if (expectedSize > batch.size) { - val newBatch = IntArray(expectedSize) - batch.copyInto(newBatch) - this.batch = newBatch - } + binder.ensureCapacity(count) } fun addBatch() { - batchPosition += paramCount - batchQueryCount++ - val batch = batch - if ((batchPosition + paramCount) > batch.size) { - val newBatch = IntArray(batch.size * 2) - batch.copyInto(newBatch) - this.batch = newBatch - } + binder.addBatch() } override fun executeBatch() { - if (batchQueryCount == 0) { + if (binder.batchQueryCount == 0) { return } try { connection.useDb { db -> - val pointer = pointer ?: throw IllegalStateException("The statement pointer is closed") pointer.ensureOpen() - for (batchIndex in 0 until batchQueryCount) { - db.reset(pointer.pointer) - for (position in 0 until paramCount) { - val status = db.bind_int(pointer.pointer, position + 1, batch[batchIndex * paramCount + position]) and 0xFF - if (status != SqliteCodes.SQLITE_OK) { - throw db.newException(status) - } - } - - stepInBatch(statementPointer = pointer.pointer, db = db, batchIndex = batchIndex) - } - db.reset(pointer.pointer) + binder.executeBatch(pointer.pointer, db) } } finally { - clearBatch() + binder.clearBatch() } } - - private fun clearBatch() { - batchPosition = 0 - batchQueryCount = 0 - } - - fun setInt(position: Int, value: Int) { - batch[(batchPosition + position) - 1] = value - } } diff --git a/platform/sqlite/test/org/jetbrains/sqlite/SqliteTest.kt b/platform/sqlite/test/org/jetbrains/sqlite/SqliteTest.kt index 656d3b0cc38a..8516c381353a 100644 --- a/platform/sqlite/test/org/jetbrains/sqlite/SqliteTest.kt +++ b/platform/sqlite/test/org/jetbrains/sqlite/SqliteTest.kt @@ -7,6 +7,7 @@ import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test +import java.util.* class SqliteTest { private lateinit var connection: SqliteConnection @@ -54,6 +55,84 @@ class SqliteTest { } } + @Test + fun intPreparedStatement() { + connection.execute(""" + create table log ( + a integer not null, + b integer not null, + c integer not null + ) strict + """) + + val rowCount = 100 + val columnCount = 3 + + val statementCollection = StatementCollection(connection) + try { + val statement = statementCollection.prepareIntStatement("insert into log(a, b, c) values(?, ?, ?)") + val random = Random(42) + repeat(rowCount) { + statement.binder.bind(random.nextInt(), random.nextInt(), random.nextInt()) + statement.addBatch() + } + statement.executeBatch() + } + finally { + statementCollection.close(true) + } + + connection.prepareStatement("select a, b, c from log order by rowid", EmptyBinder).use { statement -> + val random = Random(42) + val resultSet = statement.executeQuery() + var count = 0 + while (resultSet.next()) { + count++ + repeat(columnCount) { + assertThat(resultSet.getInt(it)).isEqualTo(random.nextInt()) + } + } + assertThat(count == rowCount) + } + } + + @Test + fun intBinder() { + connection.execute(""" + create table log ( + a integer not null, + b integer not null, + c integer not null + ) strict + """) + + val rowCount = 100 + val columnCount = 3 + + val binder = IntBinder(3) + connection.prepareStatement("insert into log(a, b, c) values(?, ?, ?)", binder).use { statement -> + val random = Random(42) + repeat(rowCount) { + binder.bind(random.nextInt(), random.nextInt(), random.nextInt()) + binder.addBatch() + } + statement.executeBatch() + } + + connection.prepareStatement("select a, b, c from log order by rowid", EmptyBinder).use { statement -> + val random = Random(42) + val resultSet = statement.executeQuery() + var count = 0 + while (resultSet.next()) { + count++ + repeat(columnCount) { + assertThat(resultSet.getInt(it)).isEqualTo(random.nextInt()) + } + } + assertThat(count == rowCount) + } + } + @Test fun longBinder() { connection.execute(""" @@ -63,10 +142,7 @@ class SqliteTest { ) strict """) - connection.prepareStatement(""" - insert into log(commitId, authorTime) - values(?, ?) - """, LongBinder(2)).use { statement -> + connection.prepareStatement("insert into log(commitId, authorTime) values(?, ?)", LongBinder(2)).use { statement -> statement.binder.bind(12, 42) statement.binder.addBatch() diff --git a/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/SqliteVcsLogStorageBackend.kt b/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/SqliteVcsLogStorageBackend.kt index af8e5468f91b..f5e52eefc52b 100644 --- a/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/SqliteVcsLogStorageBackend.kt +++ b/platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/SqliteVcsLogStorageBackend.kt @@ -696,25 +696,21 @@ private class SqliteVcsLogWriter(private val connection: SqliteConnection, priva private fun putParents(commitId: Int, root: VirtualFile, parents: List) { // clear old if any - parentDeleteStatement.setInt(1, commitId) + parentDeleteStatement.binder.bind(commitId) parentDeleteStatement.addBatch() for (parent in parents) { - parentStatement.setInt(1, commitId) - parentStatement.setInt(2, storage.getCommitIndex(parent, root)) + parentStatement.binder.bind(commitId, storage.getCommitIndex(parent, root)) parentStatement.addBatch() } } private fun putRename(parent: Int, child: Int, renames: IntArray) { - renameDeleteStatement.setInt(1, parent) - renameDeleteStatement.setInt(2, child) + renameDeleteStatement.binder.bind(parent, child) renameDeleteStatement.addBatch() for (rename in renames) { - renameStatement.setInt(1, parent) - renameStatement.setInt(2, child) - renameStatement.setInt(3, rename) + renameStatement.binder.bind(parent, child, rename) renameStatement.addBatch() } } @@ -727,9 +723,7 @@ private class SqliteVcsLogWriter(private val connection: SqliteConnection, priva val changes = entry.value for (change in changes) { - changeStatement.setInt(1, commitId) - changeStatement.setInt(2, pathId) - changeStatement.setInt(3, change.id.toInt()) + changeStatement.binder.bind(commitId, pathId, change.id.toInt()) changeStatement.addBatch() } }