sqlite - introduce executeBatch to reduce JNI calls overhead

GitOrigin-RevId: ef7d95f263e89c0bf843b86379f1fce4acbdb5fd

.idea/libraries/sqlite_native.xml

@@ -1,8 +1,8 @@
 <component name="libraryTable">
   <library name="sqlite-native" type="repository">
-    <properties include-transitive-deps="false" maven-id="org.sqlite:native:3.42.0-jb.0" />
+    <properties include-transitive-deps="false" maven-id="org.sqlite:native:3.42.0-jb.1" />
     <CLASSES>
-      <root url="jar://$MAVEN_REPOSITORY$/org/sqlite/native/3.42.0-jb.0/native-3.42.0-jb.0.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/sqlite/native/3.42.0-jb.1/native-3.42.0-jb.1.jar!/" />
     </CLASSES>
     <JAVADOC />
     <SOURCES />

platform/sqlite/Makefile

@@ -1,4 +1,4 @@
-VERSION := 3.42.0-jb.0
+VERSION := 3.42.0-jb.1
 
 .phony: archive-native install-native
 

platform/sqlite/build.sh

@@ -31,16 +31,18 @@ createSysRoot() {
 createSysRoot x86_64
 #createSysRoot aarch64
 
-OS=mac ARCH=aarch64 ./make.sh
-OS=mac ARCH=x86_64 ./make.sh
-OS=linux ARCH=aarch64 ./make.sh
-OS=linux ARCH=x86_64 ./make.sh
+OS=mac ARCH=aarch64 ./make.sh &
+OS=mac ARCH=x86_64 ./make.sh &
+OS=linux ARCH=aarch64 ./make.sh &
+OS=linux ARCH=x86_64 ./make.sh &
 
 nerdctl run --rm --platform linux/amd64 -v "$SCRIPT_DIR":/work --workdir=/work \
-  dockcross/windows-static-x64@sha256:8a53628099d9ce085303aa962120cd45b8f7a2b58b86fefc9797e9cbf43ce906 bash -c 'OS=win ARCH=x86_64 CC=gcc CROSS_PREFIX=x86_64-w64-mingw32.static- ./make.sh'
+  dockcross/windows-static-x64@sha256:8a53628099d9ce085303aa962120cd45b8f7a2b58b86fefc9797e9cbf43ce906 bash -c 'OS=win ARCH=x86_64 CC=gcc CROSS_PREFIX=x86_64-w64-mingw32.static- ./make.sh' &
 
 nerdctl run --rm --platform linux/amd64 -v "$SCRIPT_DIR":/work --workdir=/work \
-  dockcross/windows-arm64@sha256:345e3c190fbdf44384ce256dd09f5ca9ace831a5344308c8dccb36b78c382d95 bash -c 'OS=win ARCH=aarch64 CC=clang CROSS_PREFIX=aarch64-w64-mingw32- ./make.sh'
+  dockcross/windows-arm64@sha256:345e3c190fbdf44384ce256dd09f5ca9ace831a5344308c8dccb36b78c382d95 bash -c 'OS=win ARCH=aarch64 CC=clang CROSS_PREFIX=aarch64-w64-mingw32- ./make.sh' &
 
 nerdctl run --rm --platform linux/amd64 -v "$SCRIPT_DIR":/work --workdir=/work \
-  dockcross/linux-arm64-lts@sha256:3bbb880b002f6cc1b5332719bbb0c2ba5c646260140c2ff15bff8d63a16187ba bash -c 'OS=linux ARCH=aarch64 CC=gcc CROSS_PREFIX=aarch64-unknown-linux-gnu- ./make.sh'
+  dockcross/linux-arm64-lts@sha256:3bbb880b002f6cc1b5332719bbb0c2ba5c646260140c2ff15bff8d63a16187ba bash -c 'OS=linux ARCH=aarch64 CC=gcc CROSS_PREFIX=aarch64-unknown-linux-gnu- ./make.sh' &
+
+wait

platform/sqlite/sqlite/NativeDB.c

@@ -435,7 +435,6 @@ JNIEXPORT jlong JNICALL Java_org_jetbrains_sqlite_NativeDB_prepare_1utf8(
     return fromref(stmt);
 }
 
-
 JNIEXPORT jint JNICALL Java_org_jetbrains_sqlite_NativeDB__1exec_1utf8(
         JNIEnv *env, jobject this, jbyteArray sql)
 {
@@ -777,6 +776,48 @@ JNIEXPORT jint JNICALL Java_org_jetbrains_sqlite_NativeDB_bind_1int(
     return sqlite3_bind_int(toref(stmt), pos, v);
 }
 
+JNIEXPORT jint JNICALL Java_org_jetbrains_sqlite_NativeDB_executeBatch
+(
+  JNIEnv *env, jobject this, jlong statement, jint queryCount, jint paramCount, jintArray data
+)
+{
+  if (!statement)
+  {
+      throwex_stmt_finalized(env);
+      return SQLITE_MISUSE;
+  }
+
+  int batchIndex;
+  int position;
+  int status;
+  jint *body = (*env)->GetIntArrayElements(env, data, NULL);
+  for (batchIndex = 0; batchIndex < queryCount; batchIndex++) {
+    sqlite3_reset(toref(statement));
+
+    for (position = 0; position < paramCount; position++) {
+      status = sqlite3_bind_int(toref(statement), position + 1, body[(batchIndex * paramCount) + position]);
+      if (status != SQLITE_OK) {
+        (*env)->ReleaseIntArrayElements(env, data, body, JNI_ABORT);
+        throwex(env, this);
+        return SQLITE_MISUSE;
+      }
+    }
+
+    status = sqlite3_step(toref(statement));
+    if (status != SQLITE_DONE) {
+      (*env)->ReleaseIntArrayElements(env, data, body, JNI_ABORT);
+      sqlite3_reset(toref(statement));
+      throwex(env, this);
+      return SQLITE_MISUSE;
+    }
+  }
+  (*env)->ReleaseIntArrayElements(env, data, body, JNI_ABORT);
+  sqlite3_reset(toref(statement));
+  sqlite3_clear_bindings(toref(statement));
+
+  return SQLITE_OK;
+}
+
 JNIEXPORT jint JNICALL Java_org_jetbrains_sqlite_NativeDB_bind_1long(
         JNIEnv *env, jobject this, jlong stmt, jint pos, jlong v)
 {

platform/sqlite/sqlite/NativeDB.h

@@ -244,6 +244,8 @@ JNIEXPORT jint JNICALL Java_org_jetbrains_sqlite_NativeDB_bind_1null
 JNIEXPORT jint JNICALL Java_org_jetbrains_sqlite_NativeDB_bind_1int
   (JNIEnv *, jobject, jlong, jint, jint);
 
+JNIEXPORT jint JNICALL Java_org_jetbrains_sqlite_NativeDB_executeBatch(JNIEnv *, jobject, jlong, jint, jint, jintArray);
+
 /*
  * Class:     org_jetbrains_sqlite_NativeDB
  * Method:    bind_long

platform/sqlite/src/org/jetbrains/sqlite/IntBinder.kt

@@ -24,6 +24,15 @@ class IntBinder(paramCount: Int, batchCountHint: Int = 1) : BaseBinder(paramCoun
     batch[batchPosition + 2] = v3
   }
 
+  internal fun ensureCapacity(count: Int) {
+    val expectedSize = count * paramCount
+    if (expectedSize > batch.size) {
+      val newBatch = IntArray(expectedSize)
+      batch.copyInto(newBatch)
+      this.batch = newBatch
+    }
+  }
+
   override fun bindParams(pointer: Long, db: SqliteDb) {
     assert(batchQueryCount == 0)
     for ((index, value) in batch.withIndex()) {
@@ -47,22 +56,12 @@ class IntBinder(paramCount: Int, batchCountHint: Int = 1) : BaseBinder(paramCoun
   }
 
   override fun executeBatch(pointer: Long, db: NativeDB) {
-    for (batchIndex in 0 until batchQueryCount) {
-      db.reset(pointer)
-      for (index in 0 until paramCount) {
-        val status = db.bind_int(pointer, index + 1, batch[batchIndex * paramCount + index]) and 0xFF
-        if (status != SqliteCodes.SQLITE_OK) {
-          throw db.newException(status)
-        }
-      }
-
-      stepInBatch(statementPointer = pointer, db = db, batchIndex = batchIndex)
-    }
+    db.executeBatch(statementPointer = pointer, queryCount = batchQueryCount, paramCount = paramCount, data = batch)
   }
 }
 
 class LongBinder(paramCount: Int, batchCountHint: Int = 1) : BaseBinder(paramCount) {
-  private var batch: LongArray = LongArray(paramCount * batchCountHint)
+  private var batch = LongArray(paramCount * batchCountHint)
 
   fun bind(v1: Long) {
     assert(paramCount == 1)

platform/sqlite/src/org/jetbrains/sqlite/NativeDB.kt

@@ -177,6 +177,8 @@ internal class NativeDB : SqliteDb() {
   @Synchronized
   external override fun bind_int(stmt: Long, oneBasedColumnIndex: Int, v: Int): Int
 
+  external fun executeBatch(statementPointer: Long, queryCount: Int, paramCount: Int, data: IntArray)
+
   @Synchronized
   external override fun bind_long(stmt: Long, oneBasedColumnIndex: Int, v: Long): Int
 

platform/sqlite/src/org/jetbrains/sqlite/SafeStatementPointer.kt

@@ -5,11 +5,7 @@ package org.jetbrains.sqlite
  * A class for safely wrapping calls to a native pointer to a statement, ensuring no other thread
  * has access to the pointer while it is run
  */
-internal class SafeStatementPointer(
-  private val connection: SqliteConnection,
-  @JvmField
-  internal val pointer: Long,
-) {
+internal class SafeStatementPointer(private val connection: SqliteConnection, @JvmField internal val pointer: Long) {
   /**
    * Check whether this pointer has been closed
    */
@@ -18,7 +14,6 @@ internal class SafeStatementPointer(
     private set
 
   internal fun close(db: SqliteDb) {
-    // if this is already closed, return or throw the previous result
     if (isClosed) {
       return
     }
@@ -28,7 +23,6 @@ internal class SafeStatementPointer(
       if (status != SqliteCodes.SQLITE_OK && status != SqliteCodes.SQLITE_MISUSE) {
         throw db.newException(status)
       }
-      return
     }
     finally {
       isClosed = true

platform/sqlite/src/org/jetbrains/sqlite/SqliteIntPreparedStatement.kt

@@ -1,44 +1,26 @@
 // Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
 package org.jetbrains.sqlite
 
-class SqliteIntPreparedStatement internal constructor(private val connection: SqliteConnection,
-                                                      private val sql: String) : SqliteStatement {
-  private var pointer: SafeStatementPointer?
-
-  private var batchPosition = 0
-  private var batch: IntArray
-
-  private val columnCount: Int
-  private val paramCount: Int
-  private var batchQueryCount: Int
+class SqliteIntPreparedStatement internal constructor(private val connection: SqliteConnection, private val sql: String) : SqliteStatement {
+  private val pointer: SafeStatementPointer
+  val binder: IntBinder
 
   init {
     lateinit var pointer: SafeStatementPointer
-    var columnCount = 0
     var paramCount = 0
     connection.useDb { db ->
       pointer = db.addStatement(SafeStatementPointer(connection = connection, pointer = db.prepare_utf8(sql.encodeToByteArray())))
-      columnCount = db.column_count(pointer.pointer)
       paramCount = db.bind_parameter_count(pointer.pointer)
     }
-
-    this.pointer = pointer
-    this.columnCount = columnCount
-    this.paramCount = paramCount
-
     require(paramCount > 0)
 
-    batchQueryCount = 0
+    this.pointer = pointer
     // pre-allocate twice more than needed to reduce overall allocations
-    batch = IntArray(paramCount * 2)
-    batchPosition = 0
+    binder = IntBinder(paramCount, paramCount * 2)
   }
 
   override fun close() {
     connection.useDb { db ->
-      val pointer = pointer ?: return
-      this.pointer = null
-      batchPosition = 0
       pointer.close(db)
     }
   }
@@ -46,59 +28,26 @@ class SqliteIntPreparedStatement internal constructor(private val connection: Sq
   override fun toString(): String = sql
 
   fun ensureCapacity(count: Int) {
-    val expectedSize = count * paramCount
-    if (expectedSize > batch.size) {
-      val newBatch = IntArray(expectedSize)
-      batch.copyInto(newBatch)
-      this.batch = newBatch
-    }
+    binder.ensureCapacity(count)
   }
 
   fun addBatch() {
-    batchPosition += paramCount
-    batchQueryCount++
-    val batch = batch
-    if ((batchPosition + paramCount) > batch.size) {
-      val newBatch = IntArray(batch.size * 2)
-      batch.copyInto(newBatch)
-      this.batch = newBatch
-    }
+    binder.addBatch()
   }
 
   override fun executeBatch() {
-    if (batchQueryCount == 0) {
+    if (binder.batchQueryCount == 0) {
       return
     }
 
     try {
       connection.useDb { db ->
-        val pointer = pointer ?: throw IllegalStateException("The statement pointer is closed")
         pointer.ensureOpen()
-        for (batchIndex in 0 until batchQueryCount) {
-          db.reset(pointer.pointer)
-          for (position in 0 until paramCount) {
-            val status = db.bind_int(pointer.pointer, position + 1, batch[batchIndex * paramCount + position]) and 0xFF
-            if (status != SqliteCodes.SQLITE_OK) {
-              throw db.newException(status)
-            }
-          }
-
-          stepInBatch(statementPointer = pointer.pointer, db = db, batchIndex = batchIndex)
-        }
-        db.reset(pointer.pointer)
+        binder.executeBatch(pointer.pointer, db)
       }
     }
     finally {
-      clearBatch()
+      binder.clearBatch()
     }
   }
-
-  private fun clearBatch() {
-    batchPosition = 0
-    batchQueryCount = 0
-  }
-
-  fun setInt(position: Int, value: Int) {
-    batch[(batchPosition + position) - 1] = value
-  }
 }

platform/sqlite/test/org/jetbrains/sqlite/SqliteTest.kt

@@ -7,6 +7,7 @@ import org.assertj.core.api.Assertions.assertThat
 import org.junit.jupiter.api.AfterEach
 import org.junit.jupiter.api.BeforeEach
 import org.junit.jupiter.api.Test
+import java.util.*
 
 class SqliteTest {
   private lateinit var connection: SqliteConnection
@@ -54,6 +55,84 @@ class SqliteTest {
     }
   }
 
+  @Test
+  fun intPreparedStatement() {
+    connection.execute("""
+      create table log (
+        a integer not null,
+        b integer not null,
+        c integer not null
+      ) strict
+    """)
+
+    val rowCount = 100
+    val columnCount = 3
+
+    val statementCollection = StatementCollection(connection)
+    try {
+      val statement = statementCollection.prepareIntStatement("insert into log(a, b, c) values(?, ?, ?)")
+      val random = Random(42)
+      repeat(rowCount) {
+        statement.binder.bind(random.nextInt(), random.nextInt(), random.nextInt())
+        statement.addBatch()
+      }
+      statement.executeBatch()
+    }
+    finally {
+      statementCollection.close(true)
+    }
+
+    connection.prepareStatement("select a, b, c from log order by rowid", EmptyBinder).use { statement ->
+      val random = Random(42)
+      val resultSet = statement.executeQuery()
+      var count = 0
+      while (resultSet.next()) {
+        count++
+        repeat(columnCount) {
+          assertThat(resultSet.getInt(it)).isEqualTo(random.nextInt())
+        }
+      }
+      assertThat(count == rowCount)
+    }
+  }
+
+  @Test
+  fun intBinder() {
+    connection.execute("""
+      create table log (
+        a integer not null,
+        b integer not null,
+        c integer not null
+      ) strict
+    """)
+
+    val rowCount = 100
+    val columnCount = 3
+
+    val binder = IntBinder(3)
+    connection.prepareStatement("insert into log(a, b, c) values(?, ?, ?)", binder).use { statement ->
+      val random = Random(42)
+      repeat(rowCount) {
+        binder.bind(random.nextInt(), random.nextInt(), random.nextInt())
+        binder.addBatch()
+      }
+      statement.executeBatch()
+    }
+
+    connection.prepareStatement("select a, b, c from log order by rowid", EmptyBinder).use { statement ->
+      val random = Random(42)
+      val resultSet = statement.executeQuery()
+      var count = 0
+      while (resultSet.next()) {
+        count++
+        repeat(columnCount) {
+          assertThat(resultSet.getInt(it)).isEqualTo(random.nextInt())
+        }
+      }
+      assertThat(count == rowCount)
+    }
+  }
+
   @Test
   fun longBinder() {
     connection.execute("""
@@ -63,10 +142,7 @@ class SqliteTest {
       ) strict
     """)
 
-    connection.prepareStatement("""
-      insert into log(commitId, authorTime) 
-      values(?, ?)
-    """, LongBinder(2)).use { statement ->
+    connection.prepareStatement("insert into log(commitId, authorTime) values(?, ?)", LongBinder(2)).use { statement ->
       statement.binder.bind(12, 42)
       statement.binder.addBatch()
 

platform/vcs-log/impl/src/com/intellij/vcs/log/data/index/SqliteVcsLogStorageBackend.kt

@@ -696,25 +696,21 @@ private class SqliteVcsLogWriter(private val connection: SqliteConnection, priva
 
   private fun putParents(commitId: Int, root: VirtualFile, parents: List<Hash>) {
     // clear old if any
-    parentDeleteStatement.setInt(1, commitId)
+    parentDeleteStatement.binder.bind(commitId)
     parentDeleteStatement.addBatch()
 
     for (parent in parents) {
-      parentStatement.setInt(1, commitId)
-      parentStatement.setInt(2, storage.getCommitIndex(parent, root))
+      parentStatement.binder.bind(commitId, storage.getCommitIndex(parent, root))
       parentStatement.addBatch()
     }
   }
 
   private fun putRename(parent: Int, child: Int, renames: IntArray) {
-    renameDeleteStatement.setInt(1, parent)
-    renameDeleteStatement.setInt(2, child)
+    renameDeleteStatement.binder.bind(parent, child)
     renameDeleteStatement.addBatch()
 
     for (rename in renames) {
-      renameStatement.setInt(1, parent)
-      renameStatement.setInt(2, child)
-      renameStatement.setInt(3, rename)
+      renameStatement.binder.bind(parent, child, rename)
       renameStatement.addBatch()
     }
   }
@@ -727,9 +723,7 @@ private class SqliteVcsLogWriter(private val connection: SqliteConnection, priva
       val changes = entry.value
 
       for (change in changes) {
-        changeStatement.setInt(1, commitId)
-        changeStatement.setInt(2, pathId)
-        changeStatement.setInt(3, change.id.toInt())
+        changeStatement.binder.bind(commitId, pathId, change.id.toInt())
         changeStatement.addBatch()
       }
     }