From 89eeb76c465b29c65bf31d9c3ff802e3d8859941 Mon Sep 17 00:00:00 2001 From: hrj Date: Mon, 12 Apr 2021 07:00:53 +0530 Subject: [PATCH 1/9] Use a fork join pool executor with parallelism = 4 --- src/main/scala/lc/server/Server.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/scala/lc/server/Server.scala b/src/main/scala/lc/server/Server.scala index e1d048a..7d45f03 100644 --- a/src/main/scala/lc/server/Server.scala +++ b/src/main/scala/lc/server/Server.scala @@ -14,6 +14,7 @@ class Server(port: Int) { implicit val formats: DefaultFormats.type = DefaultFormats val server: HttpServer = HttpServer.create(new InetSocketAddress(port), 32) + server.setExecutor(new java.util.concurrent.ForkJoinPool(Runtime.getRuntime().availableProcessors()*4)) private def getRequestJson(ex: HttpExchange): JValue = { val requestBody = ex.getRequestBody From 6fd34c16ac03122e2d0c20d4439b92af25e7bc7d Mon Sep 17 00:00:00 2001 From: hrj Date: Mon, 12 Apr 2021 07:06:44 +0530 Subject: [PATCH 2/9] optimise string operations --- src/main/scala/lc/server/Server.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/scala/lc/server/Server.scala b/src/main/scala/lc/server/Server.scala index 7d45f03..498c3ad 100644 --- a/src/main/scala/lc/server/Server.scala +++ b/src/main/scala/lc/server/Server.scala @@ -19,15 +19,15 @@ class Server(port: Int) { private def getRequestJson(ex: HttpExchange): JValue = { val requestBody = ex.getRequestBody val bytes = requestBody.readAllBytes - val string = bytes.map(_.toChar).mkString + val string = new String(bytes) parse(string) } + private val eqPattern = java.util.regex.Pattern.compile("=") private def getPathParameter(ex: HttpExchange): String = { try { - val uri = ex.getRequestURI.toString - val param = uri.split("\\?")(1) - param.split("=")(1) + val query = ex.getRequestURI.getQuery + eqPattern.split(query)(1) } catch { case exception: ArrayIndexOutOfBoundsException => { println(exception.getStackTrace) From fb400a6aea2f290a84f3d841fc39ca018bd26922 Mon Sep 17 00:00:00 2001 From: hrj Date: Mon, 12 Apr 2021 07:10:00 +0530 Subject: [PATCH 3/9] remove debug print --- src/main/scala/lc/core/captcha.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/lc/core/captcha.scala b/src/main/scala/lc/core/captcha.scala index 15d32a9..9ccd39a 100644 --- a/src/main/scala/lc/core/captcha.scala +++ b/src/main/scala/lc/core/captcha.scala @@ -49,7 +49,7 @@ object Captcha { val token = if (rs.next()) { rs.getInt("token") } - println("Added new challenge: " + token.toString) + // println("Added new challenge: " + token.toString) token.asInstanceOf[Int] } From 43e1a379ac1eff32e947f9abd0dcb784a39e8ef2 Mon Sep 17 00:00:00 2001 From: hrj Date: Mon, 12 Apr 2021 07:22:53 +0530 Subject: [PATCH 4/9] use an unbounded thread pool mimics the performance with JLHttpServer --- src/main/scala/lc/server/Server.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/lc/server/Server.scala b/src/main/scala/lc/server/Server.scala index 498c3ad..26089f2 100644 --- a/src/main/scala/lc/server/Server.scala +++ b/src/main/scala/lc/server/Server.scala @@ -14,7 +14,7 @@ class Server(port: Int) { implicit val formats: DefaultFormats.type = DefaultFormats val server: HttpServer = HttpServer.create(new InetSocketAddress(port), 32) - server.setExecutor(new java.util.concurrent.ForkJoinPool(Runtime.getRuntime().availableProcessors()*4)) + server.setExecutor(java.util.concurrent.Executors.newCachedThreadPool()) private def getRequestJson(ex: HttpExchange): JValue = { val requestBody = ex.getRequestBody From 8840a13a8f5ecb22e36aba2eaf027bcc51f907d1 Mon Sep 17 00:00:00 2001 From: hrj Date: Mon, 12 Apr 2021 09:25:33 +0530 Subject: [PATCH 5/9] minor: moved blob variable to inner scope --- src/main/scala/lc/core/captcha.scala | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/main/scala/lc/core/captcha.scala b/src/main/scala/lc/core/captcha.scala index 9ccd39a..146761b 100644 --- a/src/main/scala/lc/core/captcha.scala +++ b/src/main/scala/lc/core/captcha.scala @@ -5,20 +5,17 @@ import java.util.UUID import java.io.ByteArrayInputStream import lc.database.Statements import lc.core.CaptchaProviders -import java.sql.Blob - object Captcha { def getCaptcha(id: Id): Array[Byte] = { var image: Array[Byte] = null - var blob: Blob = null try { val imagePstmt = Statements.tlStmts.get.imagePstmt imagePstmt.setString(1, id.id) val rs: ResultSet = imagePstmt.executeQuery() if (rs.next()) { - blob = rs.getBlob("image") + val blob = rs.getBlob("image") if (blob != null) { image = blob.getBytes(1, blob.length().toInt) } From e8416ff70ef7757f2c459fcf8ef357cf35149f05 Mon Sep 17 00:00:00 2001 From: hrj Date: Mon, 12 Apr 2021 16:19:06 +0530 Subject: [PATCH 6/9] internal change: make maxAttempts easier to specify --- src/main/scala/lc/database/statements.scala | 26 +++++++++++---------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/main/scala/lc/database/statements.scala b/src/main/scala/lc/database/statements.scala index 1439b56..ec0ae24 100644 --- a/src/main/scala/lc/database/statements.scala +++ b/src/main/scala/lc/database/statements.scala @@ -4,7 +4,7 @@ import lc.database.DBConn import java.sql.Statement import java.sql.PreparedStatement -class Statements(dbConn: DBConn) { +class Statements(dbConn: DBConn, maxAttempts: Int) { private val stmt = dbConn.getStatement() @@ -71,13 +71,14 @@ class Statements(dbConn: DBConn) { ) val tokenPstmt: PreparedStatement = dbConn.con.prepareStatement( - "SELECT token " + - "FROM challenge " + - "WHERE attempted < 10 AND " + - "contentLevel = ? AND " + - "contentType = ? AND " + - "contentInput = ? " + - "ORDER BY RAND() LIMIT 1" + s""" + SELECT token + FROM challenge + WHERE attempted < $maxAttempts AND + contentLevel = ? AND + contentType = ? AND + contentInput = ? + ORDER BY RAND() LIMIT 1""" ) val deleteAnswerPstmt: PreparedStatement = dbConn.con.prepareStatement( @@ -85,9 +86,9 @@ class Statements(dbConn: DBConn) { ) val challengeGCPstmt: PreparedStatement = dbConn.con.prepareStatement( - "DELETE FROM challenge " + - "WHERE attempted >= 10 AND " + - "token NOT IN (SELECT token FROM mapId)" + s"""DELETE FROM challenge + WHERE attempted >= $maxAttempts AND + token NOT IN (SELECT token FROM mapId)""" ) val mapIdGCPstmt: PreparedStatement = dbConn.con.prepareStatement( @@ -110,5 +111,6 @@ class Statements(dbConn: DBConn) { object Statements { private val dbConn: DBConn = new DBConn() - val tlStmts: ThreadLocal[Statements] = ThreadLocal.withInitial(() => new Statements(dbConn)) + private val maxAttempts = 10 + val tlStmts: ThreadLocal[Statements] = ThreadLocal.withInitial(() => new Statements(dbConn, maxAttempts)) } From 79428bc5e7302551deb3bcd04f71d9da3b7c7ad5 Mon Sep 17 00:00:00 2001 From: hrj Date: Mon, 12 Apr 2021 16:56:59 +0530 Subject: [PATCH 7/9] reduce number of users in locust stress test by reducing wait time, we can generate same load with fewer users --- tests/locustfile.py | 4 ++-- tests/run.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/locustfile.py b/tests/locustfile.py index df55b4c..941c45e 100644 --- a/tests/locustfile.py +++ b/tests/locustfile.py @@ -19,7 +19,7 @@ def _(environment, **kw): environment.process_exit_code = 0 class QuickStartUser(SequentialTaskSet): - wait_time = between(0.1,1) + wait_time = between(0.1,0.2) @task def captcha(self): @@ -52,6 +52,6 @@ class QuickStartUser(SequentialTaskSet): class User(FastHttpUser): - wait_time = between(0.1,1) + wait_time = between(0.1,0.2) tasks = [QuickStartUser] host = "http://localhost:8888" diff --git a/tests/run.sh b/tests/run.sh index 34ed4fd..0d10e93 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -7,7 +7,7 @@ java -jar target/scala-2.13/LibreCaptcha.jar & JAVA_PID=$! sleep 4 -locust --headless -u 1000 -r 100 --run-time 4m --stop-timeout 30 -f tests/locustfile.py +locust --headless -u 300 -r 100 --run-time 4m --stop-timeout 30 -f tests/locustfile.py status=$? kill $JAVA_PID From f8de5a59328666151ca3a98bce34094409eabd87 Mon Sep 17 00:00:00 2001 From: hrj Date: Mon, 12 Apr 2021 17:01:24 +0530 Subject: [PATCH 8/9] minor: added comment about per thread connections --- src/main/scala/lc/database/statements.scala | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/main/scala/lc/database/statements.scala b/src/main/scala/lc/database/statements.scala index ec0ae24..d23d6e2 100644 --- a/src/main/scala/lc/database/statements.scala +++ b/src/main/scala/lc/database/statements.scala @@ -110,6 +110,13 @@ class Statements(dbConn: DBConn, maxAttempts: Int) { } object Statements { + /* Note: h2 documentation recommends using a separate DB connection per thread + But in practice, as of version 1.4.200, multiple connections occassionally shows error on the console of the form + ``` + org.h2.jdbc.JdbcSQLNonTransientException: General error: "java.lang.NullPointerException"; SQL statement: + SELECT image FROM challenge c, mapId m WHERE c.token=m.token AND m.uuid = ? [50000-200] + ``` + */ private val dbConn: DBConn = new DBConn() private val maxAttempts = 10 val tlStmts: ThreadLocal[Statements] = ThreadLocal.withInitial(() => new Statements(dbConn, maxAttempts)) From 2d7f229d30c94db18b5bfb8cdf439be99edeac2c Mon Sep 17 00:00:00 2001 From: hrj Date: Mon, 12 Apr 2021 19:55:27 +0530 Subject: [PATCH 9/9] bump to latest sbt version --- build.sbt | 10 +++++----- project/build.properties | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/build.sbt b/build.sbt index c9d5e50..01eeee4 100644 --- a/build.sbt +++ b/build.sbt @@ -15,7 +15,7 @@ lazy val root = (project in file(".")).settings( libraryDependencies += "org.json4s" % "json4s-jackson_2.13" % "3.6.11" ) -unmanagedResourceDirectories in Compile += { baseDirectory.value / "lib" } +Compile / unmanagedResourceDirectories += { baseDirectory.value / "lib" } scalacOptions ++= List( "-Yrangepos", "-Ywarn-unused", @@ -23,8 +23,8 @@ scalacOptions ++= List( ) javacOptions += "-g:none" compileOrder := CompileOrder.JavaThenScala -mainClass in assembly := Some("lc.LCFramework") -mainClass in (Compile, run) := Some("lc.LCFramework") -assemblyJarName in assembly := "LibreCaptcha.jar" +assembly / mainClass := Some("lc.LCFramework") +Compile / run / mainClass := Some("lc.LCFramework") +assembly / assemblyJarName := "LibreCaptcha.jar" -fork in run := true +run / fork := true diff --git a/project/build.properties b/project/build.properties index dbae93b..e67343a 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.4.9 +sbt.version=1.5.0