From 1c6500c131e0639fc819b2b329f2da054c804fa4 Mon Sep 17 00:00:00 2001 From: Thanh Le Date: Sat, 26 Jul 2025 10:14:39 +0200 Subject: [PATCH 01/20] Setting version to 3.2.2-SNAPSHOT --- version.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.sbt b/version.sbt index 310864c6..2ccaca76 100644 --- a/version.sbt +++ b/version.sbt @@ -1 +1 @@ -ThisBuild / version := "3.2.1" +ThisBuild / version := "3.2.2-SNAPSHOT" From 8c84eff264794337b4abb18feb8ce66ba0c0bd30 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Sat, 2 Aug 2025 18:16:03 +0000 Subject: [PATCH 02/20] Update smithy4s-core, smithy4s-http4s, ... to 0.18.41 --- project/plugins.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/plugins.sbt b/project/plugins.sbt index c7240eff..855616c6 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,6 +1,6 @@ addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.14.3") -addSbtPlugin("com.disneystreaming.smithy4s" % "smithy4s-sbt-codegen" % "0.18.40") +addSbtPlugin("com.disneystreaming.smithy4s" % "smithy4s-sbt-codegen" % "0.18.41") addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.13.1") From 28ec9fe45cbbc41e708b9afaa92cfd5f1abb4e51 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Sat, 2 Aug 2025 18:16:14 +0000 Subject: [PATCH 03/20] Update scalachess to 17.9.3 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 865b0472..34b25baf 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -9,7 +9,7 @@ object Dependencies { object V { val catsEffect = "3.6.3" - val chess = "17.9.1" + val chess = "17.9.3" val ciris = "3.9.0" val decline = "2.5.0" val elastic4s = "9.0.0" From 061216b508a296a69e4b7ac953340110ae0aa27e Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Sat, 2 Aug 2025 18:16:20 +0000 Subject: [PATCH 04/20] Update elastic4s-client-http4s to 9.1.0 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 865b0472..f6748b28 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -12,7 +12,7 @@ object Dependencies { val chess = "17.9.1" val ciris = "3.9.0" val decline = "2.5.0" - val elastic4s = "9.0.0" + val elastic4s = "9.1.0" val fs2 = "3.12.0" val http4s = "0.23.30" val iron = "2.5.0" From 0c7da32fffb49c6b7dd888655d55aabf8166cbe5 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Sat, 2 Aug 2025 18:16:33 +0000 Subject: [PATCH 05/20] Update scala3-library to 3.7.2 --- build.sbt | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/build.sbt b/build.sbt index 7defaf08..f7097037 100644 --- a/build.sbt +++ b/build.sbt @@ -3,15 +3,15 @@ import org.typelevel.scalacoptions.ScalacOptions inThisBuild( Seq( - scalaVersion := "3.7.1", + scalaVersion := "3.7.2", versionScheme := Some("early-semver"), - organization := "org.lichess.search", - run / fork := true, + organization := "org.lichess.search", + run / fork := true, run / javaOptions += "-Dconfig.override_with_env_vars=true", semanticdbEnabled := true, // for scalafix resolvers ++= ourResolvers, Compile / doc / sources := Seq.empty, - publishTo := Option(Resolver.file("file", new File(sys.props.getOrElse("publishTo", "")))) + publishTo := Option(Resolver.file("file", new File(sys.props.getOrElse("publishTo", "")))) ) ) @@ -65,7 +65,7 @@ lazy val elastic = project .settings( name := "elastic", commonSettings, - publish := {}, + publish := {}, publish / skip := true, libraryDependencies ++= Seq( catsCore, @@ -85,8 +85,8 @@ lazy val ingestor = project commonSettings, buildInfoSettings, dockerBaseImage := "docker.io/library/eclipse-temurin:21-jdk", - publish := {}, - publish / skip := true, + publish := {}, + publish / skip := true, libraryDependencies ++= Seq( chess, catsCore, @@ -117,7 +117,7 @@ lazy val ingestor = project weaverScalaCheck ), Compile / doc / sources := Seq.empty, - Compile / run / fork := true + Compile / run / fork := true ) .dependsOn(elastic, core) @@ -143,8 +143,8 @@ lazy val app = project commonSettings, buildInfoSettings, dockerBaseImage := "docker.io/library/eclipse-temurin:21-jdk", - publish := {}, - publish / skip := true, + publish := {}, + publish / skip := true, libraryDependencies ++= Seq( smithy4sHttp4s, jsoniterCore, @@ -165,14 +165,14 @@ lazy val app = project otel4sInstrumentationMetrics ), Compile / doc / sources := Seq.empty, - Compile / run / fork := true + Compile / run / fork := true ) .dependsOn(api, elastic) val e2e = project .in(file("modules/e2e")) .settings( - publish := {}, + publish := {}, publish / skip := true, libraryDependencies ++= Seq(testContainers, weaver) ) From be6b673f1c3ad1cfb144a4836f5c43f68a90c5c5 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Sat, 2 Aug 2025 18:16:39 +0000 Subject: [PATCH 06/20] Update scalafmt-core to 3.9.9 --- .scalafmt.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.scalafmt.conf b/.scalafmt.conf index ca23448d..d1045b0f 100644 --- a/.scalafmt.conf +++ b/.scalafmt.conf @@ -1,4 +1,4 @@ -version = "3.9.8" +version = "3.9.9" runner.dialect = scala3 align.preset = none From 6dfb7ed8ffa2f5df0f5c36bee8d3497d29016495 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Sat, 2 Aug 2025 18:16:50 +0000 Subject: [PATCH 07/20] Reformat with scalafmt 3.9.9 Executed command: scalafmt --non-interactive --- build.sbt | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/build.sbt b/build.sbt index 7defaf08..858a2efa 100644 --- a/build.sbt +++ b/build.sbt @@ -3,15 +3,15 @@ import org.typelevel.scalacoptions.ScalacOptions inThisBuild( Seq( - scalaVersion := "3.7.1", + scalaVersion := "3.7.1", versionScheme := Some("early-semver"), - organization := "org.lichess.search", - run / fork := true, + organization := "org.lichess.search", + run / fork := true, run / javaOptions += "-Dconfig.override_with_env_vars=true", semanticdbEnabled := true, // for scalafix resolvers ++= ourResolvers, Compile / doc / sources := Seq.empty, - publishTo := Option(Resolver.file("file", new File(sys.props.getOrElse("publishTo", "")))) + publishTo := Option(Resolver.file("file", new File(sys.props.getOrElse("publishTo", "")))) ) ) @@ -65,7 +65,7 @@ lazy val elastic = project .settings( name := "elastic", commonSettings, - publish := {}, + publish := {}, publish / skip := true, libraryDependencies ++= Seq( catsCore, @@ -85,8 +85,8 @@ lazy val ingestor = project commonSettings, buildInfoSettings, dockerBaseImage := "docker.io/library/eclipse-temurin:21-jdk", - publish := {}, - publish / skip := true, + publish := {}, + publish / skip := true, libraryDependencies ++= Seq( chess, catsCore, @@ -117,7 +117,7 @@ lazy val ingestor = project weaverScalaCheck ), Compile / doc / sources := Seq.empty, - Compile / run / fork := true + Compile / run / fork := true ) .dependsOn(elastic, core) @@ -143,8 +143,8 @@ lazy val app = project commonSettings, buildInfoSettings, dockerBaseImage := "docker.io/library/eclipse-temurin:21-jdk", - publish := {}, - publish / skip := true, + publish := {}, + publish / skip := true, libraryDependencies ++= Seq( smithy4sHttp4s, jsoniterCore, @@ -165,14 +165,14 @@ lazy val app = project otel4sInstrumentationMetrics ), Compile / doc / sources := Seq.empty, - Compile / run / fork := true + Compile / run / fork := true ) .dependsOn(api, elastic) val e2e = project .in(file("modules/e2e")) .settings( - publish := {}, + publish := {}, publish / skip := true, libraryDependencies ++= Seq(testContainers, weaver) ) From 6c21afa9c626418e50cef309203ad346f3f2ddd7 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Sat, 2 Aug 2025 18:16:50 +0000 Subject: [PATCH 08/20] Add 'Reformat with scalafmt 3.9.9' to .git-blame-ignore-revs --- .git-blame-ignore-revs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index fef72f09..7f80fb21 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -18,3 +18,6 @@ ed503a2852d8148f39ac99b7188e47f569bdf01a # scalafmt align.preset = none adcf303d99abee49f1266dad223bb3e5e34bd73f + +# Scala Steward: Reformat with scalafmt 3.9.9 +6dfb7ed8ffa2f5df0f5c36bee8d3497d29016495 From 638b7965b8277f9424d401bd08ae6cae9e70eb56 Mon Sep 17 00:00:00 2001 From: Thanh Le Date: Mon, 4 Aug 2025 15:34:20 +0200 Subject: [PATCH 09/20] Fix build --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index bb927161..f7097037 100644 --- a/build.sbt +++ b/build.sbt @@ -3,7 +3,7 @@ import org.typelevel.scalacoptions.ScalacOptions inThisBuild( Seq( - scalaVersion := "3.7.2" + scalaVersion := "3.7.2", versionScheme := Some("early-semver"), organization := "org.lichess.search", run / fork := true, From 1a272a59aca61eaab0414c641deebcd8460e2ad7 Mon Sep 17 00:00:00 2001 From: Thanh Le Date: Mon, 4 Aug 2025 17:44:17 +0200 Subject: [PATCH 10/20] Use new fancy cats-mtl submarine error propagation Instead of throw an exception, we raise ElasticError instead and in the callsite we handle them by allow/rescue syntax --- build.sbt | 1 + .../app/src/main/scala/service.health.scala | 14 ++++--- .../app/src/main/scala/service.search.scala | 25 +++++------ modules/e2e/src/test/scala/CompatSuite.scala | 14 +++---- .../e2e/src/test/scala/IntegrationSuite.scala | 8 ++++ modules/elastic/src/main/scala/ESClient.scala | 42 ++++++++++--------- modules/elastic/src/main/scala/package.scala | 16 +++---- .../ingestor/src/main/scala/ingestor.scala | 25 +++++------ project/Dependencies.scala | 4 +- 9 files changed, 84 insertions(+), 65 deletions(-) diff --git a/build.sbt b/build.sbt index f7097037..5c71be82 100644 --- a/build.sbt +++ b/build.sbt @@ -70,6 +70,7 @@ lazy val elastic = project libraryDependencies ++= Seq( catsCore, catsEffect, + catsMtl, http4sClient, elastic4sHttp4sClient, otel4sCore diff --git a/modules/app/src/main/scala/service.health.scala b/modules/app/src/main/scala/service.health.scala index 3c3f8bfa..254f8514 100644 --- a/modules/app/src/main/scala/service.health.scala +++ b/modules/app/src/main/scala/service.health.scala @@ -2,6 +2,7 @@ package lila.search package app import cats.effect.* +import cats.mtl.Handle.* import cats.syntax.all.* import lila.search.spec.* import org.typelevel.log4cats.{ Logger, LoggerFactory } @@ -11,12 +12,13 @@ class HealthServiceImpl(esClient: ESClient[IO])(using LoggerFactory[IO]) extends given Logger[IO] = LoggerFactory[IO].getLogger override def healthCheck(): IO[HealthCheckOutput] = - esClient.status - .flatMap(transform) - .map(HealthCheckOutput(_)) - .handleErrorWith: e => - Logger[IO].error(e)("Error in health check") *> - IO.raiseError(InternalServerError(s"Internal server error ${e.getMessage}")) + allow: + esClient.status + .flatMap(transform) + .map(HealthCheckOutput(_)) + .rescue: e => + Logger[IO].error(e.asException)("Error in health check") *> + IO.raiseError(InternalServerError(s"Internal server error ${e.reason}")) private def transform(status: String): IO[ElasticStatus] = status match diff --git a/modules/app/src/main/scala/service.search.scala b/modules/app/src/main/scala/service.search.scala index ed9ff73f..25fc9a8a 100644 --- a/modules/app/src/main/scala/service.search.scala +++ b/modules/app/src/main/scala/service.search.scala @@ -2,6 +2,7 @@ package lila.search package app import cats.effect.* +import cats.mtl.Handle.* import io.github.arainko.ducktape.* import lila.search.forum.Forum import lila.search.game.Game @@ -51,21 +52,21 @@ class SearchServiceImpl(esClient: ESClient[IO], metric: Histogram[IO, Double])(u override def count(query: Query): IO[CountOutput] = countRecord: - esClient - .count(query) - .map(CountOutput.apply) - .handleErrorWith: e => - logger.error(e)(s"Error in count: query=${query.toString}") *> - IO.raiseError(InternalServerError("Internal server error")) + allow: + esClient.count(query) + .rescue: e => + logger.error(e.asException)(s"Error in count: query=${query.toString}") *> + IO.raiseError(InternalServerError("Internal server error")) + .map(CountOutput.apply) override def search(query: Query, from: From, size: Size): IO[SearchOutput] = searchRecord: - esClient - .search(query, from, size) - .map(SearchOutput.apply) - .handleErrorWith: e => - logger.error(e)(s"Error in search: query=${query.toString}, from=$from, size=$size") *> - IO.raiseError(InternalServerError("Internal server error")) + allow: + esClient.search(query, from, size) + .rescue: e => + logger.error(e.asException)(s"Error in search: query=${query.toString}, from=$from, size=$size") *> + IO.raiseError(InternalServerError("Internal server error")) + .map(SearchOutput.apply) object SearchServiceImpl: diff --git a/modules/e2e/src/test/scala/CompatSuite.scala b/modules/e2e/src/test/scala/CompatSuite.scala index e628eed6..071942c7 100644 --- a/modules/e2e/src/test/scala/CompatSuite.scala +++ b/modules/e2e/src/test/scala/CompatSuite.scala @@ -67,18 +67,18 @@ object CompatSuite extends weaver.IOSuite: def fakeClient: ESClient[IO] = new: - override def store[A](index: Index, id: Id, obj: A)(using Indexable[A]): IO[Unit] = IO.unit + override def store[A](index: Index, id: Id, obj: A)(using Indexable[A]) = IO.unit - override def storeBulk[A](index: Index, objs: Seq[SourceWithId[A]])(using Indexable[A]): IO[Unit] = + override def storeBulk[A](index: Index, objs: Seq[SourceWithId[A]])(using Indexable[A]) = IO.unit - override def putMapping(index: Index): IO[Unit] = IO.unit + override def putMapping(index: Index) = IO.unit - override def refreshIndex(index: Index): IO[Unit] = IO.unit + override def refreshIndex(index: Index) = IO.unit - override def deleteOne(index: Index, id: Id): IO[Unit] = IO.unit + override def deleteOne(index: Index, id: Id) = IO.unit - override def deleteMany(index: Index, ids: List[Id]): IO[Unit] = IO.unit + override def deleteMany(index: Index, ids: List[Id]) = IO.unit override def count[A](query: A)(using Queryable[A]) = IO.pure(0) @@ -86,7 +86,7 @@ object CompatSuite extends weaver.IOSuite: override def search[A](query: A, from: From, size: Size)(using Queryable[A]) = IO.pure(Nil) - override def status: IO[String] = IO.pure("yellow") + override def status = IO.pure("yellow") given system: ActorSystem = ActorSystem() diff --git a/modules/e2e/src/test/scala/IntegrationSuite.scala b/modules/e2e/src/test/scala/IntegrationSuite.scala index 59abc78b..9a19e46f 100644 --- a/modules/e2e/src/test/scala/IntegrationSuite.scala +++ b/modules/e2e/src/test/scala/IntegrationSuite.scala @@ -1,9 +1,13 @@ package lila.search package app package test + +import cats.Functor import cats.effect.{ IO, Resource } +import cats.mtl.Raise import cats.syntax.all.* import com.comcast.ip4s.* +import com.sksamuel.elastic4s.ElasticError import lila.search.ingestor.Ingestor.given import lila.search.spec.* import org.http4s.Uri @@ -22,6 +26,10 @@ object IntegrationSuite extends IOSuite: given Logger[IO] = NoOpLogger[IO] given LoggerFactory[IO] = NoOpFactory[IO] given Meter[IO] = Meter.noop[IO] + private given Raise[IO, ElasticError]: + def functor: Functor[IO] = Functor[IO] + def raise[E <: ElasticError, A](e: E): IO[A] = + IO.raiseError(e.asException) private val uri = Uri.unsafeFromString("http://localhost:9999") diff --git a/modules/elastic/src/main/scala/ESClient.scala b/modules/elastic/src/main/scala/ESClient.scala index 6ee6751f..06c00318 100644 --- a/modules/elastic/src/main/scala/ESClient.scala +++ b/modules/elastic/src/main/scala/ESClient.scala @@ -1,10 +1,11 @@ package lila.search import cats.effect.* +import cats.mtl.Raise import cats.syntax.all.* import com.sksamuel.elastic4s.ElasticDsl.* import com.sksamuel.elastic4s.http4s.Http4sClient -import com.sksamuel.elastic4s.{ ElasticClient, ElasticDsl, Index as ESIndex, Indexable } +import com.sksamuel.elastic4s.{ ElasticClient, ElasticDsl, ElasticError, Index as ESIndex, Indexable } import lila.search.ESClient.MetricKeys.* import org.http4s.Uri import org.http4s.client.Client @@ -14,16 +15,17 @@ import org.typelevel.otel4s.{ Attribute, AttributeKey, Attributes } import java.util.concurrent.TimeUnit trait ESClient[F[_]]: - - def search[A](query: A, from: From, size: Size)(using Queryable[A]): F[List[Id]] - def count[A](query: A)(using Queryable[A]): F[Long] - def store[A](index: Index, id: Id, obj: A)(using Indexable[A]): F[Unit] - def storeBulk[A](index: Index, objs: Seq[SourceWithId[A]])(using Indexable[A]): F[Unit] - def deleteOne(index: Index, id: Id): F[Unit] - def deleteMany(index: Index, ids: List[Id]): F[Unit] - def putMapping(index: Index): F[Unit] - def refreshIndex(index: Index): F[Unit] - def status: F[String] + type RaiseF[A] = Raise[F, ElasticError] ?=> F[A] + + def search[A](query: A, from: From, size: Size)(using Queryable[A]): RaiseF[List[Id]] + def count[A](query: A)(using Queryable[A]): RaiseF[Long] + def store[A](index: Index, id: Id, obj: A)(using Indexable[A]): RaiseF[Unit] + def storeBulk[A](index: Index, objs: Seq[SourceWithId[A]])(using Indexable[A]): RaiseF[Unit] + def deleteOne(index: Index, id: Id): RaiseF[Unit] + def deleteMany(index: Index, ids: List[Id]): RaiseF[Unit] + def putMapping(index: Index): RaiseF[Unit] + def refreshIndex(index: Index): RaiseF[Unit] + def status: RaiseF[String] object ESClient: @@ -43,13 +45,13 @@ object ESClient: metric: Histogram[F, Double] ) = new ESClient[F]: - def status: F[String] = + def status: RaiseF[String] = client .execute(clusterHealth()) .flatMap(_.toResult) .map(_.status) - def search[A](query: A, from: From, size: Size)(using Queryable[A]): F[List[Id]] = + def search[A](query: A, from: From, size: Size)(using Queryable[A]): RaiseF[List[Id]] = metric .recordDuration( TimeUnit.MILLISECONDS, @@ -65,7 +67,7 @@ object ESClient: .flatMap(_.toResult) .map(_.hits.hits.toList.map(h => Id(h.id))) - def count[A](query: A)(using Queryable[A]): F[Long] = + def count[A](query: A)(using Queryable[A]): RaiseF[Long] = metric .recordDuration( TimeUnit.MILLISECONDS, @@ -81,7 +83,7 @@ object ESClient: .flatMap(_.toResult) .map(_.count) - def store[A](index: Index, id: Id, obj: A)(using Indexable[A]): F[Unit] = + def store[A](index: Index, id: Id, obj: A)(using Indexable[A]): RaiseF[Unit] = metric .recordDuration( TimeUnit.MILLISECONDS, @@ -96,7 +98,7 @@ object ESClient: .execute(indexInto(index.value).source(obj).id(id.value)) .flatMap(_.unitOrFail) - def storeBulk[A](index: Index, objs: Seq[SourceWithId[A]])(using Indexable[A]): F[Unit] = + def storeBulk[A](index: Index, objs: Seq[SourceWithId[A]])(using Indexable[A]): RaiseF[Unit] = val request = indexInto(index.value) val requests = bulk(objs.map { case (id, source) => request.source(source).id(id) }) metric @@ -115,7 +117,7 @@ object ESClient: .flatMap(_.unitOrFail) .whenA(objs.nonEmpty) - def deleteOne(index: Index, id: Id): F[Unit] = + def deleteOne(index: Index, id: Id): RaiseF[Unit] = metric .recordDuration( TimeUnit.MILLISECONDS, @@ -130,7 +132,7 @@ object ESClient: .execute(deleteById(index.toES, id.value)) .flatMap(_.unitOrFail) - def deleteMany(index: Index, ids: List[Id]): F[Unit] = + def deleteMany(index: Index, ids: List[Id]): RaiseF[Unit] = metric .recordDuration( TimeUnit.MILLISECONDS, @@ -147,7 +149,7 @@ object ESClient: .flatMap(_.unitOrFail) .whenA(ids.nonEmpty) - def putMapping(index: Index): F[Unit] = + def putMapping(index: Index): RaiseF[Unit] = dropIndex(index) *> client .execute: createIndex(index.value) @@ -157,7 +159,7 @@ object ESClient: .refreshInterval(index.refreshInterval) .flatMap(_.unitOrFail) - def refreshIndex(index: Index): F[Unit] = + def refreshIndex(index: Index): RaiseF[Unit] = client .execute(ElasticDsl.refreshIndex(index.value)) .flatMap(_.unitOrFail) diff --git a/modules/elastic/src/main/scala/package.scala b/modules/elastic/src/main/scala/package.scala index 616e6fae..5ca6ad7a 100644 --- a/modules/elastic/src/main/scala/package.scala +++ b/modules/elastic/src/main/scala/package.scala @@ -1,10 +1,12 @@ package lila.search -import cats.MonadThrow +import cats.Monad +import cats.mtl.Raise +import cats.mtl.implicits.* import cats.syntax.all.* import com.sksamuel.elastic4s.ElasticDsl.* import com.sksamuel.elastic4s.requests.searches.queries.Query -import com.sksamuel.elastic4s.{ Index as ESIndex, Response } +import com.sksamuel.elastic4s.{ ElasticError, Index as ESIndex, Response } type SourceWithId[A] = (id: String, source: A) @@ -31,8 +33,8 @@ extension (index: Index) case Index.Study => "10s" case _ => "300s" -extension [F[_]: MonadThrow, A](response: Response[A]) - def toResult: F[A] = - response.fold(response.error.asException.raiseError)(r => r.pure[F]) - def unitOrFail: F[Unit] = - response.fold(response.error.asException.raiseError)(_ => ().pure[F]) +extension [F[_]: Monad, A](response: Response[A]) + def toResult: Raise[F, ElasticError] ?=> F[A] = + response.fold(response.error.raise)(_.pure[F]) + def unitOrFail: Raise[F, ElasticError] ?=> F[Unit] = + response.fold(response.error.raise)(_ => ().pure[F]) diff --git a/modules/ingestor/src/main/scala/ingestor.scala b/modules/ingestor/src/main/scala/ingestor.scala index 6d92d6c5..d1eab472 100644 --- a/modules/ingestor/src/main/scala/ingestor.scala +++ b/modules/ingestor/src/main/scala/ingestor.scala @@ -2,6 +2,7 @@ package lila.search package ingestor import cats.effect.* +import cats.mtl.Handle.* import cats.syntax.all.* import com.github.plokhotnyuk.jsoniter_scala.core.* import com.sksamuel.elastic4s.Indexable @@ -31,7 +32,7 @@ object Ingestor: elastic: ESClient[IO], defaultStartAt: Option[Instant] )(using LoggerFactory[IO]): Ingestor = new: - given Logger[IO] = LoggerFactory[IO].getLoggerFromName(s"${index.value}.ingestor") + given logger: Logger[IO] = LoggerFactory[IO].getLoggerFromName(s"${index.value}.ingestor") def watch: IO[Unit] = fs2.Stream @@ -71,21 +72,21 @@ object Ingestor: .flatTap(since => info"Starting ${index.value} ingestor from $since") private def deleteMany(index: Index, ids: List[Id]): IO[Unit] = - elastic - .deleteMany(index, ids) - .flatTap(_ => Logger[IO].info(s"Deleted ${ids.size} ${index.value}s")) - .handleErrorWith: e => - Logger[IO].error(e)(s"Failed to delete ${index.value}: ${ids.map(_.value).mkString(", ")}") + allow: + elastic.deleteMany(index, ids) + .rescue: e => + logger.error(e.asException)(s"Failed to delete ${index.value}: ${ids.map(_.value).mkString(", ")}") + .flatTap(_ => Logger[IO].info(s"Deleted ${ids.size} ${index.value}s")) .whenA(ids.nonEmpty) private def storeBulk(index: Index, sources: List[SourceWithId[A]]): IO[Unit] = Logger[IO].info(s"Received ${sources.size} docs to ${index.value}") *> - elastic - .storeBulk(index, sources) - .handleErrorWith: e => - Logger[IO].error(e)(s"Failed to ${index.value} index: ${sources.map(_.id).mkString(", ")}") - .whenA(sources.nonEmpty) - *> Logger[IO].info(s"Indexed ${sources.size} ${index.value}s") + allow: + elastic.storeBulk(index, sources) + .rescue: e => + logger.error(e.asException)(s"Failed to ${index.value} index: ${sources.map(_.id).mkString(", ")}") + .whenA(sources.nonEmpty) + *> logger.info(s"Indexed ${sources.size} ${index.value}s") private val saveLastIndexedTimestamp: Option[Instant] => IO[Unit] = _.traverse_(time => diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 279542c9..f1b21132 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -5,10 +5,11 @@ object Dependencies { val lilaMaven = "lila-maven" at "https://raw.githubusercontent.com/lichess-org/lila-maven/master" val jitpack = "jitpack".at("https://jitpack.io") - val ourResolvers = Seq(lilaMaven, jitpack) + val ourResolvers = Seq(lilaMaven, jitpack, Resolver.sonatypeCentralSnapshots) object V { val catsEffect = "3.6.3" + val catsMtl = "1.6-6ad7882-SNAPSHOT" val chess = "17.9.3" val ciris = "3.9.0" val decline = "2.5.0" @@ -27,6 +28,7 @@ object Dependencies { val catsCore = "org.typelevel" %% "cats-core" % "2.13.0" val catsEffect = "org.typelevel" %% "cats-effect" % V.catsEffect + val catsMtl = "org.typelevel" %% "cats-mtl" % V.catsMtl val fs2 = "co.fs2" %% "fs2-core" % V.fs2 val fs2IO = "co.fs2" %% "fs2-io" % V.fs2 From c9c66f89d0af35c1b26692f67df5ab2bbf9f4656 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Sat, 16 Aug 2025 16:10:55 +0000 Subject: [PATCH 11/20] Update scalachess to 17.9.5 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index f1b21132..edc2c4d4 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -10,7 +10,7 @@ object Dependencies { object V { val catsEffect = "3.6.3" val catsMtl = "1.6-6ad7882-SNAPSHOT" - val chess = "17.9.3" + val chess = "17.9.5" val ciris = "3.9.0" val decline = "2.5.0" val elastic4s = "9.1.0" From 5e1110c831afaab9c4f09486cbde34fc019f5e9e Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Sat, 16 Aug 2025 16:10:59 +0000 Subject: [PATCH 12/20] Update jsoniter-scala-core, ... to 2.37.6 --- project/Dependencies.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index f1b21132..40478aa9 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -47,8 +47,8 @@ object Dependencies { lazy val smithy4sHttp4sSwagger = smithy4s("http4s-swagger") lazy val smithy4sJson = smithy4s("json") - val jsoniterCore = "com.github.plokhotnyuk.jsoniter-scala" %% "jsoniter-scala-core" % "2.37.0" - val jsoniterMacro = "com.github.plokhotnyuk.jsoniter-scala" %% "jsoniter-scala-macros" % "2.37.0" + val jsoniterCore = "com.github.plokhotnyuk.jsoniter-scala" %% "jsoniter-scala-core" % "2.37.6" + val jsoniterMacro = "com.github.plokhotnyuk.jsoniter-scala" %% "jsoniter-scala-macros" % "2.37.6" val playWS = "com.typesafe.play" %% "play-ahc-ws-standalone" % "2.2.11" From b1fafdf1e29dcc0014382ad9a5b6cb55c7f149e6 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Tue, 26 Aug 2025 17:10:31 +0000 Subject: [PATCH 13/20] Update sbt, scripted-plugin to 1.11.5 --- project/build.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/build.properties b/project/build.properties index c02c575f..e480c675 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.11.3 +sbt.version=1.11.5 From 78d6c1ba4444f71bdbea4ccd26db419d5db4aa08 Mon Sep 17 00:00:00 2001 From: Thanh Le Date: Sat, 13 Sep 2025 17:11:13 +0200 Subject: [PATCH 14/20] Bum dependencies --- build.sbt | 2 +- project/Dependencies.scala | 21 +++++++++------------ project/build.properties | 2 +- project/plugins.sbt | 4 ++-- 4 files changed, 13 insertions(+), 16 deletions(-) diff --git a/build.sbt b/build.sbt index 5c71be82..81cb96c1 100644 --- a/build.sbt +++ b/build.sbt @@ -3,7 +3,7 @@ import org.typelevel.scalacoptions.ScalacOptions inThisBuild( Seq( - scalaVersion := "3.7.2", + scalaVersion := "3.7.3", versionScheme := Some("early-semver"), organization := "org.lichess.search", run / fork := true, diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 13866fe8..4d12e5be 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -9,14 +9,13 @@ object Dependencies { object V { val catsEffect = "3.6.3" - val catsMtl = "1.6-6ad7882-SNAPSHOT" - val chess = "17.9.5" - val ciris = "3.9.0" + val catsMtl = "1.6.0" + val chess = "17.9.6" + val ciris = "3.10.0" val decline = "2.5.0" val elastic4s = "9.1.0" - val fs2 = "3.12.0" + val fs2 = "3.12.2" val http4s = "0.23.30" - val iron = "2.5.0" val mongo4cats = "0.7.13" val otel4s = "0.13.1" } @@ -35,8 +34,6 @@ object Dependencies { val cirisCore = "is.cir" %% "ciris" % V.ciris val cirisHtt4s = "is.cir" %% "ciris-http4s" % V.ciris - val iron = "io.github.iltotore" %% "iron" % V.iron - val ironCiris = "io.github.iltotore" %% "iron-ciris" % V.iron val http4sServer = http4s("ember-server") val http4sClient = http4s("client") @@ -47,10 +44,10 @@ object Dependencies { lazy val smithy4sHttp4sSwagger = smithy4s("http4s-swagger") lazy val smithy4sJson = smithy4s("json") - val jsoniterCore = "com.github.plokhotnyuk.jsoniter-scala" %% "jsoniter-scala-core" % "2.37.6" - val jsoniterMacro = "com.github.plokhotnyuk.jsoniter-scala" %% "jsoniter-scala-macros" % "2.37.6" + val jsoniterCore = "com.github.plokhotnyuk.jsoniter-scala" %% "jsoniter-scala-core" % "2.37.10" + val jsoniterMacro = "com.github.plokhotnyuk.jsoniter-scala" %% "jsoniter-scala-macros" % "2.37.10" - val playWS = "com.typesafe.play" %% "play-ahc-ws-standalone" % "2.2.11" + val playWS = "com.typesafe.play" %% "play-ahc-ws-standalone" % "2.2.12" val elastic4sHttp4sClient = "nl.gn0s1s" %% "elastic4s-client-http4s" % V.elastic4s @@ -67,13 +64,13 @@ object Dependencies { val log4Cats = "org.typelevel" %% "log4cats-slf4j" % "2.7.1" val logback = "ch.qos.logback" % "logback-classic" % "1.5.18" - val ducktape = "io.github.arainko" %% "ducktape" % "0.2.9" + val ducktape = "io.github.arainko" %% "ducktape" % "0.2.10" val declineCore = "com.monovore" %% "decline" % V.decline val declineCatsEffect = "com.monovore" %% "decline-effect" % V.decline val testContainers = "com.dimafeng" %% "testcontainers-scala-core" % "0.43.0" % Test - val weaver = "org.typelevel" %% "weaver-cats" % "0.9.3" % Test + val weaver = "org.typelevel" %% "weaver-cats" % "0.10.1" % Test val weaverScalaCheck = "org.typelevel" %% "weaver-scalacheck" % "0.9.3" % Test val catsEffectTestKit = "org.typelevel" %% "cats-effect-testkit" % V.catsEffect % Test val scalacheck = "org.scalacheck" %% "scalacheck" % "1.17.0" % Test diff --git a/project/build.properties b/project/build.properties index e480c675..5e6884d3 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.11.5 +sbt.version=1.11.6 diff --git a/project/plugins.sbt b/project/plugins.sbt index 855616c6..31f32462 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,12 +1,12 @@ addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.14.3") -addSbtPlugin("com.disneystreaming.smithy4s" % "smithy4s-sbt-codegen" % "0.18.41") +addSbtPlugin("com.disneystreaming.smithy4s" % "smithy4s-sbt-codegen" % "0.18.42") addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.13.1") addSbtPlugin("com.github.sbt" % "sbt-git" % "2.1.0") -addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.11.1") +addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.11.3") addSbtPlugin("com.github.sbt" % "sbt-release" % "1.4.0") From aef0d7ca4e25b405879a21c32d1e649a9a29268f Mon Sep 17 00:00:00 2001 From: Thanh Le Date: Sat, 13 Sep 2025 18:05:30 +0200 Subject: [PATCH 15/20] Split app and cli into its own module This will help setup docker image for both of them --- README.md | 4 +- build.sbt | 69 ++++++++++++++----- .../src/main/resources/logback.xml | 0 .../src/main/scala/app.scala | 0 .../src/main/resources/logback.xml | 27 ++++++++ .../src/main/scala/cli.scala | 0 .../src/test/scala/ClITest.scala | 0 .../src/main/scala/HasDocId.scala | 0 .../src/main/scala/Repo.scala | 0 .../src/main/scala/config.scala} | 1 + .../src/main/scala/ingestor.scala | 0 .../src/main/scala/ingestors.scala | 0 .../src/main/scala/kvstore.scala | 0 .../src/main/scala/mongo.chapter.scala | 0 .../src/main/scala/mongo.forum.scala | 0 .../src/main/scala/mongo.game.scala | 0 .../src/main/scala/mongo.study.scala | 0 .../src/main/scala/mongo.team.scala | 0 .../src/main/scala/mongo.ublog.scala | 0 .../src/main/scala/resources.scala} | 0 .../src/main/smithy/model.smithy | 0 .../src/test/scala/HasDocIdTest.scala | 0 project/Dependencies.scala | 2 +- 23 files changed, 82 insertions(+), 21 deletions(-) rename modules/{ingestor => ingestor-app}/src/main/resources/logback.xml (100%) rename modules/{ingestor => ingestor-app}/src/main/scala/app.scala (100%) create mode 100644 modules/ingestor-cli/src/main/resources/logback.xml rename modules/{ingestor => ingestor-cli}/src/main/scala/cli.scala (100%) rename modules/{ingestor => ingestor-cli}/src/test/scala/ClITest.scala (100%) rename modules/{ingestor => ingestor-core}/src/main/scala/HasDocId.scala (100%) rename modules/{ingestor => ingestor-core}/src/main/scala/Repo.scala (100%) rename modules/{ingestor/src/main/scala/app.config.scala => ingestor-core/src/main/scala/config.scala} (99%) rename modules/{ingestor => ingestor-core}/src/main/scala/ingestor.scala (100%) rename modules/{ingestor => ingestor-core}/src/main/scala/ingestors.scala (100%) rename modules/{ingestor => ingestor-core}/src/main/scala/kvstore.scala (100%) rename modules/{ingestor => ingestor-core}/src/main/scala/mongo.chapter.scala (100%) rename modules/{ingestor => ingestor-core}/src/main/scala/mongo.forum.scala (100%) rename modules/{ingestor => ingestor-core}/src/main/scala/mongo.game.scala (100%) rename modules/{ingestor => ingestor-core}/src/main/scala/mongo.study.scala (100%) rename modules/{ingestor => ingestor-core}/src/main/scala/mongo.team.scala (100%) rename modules/{ingestor => ingestor-core}/src/main/scala/mongo.ublog.scala (100%) rename modules/{ingestor/src/main/scala/app.resources.scala => ingestor-core/src/main/scala/resources.scala} (100%) rename modules/{ingestor => ingestor-core}/src/main/smithy/model.smithy (100%) rename modules/{ingestor => ingestor-core}/src/test/scala/HasDocIdTest.scala (100%) diff --git a/README.md b/README.md index f929070e..ad7adf64 100644 --- a/README.md +++ b/README.md @@ -35,12 +35,12 @@ prepare Start ingestor service: ```sh -ingestor/runMain lila.search.ingestor.App +ingestor-app/run ``` Start ingestor cli tool ```sh -ingestor/runMain lila.search.ingestor.cli --help +ingestor-cli/run --help ``` #### CLI tool diff --git a/build.sbt b/build.sbt index 81cb96c1..c6e89f6a 100644 --- a/build.sbt +++ b/build.sbt @@ -78,47 +78,80 @@ lazy val elastic = project ) .dependsOn(api, core) -lazy val ingestor = project - .in(file("modules/ingestor")) - .enablePlugins(JavaAppPackaging, Smithy4sCodegenPlugin, BuildInfoPlugin) +lazy val `ingestor-app` = project + .in(file("modules/ingestor-app")) + .enablePlugins(JavaAppPackaging, BuildInfoPlugin) .settings( - name := "ingestor", + name := "ingestor-app", commonSettings, buildInfoSettings, dockerBaseImage := "docker.io/library/eclipse-temurin:21-jdk", publish := {}, publish / skip := true, + libraryDependencies ++= Seq( + logback % Runtime, + otel4sSdk, + otel4sMetrics, + otel4sPrometheusExporter, + otel4sInstrumentationMetrics + ), + Compile / doc / sources := Seq.empty, + Compile / run / fork := true + ) + .dependsOn(`ingestor-core`) + +lazy val `ingestor-cli` = project + .in(file("modules/ingestor-cli")) + .enablePlugins(JavaAppPackaging, BuildInfoPlugin) + .settings( + name := "ingestor-cli", + commonSettings, + buildInfoSettings, + dockerBaseImage := "docker.io/library/eclipse-temurin:21-jdk", + publish := {}, + publish / skip := true, + libraryDependencies ++= Seq( + declineCore, + declineCatsEffect, + otel4sCore, + logback % Runtime, + weaver + ), + Compile / doc / sources := Seq.empty, + Compile / run / fork := true + ) + .dependsOn(elastic, core, `ingestor-core`) + +lazy val `ingestor-core` = project + .in(file("modules/ingestor-core")) + .enablePlugins(Smithy4sCodegenPlugin) + .settings( + name := "ingestor-core", + commonSettings, + publish := {}, + publish / skip := true, libraryDependencies ++= Seq( chess, catsCore, fs2, fs2IO, catsEffect, - declineCore, - declineCatsEffect, - ducktape, cirisCore, cirisHtt4s, + ducktape, smithy4sCore, smithy4sJson, jsoniterCore, jsoniterMacro, circe, - http4sServer, - http4sEmberClient, mongo4catsCore, mongo4catsCirce, + http4sEmberClient, log4Cats, - logback % Runtime, - otel4sSdk, - otel4sMetrics, - otel4sPrometheusExporter, - otel4sInstrumentationMetrics, weaver, weaverScalaCheck ), - Compile / doc / sources := Seq.empty, - Compile / run / fork := true + Compile / doc / sources := Seq.empty ) .dependsOn(elastic, core) @@ -177,12 +210,12 @@ val e2e = project publish / skip := true, libraryDependencies ++= Seq(testContainers, weaver) ) - .dependsOn(client, app, ingestor) + .dependsOn(client, app, `ingestor-core`) lazy val root = project .in(file(".")) .settings(publish := {}, publish / skip := true) - .aggregate(core, api, app, client, e2e, elastic, ingestor) + .aggregate(core, api, app, client, e2e, elastic, `ingestor-core`, `ingestor-app`, `ingestor-cli`) addCommandAlias("prepare", "scalafixAll; scalafmtAll") addCommandAlias("check", "; scalafixAll --check ; scalafmtCheckAll") diff --git a/modules/ingestor/src/main/resources/logback.xml b/modules/ingestor-app/src/main/resources/logback.xml similarity index 100% rename from modules/ingestor/src/main/resources/logback.xml rename to modules/ingestor-app/src/main/resources/logback.xml diff --git a/modules/ingestor/src/main/scala/app.scala b/modules/ingestor-app/src/main/scala/app.scala similarity index 100% rename from modules/ingestor/src/main/scala/app.scala rename to modules/ingestor-app/src/main/scala/app.scala diff --git a/modules/ingestor-cli/src/main/resources/logback.xml b/modules/ingestor-cli/src/main/resources/logback.xml new file mode 100644 index 00000000..381f715b --- /dev/null +++ b/modules/ingestor-cli/src/main/resources/logback.xml @@ -0,0 +1,27 @@ + + + + + + + + logs/lila-search-ingestor-cli.log + + %date [%thread] %-5level %logger{20} - %msg%n%xException + + + + + + %date [%thread] %-5level %logger{20} - %msg%n%xException + + + + + + + + + + + diff --git a/modules/ingestor/src/main/scala/cli.scala b/modules/ingestor-cli/src/main/scala/cli.scala similarity index 100% rename from modules/ingestor/src/main/scala/cli.scala rename to modules/ingestor-cli/src/main/scala/cli.scala diff --git a/modules/ingestor/src/test/scala/ClITest.scala b/modules/ingestor-cli/src/test/scala/ClITest.scala similarity index 100% rename from modules/ingestor/src/test/scala/ClITest.scala rename to modules/ingestor-cli/src/test/scala/ClITest.scala diff --git a/modules/ingestor/src/main/scala/HasDocId.scala b/modules/ingestor-core/src/main/scala/HasDocId.scala similarity index 100% rename from modules/ingestor/src/main/scala/HasDocId.scala rename to modules/ingestor-core/src/main/scala/HasDocId.scala diff --git a/modules/ingestor/src/main/scala/Repo.scala b/modules/ingestor-core/src/main/scala/Repo.scala similarity index 100% rename from modules/ingestor/src/main/scala/Repo.scala rename to modules/ingestor-core/src/main/scala/Repo.scala diff --git a/modules/ingestor/src/main/scala/app.config.scala b/modules/ingestor-core/src/main/scala/config.scala similarity index 99% rename from modules/ingestor/src/main/scala/app.config.scala rename to modules/ingestor-core/src/main/scala/config.scala index 8763b846..eec04dca 100644 --- a/modules/ingestor/src/main/scala/app.config.scala +++ b/modules/ingestor-core/src/main/scala/config.scala @@ -58,6 +58,7 @@ case class IngestorConfig( ) object IngestorConfig: + case class Forum(batchSize: Int, timeWindows: Int, startAt: Option[Instant], maxPostLength: Int) case class Ublog(batchSize: Int, timeWindows: Int, startAt: Option[Instant]) case class Team(batchSize: Int, timeWindows: Int, startAt: Option[Instant]) diff --git a/modules/ingestor/src/main/scala/ingestor.scala b/modules/ingestor-core/src/main/scala/ingestor.scala similarity index 100% rename from modules/ingestor/src/main/scala/ingestor.scala rename to modules/ingestor-core/src/main/scala/ingestor.scala diff --git a/modules/ingestor/src/main/scala/ingestors.scala b/modules/ingestor-core/src/main/scala/ingestors.scala similarity index 100% rename from modules/ingestor/src/main/scala/ingestors.scala rename to modules/ingestor-core/src/main/scala/ingestors.scala diff --git a/modules/ingestor/src/main/scala/kvstore.scala b/modules/ingestor-core/src/main/scala/kvstore.scala similarity index 100% rename from modules/ingestor/src/main/scala/kvstore.scala rename to modules/ingestor-core/src/main/scala/kvstore.scala diff --git a/modules/ingestor/src/main/scala/mongo.chapter.scala b/modules/ingestor-core/src/main/scala/mongo.chapter.scala similarity index 100% rename from modules/ingestor/src/main/scala/mongo.chapter.scala rename to modules/ingestor-core/src/main/scala/mongo.chapter.scala diff --git a/modules/ingestor/src/main/scala/mongo.forum.scala b/modules/ingestor-core/src/main/scala/mongo.forum.scala similarity index 100% rename from modules/ingestor/src/main/scala/mongo.forum.scala rename to modules/ingestor-core/src/main/scala/mongo.forum.scala diff --git a/modules/ingestor/src/main/scala/mongo.game.scala b/modules/ingestor-core/src/main/scala/mongo.game.scala similarity index 100% rename from modules/ingestor/src/main/scala/mongo.game.scala rename to modules/ingestor-core/src/main/scala/mongo.game.scala diff --git a/modules/ingestor/src/main/scala/mongo.study.scala b/modules/ingestor-core/src/main/scala/mongo.study.scala similarity index 100% rename from modules/ingestor/src/main/scala/mongo.study.scala rename to modules/ingestor-core/src/main/scala/mongo.study.scala diff --git a/modules/ingestor/src/main/scala/mongo.team.scala b/modules/ingestor-core/src/main/scala/mongo.team.scala similarity index 100% rename from modules/ingestor/src/main/scala/mongo.team.scala rename to modules/ingestor-core/src/main/scala/mongo.team.scala diff --git a/modules/ingestor/src/main/scala/mongo.ublog.scala b/modules/ingestor-core/src/main/scala/mongo.ublog.scala similarity index 100% rename from modules/ingestor/src/main/scala/mongo.ublog.scala rename to modules/ingestor-core/src/main/scala/mongo.ublog.scala diff --git a/modules/ingestor/src/main/scala/app.resources.scala b/modules/ingestor-core/src/main/scala/resources.scala similarity index 100% rename from modules/ingestor/src/main/scala/app.resources.scala rename to modules/ingestor-core/src/main/scala/resources.scala diff --git a/modules/ingestor/src/main/smithy/model.smithy b/modules/ingestor-core/src/main/smithy/model.smithy similarity index 100% rename from modules/ingestor/src/main/smithy/model.smithy rename to modules/ingestor-core/src/main/smithy/model.smithy diff --git a/modules/ingestor/src/test/scala/HasDocIdTest.scala b/modules/ingestor-core/src/test/scala/HasDocIdTest.scala similarity index 100% rename from modules/ingestor/src/test/scala/HasDocIdTest.scala rename to modules/ingestor-core/src/test/scala/HasDocIdTest.scala diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 4d12e5be..d79b99b8 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -71,7 +71,7 @@ object Dependencies { val testContainers = "com.dimafeng" %% "testcontainers-scala-core" % "0.43.0" % Test val weaver = "org.typelevel" %% "weaver-cats" % "0.10.1" % Test - val weaverScalaCheck = "org.typelevel" %% "weaver-scalacheck" % "0.9.3" % Test + val weaverScalaCheck = "org.typelevel" %% "weaver-scalacheck" % "0.10.1" % Test val catsEffectTestKit = "org.typelevel" %% "cats-effect-testkit" % V.catsEffect % Test val scalacheck = "org.scalacheck" %% "scalacheck" % "1.17.0" % Test } From fdbc6961078cac24b90358f43696b6061804b9e9 Mon Sep 17 00:00:00 2001 From: Thanh Le Date: Sat, 13 Sep 2025 20:18:01 +0200 Subject: [PATCH 16/20] Remove unneded given in cli --- modules/ingestor-cli/src/main/scala/cli.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/modules/ingestor-cli/src/main/scala/cli.scala b/modules/ingestor-cli/src/main/scala/cli.scala index 0bc99ed2..44c7049a 100644 --- a/modules/ingestor-cli/src/main/scala/cli.scala +++ b/modules/ingestor-cli/src/main/scala/cli.scala @@ -3,7 +3,6 @@ package ingestor import cats.data.Validated import cats.effect.* -import cats.effect.unsafe.IORuntime import cats.syntax.all.* import com.monovore.decline.* import com.monovore.decline.effect.* @@ -24,7 +23,6 @@ object cli given LoggerFactory[IO] = Slf4jFactory.create[IO] given Logger[IO] = LoggerFactory[IO].getLogger given Meter[IO] = Meter.noop[IO] - given IORuntime = runtime override def main: Opts[IO[ExitCode]] = opts.parse.map: opts => From a0770d7f9901fcdf723bc9c3f8bd88d244d871cf Mon Sep 17 00:00:00 2001 From: Thanh Le Date: Sat, 13 Sep 2025 20:44:37 +0200 Subject: [PATCH 17/20] Support publish docker images --- .github/workflows/docker.yml | 53 ++++++++++++++++++++++++++++++++++++ build.sbt | 19 ++++++++----- 2 files changed, 65 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/docker.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 00000000..5c54b5f5 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,53 @@ +name: Create and publish a Docker image +on: + push: + tags: + - v* + pull_request: + branches: ['**'] +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + + permissions: + contents: read + packages: write + + steps: + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup JVM + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 21 + cache: sbt + + - name: Install sbt + uses: sbt/setup-sbt@v1 + + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build and push Docker image + run: sbt Docker/publish + diff --git a/build.sbt b/build.sbt index c6e89f6a..c02e084e 100644 --- a/build.sbt +++ b/build.sbt @@ -11,7 +11,12 @@ inThisBuild( semanticdbEnabled := true, // for scalafix resolvers ++= ourResolvers, Compile / doc / sources := Seq.empty, - publishTo := Option(Resolver.file("file", new File(sys.props.getOrElse("publishTo", "")))) + publishTo := Option(Resolver.file("file", new File(sys.props.getOrElse("publishTo", "")))), + dockerBaseImage := "eclipse-temurin:21-jdk-noble", + dockerUpdateLatest := true, + dockerBuildxPlatforms := Seq("linux/amd64", "linux/arm64"), + Docker / maintainer := "lichess.org", + Docker / dockerRepository := Some("ghcr.io"), ) ) @@ -80,12 +85,12 @@ lazy val elastic = project lazy val `ingestor-app` = project .in(file("modules/ingestor-app")) - .enablePlugins(JavaAppPackaging, BuildInfoPlugin) + .enablePlugins(JavaAppPackaging, BuildInfoPlugin, DockerPlugin) .settings( name := "ingestor-app", commonSettings, buildInfoSettings, - dockerBaseImage := "docker.io/library/eclipse-temurin:21-jdk", + Docker / packageName := "lichess-org/lila-search-ingestor-app", publish := {}, publish / skip := true, libraryDependencies ++= Seq( @@ -102,12 +107,12 @@ lazy val `ingestor-app` = project lazy val `ingestor-cli` = project .in(file("modules/ingestor-cli")) - .enablePlugins(JavaAppPackaging, BuildInfoPlugin) + .enablePlugins(JavaAppPackaging, BuildInfoPlugin, DockerPlugin) .settings( name := "ingestor-cli", commonSettings, buildInfoSettings, - dockerBaseImage := "docker.io/library/eclipse-temurin:21-jdk", + Docker / packageName := "lichess-org/lila-search-ingestor-cli", publish := {}, publish / skip := true, libraryDependencies ++= Seq( @@ -170,13 +175,13 @@ lazy val client = project .dependsOn(api, core) lazy val app = project - .enablePlugins(JavaAppPackaging, BuildInfoPlugin) + .enablePlugins(JavaAppPackaging, BuildInfoPlugin, DockerPlugin) .in(file("modules/app")) .settings( name := "lila-search", commonSettings, buildInfoSettings, - dockerBaseImage := "docker.io/library/eclipse-temurin:21-jdk", + Docker / packageName := "lichess-org/lila-search-app", publish := {}, publish / skip := true, libraryDependencies ++= Seq( From f231883d8121ff2095489269110906d31111bd51 Mon Sep 17 00:00:00 2001 From: Thanh Le Date: Sat, 13 Sep 2025 20:56:09 +0200 Subject: [PATCH 18/20] Adapt deploy scripts --- bin/deploy | 2 +- bin/deploy-cli | 31 +++++++++++++++++++++++++++++++ bin/deploy-ingestor | 6 +++--- 3 files changed, 35 insertions(+), 4 deletions(-) create mode 100755 bin/deploy-cli diff --git a/bin/deploy b/bin/deploy index c54fcfb0..b1df84d0 100755 --- a/bin/deploy +++ b/bin/deploy @@ -24,7 +24,7 @@ RSYNC_OPTIONS=" \ --exclude '.git/'" stage="modules/app/target/universal/stage" -include="bin $stage/bin $stage/lib" +include="$stage/bin $stage/lib" rsync_command="rsync $RSYNC_OPTIONS $include $REMOTE:$REMOTE_DIR" echo "$rsync_command" $rsync_command diff --git a/bin/deploy-cli b/bin/deploy-cli new file mode 100755 index 00000000..a0212a95 --- /dev/null +++ b/bin/deploy-cli @@ -0,0 +1,31 @@ +#!/bin/sh + +REMOTE=$1 +REMOTE_DIR="/home/lila-search-cli" + +echo "Deploy to server $REMOTE:$REMOTE_DIR" + +sbt ";ingestor-cli/stage;exit" +if [ $? != 0 ]; then + echo "Deploy canceled" + exit 1 +fi + +RSYNC_OPTIONS=" \ + --archive \ + --no-o --no-g \ + --force \ + --delete \ + --progress \ + --compress \ + --checksum \ + --verbose \ + --exclude RUNNING_PID \ + --exclude '.git/'" + +stage="modules/ingestor-cli/target/universal/stage" +include="$stage/bin $stage/lib" +rsync_command="rsync $RSYNC_OPTIONS $include $REMOTE:$REMOTE_DIR" +echo "$rsync_command" +$rsync_command +echo "rsync complete" diff --git a/bin/deploy-ingestor b/bin/deploy-ingestor index 797747fd..dd4d2d0b 100755 --- a/bin/deploy-ingestor +++ b/bin/deploy-ingestor @@ -5,7 +5,7 @@ REMOTE_DIR="/home/lila-search-ingestor" echo "Deploy to server $REMOTE:$REMOTE_DIR" -sbt ";ingestor/stage;exit" +sbt ";ingestor-app/stage;exit" if [ $? != 0 ]; then echo "Deploy canceled" exit 1 @@ -23,8 +23,8 @@ RSYNC_OPTIONS=" \ --exclude RUNNING_PID \ --exclude '.git/'" -stage="modules/ingestor/target/universal/stage" -include="bin $stage/bin $stage/lib" +stage="modules/ingestor-app/target/universal/stage" +include="$stage/bin $stage/lib" rsync_command="rsync $RSYNC_OPTIONS $include $REMOTE:$REMOTE_DIR" echo "$rsync_command" $rsync_command From 00b0d5032504b716a459f08ecde915eff92fd402 Mon Sep 17 00:00:00 2001 From: Thanh Le Date: Sun, 14 Sep 2025 08:54:23 +0200 Subject: [PATCH 19/20] Only publish docker images on tag --- .github/workflows/docker.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 5c54b5f5..d5701f4e 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -3,8 +3,7 @@ on: push: tags: - v* - pull_request: - branches: ['**'] + env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} From b30f6965f3379bee48ad337765daeeb5697da75d Mon Sep 17 00:00:00 2001 From: Thanh Le Date: Sun, 14 Sep 2025 09:05:34 +0200 Subject: [PATCH 20/20] Setting version to 3.2.2 --- version.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.sbt b/version.sbt index 2ccaca76..14b63d3c 100644 --- a/version.sbt +++ b/version.sbt @@ -1 +1 @@ -ThisBuild / version := "3.2.2-SNAPSHOT" +ThisBuild / version := "3.2.2"