diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index fef72f09..7f80fb21 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -18,3 +18,6 @@ ed503a2852d8148f39ac99b7188e47f569bdf01a # scalafmt align.preset = none adcf303d99abee49f1266dad223bb3e5e34bd73f + +# Scala Steward: Reformat with scalafmt 3.9.9 +6dfb7ed8ffa2f5df0f5c36bee8d3497d29016495 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 00000000..d5701f4e --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,52 @@ +name: Create and publish a Docker image +on: + push: + tags: + - v* + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + + permissions: + contents: read + packages: write + + steps: + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup JVM + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 21 + cache: sbt + + - name: Install sbt + uses: sbt/setup-sbt@v1 + + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build and push Docker image + run: sbt Docker/publish + diff --git a/.scalafmt.conf b/.scalafmt.conf index ca23448d..d1045b0f 100644 --- a/.scalafmt.conf +++ b/.scalafmt.conf @@ -1,4 +1,4 @@ -version = "3.9.8" +version = "3.9.9" runner.dialect = scala3 align.preset = none diff --git a/README.md b/README.md index f929070e..ad7adf64 100644 --- a/README.md +++ b/README.md @@ -35,12 +35,12 @@ prepare Start ingestor service: ```sh -ingestor/runMain lila.search.ingestor.App +ingestor-app/run ``` Start ingestor cli tool ```sh -ingestor/runMain lila.search.ingestor.cli --help +ingestor-cli/run --help ``` #### CLI tool diff --git a/bin/deploy b/bin/deploy index c54fcfb0..b1df84d0 100755 --- a/bin/deploy +++ b/bin/deploy @@ -24,7 +24,7 @@ RSYNC_OPTIONS=" \ --exclude '.git/'" stage="modules/app/target/universal/stage" -include="bin $stage/bin $stage/lib" +include="$stage/bin $stage/lib" rsync_command="rsync $RSYNC_OPTIONS $include $REMOTE:$REMOTE_DIR" echo "$rsync_command" $rsync_command diff --git a/bin/deploy-cli b/bin/deploy-cli new file mode 100755 index 00000000..a0212a95 --- /dev/null +++ b/bin/deploy-cli @@ -0,0 +1,31 @@ +#!/bin/sh + +REMOTE=$1 +REMOTE_DIR="/home/lila-search-cli" + +echo "Deploy to server $REMOTE:$REMOTE_DIR" + +sbt ";ingestor-cli/stage;exit" +if [ $? != 0 ]; then + echo "Deploy canceled" + exit 1 +fi + +RSYNC_OPTIONS=" \ + --archive \ + --no-o --no-g \ + --force \ + --delete \ + --progress \ + --compress \ + --checksum \ + --verbose \ + --exclude RUNNING_PID \ + --exclude '.git/'" + +stage="modules/ingestor-cli/target/universal/stage" +include="$stage/bin $stage/lib" +rsync_command="rsync $RSYNC_OPTIONS $include $REMOTE:$REMOTE_DIR" +echo "$rsync_command" +$rsync_command +echo "rsync complete" diff --git a/bin/deploy-ingestor b/bin/deploy-ingestor index 797747fd..dd4d2d0b 100755 --- a/bin/deploy-ingestor +++ b/bin/deploy-ingestor @@ -5,7 +5,7 @@ REMOTE_DIR="/home/lila-search-ingestor" echo "Deploy to server $REMOTE:$REMOTE_DIR" -sbt ";ingestor/stage;exit" +sbt ";ingestor-app/stage;exit" if [ $? != 0 ]; then echo "Deploy canceled" exit 1 @@ -23,8 +23,8 @@ RSYNC_OPTIONS=" \ --exclude RUNNING_PID \ --exclude '.git/'" -stage="modules/ingestor/target/universal/stage" -include="bin $stage/bin $stage/lib" +stage="modules/ingestor-app/target/universal/stage" +include="$stage/bin $stage/lib" rsync_command="rsync $RSYNC_OPTIONS $include $REMOTE:$REMOTE_DIR" echo "$rsync_command" $rsync_command diff --git a/build.sbt b/build.sbt index 7defaf08..c02e084e 100644 --- a/build.sbt +++ b/build.sbt @@ -3,15 +3,20 @@ import org.typelevel.scalacoptions.ScalacOptions inThisBuild( Seq( - scalaVersion := "3.7.1", + scalaVersion := "3.7.3", versionScheme := Some("early-semver"), - organization := "org.lichess.search", - run / fork := true, + organization := "org.lichess.search", + run / fork := true, run / javaOptions += "-Dconfig.override_with_env_vars=true", semanticdbEnabled := true, // for scalafix resolvers ++= ourResolvers, Compile / doc / sources := Seq.empty, - publishTo := Option(Resolver.file("file", new File(sys.props.getOrElse("publishTo", "")))) + publishTo := Option(Resolver.file("file", new File(sys.props.getOrElse("publishTo", "")))), + dockerBaseImage := "eclipse-temurin:21-jdk-noble", + dockerUpdateLatest := true, + dockerBuildxPlatforms := Seq("linux/amd64", "linux/arm64"), + Docker / maintainer := "lichess.org", + Docker / dockerRepository := Some("ghcr.io"), ) ) @@ -65,11 +70,12 @@ lazy val elastic = project .settings( name := "elastic", commonSettings, - publish := {}, + publish := {}, publish / skip := true, libraryDependencies ++= Seq( catsCore, catsEffect, + catsMtl, http4sClient, elastic4sHttp4sClient, otel4sCore @@ -77,47 +83,80 @@ lazy val elastic = project ) .dependsOn(api, core) -lazy val ingestor = project - .in(file("modules/ingestor")) - .enablePlugins(JavaAppPackaging, Smithy4sCodegenPlugin, BuildInfoPlugin) +lazy val `ingestor-app` = project + .in(file("modules/ingestor-app")) + .enablePlugins(JavaAppPackaging, BuildInfoPlugin, DockerPlugin) .settings( - name := "ingestor", + name := "ingestor-app", commonSettings, buildInfoSettings, - dockerBaseImage := "docker.io/library/eclipse-temurin:21-jdk", - publish := {}, - publish / skip := true, + Docker / packageName := "lichess-org/lila-search-ingestor-app", + publish := {}, + publish / skip := true, + libraryDependencies ++= Seq( + logback % Runtime, + otel4sSdk, + otel4sMetrics, + otel4sPrometheusExporter, + otel4sInstrumentationMetrics + ), + Compile / doc / sources := Seq.empty, + Compile / run / fork := true + ) + .dependsOn(`ingestor-core`) + +lazy val `ingestor-cli` = project + .in(file("modules/ingestor-cli")) + .enablePlugins(JavaAppPackaging, BuildInfoPlugin, DockerPlugin) + .settings( + name := "ingestor-cli", + commonSettings, + buildInfoSettings, + Docker / packageName := "lichess-org/lila-search-ingestor-cli", + publish := {}, + publish / skip := true, + libraryDependencies ++= Seq( + declineCore, + declineCatsEffect, + otel4sCore, + logback % Runtime, + weaver + ), + Compile / doc / sources := Seq.empty, + Compile / run / fork := true + ) + .dependsOn(elastic, core, `ingestor-core`) + +lazy val `ingestor-core` = project + .in(file("modules/ingestor-core")) + .enablePlugins(Smithy4sCodegenPlugin) + .settings( + name := "ingestor-core", + commonSettings, + publish := {}, + publish / skip := true, libraryDependencies ++= Seq( chess, catsCore, fs2, fs2IO, catsEffect, - declineCore, - declineCatsEffect, - ducktape, cirisCore, cirisHtt4s, + ducktape, smithy4sCore, smithy4sJson, jsoniterCore, jsoniterMacro, circe, - http4sServer, - http4sEmberClient, mongo4catsCore, mongo4catsCirce, + http4sEmberClient, log4Cats, - logback % Runtime, - otel4sSdk, - otel4sMetrics, - otel4sPrometheusExporter, - otel4sInstrumentationMetrics, weaver, weaverScalaCheck ), - Compile / doc / sources := Seq.empty, - Compile / run / fork := true + Compile / doc / sources := Seq.empty ) .dependsOn(elastic, core) @@ -136,15 +175,15 @@ lazy val client = project .dependsOn(api, core) lazy val app = project - .enablePlugins(JavaAppPackaging, BuildInfoPlugin) + .enablePlugins(JavaAppPackaging, BuildInfoPlugin, DockerPlugin) .in(file("modules/app")) .settings( name := "lila-search", commonSettings, buildInfoSettings, - dockerBaseImage := "docker.io/library/eclipse-temurin:21-jdk", - publish := {}, - publish / skip := true, + Docker / packageName := "lichess-org/lila-search-app", + publish := {}, + publish / skip := true, libraryDependencies ++= Seq( smithy4sHttp4s, jsoniterCore, @@ -165,23 +204,23 @@ lazy val app = project otel4sInstrumentationMetrics ), Compile / doc / sources := Seq.empty, - Compile / run / fork := true + Compile / run / fork := true ) .dependsOn(api, elastic) val e2e = project .in(file("modules/e2e")) .settings( - publish := {}, + publish := {}, publish / skip := true, libraryDependencies ++= Seq(testContainers, weaver) ) - .dependsOn(client, app, ingestor) + .dependsOn(client, app, `ingestor-core`) lazy val root = project .in(file(".")) .settings(publish := {}, publish / skip := true) - .aggregate(core, api, app, client, e2e, elastic, ingestor) + .aggregate(core, api, app, client, e2e, elastic, `ingestor-core`, `ingestor-app`, `ingestor-cli`) addCommandAlias("prepare", "scalafixAll; scalafmtAll") addCommandAlias("check", "; scalafixAll --check ; scalafmtCheckAll") diff --git a/modules/app/src/main/scala/service.health.scala b/modules/app/src/main/scala/service.health.scala index 3c3f8bfa..254f8514 100644 --- a/modules/app/src/main/scala/service.health.scala +++ b/modules/app/src/main/scala/service.health.scala @@ -2,6 +2,7 @@ package lila.search package app import cats.effect.* +import cats.mtl.Handle.* import cats.syntax.all.* import lila.search.spec.* import org.typelevel.log4cats.{ Logger, LoggerFactory } @@ -11,12 +12,13 @@ class HealthServiceImpl(esClient: ESClient[IO])(using LoggerFactory[IO]) extends given Logger[IO] = LoggerFactory[IO].getLogger override def healthCheck(): IO[HealthCheckOutput] = - esClient.status - .flatMap(transform) - .map(HealthCheckOutput(_)) - .handleErrorWith: e => - Logger[IO].error(e)("Error in health check") *> - IO.raiseError(InternalServerError(s"Internal server error ${e.getMessage}")) + allow: + esClient.status + .flatMap(transform) + .map(HealthCheckOutput(_)) + .rescue: e => + Logger[IO].error(e.asException)("Error in health check") *> + IO.raiseError(InternalServerError(s"Internal server error ${e.reason}")) private def transform(status: String): IO[ElasticStatus] = status match diff --git a/modules/app/src/main/scala/service.search.scala b/modules/app/src/main/scala/service.search.scala index ed9ff73f..25fc9a8a 100644 --- a/modules/app/src/main/scala/service.search.scala +++ b/modules/app/src/main/scala/service.search.scala @@ -2,6 +2,7 @@ package lila.search package app import cats.effect.* +import cats.mtl.Handle.* import io.github.arainko.ducktape.* import lila.search.forum.Forum import lila.search.game.Game @@ -51,21 +52,21 @@ class SearchServiceImpl(esClient: ESClient[IO], metric: Histogram[IO, Double])(u override def count(query: Query): IO[CountOutput] = countRecord: - esClient - .count(query) - .map(CountOutput.apply) - .handleErrorWith: e => - logger.error(e)(s"Error in count: query=${query.toString}") *> - IO.raiseError(InternalServerError("Internal server error")) + allow: + esClient.count(query) + .rescue: e => + logger.error(e.asException)(s"Error in count: query=${query.toString}") *> + IO.raiseError(InternalServerError("Internal server error")) + .map(CountOutput.apply) override def search(query: Query, from: From, size: Size): IO[SearchOutput] = searchRecord: - esClient - .search(query, from, size) - .map(SearchOutput.apply) - .handleErrorWith: e => - logger.error(e)(s"Error in search: query=${query.toString}, from=$from, size=$size") *> - IO.raiseError(InternalServerError("Internal server error")) + allow: + esClient.search(query, from, size) + .rescue: e => + logger.error(e.asException)(s"Error in search: query=${query.toString}, from=$from, size=$size") *> + IO.raiseError(InternalServerError("Internal server error")) + .map(SearchOutput.apply) object SearchServiceImpl: diff --git a/modules/e2e/src/test/scala/CompatSuite.scala b/modules/e2e/src/test/scala/CompatSuite.scala index e628eed6..071942c7 100644 --- a/modules/e2e/src/test/scala/CompatSuite.scala +++ b/modules/e2e/src/test/scala/CompatSuite.scala @@ -67,18 +67,18 @@ object CompatSuite extends weaver.IOSuite: def fakeClient: ESClient[IO] = new: - override def store[A](index: Index, id: Id, obj: A)(using Indexable[A]): IO[Unit] = IO.unit + override def store[A](index: Index, id: Id, obj: A)(using Indexable[A]) = IO.unit - override def storeBulk[A](index: Index, objs: Seq[SourceWithId[A]])(using Indexable[A]): IO[Unit] = + override def storeBulk[A](index: Index, objs: Seq[SourceWithId[A]])(using Indexable[A]) = IO.unit - override def putMapping(index: Index): IO[Unit] = IO.unit + override def putMapping(index: Index) = IO.unit - override def refreshIndex(index: Index): IO[Unit] = IO.unit + override def refreshIndex(index: Index) = IO.unit - override def deleteOne(index: Index, id: Id): IO[Unit] = IO.unit + override def deleteOne(index: Index, id: Id) = IO.unit - override def deleteMany(index: Index, ids: List[Id]): IO[Unit] = IO.unit + override def deleteMany(index: Index, ids: List[Id]) = IO.unit override def count[A](query: A)(using Queryable[A]) = IO.pure(0) @@ -86,7 +86,7 @@ object CompatSuite extends weaver.IOSuite: override def search[A](query: A, from: From, size: Size)(using Queryable[A]) = IO.pure(Nil) - override def status: IO[String] = IO.pure("yellow") + override def status = IO.pure("yellow") given system: ActorSystem = ActorSystem() diff --git a/modules/e2e/src/test/scala/IntegrationSuite.scala b/modules/e2e/src/test/scala/IntegrationSuite.scala index 59abc78b..9a19e46f 100644 --- a/modules/e2e/src/test/scala/IntegrationSuite.scala +++ b/modules/e2e/src/test/scala/IntegrationSuite.scala @@ -1,9 +1,13 @@ package lila.search package app package test + +import cats.Functor import cats.effect.{ IO, Resource } +import cats.mtl.Raise import cats.syntax.all.* import com.comcast.ip4s.* +import com.sksamuel.elastic4s.ElasticError import lila.search.ingestor.Ingestor.given import lila.search.spec.* import org.http4s.Uri @@ -22,6 +26,10 @@ object IntegrationSuite extends IOSuite: given Logger[IO] = NoOpLogger[IO] given LoggerFactory[IO] = NoOpFactory[IO] given Meter[IO] = Meter.noop[IO] + private given Raise[IO, ElasticError]: + def functor: Functor[IO] = Functor[IO] + def raise[E <: ElasticError, A](e: E): IO[A] = + IO.raiseError(e.asException) private val uri = Uri.unsafeFromString("http://localhost:9999") diff --git a/modules/elastic/src/main/scala/ESClient.scala b/modules/elastic/src/main/scala/ESClient.scala index 6ee6751f..06c00318 100644 --- a/modules/elastic/src/main/scala/ESClient.scala +++ b/modules/elastic/src/main/scala/ESClient.scala @@ -1,10 +1,11 @@ package lila.search import cats.effect.* +import cats.mtl.Raise import cats.syntax.all.* import com.sksamuel.elastic4s.ElasticDsl.* import com.sksamuel.elastic4s.http4s.Http4sClient -import com.sksamuel.elastic4s.{ ElasticClient, ElasticDsl, Index as ESIndex, Indexable } +import com.sksamuel.elastic4s.{ ElasticClient, ElasticDsl, ElasticError, Index as ESIndex, Indexable } import lila.search.ESClient.MetricKeys.* import org.http4s.Uri import org.http4s.client.Client @@ -14,16 +15,17 @@ import org.typelevel.otel4s.{ Attribute, AttributeKey, Attributes } import java.util.concurrent.TimeUnit trait ESClient[F[_]]: - - def search[A](query: A, from: From, size: Size)(using Queryable[A]): F[List[Id]] - def count[A](query: A)(using Queryable[A]): F[Long] - def store[A](index: Index, id: Id, obj: A)(using Indexable[A]): F[Unit] - def storeBulk[A](index: Index, objs: Seq[SourceWithId[A]])(using Indexable[A]): F[Unit] - def deleteOne(index: Index, id: Id): F[Unit] - def deleteMany(index: Index, ids: List[Id]): F[Unit] - def putMapping(index: Index): F[Unit] - def refreshIndex(index: Index): F[Unit] - def status: F[String] + type RaiseF[A] = Raise[F, ElasticError] ?=> F[A] + + def search[A](query: A, from: From, size: Size)(using Queryable[A]): RaiseF[List[Id]] + def count[A](query: A)(using Queryable[A]): RaiseF[Long] + def store[A](index: Index, id: Id, obj: A)(using Indexable[A]): RaiseF[Unit] + def storeBulk[A](index: Index, objs: Seq[SourceWithId[A]])(using Indexable[A]): RaiseF[Unit] + def deleteOne(index: Index, id: Id): RaiseF[Unit] + def deleteMany(index: Index, ids: List[Id]): RaiseF[Unit] + def putMapping(index: Index): RaiseF[Unit] + def refreshIndex(index: Index): RaiseF[Unit] + def status: RaiseF[String] object ESClient: @@ -43,13 +45,13 @@ object ESClient: metric: Histogram[F, Double] ) = new ESClient[F]: - def status: F[String] = + def status: RaiseF[String] = client .execute(clusterHealth()) .flatMap(_.toResult) .map(_.status) - def search[A](query: A, from: From, size: Size)(using Queryable[A]): F[List[Id]] = + def search[A](query: A, from: From, size: Size)(using Queryable[A]): RaiseF[List[Id]] = metric .recordDuration( TimeUnit.MILLISECONDS, @@ -65,7 +67,7 @@ object ESClient: .flatMap(_.toResult) .map(_.hits.hits.toList.map(h => Id(h.id))) - def count[A](query: A)(using Queryable[A]): F[Long] = + def count[A](query: A)(using Queryable[A]): RaiseF[Long] = metric .recordDuration( TimeUnit.MILLISECONDS, @@ -81,7 +83,7 @@ object ESClient: .flatMap(_.toResult) .map(_.count) - def store[A](index: Index, id: Id, obj: A)(using Indexable[A]): F[Unit] = + def store[A](index: Index, id: Id, obj: A)(using Indexable[A]): RaiseF[Unit] = metric .recordDuration( TimeUnit.MILLISECONDS, @@ -96,7 +98,7 @@ object ESClient: .execute(indexInto(index.value).source(obj).id(id.value)) .flatMap(_.unitOrFail) - def storeBulk[A](index: Index, objs: Seq[SourceWithId[A]])(using Indexable[A]): F[Unit] = + def storeBulk[A](index: Index, objs: Seq[SourceWithId[A]])(using Indexable[A]): RaiseF[Unit] = val request = indexInto(index.value) val requests = bulk(objs.map { case (id, source) => request.source(source).id(id) }) metric @@ -115,7 +117,7 @@ object ESClient: .flatMap(_.unitOrFail) .whenA(objs.nonEmpty) - def deleteOne(index: Index, id: Id): F[Unit] = + def deleteOne(index: Index, id: Id): RaiseF[Unit] = metric .recordDuration( TimeUnit.MILLISECONDS, @@ -130,7 +132,7 @@ object ESClient: .execute(deleteById(index.toES, id.value)) .flatMap(_.unitOrFail) - def deleteMany(index: Index, ids: List[Id]): F[Unit] = + def deleteMany(index: Index, ids: List[Id]): RaiseF[Unit] = metric .recordDuration( TimeUnit.MILLISECONDS, @@ -147,7 +149,7 @@ object ESClient: .flatMap(_.unitOrFail) .whenA(ids.nonEmpty) - def putMapping(index: Index): F[Unit] = + def putMapping(index: Index): RaiseF[Unit] = dropIndex(index) *> client .execute: createIndex(index.value) @@ -157,7 +159,7 @@ object ESClient: .refreshInterval(index.refreshInterval) .flatMap(_.unitOrFail) - def refreshIndex(index: Index): F[Unit] = + def refreshIndex(index: Index): RaiseF[Unit] = client .execute(ElasticDsl.refreshIndex(index.value)) .flatMap(_.unitOrFail) diff --git a/modules/elastic/src/main/scala/package.scala b/modules/elastic/src/main/scala/package.scala index 616e6fae..5ca6ad7a 100644 --- a/modules/elastic/src/main/scala/package.scala +++ b/modules/elastic/src/main/scala/package.scala @@ -1,10 +1,12 @@ package lila.search -import cats.MonadThrow +import cats.Monad +import cats.mtl.Raise +import cats.mtl.implicits.* import cats.syntax.all.* import com.sksamuel.elastic4s.ElasticDsl.* import com.sksamuel.elastic4s.requests.searches.queries.Query -import com.sksamuel.elastic4s.{ Index as ESIndex, Response } +import com.sksamuel.elastic4s.{ ElasticError, Index as ESIndex, Response } type SourceWithId[A] = (id: String, source: A) @@ -31,8 +33,8 @@ extension (index: Index) case Index.Study => "10s" case _ => "300s" -extension [F[_]: MonadThrow, A](response: Response[A]) - def toResult: F[A] = - response.fold(response.error.asException.raiseError)(r => r.pure[F]) - def unitOrFail: F[Unit] = - response.fold(response.error.asException.raiseError)(_ => ().pure[F]) +extension [F[_]: Monad, A](response: Response[A]) + def toResult: Raise[F, ElasticError] ?=> F[A] = + response.fold(response.error.raise)(_.pure[F]) + def unitOrFail: Raise[F, ElasticError] ?=> F[Unit] = + response.fold(response.error.raise)(_ => ().pure[F]) diff --git a/modules/ingestor/src/main/resources/logback.xml b/modules/ingestor-app/src/main/resources/logback.xml similarity index 100% rename from modules/ingestor/src/main/resources/logback.xml rename to modules/ingestor-app/src/main/resources/logback.xml diff --git a/modules/ingestor/src/main/scala/app.scala b/modules/ingestor-app/src/main/scala/app.scala similarity index 100% rename from modules/ingestor/src/main/scala/app.scala rename to modules/ingestor-app/src/main/scala/app.scala diff --git a/modules/ingestor-cli/src/main/resources/logback.xml b/modules/ingestor-cli/src/main/resources/logback.xml new file mode 100644 index 00000000..381f715b --- /dev/null +++ b/modules/ingestor-cli/src/main/resources/logback.xml @@ -0,0 +1,27 @@ + + + + + + + + logs/lila-search-ingestor-cli.log + + %date [%thread] %-5level %logger{20} - %msg%n%xException + + + + + + %date [%thread] %-5level %logger{20} - %msg%n%xException + + + + + + + + + + + diff --git a/modules/ingestor/src/main/scala/cli.scala b/modules/ingestor-cli/src/main/scala/cli.scala similarity index 98% rename from modules/ingestor/src/main/scala/cli.scala rename to modules/ingestor-cli/src/main/scala/cli.scala index 0bc99ed2..44c7049a 100644 --- a/modules/ingestor/src/main/scala/cli.scala +++ b/modules/ingestor-cli/src/main/scala/cli.scala @@ -3,7 +3,6 @@ package ingestor import cats.data.Validated import cats.effect.* -import cats.effect.unsafe.IORuntime import cats.syntax.all.* import com.monovore.decline.* import com.monovore.decline.effect.* @@ -24,7 +23,6 @@ object cli given LoggerFactory[IO] = Slf4jFactory.create[IO] given Logger[IO] = LoggerFactory[IO].getLogger given Meter[IO] = Meter.noop[IO] - given IORuntime = runtime override def main: Opts[IO[ExitCode]] = opts.parse.map: opts => diff --git a/modules/ingestor/src/test/scala/ClITest.scala b/modules/ingestor-cli/src/test/scala/ClITest.scala similarity index 100% rename from modules/ingestor/src/test/scala/ClITest.scala rename to modules/ingestor-cli/src/test/scala/ClITest.scala diff --git a/modules/ingestor/src/main/scala/HasDocId.scala b/modules/ingestor-core/src/main/scala/HasDocId.scala similarity index 100% rename from modules/ingestor/src/main/scala/HasDocId.scala rename to modules/ingestor-core/src/main/scala/HasDocId.scala diff --git a/modules/ingestor/src/main/scala/Repo.scala b/modules/ingestor-core/src/main/scala/Repo.scala similarity index 100% rename from modules/ingestor/src/main/scala/Repo.scala rename to modules/ingestor-core/src/main/scala/Repo.scala diff --git a/modules/ingestor/src/main/scala/app.config.scala b/modules/ingestor-core/src/main/scala/config.scala similarity index 99% rename from modules/ingestor/src/main/scala/app.config.scala rename to modules/ingestor-core/src/main/scala/config.scala index 8763b846..eec04dca 100644 --- a/modules/ingestor/src/main/scala/app.config.scala +++ b/modules/ingestor-core/src/main/scala/config.scala @@ -58,6 +58,7 @@ case class IngestorConfig( ) object IngestorConfig: + case class Forum(batchSize: Int, timeWindows: Int, startAt: Option[Instant], maxPostLength: Int) case class Ublog(batchSize: Int, timeWindows: Int, startAt: Option[Instant]) case class Team(batchSize: Int, timeWindows: Int, startAt: Option[Instant]) diff --git a/modules/ingestor/src/main/scala/ingestor.scala b/modules/ingestor-core/src/main/scala/ingestor.scala similarity index 80% rename from modules/ingestor/src/main/scala/ingestor.scala rename to modules/ingestor-core/src/main/scala/ingestor.scala index 6d92d6c5..d1eab472 100644 --- a/modules/ingestor/src/main/scala/ingestor.scala +++ b/modules/ingestor-core/src/main/scala/ingestor.scala @@ -2,6 +2,7 @@ package lila.search package ingestor import cats.effect.* +import cats.mtl.Handle.* import cats.syntax.all.* import com.github.plokhotnyuk.jsoniter_scala.core.* import com.sksamuel.elastic4s.Indexable @@ -31,7 +32,7 @@ object Ingestor: elastic: ESClient[IO], defaultStartAt: Option[Instant] )(using LoggerFactory[IO]): Ingestor = new: - given Logger[IO] = LoggerFactory[IO].getLoggerFromName(s"${index.value}.ingestor") + given logger: Logger[IO] = LoggerFactory[IO].getLoggerFromName(s"${index.value}.ingestor") def watch: IO[Unit] = fs2.Stream @@ -71,21 +72,21 @@ object Ingestor: .flatTap(since => info"Starting ${index.value} ingestor from $since") private def deleteMany(index: Index, ids: List[Id]): IO[Unit] = - elastic - .deleteMany(index, ids) - .flatTap(_ => Logger[IO].info(s"Deleted ${ids.size} ${index.value}s")) - .handleErrorWith: e => - Logger[IO].error(e)(s"Failed to delete ${index.value}: ${ids.map(_.value).mkString(", ")}") + allow: + elastic.deleteMany(index, ids) + .rescue: e => + logger.error(e.asException)(s"Failed to delete ${index.value}: ${ids.map(_.value).mkString(", ")}") + .flatTap(_ => Logger[IO].info(s"Deleted ${ids.size} ${index.value}s")) .whenA(ids.nonEmpty) private def storeBulk(index: Index, sources: List[SourceWithId[A]]): IO[Unit] = Logger[IO].info(s"Received ${sources.size} docs to ${index.value}") *> - elastic - .storeBulk(index, sources) - .handleErrorWith: e => - Logger[IO].error(e)(s"Failed to ${index.value} index: ${sources.map(_.id).mkString(", ")}") - .whenA(sources.nonEmpty) - *> Logger[IO].info(s"Indexed ${sources.size} ${index.value}s") + allow: + elastic.storeBulk(index, sources) + .rescue: e => + logger.error(e.asException)(s"Failed to ${index.value} index: ${sources.map(_.id).mkString(", ")}") + .whenA(sources.nonEmpty) + *> logger.info(s"Indexed ${sources.size} ${index.value}s") private val saveLastIndexedTimestamp: Option[Instant] => IO[Unit] = _.traverse_(time => diff --git a/modules/ingestor/src/main/scala/ingestors.scala b/modules/ingestor-core/src/main/scala/ingestors.scala similarity index 100% rename from modules/ingestor/src/main/scala/ingestors.scala rename to modules/ingestor-core/src/main/scala/ingestors.scala diff --git a/modules/ingestor/src/main/scala/kvstore.scala b/modules/ingestor-core/src/main/scala/kvstore.scala similarity index 100% rename from modules/ingestor/src/main/scala/kvstore.scala rename to modules/ingestor-core/src/main/scala/kvstore.scala diff --git a/modules/ingestor/src/main/scala/mongo.chapter.scala b/modules/ingestor-core/src/main/scala/mongo.chapter.scala similarity index 100% rename from modules/ingestor/src/main/scala/mongo.chapter.scala rename to modules/ingestor-core/src/main/scala/mongo.chapter.scala diff --git a/modules/ingestor/src/main/scala/mongo.forum.scala b/modules/ingestor-core/src/main/scala/mongo.forum.scala similarity index 100% rename from modules/ingestor/src/main/scala/mongo.forum.scala rename to modules/ingestor-core/src/main/scala/mongo.forum.scala diff --git a/modules/ingestor/src/main/scala/mongo.game.scala b/modules/ingestor-core/src/main/scala/mongo.game.scala similarity index 100% rename from modules/ingestor/src/main/scala/mongo.game.scala rename to modules/ingestor-core/src/main/scala/mongo.game.scala diff --git a/modules/ingestor/src/main/scala/mongo.study.scala b/modules/ingestor-core/src/main/scala/mongo.study.scala similarity index 100% rename from modules/ingestor/src/main/scala/mongo.study.scala rename to modules/ingestor-core/src/main/scala/mongo.study.scala diff --git a/modules/ingestor/src/main/scala/mongo.team.scala b/modules/ingestor-core/src/main/scala/mongo.team.scala similarity index 100% rename from modules/ingestor/src/main/scala/mongo.team.scala rename to modules/ingestor-core/src/main/scala/mongo.team.scala diff --git a/modules/ingestor/src/main/scala/mongo.ublog.scala b/modules/ingestor-core/src/main/scala/mongo.ublog.scala similarity index 100% rename from modules/ingestor/src/main/scala/mongo.ublog.scala rename to modules/ingestor-core/src/main/scala/mongo.ublog.scala diff --git a/modules/ingestor/src/main/scala/app.resources.scala b/modules/ingestor-core/src/main/scala/resources.scala similarity index 100% rename from modules/ingestor/src/main/scala/app.resources.scala rename to modules/ingestor-core/src/main/scala/resources.scala diff --git a/modules/ingestor/src/main/smithy/model.smithy b/modules/ingestor-core/src/main/smithy/model.smithy similarity index 100% rename from modules/ingestor/src/main/smithy/model.smithy rename to modules/ingestor-core/src/main/smithy/model.smithy diff --git a/modules/ingestor/src/test/scala/HasDocIdTest.scala b/modules/ingestor-core/src/test/scala/HasDocIdTest.scala similarity index 100% rename from modules/ingestor/src/test/scala/HasDocIdTest.scala rename to modules/ingestor-core/src/test/scala/HasDocIdTest.scala diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 865b0472..d79b99b8 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -5,17 +5,17 @@ object Dependencies { val lilaMaven = "lila-maven" at "https://raw.githubusercontent.com/lichess-org/lila-maven/master" val jitpack = "jitpack".at("https://jitpack.io") - val ourResolvers = Seq(lilaMaven, jitpack) + val ourResolvers = Seq(lilaMaven, jitpack, Resolver.sonatypeCentralSnapshots) object V { val catsEffect = "3.6.3" - val chess = "17.9.1" - val ciris = "3.9.0" + val catsMtl = "1.6.0" + val chess = "17.9.6" + val ciris = "3.10.0" val decline = "2.5.0" - val elastic4s = "9.0.0" - val fs2 = "3.12.0" + val elastic4s = "9.1.0" + val fs2 = "3.12.2" val http4s = "0.23.30" - val iron = "2.5.0" val mongo4cats = "0.7.13" val otel4s = "0.13.1" } @@ -27,14 +27,13 @@ object Dependencies { val catsCore = "org.typelevel" %% "cats-core" % "2.13.0" val catsEffect = "org.typelevel" %% "cats-effect" % V.catsEffect + val catsMtl = "org.typelevel" %% "cats-mtl" % V.catsMtl val fs2 = "co.fs2" %% "fs2-core" % V.fs2 val fs2IO = "co.fs2" %% "fs2-io" % V.fs2 val cirisCore = "is.cir" %% "ciris" % V.ciris val cirisHtt4s = "is.cir" %% "ciris-http4s" % V.ciris - val iron = "io.github.iltotore" %% "iron" % V.iron - val ironCiris = "io.github.iltotore" %% "iron-ciris" % V.iron val http4sServer = http4s("ember-server") val http4sClient = http4s("client") @@ -45,10 +44,10 @@ object Dependencies { lazy val smithy4sHttp4sSwagger = smithy4s("http4s-swagger") lazy val smithy4sJson = smithy4s("json") - val jsoniterCore = "com.github.plokhotnyuk.jsoniter-scala" %% "jsoniter-scala-core" % "2.37.0" - val jsoniterMacro = "com.github.plokhotnyuk.jsoniter-scala" %% "jsoniter-scala-macros" % "2.37.0" + val jsoniterCore = "com.github.plokhotnyuk.jsoniter-scala" %% "jsoniter-scala-core" % "2.37.10" + val jsoniterMacro = "com.github.plokhotnyuk.jsoniter-scala" %% "jsoniter-scala-macros" % "2.37.10" - val playWS = "com.typesafe.play" %% "play-ahc-ws-standalone" % "2.2.11" + val playWS = "com.typesafe.play" %% "play-ahc-ws-standalone" % "2.2.12" val elastic4sHttp4sClient = "nl.gn0s1s" %% "elastic4s-client-http4s" % V.elastic4s @@ -65,14 +64,14 @@ object Dependencies { val log4Cats = "org.typelevel" %% "log4cats-slf4j" % "2.7.1" val logback = "ch.qos.logback" % "logback-classic" % "1.5.18" - val ducktape = "io.github.arainko" %% "ducktape" % "0.2.9" + val ducktape = "io.github.arainko" %% "ducktape" % "0.2.10" val declineCore = "com.monovore" %% "decline" % V.decline val declineCatsEffect = "com.monovore" %% "decline-effect" % V.decline val testContainers = "com.dimafeng" %% "testcontainers-scala-core" % "0.43.0" % Test - val weaver = "org.typelevel" %% "weaver-cats" % "0.9.3" % Test - val weaverScalaCheck = "org.typelevel" %% "weaver-scalacheck" % "0.9.3" % Test + val weaver = "org.typelevel" %% "weaver-cats" % "0.10.1" % Test + val weaverScalaCheck = "org.typelevel" %% "weaver-scalacheck" % "0.10.1" % Test val catsEffectTestKit = "org.typelevel" %% "cats-effect-testkit" % V.catsEffect % Test val scalacheck = "org.scalacheck" %% "scalacheck" % "1.17.0" % Test } diff --git a/project/build.properties b/project/build.properties index c02c575f..5e6884d3 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.11.3 +sbt.version=1.11.6 diff --git a/project/plugins.sbt b/project/plugins.sbt index c7240eff..31f32462 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,12 +1,12 @@ addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.14.3") -addSbtPlugin("com.disneystreaming.smithy4s" % "smithy4s-sbt-codegen" % "0.18.40") +addSbtPlugin("com.disneystreaming.smithy4s" % "smithy4s-sbt-codegen" % "0.18.42") addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.13.1") addSbtPlugin("com.github.sbt" % "sbt-git" % "2.1.0") -addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.11.1") +addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.11.3") addSbtPlugin("com.github.sbt" % "sbt-release" % "1.4.0") diff --git a/version.sbt b/version.sbt index 310864c6..14b63d3c 100644 --- a/version.sbt +++ b/version.sbt @@ -1 +1 @@ -ThisBuild / version := "3.2.1" +ThisBuild / version := "3.2.2"