diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 1f59e77a..27f2ad91 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -12,3 +12,15 @@ d233965417545163fb51ec91988f1a654a36b1b0 # Scala Steward: Reformat with scalafmt 3.9.7 7b0312660489cfe4eb32241e6d569957c4e70b4a + +# Scala Steward: Reformat with scalafmt 3.9.8 +ed503a2852d8148f39ac99b7188e47f569bdf01a + +# scalafmt align.preset = none +adcf303d99abee49f1266dad223bb3e5e34bd73f + +# Scala Steward: Reformat with scalafmt 3.9.9 +6dfb7ed8ffa2f5df0f5c36bee8d3497d29016495 + +# Scala Steward: Reformat with scalafmt 3.9.10 +17147bfe8689d7fe7487912385c864ca64a9e10f diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..fb2b0d6d --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,10 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: monthly + groups: + ci-dependencies: + patterns: + - "*" # Match all CI dependencies to one PR. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e888c3da..0bd143cb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,41 +5,41 @@ on: branches: ['**'] push: branches: ['master'] + tags: [v*] -jobs: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: 21 - - name: Setup sbt - uses: sbt/setup-sbt@v1 - - run: sbt compile +jobs: - lint: + ci: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + script: + - compile + - check + - test + java: + - 21 + - 25 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 + - uses: actions/checkout@v5 + - uses: actions/setup-java@v5 with: distribution: temurin - java-version: 21 + java-version: ${{ matrix.java }} - name: Setup sbt uses: sbt/setup-sbt@v1 - - run: sbt check + - run: sbt ${{ matrix.script }} - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: 21 - - name: Setup sbt - uses: sbt/setup-sbt@v1 - - run: sbt test + deploy: + uses: ./.github/workflows/deploy.yml + with: + env: sirch + secrets: inherit + if: github.event_name != 'pull_request' && startsWith(github.ref, 'refs/tags/v') + needs: + - ci diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index f09af5a6..705a5f33 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,55 +1,82 @@ name: Deploy on: - push: - pull_request: workflow_dispatch: inputs: - environment: - description: Target environment for deployment + env: required: true type: choice options: - sirch + workflow_call: + inputs: + env: + description: Target environment for deployment + required: true + type: string jobs: - # todo - # - split between lila-search / lila-search-ingestor - # - only build when have success build - # - auto matically run on tag or workflow_dispatch - # - tpolecat release mode stage: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 + - uses: actions/checkout@v5 + - uses: actions/setup-java@v5 with: distribution: temurin java-version: 21 - name: Setup sbt uses: sbt/setup-sbt@v1 - run: sbt stage - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v5 with: - name: lila-search + name: lila-search-app path: modules/app/target/universal/stage + - uses: actions/upload-artifact@v5 + with: + name: lila-search-ingestor + path: modules/ingestor-app/target/universal/stage + - uses: actions/upload-artifact@v5 + with: + name: lila-search-cli + path: modules/ingestor-cli/target/universal/stage - deploy-lila-search: + deploy-components: runs-on: ubuntu-latest + needs: stage environment: - name: ${{ inputs.environment }} + name: ${{ inputs.env }} + strategy: + fail-fast: false + max-parallel: 1 + matrix: + include: + - artifact: lila-search-app + remote_dir: /home/lila-search + user: lila-search + restart_cmd: systemctl restart lila-search + - artifact: lila-search-ingestor + remote_dir: /home/lila-search-ingestor + user: lila-search-ingestor + restart_cmd: systemctl restart lila-search-ingestor + - artifact: lila-search-cli + remote_dir: /home/lila-search-cli + user: "" + restart_cmd: "" # no restart needed concurrency: - group: ${{ inputs.environment }} - needs: stage - if: github.event_name == 'workflow_dispatch' + group: deploy-${{ inputs.env }} steps: - - uses: actions/download-artifact@v4 + - name: Download artifact + uses: actions/download-artifact@v6 with: - name: lila-search + name: ${{ matrix.artifact }} + path: stage + - name: Configure SSH run: | + set -euo pipefail mkdir -p ~/.ssh + chmod 700 ~/.ssh touch ~/.ssh/id_deploy chmod 600 ~/.ssh/id_deploy echo "$SSH_KEY" > ~/.ssh/id_deploy @@ -66,8 +93,26 @@ jobs: SSH_HOST: ${{ secrets.SSH_HOST }} SSH_USER: ${{ secrets.SSH_USER }} SSH_HOST_KEY: ${{ secrets.SSH_HOST_KEY }} + - name: Deploy via SSH + run: | + set -euo pipefail + if [ ! -d stage/bin ]; then + echo "ERROR: Expected staged artifact (stage/bin) not found. You skipped build jobs via workflow_dispatch." + echo "Either (a) run a normal CI build first, or (b) modify deploy job to build artifacts." + exit 1 + fi + RSYNC_OPTIONS="--archive --no-o --no-g --force --delete --progress --compress --checksum --verbose --exclude RUNNING_PID --exclude '.git/'" + include="stage/bin stage/lib" + echo "Deploying ${{ matrix.artifact }} to ${{ matrix.remote_dir }}" + rsync $RSYNC_OPTIONS $include deploy-host:${{ matrix.remote_dir }} + echo "rsync complete (${{ matrix.artifact }})" + + if [ -n "${{ matrix.restart_cmd }}" ]; then + echo "run restart: ${{ matrix.restart_cmd }}" + ssh deploy-host "chmod +x ${{ matrix.remote_dir }}/bin/${{ matrix.artifact }} && chown -R ${{ matrix.user }}:${{ matrix.user }} ${{ matrix.remote_dir }} && ${{ matrix.restart_cmd }}" + else + echo "No restart command defined for ${{ matrix.artifact }}." + fi - # ssh $REMOTE "chown -R lila-search:lila-search /home/lila-search && systemctl restart lila-search" - run: cat lila-search | ssh deploy-host "cat - > /home/lichess-search" - # run: cat lila-search | ssh deploy-host "cat - > /home/lila-search && chown -R lila-search:lila-search && systemctl restart lila-search" + echo "Deploy complete (${{ matrix.artifact }})" diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 00000000..17b9ace7 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,52 @@ +name: Create and publish a Docker image +on: + push: + tags: + - v* + workflow_dispatch: + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + + permissions: + contents: read + packages: write + + steps: + + - name: Checkout repository + uses: actions/checkout@v5 + + - name: Setup JVM + uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: 25 + cache: sbt + + - name: Install sbt + uses: sbt/setup-sbt@v1 + + - name: Log in to the Container registry + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@032a4b3bda1b716928481836ac5bfe36e1feaad6 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build and push Docker image + run: sbt Docker/publish diff --git a/.sbtops.example b/.sbtops.example deleted file mode 100644 index 27f4f7b4..00000000 --- a/.sbtops.example +++ /dev/null @@ -1,2 +0,0 @@ --J--add-opens=java.base/java.util=ALL-UNNAMED --J--add-opens=java.base/java.lang=ALL-UNNAMED diff --git a/.scalafmt.conf b/.scalafmt.conf index eb9551f2..ee9bd960 100644 --- a/.scalafmt.conf +++ b/.scalafmt.conf @@ -1,7 +1,7 @@ -version = "3.9.7" +version = "3.10.1" runner.dialect = scala3 -align.preset = more +align.preset = none maxColumn = 110 spaces.inImportCurlyBraces = true @@ -12,10 +12,10 @@ rewrite.scala3.convertToNewSyntax = yes fileOverride { "glob:**/build.sbt" { - runner.dialect = scala213 + runner.dialect = Scala213Source3 } "glob:**/project/**" { - runner.dialect = scala213 + runner.dialect = Scala213Source3 } } diff --git a/README.md b/README.md index f929070e..a3da1242 100644 --- a/README.md +++ b/README.md @@ -6,11 +6,6 @@ ### Start sbt -Copy default settings -```sh -cp .sbtops.example .sbtopts -``` - Start sbt: ```sh sbt @@ -35,22 +30,22 @@ prepare Start ingestor service: ```sh -ingestor/runMain lila.search.ingestor.App +ingestor-app/run ``` Start ingestor cli tool ```sh -ingestor/runMain lila.search.ingestor.cli --help +ingestor-cli/run --help ``` #### CLI tool ```sh # index all documents for specific index -sbt 'ingestor/runMain lila.search.ingestor.cli index --index team --since 0' +sbt 'ingestor-cli/run index --index team --since 0' # index all documents for all indexes -sbt 'ingestor/runMain lila.search.ingestor.cli index --all --since 0' +sbt 'ingestor-cli/run index --all --since 0' ``` ### release diff --git a/bin/deploy b/bin/deploy index c54fcfb0..b1df84d0 100755 --- a/bin/deploy +++ b/bin/deploy @@ -24,7 +24,7 @@ RSYNC_OPTIONS=" \ --exclude '.git/'" stage="modules/app/target/universal/stage" -include="bin $stage/bin $stage/lib" +include="$stage/bin $stage/lib" rsync_command="rsync $RSYNC_OPTIONS $include $REMOTE:$REMOTE_DIR" echo "$rsync_command" $rsync_command diff --git a/bin/deploy-cli b/bin/deploy-cli new file mode 100755 index 00000000..a0212a95 --- /dev/null +++ b/bin/deploy-cli @@ -0,0 +1,31 @@ +#!/bin/sh + +REMOTE=$1 +REMOTE_DIR="/home/lila-search-cli" + +echo "Deploy to server $REMOTE:$REMOTE_DIR" + +sbt ";ingestor-cli/stage;exit" +if [ $? != 0 ]; then + echo "Deploy canceled" + exit 1 +fi + +RSYNC_OPTIONS=" \ + --archive \ + --no-o --no-g \ + --force \ + --delete \ + --progress \ + --compress \ + --checksum \ + --verbose \ + --exclude RUNNING_PID \ + --exclude '.git/'" + +stage="modules/ingestor-cli/target/universal/stage" +include="$stage/bin $stage/lib" +rsync_command="rsync $RSYNC_OPTIONS $include $REMOTE:$REMOTE_DIR" +echo "$rsync_command" +$rsync_command +echo "rsync complete" diff --git a/bin/deploy-ingestor b/bin/deploy-ingestor index 797747fd..dd4d2d0b 100755 --- a/bin/deploy-ingestor +++ b/bin/deploy-ingestor @@ -5,7 +5,7 @@ REMOTE_DIR="/home/lila-search-ingestor" echo "Deploy to server $REMOTE:$REMOTE_DIR" -sbt ";ingestor/stage;exit" +sbt ";ingestor-app/stage;exit" if [ $? != 0 ]; then echo "Deploy canceled" exit 1 @@ -23,8 +23,8 @@ RSYNC_OPTIONS=" \ --exclude RUNNING_PID \ --exclude '.git/'" -stage="modules/ingestor/target/universal/stage" -include="bin $stage/bin $stage/lib" +stage="modules/ingestor-app/target/universal/stage" +include="$stage/bin $stage/lib" rsync_command="rsync $RSYNC_OPTIONS $include $REMOTE:$REMOTE_DIR" echo "$rsync_command" $rsync_command diff --git a/build.sbt b/build.sbt index 478c57db..ac7a5e69 100644 --- a/build.sbt +++ b/build.sbt @@ -3,15 +3,20 @@ import org.typelevel.scalacoptions.ScalacOptions inThisBuild( Seq( - scalaVersion := "3.7.1", + scalaVersion := "3.7.3", versionScheme := Some("early-semver"), - organization := "org.lichess.search", - run / fork := true, + organization := "org.lichess.search", + run / fork := true, run / javaOptions += "-Dconfig.override_with_env_vars=true", semanticdbEnabled := true, // for scalafix resolvers ++= ourResolvers, Compile / doc / sources := Seq.empty, - publishTo := Option(Resolver.file("file", new File(sys.props.getOrElse("publishTo", "")))) + publishTo := Option(Resolver.file("file", new File(sys.props.getOrElse("publishTo", "")))), + dockerBaseImage := "eclipse-temurin:25-jdk-noble", + dockerUpdateLatest := true, + dockerBuildxPlatforms := Seq("linux/amd64", "linux/arm64"), + Docker / maintainer := "lichess.org", + Docker / dockerRepository := Some("ghcr.io") ) ) @@ -21,7 +26,7 @@ val commonSettings = Seq( ScalacOptions.other("-indent"), ScalacOptions.explain, ScalacOptions.release("21"), - ScalacOptions.other("-Wall"), + ScalacOptions.other("-Wall") ), resolvers += "jitpack".at("https://jitpack.io") ) @@ -46,45 +51,88 @@ lazy val core = project ) ) +lazy val api = project + .in(file("modules/api")) + .enablePlugins(Smithy4sCodegenPlugin) + .settings( + name := "api", + commonSettings, + smithy4sWildcardArgument := "?", + libraryDependencies ++= Seq( + catsCore, + smithy4sCore + ) + ) + .dependsOn(core) + lazy val elastic = project .in(file("modules/elastic")) .settings( name := "elastic", commonSettings, - publish := {}, + publish := {}, publish / skip := true, libraryDependencies ++= Seq( catsCore, catsEffect, + catsMtl, http4sClient, - elastic4sHttp4sClient, - otel4sCore + elastic4sHttp4sClient ) ) .dependsOn(core) -lazy val api = project - .in(file("modules/api")) - .enablePlugins(Smithy4sCodegenPlugin) +lazy val `ingestor-app` = project + .in(file("modules/ingestor-app")) + .enablePlugins(JavaAppPackaging, BuildInfoPlugin, DockerPlugin) .settings( - name := "api", + name := "lila-search-ingestor", commonSettings, - smithy4sWildcardArgument := "?", + buildInfoSettings, + Docker / packageName := "lichess-org/lila-search-ingestor-app", + publish := {}, + publish / skip := true, libraryDependencies ++= Seq( - catsCore, - smithy4sCore - ) + logback % Runtime, + otel4sSdk, + otel4sMetrics, + otel4sPrometheusExporter, + otel4sInstrumentationMetrics + ), + Compile / doc / sources := Seq.empty, + Compile / run / fork := true ) - .dependsOn(core) + .dependsOn(`ingestor-core`) -lazy val ingestor = project - .in(file("modules/ingestor")) - .enablePlugins(JavaAppPackaging, Smithy4sCodegenPlugin, BuildInfoPlugin) +lazy val `ingestor-cli` = project + .in(file("modules/ingestor-cli")) + .enablePlugins(JavaAppPackaging, BuildInfoPlugin, DockerPlugin) .settings( - name := "ingestor", + name := "lila-search-cli", commonSettings, buildInfoSettings, - publish := {}, + Docker / packageName := "lichess-org/lila-search-ingestor-cli", + publish := {}, + publish / skip := true, + libraryDependencies ++= Seq( + declineCore, + declineCatsEffect, + otel4sCore, + logback % Runtime, + weaver + ), + Compile / doc / sources := Seq.empty, + Compile / run / fork := true + ) + .dependsOn(elastic, core, `ingestor-core`) + +lazy val `ingestor-core` = project + .in(file("modules/ingestor-core")) + .enablePlugins(Smithy4sCodegenPlugin) + .settings( + name := "ingestor-core", + commonSettings, + publish := {}, publish / skip := true, libraryDependencies ++= Seq( chess, @@ -92,31 +140,25 @@ lazy val ingestor = project fs2, fs2IO, catsEffect, - declineCore, - declineCatsEffect, - ducktape, cirisCore, cirisHtt4s, + ducktape, smithy4sCore, smithy4sJson, jsoniterCore, jsoniterMacro, circe, - http4sServer, - http4sEmberClient, mongo4catsCore, mongo4catsCirce, + otel4sCore, + otel4sHttp4sCore, + otel4sHttp4sMetrics, + http4sEmberClient, log4Cats, - logback % Runtime, - otel4sSdk, - otel4sMetrics, - otel4sPrometheusExporter, - otel4sInstrumentationMetrics, weaver, weaverScalaCheck ), - Compile / doc / sources := Seq.empty, - Compile / run / fork := true + Compile / doc / sources := Seq.empty ) .dependsOn(elastic, core) @@ -135,13 +177,14 @@ lazy val client = project .dependsOn(api, core) lazy val app = project - .enablePlugins(JavaAppPackaging, BuildInfoPlugin) + .enablePlugins(JavaAppPackaging, BuildInfoPlugin, DockerPlugin) .in(file("modules/app")) .settings( - name := "lila-search", + name := "lila-search-app", commonSettings, buildInfoSettings, - publish := {}, + Docker / packageName := "lichess-org/lila-search-app", + publish := {}, publish / skip := true, libraryDependencies ++= Seq( smithy4sHttp4s, @@ -160,26 +203,28 @@ lazy val app = project otel4sSdk, otel4sMetrics, otel4sPrometheusExporter, - otel4sInstrumentationMetrics + otel4sInstrumentationMetrics, + otel4sHttp4sCore, + otel4sHttp4sMetrics ), Compile / doc / sources := Seq.empty, - Compile / run / fork := true + Compile / run / fork := true ) .dependsOn(api, elastic) val e2e = project .in(file("modules/e2e")) .settings( - publish := {}, + publish := {}, publish / skip := true, libraryDependencies ++= Seq(testContainers, weaver) ) - .dependsOn(client, app, ingestor) + .dependsOn(client, app, `ingestor-core`) lazy val root = project .in(file(".")) .settings(publish := {}, publish / skip := true) - .aggregate(core, api, app, client, e2e, elastic, ingestor) + .aggregate(core, api, app, client, e2e, elastic, `ingestor-core`, `ingestor-app`, `ingestor-cli`) addCommandAlias("prepare", "scalafixAll; scalafmtAll") addCommandAlias("check", "; scalafixAll --check ; scalafmtCheckAll") diff --git a/modules/api/src/main/smithy/search.smithy b/modules/api/src/main/smithy/search.smithy index 0dd78674..751355b2 100644 --- a/modules/api/src/main/smithy/search.smithy +++ b/modules/api/src/main/smithy/search.smithy @@ -70,6 +70,15 @@ structure Forum { troll: Boolean = false } +structure Ublog { + @required + queryText: String + @required + by: SortBlogsBy + minQuality: Integer + language: String +} + structure Team { @required text: String @@ -78,6 +87,7 @@ structure Team { structure Study { @required text: String + sorting: StudySorting userId: String } @@ -104,7 +114,7 @@ structure Game { @required duration: IntRange @required - sorting: Sorting + sorting: GameSorting analysed: Boolean whiteUser: String blackUser: String @@ -122,7 +132,7 @@ structure DateRange { b: Timestamp } -structure Sorting { +structure GameSorting { @required f: String @required @@ -133,9 +143,37 @@ list Perfs { member: Integer } +enum SortBlogsBy { + newest + oldest + score + likes +} + +enum Order { + Asc = "asc" + Desc = "desc" +} + +enum StudySortField { + Name = "name" + Likes = "like" + CreatedAt = "createdAt" + UpdatedAt = "updatedAt" + Hot = "hot" +} + +structure StudySorting { + @required + field: StudySortField + @required + order: Order +} + @adt union Query { forum: Forum + ublog: Ublog game: Game study: Study team: Team diff --git a/modules/app/src/main/scala/app.config.scala b/modules/app/src/main/scala/app.config.scala index d45f4380..66a14127 100644 --- a/modules/app/src/main/scala/app.config.scala +++ b/modules/app/src/main/scala/app.config.scala @@ -32,13 +32,13 @@ case class HttpServerConfig( ) object HttpServerConfig: - private def host = env("HTTP_HOST").or(prop("http.host")).as[Host].default(ip"0.0.0.0") - private def port = env("HTTP_PORT").or(prop("http.port")).as[Port].default(port"9673") - private def logger = env("HTTP_API_LOGGER").or(prop("http.api.logger")).as[Boolean].default(false) + private def host = env("HTTP_HOST").or(prop("http.host")).as[Host].default(ip"0.0.0.0") + private def port = env("HTTP_PORT").or(prop("http.port")).as[Port].default(port"9673") + private def logger = env("HTTP_API_LOGGER").or(prop("http.api.logger")).as[Boolean].default(false) private def shutdownTimeout = env("HTTP_SHUTDOWN_TIMEOUT").or(prop("http.shutdown.timeout")).as[Int].default(30) private def enableDocs = env("HTTP_ENABLE_DOCS").or(prop("http.enable.docs")).as[Boolean].default(false) - def config = (host, port, logger, shutdownTimeout, enableDocs).parMapN(HttpServerConfig.apply) + def config = (host, port, logger, shutdownTimeout, enableDocs).parMapN(HttpServerConfig.apply) case class ElasticConfig(uri: Uri) diff --git a/modules/app/src/main/scala/app.resources.scala b/modules/app/src/main/scala/app.resources.scala index 95b0f1c8..8d2fc099 100644 --- a/modules/app/src/main/scala/app.resources.scala +++ b/modules/app/src/main/scala/app.resources.scala @@ -2,16 +2,22 @@ package lila.search package app import cats.effect.* +import cats.syntax.all.* +import org.http4s.client.Client import org.http4s.ember.client.EmberClientBuilder -import org.typelevel.otel4s.metrics.Meter +import org.http4s.otel4s.middleware.metrics.OtelMetrics +import org.typelevel.otel4s.metrics.MeterProvider class AppResources(val esClient: ESClient[IO]) object AppResources: - def instance(conf: AppConfig)(using Meter[IO]): Resource[IO, AppResources] = - EmberClientBuilder - .default[IO] - .build - .evalMap(ESClient(conf.elastic.uri)) + def instance(conf: AppConfig)(using MeterProvider[IO]): Resource[IO, AppResources] = + val metrics = OtelMetrics + .clientMetricsOps[IO]() + .map(org.http4s.client.middleware.Metrics[IO](_, _.uri.renderString.some)) + + (metrics.toResource, EmberClientBuilder.default[IO].build) + .mapN(_.apply(_)) + .map(ESClient(conf.elastic.uri)) .map(AppResources.apply) diff --git a/modules/app/src/main/scala/app.scala b/modules/app/src/main/scala/app.scala index c46b6543..006a2a0d 100644 --- a/modules/app/src/main/scala/app.scala +++ b/modules/app/src/main/scala/app.scala @@ -10,40 +10,52 @@ import org.typelevel.otel4s.instrumentation.ce.IORuntimeMetrics import org.typelevel.otel4s.metrics.{ Meter, MeterProvider } import org.typelevel.otel4s.sdk.exporter.prometheus.PrometheusMetricExporter import org.typelevel.otel4s.sdk.metrics.SdkMetrics +import org.typelevel.otel4s.sdk.metrics.SdkMetrics.AutoConfigured.Builder import org.typelevel.otel4s.sdk.metrics.exporter.MetricExporter object App extends IOApp.Simple: given LoggerFactory[IO] = Slf4jFactory.create[IO] - given Logger[IO] = LoggerFactory[IO].getLogger + given Logger[IO] = LoggerFactory[IO].getLogger override def run: IO[Unit] = app.useForever def app: Resource[IO, Unit] = for given MetricExporter.Pull[IO] <- PrometheusMetricExporter.builder[IO].build.toResource - given Meter[IO] <- mkMeter - _ <- RuntimeMetrics.register[IO] - config <- AppConfig.load.toResource - _ <- Logger[IO].info(s"Starting lila-search with config: ${config.toString}").toResource + otel4s <- SdkMetrics.autoConfigured[IO](configBuilder) + given MeterProvider[IO] = otel4s.meterProvider + _ <- registerRuntimeMetrics + config <- AppConfig.load.toResource + _ <- Logger[IO].info(s"Starting lila-search with config: ${config.toString}").toResource res <- AppResources.instance(config) - _ <- mkServer(res, config) + _ <- mkServer(res, config) yield () - def mkMeter(using exporter: MetricExporter.Pull[IO]) = SdkMetrics - .autoConfigured[IO](_.addMeterProviderCustomizer((b, _) => b.registerMetricReader(exporter.metricReader))) - .flatMap: sdk => - given meterProvider: MeterProvider[IO] = sdk.meterProvider - IORuntimeMetrics.register[IO](runtime.metrics, IORuntimeMetrics.Config.default) *> - meterProvider.get("lila-search").toResource - def mkServer(res: AppResources, config: AppConfig)(using - Meter[IO], - MetricExporter.Pull[IO] + MetricExporter.Pull[IO], + MeterProvider[IO] ): Resource[IO, Unit] = for apiRoutes <- Routes(res, config.server) - httpRoutes = apiRoutes <+> mkPrometheusRoutes - _ <- MkHttpServer().newEmber(config.server, httpRoutes.orNotFound) + httpRoutes = apiRoutes <+> MkPrometheusRoutes + _ <- MkHttpServer.newEmber(config.server, httpRoutes.orNotFound) _ <- Logger[IO].info(s"BuildInfo: ${BuildInfo.toString}").toResource yield () + + private def registerRuntimeMetrics(using MeterProvider[IO]): Resource[IO, Unit] = + for + _ <- IORuntimeMetrics.register[IO](runtime.metrics, IORuntimeMetrics.Config.default) + given Meter[IO] <- MeterProvider[IO].get("jvm.runtime").toResource + _ <- RuntimeMetrics.register[IO] + yield () + + private def configBuilder(builder: Builder[IO])(using exporter: MetricExporter.Pull[IO]) = + builder + .addPropertiesCustomizer(_ => + Map( + "otel.metrics.exporter" -> "none", + "otel.traces.exporter" -> "none" + ) + ) + .addMeterProviderCustomizer((b, _) => b.registerMetricReader(exporter.metricReader)) diff --git a/modules/app/src/main/scala/http.middleware.scala b/modules/app/src/main/scala/http.middleware.scala index 585477aa..4abbcafc 100644 --- a/modules/app/src/main/scala/http.middleware.scala +++ b/modules/app/src/main/scala/http.middleware.scala @@ -3,13 +3,15 @@ package app import cats.effect.IO import org.http4s.* +import org.http4s.otel4s.middleware.metrics.OtelMetrics import org.http4s.server.middleware.* import org.typelevel.log4cats.LoggerFactory +import org.typelevel.otel4s.metrics.MeterProvider import scala.concurrent.duration.* type Middleware = HttpRoutes[IO] => HttpRoutes[IO] -def MkMiddleware(config: HttpServerConfig)(using LoggerFactory[IO]): Middleware = +def MkMiddleware(config: HttpServerConfig)(using LoggerFactory[IO], MeterProvider[IO]): IO[Middleware] = def verboseLogger = RequestLogger.httpRoutes[IO](true, true).andThen(ResponseLogger.httpRoutes[IO, Request[IO]](true, true)) @@ -18,4 +20,8 @@ def MkMiddleware(config: HttpServerConfig)(using LoggerFactory[IO]): Middleware if config.apiLogger then verboseLogger else ApiErrorLogger.instance(using LoggerFactory[IO].getLogger) - logger.andThen(AutoSlash(_)).andThen(Timeout(60.seconds)) + OtelMetrics + .serverMetricsOps[IO]() + .map(org.http4s.server.middleware.Metrics[IO](_)) + .map: metrics => + logger.andThen(AutoSlash(_)).andThen(Timeout(60.seconds)).andThen(metrics) diff --git a/modules/app/src/main/scala/http.routes.prometheus.scala b/modules/app/src/main/scala/http.routes.prometheus.scala index e33763e9..6b27d586 100644 --- a/modules/app/src/main/scala/http.routes.prometheus.scala +++ b/modules/app/src/main/scala/http.routes.prometheus.scala @@ -7,7 +7,7 @@ import org.http4s.server.Router import org.typelevel.otel4s.sdk.exporter.prometheus.* import org.typelevel.otel4s.sdk.metrics.exporter.MetricExporter -def mkPrometheusRoutes(using exporter: MetricExporter.Pull[IO]): HttpRoutes[IO] = - val writerConfig = PrometheusWriter.Config.default +def MkPrometheusRoutes(using exporter: MetricExporter.Pull[IO]): HttpRoutes[IO] = + val writerConfig = PrometheusWriter.Config.default val prometheusRoutes = PrometheusHttpRoutes.routes[IO](exporter, writerConfig) Router("/metrics" -> prometheusRoutes) diff --git a/modules/app/src/main/scala/http.routes.scala b/modules/app/src/main/scala/http.routes.scala index 45f7311f..024d27ec 100644 --- a/modules/app/src/main/scala/http.routes.scala +++ b/modules/app/src/main/scala/http.routes.scala @@ -7,19 +7,19 @@ import cats.syntax.all.* import lila.search.spec.* import org.http4s.HttpRoutes import org.typelevel.log4cats.LoggerFactory -import org.typelevel.otel4s.metrics.Meter +import org.typelevel.otel4s.metrics.MeterProvider import smithy4s.http4s.SimpleRestJsonBuilder -def Routes(resources: AppResources, config: HttpServerConfig)(using - LoggerFactory[IO], - Meter[IO] -): Resource[IO, HttpRoutes[IO]] = +def Routes( + resources: AppResources, + config: HttpServerConfig +)(using LoggerFactory[IO], MeterProvider[IO]): Resource[IO, HttpRoutes[IO]] = val healthServiceImpl = HealthServiceImpl(resources.esClient) + val searchServiceImpl = SearchServiceImpl(resources.esClient) val search: Resource[IO, HttpRoutes[IO]] = - SearchServiceImpl(resources.esClient).toResource - .flatMap(SimpleRestJsonBuilder.routes(_).resource) + SimpleRestJsonBuilder.routes(searchServiceImpl).resource val health: Resource[IO, HttpRoutes[IO]] = SimpleRestJsonBuilder.routes(healthServiceImpl).resource @@ -36,4 +36,4 @@ def Routes(resources: AppResources, config: HttpServerConfig)(using if config.enableDocs then apiRoutes.map(_ <+> docs) else apiRoutes - allRoutes.map(MkMiddleware(config)) + allRoutes.evalMap(routes => MkMiddleware(config).map(md => md.apply(routes))) diff --git a/modules/app/src/main/scala/http.server.scala b/modules/app/src/main/scala/http.server.scala index 6c8791b6..4eaff1d1 100644 --- a/modules/app/src/main/scala/http.server.scala +++ b/modules/app/src/main/scala/http.server.scala @@ -10,16 +10,13 @@ import org.typelevel.log4cats.Logger import scala.concurrent.duration.* -trait MkHttpServer: - def newEmber(cfg: HttpServerConfig, httpApp: HttpApp[IO]): Resource[IO, Server] - object MkHttpServer: - def apply()(using server: MkHttpServer): MkHttpServer = server - - given Logger[IO] => MkHttpServer = new: + def newEmber(cfg: HttpServerConfig, httpApp: HttpApp[IO])(using Logger[IO]): Resource[IO, Server] = + def showBanner(s: Server): IO[Unit] = + Logger[IO].info(s"lila-search started at ${s.address.toString}") - def newEmber(cfg: HttpServerConfig, httpApp: HttpApp[IO]): Resource[IO, Server] = EmberServerBuilder + EmberServerBuilder .default[IO] .withHost(cfg.host) .withPort(cfg.port) @@ -27,6 +24,3 @@ object MkHttpServer: .withShutdownTimeout(cfg.shutdownTimeout.seconds) .build .evalTap(showBanner) - - private def showBanner(s: Server): IO[Unit] = - Logger[IO].info(s"lila-search started at ${s.address.toString}") diff --git a/modules/app/src/main/scala/service.health.scala b/modules/app/src/main/scala/service.health.scala index cea3ca1d..254f8514 100644 --- a/modules/app/src/main/scala/service.health.scala +++ b/modules/app/src/main/scala/service.health.scala @@ -2,6 +2,7 @@ package lila.search package app import cats.effect.* +import cats.mtl.Handle.* import cats.syntax.all.* import lila.search.spec.* import org.typelevel.log4cats.{ Logger, LoggerFactory } @@ -11,16 +12,17 @@ class HealthServiceImpl(esClient: ESClient[IO])(using LoggerFactory[IO]) extends given Logger[IO] = LoggerFactory[IO].getLogger override def healthCheck(): IO[HealthCheckOutput] = - esClient.status - .flatMap(transform) - .map(HealthCheckOutput(_)) - .handleErrorWith: e => - Logger[IO].error(e)("Error in health check") *> - IO.raiseError(InternalServerError(s"Internal server error ${e.getMessage}")) + allow: + esClient.status + .flatMap(transform) + .map(HealthCheckOutput(_)) + .rescue: e => + Logger[IO].error(e.asException)("Error in health check") *> + IO.raiseError(InternalServerError(s"Internal server error ${e.reason}")) private def transform(status: String): IO[ElasticStatus] = status match - case "green" => ElasticStatus.green.pure[IO] + case "green" => ElasticStatus.green.pure[IO] case "yellow" => ElasticStatus.yellow.pure[IO] - case "red" => ElasticStatus.red.pure[IO] - case _ => IO.raiseError(new Exception(s"Unknown status: $status")) + case "red" => ElasticStatus.red.pure[IO] + case _ => IO.raiseError(new Exception(s"Unknown status: $status")) diff --git a/modules/app/src/main/scala/service.search.scala b/modules/app/src/main/scala/service.search.scala index 6198ac4a..a45c6121 100644 --- a/modules/app/src/main/scala/service.search.scala +++ b/modules/app/src/main/scala/service.search.scala @@ -2,69 +2,40 @@ package lila.search package app import cats.effect.* +import cats.mtl.Handle.* import io.github.arainko.ducktape.* import lila.search.forum.Forum import lila.search.game.Game import lila.search.spec.* import lila.search.study.Study import lila.search.team.Team +import lila.search.ublog.Ublog import org.typelevel.log4cats.{ Logger, LoggerFactory } -import org.typelevel.otel4s.metrics.{ Histogram, Meter } -import org.typelevel.otel4s.{ Attribute, AttributeKey, Attributes } import smithy4s.Timestamp import java.time.Instant -import java.util.concurrent.TimeUnit -class SearchServiceImpl(esClient: ESClient[IO], metric: Histogram[IO, Double])(using - LoggerFactory[IO] -) extends SearchService[IO]: +class SearchServiceImpl(esClient: ESClient[IO])(using LoggerFactory[IO]) extends SearchService[IO]: - import SearchServiceImpl.{ *, given } + import SearchServiceImpl.given private val logger: Logger[IO] = LoggerFactory[IO].getLogger - private val baseAttributes = Attributes(Attribute("http.request.method", "POST")) - private val countMetric = - metric - .recordDuration( - TimeUnit.MILLISECONDS, - withErrorType( - baseAttributes - .added(MetricKeys.httpRoute, s"/api/count/") - ) - ) - - private val searchMetric = - metric - .recordDuration( - TimeUnit.MILLISECONDS, - withErrorType( - baseAttributes - .added(MetricKeys.httpRoute, s"/api/search/") - ) - ) - - private def countRecord[A](f: IO[A]) = countMetric.surround(f) - private def searchRecord[A](f: IO[A]) = searchMetric.surround(f) - override def count(query: Query): IO[CountOutput] = - countRecord: - esClient - .count(query) - .map(CountOutput.apply) - .handleErrorWith: e => - logger.error(e)(s"Error in count: query=${query.toString}") *> - IO.raiseError(InternalServerError("Internal server error")) + allow: + esClient.count(query) + .rescue: e => + logger.error(e.asException)(s"Error in count: query=${query.toString}") *> + IO.raiseError(InternalServerError("Internal server error")) + .map(CountOutput.apply) override def search(query: Query, from: From, size: Size): IO[SearchOutput] = - searchRecord: - esClient - .search(query, from, size) - .map(SearchOutput.apply) - .handleErrorWith: e => - logger.error(e)(s"Error in search: query=${query.toString}, from=$from, size=$size") *> - IO.raiseError(InternalServerError("Internal server error")) + allow: + esClient.search(query, from, size) + .rescue: e => + logger.error(e.asException)(s"Error in search: query=${query.toString}, from=$from, size=$size") *> + IO.raiseError(InternalServerError("Internal server error")) + .map(SearchOutput.apply) object SearchServiceImpl: @@ -82,39 +53,22 @@ object SearchServiceImpl: def searchDef(from: From, size: Size) = query match case q: Query.Forum => q.to[Forum].searchDef(from, size) - case q: Query.Game => q.to[Game].searchDef(from, size) + case q: Query.Ublog => q.to[Ublog].searchDef(from, size) + case q: Query.Game => q.to[Game].searchDef(from, size) case q: Query.Study => q.to[Study].searchDef(from, size) - case q: Query.Team => q.to[Team].searchDef(from, size) + case q: Query.Team => q.to[Team].searchDef(from, size) def countDef = query match case q: Query.Forum => q.to[Forum].countDef - case q: Query.Game => q.to[Game].countDef + case q: Query.Ublog => q.to[Ublog].countDef + case q: Query.Game => q.to[Game].countDef case q: Query.Study => q.to[Study].countDef - case q: Query.Team => q.to[Team].countDef + case q: Query.Team => q.to[Team].countDef def index = query match case _: Query.Forum => Index.Forum - case _: Query.Game => Index.Game + case _: Query.Ublog => Index.Ublog + case _: Query.Game => Index.Game case _: Query.Study => Index.Study - case _: Query.Team => Index.Team - - def apply(elastic: ESClient[IO])(using Meter[IO], LoggerFactory[IO]): IO[SearchService[IO]] = - Meter[IO] - .histogram[Double]("http.server.request.duration") - .withUnit("ms") - .create - .map(new SearchServiceImpl(elastic, _)) - - object MetricKeys: - val httpRoute = AttributeKey.string("http.route") - val errorType = AttributeKey.string("error.type") - - import lila.search.ESClient.MetricKeys.* - def withErrorType(static: Attributes)(ec: Resource.ExitCase): Attributes = ec match - case Resource.ExitCase.Succeeded => - static - case Resource.ExitCase.Errored(e) => - static.added(errorType, e.getClass.getName) - case Resource.ExitCase.Canceled => - static.added(errorType, "canceled") + case _: Query.Team => Index.Team diff --git a/modules/client/src/main/scala/PlaySearchClient.scala b/modules/client/src/main/scala/PlaySearchClient.scala index 329a39a6..fd52d37b 100644 --- a/modules/client/src/main/scala/PlaySearchClient.scala +++ b/modules/client/src/main/scala/PlaySearchClient.scala @@ -14,18 +14,21 @@ import scala.util.control.NoStackTrace /** * This error is thrown when a search request fails. */ -enum SearchError extends NoStackTrace: - case BadRequest(message: String) - case InternalServerError(message: String) +enum SearchError(message: String) extends NoStackTrace: + case BadRequest(message: String) extends SearchError(message) + case InternalServerError(message: String) extends SearchError(message) /** * This error is thrown when object serialization fails. */ - case JsonWriterError(message: String) + case JsonWriterError(message: String) extends SearchError(message) + + override def getMessage: String = message class PlaySearchClient(client: StandaloneWSClient, baseUrl: String)(using ExecutionContext) extends SearchClient: + import PlaySearchClient.* import implicits.given override def count(query: Query): Future[CountOutput] = @@ -46,11 +49,15 @@ class PlaySearchClient(client: StandaloneWSClient, baseUrl: String)(using Execut case res => Future.failed(SearchError.InternalServerError(s"$url ${res.status} ${res.body}")) catch case e: JsonWriterException => Future.failed(SearchError.JsonWriterError(e.toString)) -final private case class SearchInput(query: Query) -final private case class IdsInput(ids: List[Id]) +object PlaySearchClient: + opaque type SearchInput = Query + object SearchInput: + inline def apply(query: Query): SearchInput = query + extension (s: SearchInput) inline def query: Query = s object implicits: + import PlaySearchClient.* import smithy4s.schema.Schema.struct given Schema[SearchInput] = struct( diff --git a/modules/core/src/main/scala/models.scala b/modules/core/src/main/scala/models.scala index 403ca382..e6d5f638 100644 --- a/modules/core/src/main/scala/models.scala +++ b/modules/core/src/main/scala/models.scala @@ -17,30 +17,32 @@ object SearchDateTime: def fromInstant(value: java.time.Instant): SearchDateTime = formatter.format(value) - val format = "yyyy-MM-dd HH:mm:ss" + val format = "yyyy-MM-dd HH:mm:ss" val formatter = java.time.format.DateTimeFormatter.ofPattern(format).withZone(ZoneId.systemDefault()) extension (x: SearchDateTime) def value: String = x opaque type Id = String object Id: - def apply(value: String): Id = value + def apply(value: String): Id = value extension (x: Id) def value: String = x enum Index(val value: String): case Forum extends Index("forum") - case Game extends Index("game") + case Ublog extends Index("ublog") + case Game extends Index("game") case Study extends Index("study") - case Team extends Index("team") + case Team extends Index("team") object Index: def fromString(value: String): Either[String, Index] = value match case "forum" => Index.Forum.asRight - case "game" => Index.Game.asRight + case "ublog" => Index.Ublog.asRight + case "game" => Index.Game.asRight case "study" => Index.Study.asRight - case "team" => Index.Team.asRight - case _ => s"Invalid index: $value. It must be in ${Index.valuesStrings}".asLeft + case "team" => Index.Team.asRight + case _ => s"Invalid index: $value. It must be in ${Index.valuesStrings}".asLeft private def valuesStrings = Index.values.map(_.value).toList.mkString_("{", ", ", "}") diff --git a/modules/e2e/src/test/scala/CompatSuite.scala b/modules/e2e/src/test/scala/CompatSuite.scala index 68492e7d..92128139 100644 --- a/modules/e2e/src/test/scala/CompatSuite.scala +++ b/modules/e2e/src/test/scala/CompatSuite.scala @@ -11,7 +11,7 @@ import lila.search.spec.{ CountOutput, Query, SearchOutput } import org.http4s.implicits.* import org.typelevel.log4cats.noop.{ NoOpFactory, NoOpLogger } import org.typelevel.log4cats.{ Logger, LoggerFactory } -import org.typelevel.otel4s.metrics.Meter +import org.typelevel.otel4s.metrics.MeterProvider import org.typelevel.otel4s.sdk.exporter.prometheus.PrometheusMetricExporter import org.typelevel.otel4s.sdk.metrics.exporter.MetricExporter import play.api.libs.ws.ahc.* @@ -20,21 +20,18 @@ import scala.concurrent.ExecutionContext.Implicits.* object CompatSuite extends weaver.IOSuite: - given Logger[IO] = NoOpLogger[IO] + given Logger[IO] = NoOpLogger[IO] given LoggerFactory[IO] = NoOpFactory[IO] - given Meter[IO] = Meter.noop[IO] + given MeterProvider[IO] = MeterProvider.noop[IO] override type Res = SearchClient override def sharedResource: Resource[IO, Res] = - val res = AppResources(fakeClient) for given MetricExporter.Pull[IO] <- PrometheusMetricExporter.builder[IO].build.toResource - res <- App - .mkServer(res, testAppConfig) - .flatMap(_ => wsClient) - .map(SearchClient.play(_, "http://localhost:9999/api")) - yield res + _ <- App.mkServer(AppResources(fakeESClient), testAppConfig) + wsClient <- makeWSClient + yield SearchClient.play(wsClient, "http://localhost:9999/api") val from = From(0) val size = Size(12) @@ -53,7 +50,7 @@ object CompatSuite extends weaver.IOSuite: IO.fromFuture(IO(client.search(query, from, size))) .handleErrorWith: case e: SearchError.JsonWriterError => - IO.pure(SearchOutput(Nil)) + IO(SearchOutput(Nil)) .map(expect.same(_, SearchOutput(Nil))) test("count endpoint"): client => @@ -65,20 +62,20 @@ object CompatSuite extends weaver.IOSuite: elastic = ElasticConfig(uri"http://0.0.0.0:9200") ) - def fakeClient: ESClient[IO] = new: + def fakeESClient: ESClient[IO] = new: - override def store[A](index: Index, id: Id, obj: A)(using Indexable[A]): IO[Unit] = IO.unit + override def store[A](index: Index, id: Id, obj: A)(using Indexable[A]) = IO.unit - override def storeBulk[A](index: Index, objs: Seq[SourceWithId[A]])(using Indexable[A]): IO[Unit] = + override def storeBulk[A](index: Index, objs: Seq[SourceWithId[A]])(using Indexable[A]) = IO.unit - override def putMapping(index: Index): IO[Unit] = IO.unit + override def putMapping(index: Index) = IO.unit - override def refreshIndex(index: Index): IO[Unit] = IO.unit + override def refreshIndex(index: Index) = IO.unit - override def deleteOne(index: Index, id: Id): IO[Unit] = IO.unit + override def deleteOne(index: Index, id: Id) = IO.unit - override def deleteMany(index: Index, ids: List[Id]): IO[Unit] = IO.unit + override def deleteMany(index: Index, ids: List[Id]) = IO.unit override def count[A](query: A)(using Queryable[A]) = IO.pure(0) @@ -86,10 +83,12 @@ object CompatSuite extends weaver.IOSuite: override def search[A](query: A, from: From, size: Size)(using Queryable[A]) = IO.pure(Nil) - override def status: IO[String] = IO.pure("yellow") + override def status = IO("yellow") + + override def indexExists(index: Index) = IO(true) given system: ActorSystem = ActorSystem() - def wsClient = Resource.make(IO(StandaloneAhcWSClient()))(x => + private def makeWSClient = Resource.make(IO(StandaloneAhcWSClient()))(x => IO(x.close()).flatMap(_ => IO.fromFuture(IO(system.terminate())).void) ) diff --git a/modules/e2e/src/test/scala/ElasticSearchContainer.scala b/modules/e2e/src/test/scala/ElasticSearchContainer.scala index 2dadbf1d..de49f90c 100644 --- a/modules/e2e/src/test/scala/ElasticSearchContainer.scala +++ b/modules/e2e/src/test/scala/ElasticSearchContainer.scala @@ -8,12 +8,12 @@ import org.testcontainers.containers.wait.strategy.Wait object ElasticSearchContainer: - private val PORT = 9200 + private val PORT = 9200 private val container = val env = Map( - "discovery.type" -> "single-node", + "discovery.type" -> "single-node", "http.cors.allow-origin" -> "/.*/", - "http.cors.enabled" -> "true", + "http.cors.enabled" -> "true", "xpack.security.enabled" -> "false" ) val start = IO( diff --git a/modules/e2e/src/test/scala/IntegrationSuite.scala b/modules/e2e/src/test/scala/IntegrationSuite.scala index c74f6711..5db11eda 100644 --- a/modules/e2e/src/test/scala/IntegrationSuite.scala +++ b/modules/e2e/src/test/scala/IntegrationSuite.scala @@ -1,15 +1,19 @@ package lila.search package app package test + +import cats.Functor import cats.effect.{ IO, Resource } +import cats.mtl.Raise import cats.syntax.all.* import com.comcast.ip4s.* +import com.sksamuel.elastic4s.ElasticError import lila.search.ingestor.Ingestor.given import lila.search.spec.* import org.http4s.Uri import org.typelevel.log4cats.noop.{ NoOpFactory, NoOpLogger } import org.typelevel.log4cats.{ Logger, LoggerFactory } -import org.typelevel.otel4s.metrics.Meter +import org.typelevel.otel4s.metrics.MeterProvider import org.typelevel.otel4s.sdk.exporter.prometheus.PrometheusMetricExporter import org.typelevel.otel4s.sdk.metrics.exporter.MetricExporter import smithy4s.Timestamp @@ -19,9 +23,13 @@ import java.time.Instant object IntegrationSuite extends IOSuite: - given Logger[IO] = NoOpLogger[IO] + given Logger[IO] = NoOpLogger[IO] given LoggerFactory[IO] = NoOpFactory[IO] - given Meter[IO] = Meter.noop[IO] + given MeterProvider[IO] = MeterProvider.noop[IO] + private given Raise[IO, ElasticError]: + def functor: Functor[IO] = Functor[IO] + def raise[E <: ElasticError, A](e: E): IO[A] = + IO.raiseError(e.asException) private val uri = Uri.unsafeFromString("http://localhost:9999") @@ -33,9 +41,9 @@ object IntegrationSuite extends IOSuite: for elastic <- ElasticSearchContainer.start config = testAppConfig(elastic) - res <- AppResources.instance(config) + res <- AppResources.instance(config) given MetricExporter.Pull[IO] <- PrometheusMetricExporter.builder[IO].build.toResource - _ <- App.mkServer(res, config) + _ <- App.mkServer(res, config) yield res def testAppConfig(elastic: ElasticConfig) = AppConfig( @@ -74,6 +82,28 @@ object IntegrationSuite extends IOSuite: y <- service.search(Query.forum("nt9", false), from, size) yield expect(x.hitIds.size == 1 && x == y) + test("ublog"): res => + Clients + .search(uri) + .use: service => + for + _ <- res.esClient.putMapping(Index.Ublog) + _ <- res.esClient.store( + Index.Ublog, + Id("abcdefgh"), + ingestor.UblogSource( + text = "lil bubber, hayo!", + language = "en", + likes = 0, + date = Instant.now().toEpochMilli(), + quality = 1.some + ) + ) + _ <- res.esClient.refreshIndex(Index.Ublog) + x <- service.search(Query.ublog("lil bubber", SortBlogsBy.score, 1.some), from, size) + y <- service.search(Query.ublog("hayo", SortBlogsBy.newest, 2.some), from, size) + yield expect(x.hitIds.size == 1 && y.hitIds.isEmpty) + test("team"): res => Clients .search(uri) @@ -120,15 +150,15 @@ object IntegrationSuite extends IOSuite: c <- service.search(Query.study("topic1"), from, size) yield expect(a.hitIds.size == 1 && b == a && c == a) - val defaultIntRange = IntRange(none, none) + val defaultIntRange = IntRange(none, none) val defaultDateRange = DateRange(none, none) - val defaultGame = Query.game( + val defaultGame = Query.game( turns = defaultIntRange, averageRating = defaultIntRange, aiLevel = defaultIntRange, date = defaultDateRange, duration = defaultIntRange, - sorting = Sorting("field", "asc") + sorting = GameSorting("field", "asc") ) test("game"): res => Clients diff --git a/modules/elastic/src/main/scala/ESClient.scala b/modules/elastic/src/main/scala/ESClient.scala index d5d80756..1f202bf9 100644 --- a/modules/elastic/src/main/scala/ESClient.scala +++ b/modules/elastic/src/main/scala/ESClient.scala @@ -1,153 +1,80 @@ package lila.search -import cats.effect.* +import cats.MonadThrow +import cats.effect.kernel.Sync +import cats.mtl.Raise import cats.syntax.all.* import com.sksamuel.elastic4s.ElasticDsl.* import com.sksamuel.elastic4s.http4s.Http4sClient -import com.sksamuel.elastic4s.{ ElasticClient, ElasticDsl, Index as ESIndex, Indexable } -import lila.search.ESClient.MetricKeys.* +import com.sksamuel.elastic4s.{ ElasticClient, ElasticDsl, ElasticError, Index as ESIndex, Indexable } +import fs2.io.file.Files import org.http4s.Uri import org.http4s.client.Client -import org.typelevel.otel4s.metrics.{ Histogram, Meter } -import org.typelevel.otel4s.{ Attribute, AttributeKey, Attributes } - -import java.util.concurrent.TimeUnit trait ESClient[F[_]]: - - def search[A](query: A, from: From, size: Size)(using Queryable[A]): F[List[Id]] - def count[A](query: A)(using Queryable[A]): F[Long] - def store[A](index: Index, id: Id, obj: A)(using Indexable[A]): F[Unit] - def storeBulk[A](index: Index, objs: Seq[SourceWithId[A]])(using Indexable[A]): F[Unit] - def deleteOne(index: Index, id: Id): F[Unit] - def deleteMany(index: Index, ids: List[Id]): F[Unit] - def putMapping(index: Index): F[Unit] - def refreshIndex(index: Index): F[Unit] - def status: F[String] + type RaiseF[A] = Raise[F, ElasticError] ?=> F[A] + + def search[A](query: A, from: From, size: Size)(using Queryable[A]): RaiseF[List[Id]] + def count[A](query: A)(using Queryable[A]): RaiseF[Long] + def store[A](index: Index, id: Id, obj: A)(using Indexable[A]): RaiseF[Unit] + def storeBulk[A](index: Index, objs: Seq[SourceWithId[A]])(using Indexable[A]): RaiseF[Unit] + def deleteOne(index: Index, id: Id): RaiseF[Unit] + def deleteMany(index: Index, ids: List[Id]): RaiseF[Unit] + def putMapping(index: Index): RaiseF[Unit] + def refreshIndex(index: Index): RaiseF[Unit] + def status: RaiseF[String] + def indexExists(index: Index): RaiseF[Boolean] object ESClient: - def apply(uri: Uri)(client: Client[IO])(using Meter[IO]): IO[ESClient[IO]] = - Meter[IO] - .histogram[Double]("db.client.operation.duration") - .withUnit("ms") - .create - .map( - apply( - ElasticClient(new Http4sClient(client, uri)), - Attributes(Attribute("db.system", "elasticsearch"), Attribute("server.address", uri.toString())) - ) - ) - - def apply[F[_]: MonadCancelThrow](client: ElasticClient[F], baseAttributes: Attributes)( - metric: Histogram[F, Double] - ) = new ESClient[F]: - - def status: F[String] = + def apply[F[_]: Sync: Files](uri: Uri)(client: Client[F]): ESClient[F] = + apply(ElasticClient(new Http4sClient(client, uri))) + + def apply[F[_]: MonadThrow](client: ElasticClient[F]) = new ESClient[F]: + + def status: RaiseF[String] = client .execute(clusterHealth()) .flatMap(_.toResult) .map(_.status) - def search[A](query: A, from: From, size: Size)(using Queryable[A]): F[List[Id]] = - metric - .recordDuration( - TimeUnit.MILLISECONDS, - withErrorType( - baseAttributes - .added(dbOperationName, "search") - .added(dbCollectionName, query.index.value) - ) - ) - .surround: - client - .execute(query.searchDef(from, size)) - .flatMap(_.toResult) - .map(_.hits.hits.toList.map(h => Id(h.id))) - - def count[A](query: A)(using Queryable[A]): F[Long] = - metric - .recordDuration( - TimeUnit.MILLISECONDS, - withErrorType( - baseAttributes - .added(dbOperationName, "count") - .added(dbCollectionName, query.index.value) - ) - ) - .surround: - client - .execute(query.countDef) - .flatMap(_.toResult) - .map(_.count) - - def store[A](index: Index, id: Id, obj: A)(using Indexable[A]): F[Unit] = - metric - .recordDuration( - TimeUnit.MILLISECONDS, - withErrorType( - baseAttributes - .added(dbOperationName, "store") - .added(dbCollectionName, index.value) - ) - ) - .surround: - client - .execute(indexInto(index.value).source(obj).id(id.value)) - .flatMap(_.unitOrFail) - - def storeBulk[A](index: Index, objs: Seq[SourceWithId[A]])(using Indexable[A]): F[Unit] = - val request = indexInto(index.value) + def search[A](query: A, from: From, size: Size)(using Queryable[A]): RaiseF[List[Id]] = + client + .execute(query.searchDef(from, size)) + .flatMap(_.toResult) + .map(_.hits.hits.toList.map(h => Id(h.id))) + + def count[A](query: A)(using Queryable[A]): RaiseF[Long] = + client + .execute(query.countDef) + .flatMap(_.toResult) + .map(_.count) + + def store[A](index: Index, id: Id, obj: A)(using Indexable[A]): RaiseF[Unit] = + client + .execute(indexInto(index.value).source(obj).id(id.value)) + .flatMap(_.unitOrFail) + + def storeBulk[A](index: Index, objs: Seq[SourceWithId[A]])(using Indexable[A]): RaiseF[Unit] = + val request = indexInto(index.value) val requests = bulk(objs.map { case (id, source) => request.source(source).id(id) }) - metric - .recordDuration( - TimeUnit.MILLISECONDS, - withErrorType( - baseAttributes - .added(dbOperationName, "store-bulk") - .added(dbCollectionName, index.value) - .added(dbBatchSize, objs.size) - ) - ) - .surround: - client - .execute(requests) - .flatMap(_.unitOrFail) + client + .execute(requests) + .flatMap(_.unitOrFail) .whenA(objs.nonEmpty) - def deleteOne(index: Index, id: Id): F[Unit] = - metric - .recordDuration( - TimeUnit.MILLISECONDS, - withErrorType( - baseAttributes - .added(dbOperationName, "delete-one") - .added(dbCollectionName, index.value) - ) - ) - .surround: - client - .execute(deleteById(index.toES, id.value)) - .flatMap(_.unitOrFail) - - def deleteMany(index: Index, ids: List[Id]): F[Unit] = - metric - .recordDuration( - TimeUnit.MILLISECONDS, - withErrorType( - baseAttributes - .added(dbOperationName, "delete-bulk") - .added(dbCollectionName, index.value) - .added(dbBatchSize, ids.size) - ) - ) - .surround: - client - .execute(bulk(ids.map(id => deleteById(index.toES, id.value)))) - .flatMap(_.unitOrFail) + def deleteOne(index: Index, id: Id): RaiseF[Unit] = + client + .execute(deleteById(index.toES, id.value)) + .flatMap(_.unitOrFail) + + def deleteMany(index: Index, ids: List[Id]): RaiseF[Unit] = + client + .execute(bulk(ids.map(id => deleteById(index.toES, id.value)))) + .flatMap(_.unitOrFail) .whenA(ids.nonEmpty) - def putMapping(index: Index): F[Unit] = + def putMapping(index: Index): RaiseF[Unit] = dropIndex(index) *> client .execute: createIndex(index.value) @@ -157,24 +84,16 @@ object ESClient: .refreshInterval(index.refreshInterval) .flatMap(_.unitOrFail) - def refreshIndex(index: Index): F[Unit] = + def refreshIndex(index: Index): RaiseF[Unit] = client .execute(ElasticDsl.refreshIndex(index.value)) .flatMap(_.unitOrFail) + def indexExists(index: Index): RaiseF[Boolean] = + client + .execute(ElasticDsl.indexExists(index.value)) + .flatMap(_.toResult) + .map(_.exists) + private def dropIndex(index: Index) = client.execute(deleteIndex(index.value)) - - object MetricKeys: - val dbCollectionName = AttributeKey.string("db.collection.name") - val dbBatchSize = AttributeKey.long("db.operation.batch.size") - val dbOperationName = AttributeKey.string("db.operation.name") - val errorType = AttributeKey.string("error.type") - - private def withErrorType(static: Attributes)(ec: Resource.ExitCase): Attributes = ec match - case Resource.ExitCase.Succeeded => - static - case Resource.ExitCase.Errored(e) => - static.added(errorType, e.getClass.getName) - case Resource.ExitCase.Canceled => - static.added(errorType, "canceled") diff --git a/modules/elastic/src/main/scala/forum.scala b/modules/elastic/src/main/scala/forum.scala index d59df1d9..20f83a99 100644 --- a/modules/elastic/src/main/scala/forum.scala +++ b/modules/elastic/src/main/scala/forum.scala @@ -25,16 +25,16 @@ case class Forum(text: String, troll: Boolean): ).flatten.compile object Forum: - val index = "forum" + val index = "forum" private val searchableFields = List(Fields.body, Fields.topic, Fields.author) object Fields: - val body = "bo" - val topic = "to" + val body = "bo" + val topic = "to" val topicId = "ti" - val author = "au" - val troll = "tr" - val date = "da" + val author = "au" + val troll = "tr" + val date = "da" object Mapping: import Fields.* diff --git a/modules/elastic/src/main/scala/game.scala b/modules/elastic/src/main/scala/game.scala index 5556203b..ed38c505 100644 --- a/modules/elastic/src/main/scala/game.scala +++ b/modules/elastic/src/main/scala/game.scala @@ -3,6 +3,7 @@ package game import com.sksamuel.elastic4s.ElasticDsl.* import com.sksamuel.elastic4s.requests.searches.queries.Query +import com.sksamuel.elastic4s.requests.searches.sort.FieldSort import com.sksamuel.elastic4s.requests.searches.term.TermQuery import java.time.Instant @@ -56,7 +57,7 @@ case class Game( def toQueries(query: Option[String | Int | Boolean], name: String): List[TermQuery] = query.toList.map: case s: String => termQuery(name, s.toLowerCase) - case x => termQuery(name, x) + case x => termQuery(name, x) List( usernames.map(termQuery(Fields.uids, _)), @@ -81,24 +82,24 @@ case class Game( ).flatten.compile object Fields: - val status = "s" - val turns = "t" - val rated = "r" - val perf = "p" - val uids = "u" - val winner = "w" - val loser = "o" - val winnerColor = "c" + val status = "s" + val turns = "t" + val rated = "r" + val perf = "p" + val uids = "u" + val winner = "w" + val loser = "o" + val winnerColor = "c" val averageRating = "a" - val ai = "i" - val date = "d" - val duration = "l" - val clockInit = "ct" - val clockInc = "ci" - val analysed = "n" - val whiteUser = "wu" - val blackUser = "bu" - val source = "so" + val ai = "i" + val date = "d" + val duration = "l" + val clockInit = "ct" + val clockInc = "ci" + val analysed = "n" + val whiteUser = "wu" + val blackUser = "bu" + val source = "so" object Mapping: import Fields.* @@ -129,11 +130,11 @@ object Game: case class Sorting(f: String, order: String): import com.sksamuel.elastic4s.requests.searches.sort.SortOrder - def definition = + def definition: FieldSort = fieldSort(Sorting.fieldKeys.contains(f).fold(f, Sorting.default.f)) .order((order.toLowerCase == "asc").fold(SortOrder.ASC, SortOrder.DESC)) object Sorting: - val default = Sorting(Fields.date, "desc") + val default = Sorting(Fields.date, "desc") val fieldKeys = List(Fields.date, Fields.turns, Fields.averageRating) diff --git a/modules/elastic/src/main/scala/package.scala b/modules/elastic/src/main/scala/package.scala index 3fa3fecd..5ca6ad7a 100644 --- a/modules/elastic/src/main/scala/package.scala +++ b/modules/elastic/src/main/scala/package.scala @@ -1,10 +1,12 @@ package lila.search -import cats.MonadThrow +import cats.Monad +import cats.mtl.Raise +import cats.mtl.implicits.* import cats.syntax.all.* import com.sksamuel.elastic4s.ElasticDsl.* import com.sksamuel.elastic4s.requests.searches.queries.Query -import com.sksamuel.elastic4s.{ Index as ESIndex, Response } +import com.sksamuel.elastic4s.{ ElasticError, Index as ESIndex, Response } type SourceWithId[A] = (id: String, source: A) @@ -12,26 +14,27 @@ extension (self: Boolean) def fold[A](t: => A, f: => A): A = if self then t else extension (queries: List[Query]) def compile: Query = queries match - case Nil => matchAllQuery() + case Nil => matchAllQuery() case q :: Nil => q - case _ => boolQuery().filter(queries) + case _ => boolQuery().filter(queries) extension (index: Index) def toES: ESIndex = ESIndex(index.value) def mapping = index match case Index.Forum => forum.Mapping.fields - case Index.Game => game.Mapping.fields + case Index.Ublog => ublog.Mapping.fields + case Index.Game => game.Mapping.fields case Index.Study => study.Mapping.fields - case Index.Team => team.Mapping.fields + case Index.Team => team.Mapping.fields def refreshInterval = index match case Index.Study => "10s" - case _ => "300s" + case _ => "300s" -extension [F[_]: MonadThrow, A](response: Response[A]) - def toResult: F[A] = - response.fold(response.error.asException.raiseError)(r => r.pure[F]) - def unitOrFail: F[Unit] = - response.fold(response.error.asException.raiseError)(_ => ().pure[F]) +extension [F[_]: Monad, A](response: Response[A]) + def toResult: Raise[F, ElasticError] ?=> F[A] = + response.fold(response.error.raise)(_.pure[F]) + def unitOrFail: Raise[F, ElasticError] ?=> F[Unit] = + response.fold(response.error.raise)(_ => ().pure[F]) diff --git a/modules/elastic/src/main/scala/study.scala b/modules/elastic/src/main/scala/study.scala index db7c1ae6..bd18c694 100644 --- a/modules/elastic/src/main/scala/study.scala +++ b/modules/elastic/src/main/scala/study.scala @@ -3,25 +3,23 @@ package study import com.sksamuel.elastic4s.ElasticDsl.* import com.sksamuel.elastic4s.requests.searches.queries.Query -import com.sksamuel.elastic4s.requests.searches.sort.SortOrder +import com.sksamuel.elastic4s.requests.searches.sort.{ FieldSort, SortOrder } +import lila.search.study.Study.Sorting -case class Study(text: String, userId: Option[String]): +case class Study(text: String, sorting: Option[Sorting], userId: Option[String]): def searchDef(from: From, size: Size) = search(Study.index) - .query(makeQuery) + .query(makeQuery()) .fetchSource(false) - .sortBy( - fieldSort("_score").order(SortOrder.DESC), - fieldSort(Fields.likes).order(SortOrder.DESC) - ) + .sortBy(sorting.map(_.toElastic) ++ Seq(fieldSort("_score").order(SortOrder.DESC))) .start(from.value) .size(size.value) - def countDef = count(Study.index).query(makeQuery) + def countDef = count(Study.index).query(makeQuery()) - private def makeQuery = { - val parsed = QueryParser(text, List("owner", "member")) + private def makeQuery() = { + val parsed = QueryParser(text, List("owner", "member")) val matcher: Query = if parsed.terms.isEmpty then matchAllQuery() else @@ -50,30 +48,36 @@ case class Study(text: String, userId: Option[String]): private def selectUserId(userId: String) = termQuery(Fields.members, userId) object Fields: - val name = "name" - val owner = "owner" - val members = "members" + val name = "name" + val nameRaw = "raw" + val owner = "owner" + val members = "members" val chapterNames = "chapterNames" val chapterTexts = "chapterTexts" - val topics = "topics" - // val createdAt = "createdAt" - // val updatedAt = "updatedAt" - // val rank = "rank" - val likes = "likes" + val topics = "topics" + val createdAt = "createdAt_date" + val updatedAt = "updatedAt_date" + val rank = "rank" + val likes = "likes" val public = "public" object Mapping: import Fields.* def fields = Seq( - textField(name).copy(boost = Some(10), analyzer = Some("english")), + textField(name) + .copy(boost = Some(10), analyzer = Some("english")) + .copy(fields = List(keywordField(nameRaw))), keywordField(owner).copy(boost = Some(2), docValues = Some(false)), keywordField(members).copy(boost = Some(1), docValues = Some(false)), textField(chapterNames).copy(boost = Some(4), analyzer = Some("english")), textField(chapterTexts).copy(boost = Some(1), analyzer = Some("english")), textField(topics).copy(boost = Some(5), analyzer = Some("english")), shortField(likes).copy(docValues = Some(true)), // sort by likes - booleanField(public).copy(docValues = Some(false)) + booleanField(public).copy(docValues = Some(false)), + dateField(rank).copy(format = Some(SearchDateTime.format)), + dateField(createdAt).copy(format = Some(SearchDateTime.format)), + dateField(updatedAt).copy(format = Some(SearchDateTime.format)) ) object Study: @@ -87,3 +91,22 @@ object Study: Fields.chapterNames, Fields.chapterTexts ) + + enum Field(val field: String): + case Name extends Field(s"${Fields.name}.${Fields.nameRaw}") + case Likes extends Field(Fields.likes) + case CreatedAt extends Field(Fields.createdAt) + case UpdatedAt extends Field(Fields.updatedAt) + case Hot extends Field(Fields.rank) + + enum Order: + case Asc, Desc + + extension (o: Order) + def toElastic: SortOrder = o match + case Order.Asc => SortOrder.ASC + case Order.Desc => SortOrder.DESC + + case class Sorting(field: Field, order: Order): + def toElastic: FieldSort = + fieldSort(field.field).order(order.toElastic) diff --git a/modules/elastic/src/main/scala/team.scala b/modules/elastic/src/main/scala/team.scala index b663f208..ea654bb5 100644 --- a/modules/elastic/src/main/scala/team.scala +++ b/modules/elastic/src/main/scala/team.scala @@ -20,9 +20,9 @@ case class Team(text: String): QueryParser(text, Nil).terms.map(term => multiMatchQuery(term).fields(Team.searchableFields*)).compile private object Fields: - val name = "na" + val name = "na" val description = "de" - val nbMembers = "nbm" + val nbMembers = "nbm" object Mapping: import Fields.* diff --git a/modules/elastic/src/main/scala/ublog.scala b/modules/elastic/src/main/scala/ublog.scala new file mode 100644 index 00000000..e5fb9c6f --- /dev/null +++ b/modules/elastic/src/main/scala/ublog.scala @@ -0,0 +1,78 @@ +package lila.search +package ublog + +import com.sksamuel.elastic4s.ElasticDsl.* +import com.sksamuel.elastic4s.requests.searches.sort.SortOrder +import lila.search.ublog.Ublog.SortBy + +case class Ublog( + queryText: String, + by: SortBy, + minQuality: Option[Int], + language: Option[String] +): + + val sanitized = queryText + .trim() + .toLowerCase() + .replaceAll("""([\-=&|> s + case s => s.replace(":", " ") // devs can use the query string until we get a ui for lang/quality + .mkString(" ") + + def searchDef(from: From, size: Size) = + val sortFields = + (if by == SortBy.score then Seq(scoreSort().order(SortOrder.DESC)) + else if by == SortBy.likes then Seq(fieldSort("likes").order(SortOrder.DESC)) + else Nil) ++ Seq( + fieldSort("quality").order(SortOrder.DESC).missing("_last"), + fieldSort("date") + .order(if by == SortBy.oldest then SortOrder.ASC else SortOrder.DESC) + .missing("_last") + ) + search(Ublog.index) + .query(makeQuery()) + .fetchSource(false) + .sortBy(sortFields*) + .start(from.value) + .size(size.value) + + def countDef = count(Ublog.index).query(makeQuery()) + + private def makeQuery() = + boolQuery() + .must(queryStringQuery(sanitized).defaultField(Fields.text)) + .filter( + List( + minQuality.map(f => rangeQuery(Fields.quality).gte(f)), + language.map(l => termQuery(Fields.language, l)) + ).flatten + ) + +object Ublog: + val index = "ublog" + + enum SortBy: + case newest, oldest, score, likes + +object Fields: + val text = "text" + val likes = "likes" + val quality = "quality" + val language = "language" + val date = "date" + +object Mapping: + import Fields.* + def fields = + Seq( + textField(text), + shortField(quality).copy(docValues = Some(true)), + keywordField(language).copy(docValues = Some(false)), + shortField(likes).copy(docValues = Some(true)), + dateField(date).copy(docValues = Some(true)) + ) diff --git a/modules/ingestor/src/main/resources/logback.xml b/modules/ingestor-app/src/main/resources/logback.xml similarity index 100% rename from modules/ingestor/src/main/resources/logback.xml rename to modules/ingestor-app/src/main/resources/logback.xml diff --git a/modules/ingestor-app/src/main/scala/app.scala b/modules/ingestor-app/src/main/scala/app.scala new file mode 100644 index 00000000..e4b16cc0 --- /dev/null +++ b/modules/ingestor-app/src/main/scala/app.scala @@ -0,0 +1,54 @@ +package lila.search +package ingestor + +import cats.effect.* +import org.typelevel.log4cats.slf4j.Slf4jFactory +import org.typelevel.log4cats.{ Logger, LoggerFactory } +import org.typelevel.otel4s.experimental.metrics.* +import org.typelevel.otel4s.instrumentation.ce.IORuntimeMetrics +import org.typelevel.otel4s.metrics.{ Meter, MeterProvider } +import org.typelevel.otel4s.sdk.exporter.prometheus.autoconfigure.PrometheusMetricExporterAutoConfigure +import org.typelevel.otel4s.sdk.metrics.SdkMetrics + +object App extends IOApp.Simple: + + given LoggerFactory[IO] = Slf4jFactory.create[IO] + given Logger[IO] = LoggerFactory[IO].getLogger + + override def run: IO[Unit] = app.useForever + + def app: Resource[IO, Unit] = + for + otel4s <- SdkMetrics.autoConfigured[IO](configBuilder) + given MeterProvider[IO] = otel4s.meterProvider + _ <- registerRuntimeMetrics + config <- AppConfig.load.toResource + _ <- Logger[IO].info(s"Starting lila-search ingestor with config: ${config.toString}").toResource + _ <- Logger[IO].info(s"BuildInfo: ${BuildInfo.toString}").toResource + res <- AppResources.instance(config) + _ <- IngestorApp(res, config).run() + yield () + + private def registerRuntimeMetrics(using MeterProvider[IO]): Resource[IO, Unit] = + for + _ <- IORuntimeMetrics.register[IO](runtime.metrics, IORuntimeMetrics.Config.default) + given Meter[IO] <- MeterProvider[IO].get("jvm.runtime").toResource + _ <- RuntimeMetrics.register[IO] + yield () + + private def configBuilder(builder: SdkMetrics.AutoConfigured.Builder[IO]) = + builder + .addPropertiesCustomizer(_ => + Map( + "otel.metrics.exporter" -> "prometheus", + "otel.traces.exporter" -> "none" + ) + ) + .addExporterConfigurer(PrometheusMetricExporterAutoConfigure[IO]) + +class IngestorApp(res: AppResources, config: AppConfig)(using Logger[IO], LoggerFactory[IO]): + def run(): Resource[IO, Unit] = + Ingestors(res.lichess, res.study, res.studyLocal, res.store, res.elastic, config.ingestor) + .flatMap(_.run()) + .toResource + .evalTap(_ => Logger[IO].info("Ingestor started")) diff --git a/modules/ingestor-cli/src/main/resources/logback.xml b/modules/ingestor-cli/src/main/resources/logback.xml new file mode 100644 index 00000000..381f715b --- /dev/null +++ b/modules/ingestor-cli/src/main/resources/logback.xml @@ -0,0 +1,27 @@ + + + + + + + + logs/lila-search-ingestor-cli.log + + %date [%thread] %-5level %logger{20} - %msg%n%xException + + + + + + %date [%thread] %-5level %logger{20} - %msg%n%xException + + + + + + + + + + + diff --git a/modules/ingestor/src/main/scala/cli.scala b/modules/ingestor-cli/src/main/scala/cli.scala similarity index 52% rename from modules/ingestor/src/main/scala/cli.scala rename to modules/ingestor-cli/src/main/scala/cli.scala index dac0a3a1..f3cc164f 100644 --- a/modules/ingestor/src/main/scala/cli.scala +++ b/modules/ingestor-cli/src/main/scala/cli.scala @@ -3,14 +3,14 @@ package ingestor import cats.data.Validated import cats.effect.* -import cats.effect.unsafe.IORuntime +import cats.mtl.Handle import cats.syntax.all.* import com.monovore.decline.* import com.monovore.decline.effect.* import lila.search.ingestor.opts.{ IndexOpts, WatchOpts } import org.typelevel.log4cats.slf4j.Slf4jFactory import org.typelevel.log4cats.{ Logger, LoggerFactory } -import org.typelevel.otel4s.metrics.Meter +import org.typelevel.otel4s.metrics.MeterProvider import java.time.Instant @@ -22,19 +22,19 @@ object cli ): given LoggerFactory[IO] = Slf4jFactory.create[IO] - given Logger[IO] = LoggerFactory[IO].getLogger - given Meter[IO] = Meter.noop[IO] - given IORuntime = runtime + given Logger[IO] = LoggerFactory[IO].getLogger + given MeterProvider[IO] = MeterProvider.noop[IO] override def main: Opts[IO[ExitCode]] = opts.parse.map: opts => - makeIngestor.use(_.execute(opts).as(ExitCode.Success)) + Logger[IO].info(s"Starting lila-search-cli with ${opts.toString}") *> + makeIngestor.use(execute(opts)).as(ExitCode.Success) - def makeIngestor: Resource[IO, Ingestors] = + private def makeIngestor = for - config <- AppConfig.load.toResource - res <- AppResources.instance(config) - ingestor <- Ingestors( + config <- AppConfig.load.toResource + res <- AppResources.instance(config) + ingestors <- Ingestors( res.lichess, res.study, res.studyLocal, @@ -42,18 +42,20 @@ object cli res.elastic, config.ingestor ).toResource - yield ingestor + yield (ingestors, res.elastic) - extension (ingestor: Ingestors) - def execute(opts: IndexOpts | WatchOpts): IO[Unit] = - opts match - case opts: IndexOpts => index(opts) - case opts: WatchOpts => watch(opts) + def execute(opts: IndexOpts | WatchOpts)(ingestor: Ingestors, elastic: ESClient[IO]): IO[Unit] = + opts match + case opts: IndexOpts => index(ingestor, elastic)(opts) + case opts: WatchOpts => watch(ingestor)(opts) - def index(opts: IndexOpts): IO[Unit] = - opts.index match + def index(ingestor: Ingestors, elastic: ESClient[IO])(opts: IndexOpts): IO[Unit] = + putMappingsIfNotExists(elastic, opts.index) *> + opts.index.match case Index.Forum => ingestor.forum.run(opts.since, opts.until, opts.dry) + case Index.Ublog => + ingestor.ublog.run(opts.since, opts.until, opts.dry) case Index.Study => ingestor.study.run(opts.since, opts.until, opts.dry) case Index.Game => @@ -62,28 +64,60 @@ object cli ingestor.team.run(opts.since, opts.until, opts.dry) case _ => ingestor.forum.run(opts.since, opts.until, opts.dry) *> + ingestor.ublog.run(opts.since, opts.until, opts.dry) *> ingestor.study.run(opts.since, opts.until, opts.dry) *> ingestor.game.run(opts.since, opts.until, opts.dry) *> ingestor.team.run(opts.since, opts.until, opts.dry) - - def watch(opts: WatchOpts): IO[Unit] = - opts.index match - case Index.Game => + *> refreshIndexes(elastic, opts.index).whenA(opts.refresh) + + private def putMappingsIfNotExists(elastic: ESClient[IO], index: Index | Unit): IO[Unit] = + def go(index: Index) = + Handle + .allow: + elastic + .indexExists(index) + .ifM(Logger[IO].info(s"Index ${index.value} exists, start indexing"), elastic.putMapping(index)) + .rescue: e => + Logger[IO].error(e.asException)(s"Failed to check or put mapping for ${index.value}") *> + e.asException.raiseError + index match + case i: Index => go(i) + case _ => Index.values.toList.traverse_(go) + + private def refreshIndexes(elastic: ESClient[IO], index: Index | Unit): IO[Unit] = + def go(index: Index) = + Handle + .allow: + elastic + .refreshIndex(index) + .rescue: e => + Logger[IO].error(e.asException)(s"Failed to check or put mapping for ${index.value}") *> + e.asException.raiseError + index.match + case i: Index => go(i) + case _ => Index.values.toList.traverse_(go) + + def watch(ingestor: Ingestors)(opts: WatchOpts): IO[Unit] = + opts.index match + case Index.Game => + ingestor.game.watch(opts.since.some, opts.dry) + case Index.Forum => + ingestor.forum.watch(opts.since.some, opts.dry) + case Index.Ublog => + ingestor.ublog.watch(opts.since.some, opts.dry) + case Index.Team => + ingestor.team.watch(opts.since.some, opts.dry) + case Index.Study => + ingestor.study.watch(opts.since.some, opts.dry) + case _ => + ingestor.forum.watch(opts.since.some, opts.dry) *> + ingestor.ublog.watch(opts.since.some, opts.dry) *> + ingestor.team.watch(opts.since.some, opts.dry) *> + ingestor.study.watch(opts.since.some, opts.dry) *> ingestor.game.watch(opts.since.some, opts.dry) - case Index.Forum => - ingestor.forum.watch(opts.since.some, opts.dry) - case Index.Team => - ingestor.team.watch(opts.since.some, opts.dry) - case Index.Study => - ingestor.study.watch(opts.since.some, opts.dry) - case _ => - ingestor.forum.watch(opts.since.some, opts.dry) *> - ingestor.team.watch(opts.since.some, opts.dry) *> - ingestor.study.watch(opts.since.some, opts.dry) *> - ingestor.game.watch(opts.since.some, opts.dry) object opts: - case class IndexOpts(index: Index | Unit, since: Instant, until: Instant, dry: Boolean) + case class IndexOpts(index: Index | Unit, since: Instant, until: Instant, refresh: Boolean, dry: Boolean) case class WatchOpts(index: Index | Unit, since: Instant, dry: Boolean) def parse = Opts.subcommand("index", "index documents")(indexOpt) <+> @@ -95,7 +129,7 @@ object opts: long = "index", help = "Target index", short = "i", - metavar = "forum|team|study|game" + metavar = "forum|ublog|team|study|game" ) val allIndexOpt = @@ -109,6 +143,12 @@ object opts: .orNone .map(_.isDefined) + val refreshOpt = + Opts + .flag(long = "refresh", help = "Refresh index(ex) after finishing index", short = "r") + .orNone + .map(_.isDefined) + val indexOpt = ( singleIndexOpt orElse allIndexOpt, Opts.option[Instant]( @@ -125,6 +165,7 @@ object opts: metavar = "time in epoch seconds" ) .orElse(Instant.now.pure[Opts]), + refreshOpt, dryOpt ).mapN(IndexOpts.apply) .mapValidated(x => diff --git a/modules/ingestor/src/test/scala/ClITest.scala b/modules/ingestor-cli/src/test/scala/ClITest.scala similarity index 94% rename from modules/ingestor/src/test/scala/ClITest.scala rename to modules/ingestor-cli/src/test/scala/ClITest.scala index 57f6afae..ca7d459b 100644 --- a/modules/ingestor/src/test/scala/ClITest.scala +++ b/modules/ingestor-cli/src/test/scala/ClITest.scala @@ -16,7 +16,7 @@ object CLITest extends weaver.FunSuite: test("index command"): expect.same( testCommand("index", "--index", "team", "--since", "0", "--until", "1", "--dry"), - IndexOpts(Index.Team, Instant.ofEpochSecond(0), Instant.ofEpochSecond(1), true).asRight + IndexOpts(Index.Team, Instant.ofEpochSecond(0), Instant.ofEpochSecond(1), false, true).asRight ) test("watch command"): diff --git a/modules/ingestor/src/main/scala/HasDocId.scala b/modules/ingestor-core/src/main/scala/HasDocId.scala similarity index 100% rename from modules/ingestor/src/main/scala/HasDocId.scala rename to modules/ingestor-core/src/main/scala/HasDocId.scala diff --git a/modules/ingestor/src/main/scala/Repo.scala b/modules/ingestor-core/src/main/scala/Repo.scala similarity index 100% rename from modules/ingestor/src/main/scala/Repo.scala rename to modules/ingestor-core/src/main/scala/Repo.scala diff --git a/modules/ingestor/src/main/scala/app.config.scala b/modules/ingestor-core/src/main/scala/config.scala similarity index 80% rename from modules/ingestor/src/main/scala/app.config.scala rename to modules/ingestor-core/src/main/scala/config.scala index e50f8516..dc73cf1e 100644 --- a/modules/ingestor/src/main/scala/app.config.scala +++ b/modules/ingestor-core/src/main/scala/config.scala @@ -17,16 +17,20 @@ object AppConfig: def load: IO[AppConfig] = appConfig.load[IO] + private def kvStorePath = env("KV_STORE_PATH").or(prop("kv.store.path")).as[String].default("store.json") + def appConfig = ( MongoConfig.config, ElasticConfig.config, - IngestorConfig.config + IngestorConfig.config, + kvStorePath ).parMapN(AppConfig.apply) case class AppConfig( mongo: MongoConfig, elastic: ElasticConfig, - ingestor: IngestorConfig + ingestor: IngestorConfig, + kvStorePath: String ) case class MongoConfig(uri: String, name: String, studyUri: String, studyName: String) @@ -35,7 +39,7 @@ private def studyDatabase = object MongoConfig: - private def uri = env("MONGO_URI").or(prop("mongo.uri")).as[String] + private def uri = env("MONGO_URI").or(prop("mongo.uri")).as[String] private def name = env("MONGO_DATABASE").or(prop("mongo.database")).as[String].default("lichess") private def studyUri = env("MONGO_STUDY_URI").or(prop("mongo.study.uri")).as[String] @@ -51,13 +55,16 @@ object ElasticConfig: case class IngestorConfig( forum: IngestorConfig.Forum, + ublog: IngestorConfig.Ublog, team: IngestorConfig.Team, study: IngestorConfig.Study, game: IngestorConfig.Game ) object IngestorConfig: + case class Forum(batchSize: Int, timeWindows: Int, startAt: Option[Instant], maxPostLength: Int) + case class Ublog(batchSize: Int, timeWindows: Int, startAt: Option[Instant]) case class Team(batchSize: Int, timeWindows: Int, startAt: Option[Instant]) case class Study(batchSize: Int, startAt: Option[Instant], interval: FiniteDuration, databaseName: String) case class Game(batchSize: Int, timeWindows: Int, startAt: Option[Instant]) @@ -73,6 +80,15 @@ object IngestorConfig: env("INGESTOR_FORUM_MAX_POST_LENGTH").or(prop("ingestor.forum.max.post.length")).as[Int].default(5_000) def config = (batchSize, timeWindows, startAt, maxPostLength).parMapN(Forum.apply) + private object Ublog: + private def batchSize = + env("INGESTOR_UBLOG_BATCH_SIZE").or(prop("ingestor.ublog.batch.size")).as[Int].default(100) + private def timeWindows = + env("INGESTOR_UBLOG_TIME_WINDOWS").or(prop("ingestor.ublog.time.windows")).as[Int].default(10) + private def startAt = + env("INGESTOR_UBLOG_START_AT").or(prop("ingestor.ublog.start.at")).as[Instant].option + def config = (batchSize, timeWindows, startAt).parMapN(Ublog.apply) + private object Team: private def batchSize = env("INGESTOR_TEAM_BATCH_SIZE").or(prop("ingestor.team.batch.size")).as[Int].default(100) @@ -104,7 +120,7 @@ object IngestorConfig: env("INGESTOR_GAME_START_AT").or(prop("ingestor.game.start.at")).as[Instant].option def config = (batchSize, timeWindows, startAt).mapN(Game.apply) - def config = (Forum.config, Team.config, Study.config, Game.config).mapN(IngestorConfig.apply) + def config = (Forum.config, Ublog.config, Team.config, Study.config, Game.config).mapN(IngestorConfig.apply) object CirisCodec: given ConfigDecoder[String, Instant] = ConfigDecoder[String] diff --git a/modules/ingestor/src/main/scala/ingestor.scala b/modules/ingestor-core/src/main/scala/ingestor.scala similarity index 80% rename from modules/ingestor/src/main/scala/ingestor.scala rename to modules/ingestor-core/src/main/scala/ingestor.scala index 6d92d6c5..d1eab472 100644 --- a/modules/ingestor/src/main/scala/ingestor.scala +++ b/modules/ingestor-core/src/main/scala/ingestor.scala @@ -2,6 +2,7 @@ package lila.search package ingestor import cats.effect.* +import cats.mtl.Handle.* import cats.syntax.all.* import com.github.plokhotnyuk.jsoniter_scala.core.* import com.sksamuel.elastic4s.Indexable @@ -31,7 +32,7 @@ object Ingestor: elastic: ESClient[IO], defaultStartAt: Option[Instant] )(using LoggerFactory[IO]): Ingestor = new: - given Logger[IO] = LoggerFactory[IO].getLoggerFromName(s"${index.value}.ingestor") + given logger: Logger[IO] = LoggerFactory[IO].getLoggerFromName(s"${index.value}.ingestor") def watch: IO[Unit] = fs2.Stream @@ -71,21 +72,21 @@ object Ingestor: .flatTap(since => info"Starting ${index.value} ingestor from $since") private def deleteMany(index: Index, ids: List[Id]): IO[Unit] = - elastic - .deleteMany(index, ids) - .flatTap(_ => Logger[IO].info(s"Deleted ${ids.size} ${index.value}s")) - .handleErrorWith: e => - Logger[IO].error(e)(s"Failed to delete ${index.value}: ${ids.map(_.value).mkString(", ")}") + allow: + elastic.deleteMany(index, ids) + .rescue: e => + logger.error(e.asException)(s"Failed to delete ${index.value}: ${ids.map(_.value).mkString(", ")}") + .flatTap(_ => Logger[IO].info(s"Deleted ${ids.size} ${index.value}s")) .whenA(ids.nonEmpty) private def storeBulk(index: Index, sources: List[SourceWithId[A]]): IO[Unit] = Logger[IO].info(s"Received ${sources.size} docs to ${index.value}") *> - elastic - .storeBulk(index, sources) - .handleErrorWith: e => - Logger[IO].error(e)(s"Failed to ${index.value} index: ${sources.map(_.id).mkString(", ")}") - .whenA(sources.nonEmpty) - *> Logger[IO].info(s"Indexed ${sources.size} ${index.value}s") + allow: + elastic.storeBulk(index, sources) + .rescue: e => + logger.error(e.asException)(s"Failed to ${index.value} index: ${sources.map(_.id).mkString(", ")}") + .whenA(sources.nonEmpty) + *> logger.info(s"Indexed ${sources.size} ${index.value}s") private val saveLastIndexedTimestamp: Option[Instant] => IO[Unit] = _.traverse_(time => diff --git a/modules/ingestor/src/main/scala/ingestors.scala b/modules/ingestor-core/src/main/scala/ingestors.scala similarity index 79% rename from modules/ingestor/src/main/scala/ingestors.scala rename to modules/ingestor-core/src/main/scala/ingestors.scala index 9a36057a..55f63f32 100644 --- a/modules/ingestor/src/main/scala/ingestors.scala +++ b/modules/ingestor-core/src/main/scala/ingestors.scala @@ -8,12 +8,13 @@ import org.typelevel.log4cats.LoggerFactory class Ingestors( val forum: Ingestor, + val ublog: Ingestor, val study: Ingestor, val game: Ingestor, val team: Ingestor ): def run(): IO[Unit] = - List(forum.watch, team.watch, study.watch, game.watch).parSequence_ + List(forum.watch, ublog.watch, team.watch, study.watch, game.watch).parSequence_ object Ingestors: @@ -27,12 +28,14 @@ object Ingestors: )(using LoggerFactory[IO]): IO[Ingestors] = ( ForumRepo(lichess, config.forum), + UblogRepo(lichess, config.ublog), StudyRepo(study, local, config.study), GameRepo(lichess, config.game), TeamRepo(lichess, config.team) - ).mapN: (forums, studies, games, teams) => + ).mapN: (forums, ublogs, studies, games, teams) => new Ingestors( Ingestor(Index.Forum, forums, store, elastic, config.forum.startAt), + Ingestor(Index.Ublog, ublogs, store, elastic, config.ublog.startAt), Ingestor(Index.Study, studies, store, elastic, config.study.startAt), Ingestor(Index.Game, games, store, elastic, config.game.startAt), Ingestor(Index.Team, teams, store, elastic, config.team.startAt) diff --git a/modules/ingestor/src/main/scala/kvstore.scala b/modules/ingestor-core/src/main/scala/kvstore.scala similarity index 94% rename from modules/ingestor/src/main/scala/kvstore.scala rename to modules/ingestor-core/src/main/scala/kvstore.scala index c1da8b6c..64647f6f 100644 --- a/modules/ingestor/src/main/scala/kvstore.scala +++ b/modules/ingestor-core/src/main/scala/kvstore.scala @@ -15,11 +15,11 @@ trait KVStore: object KVStore: - val file: String = "store.json" + val file: String = "store.json" given JsonValueCodec[Map[String, Long]] = JsonCodecMaker.make type State = Map[String, Long] - def apply(): IO[KVStore] = + def apply(file: String): IO[KVStore] = Mutex .apply[IO] .map: mutex => diff --git a/modules/ingestor/src/main/scala/mongo.chapter.scala b/modules/ingestor-core/src/main/scala/mongo.chapter.scala similarity index 92% rename from modules/ingestor/src/main/scala/mongo.chapter.scala rename to modules/ingestor-core/src/main/scala/mongo.chapter.scala index 60a0af11..834753a1 100644 --- a/modules/ingestor/src/main/scala/mongo.chapter.scala +++ b/modules/ingestor-core/src/main/scala/mongo.chapter.scala @@ -46,7 +46,7 @@ object StudyData: given Decoder[Tag] = Decoder.decodeString.emap: s => s.split(":", 2) match case Array(name, value) => Tag(name, value).asRight - case _ => "Invalid pgn tag $v".asLeft + case _ => "Invalid pgn tag $v".asLeft given Encoder[Tag] = Encoder.encodeString.contramap(t => s"${t.name.toString}:${t.value}") @@ -69,16 +69,16 @@ object ChapterRepo: object F: - val name = "name" - val studyId = "studyId" - val tags = "tags" - val conceal = "conceal" + val name = "name" + val studyId = "studyId" + val tags = "tags" + val conceal = "conceal" val description = "description" - val practice = "practice" - val gamebook = "gamebook" + val practice = "practice" + val gamebook = "gamebook" // accumulates comments into a list - val comments = "comments" + val comments = "comments" val commentTexts = "comments.v.co.text" object Query: diff --git a/modules/ingestor/src/main/scala/mongo.forum.scala b/modules/ingestor-core/src/main/scala/mongo.forum.scala similarity index 95% rename from modules/ingestor/src/main/scala/mongo.forum.scala rename to modules/ingestor-core/src/main/scala/mongo.forum.scala index 57f3de7b..a590680c 100644 --- a/modules/ingestor/src/main/scala/mongo.forum.scala +++ b/modules/ingestor-core/src/main/scala/mongo.forum.scala @@ -28,7 +28,7 @@ object ForumRepo: Filter.in("operationType", interestedOperations) && maxPostSizeFilter(maxPostLength) private val interestedFields = List(_id, F.text, F.topicId, F.troll, F.createdAt, F.userId, F.erasedAt) - private val postProjection = Projection.include(interestedFields) + private val postProjection = Projection.include(interestedFields) private val interestedEventFields = List("operationType", "clusterTime", "documentKey._id") ++ interestedFields.map("fullDocument." + _) @@ -82,7 +82,7 @@ object ForumRepo: .evalTap(_.traverse_(x => debug"received $x")) .map(_.toList.distincByDocId) .evalMap: events => - val lastEventTimestamp = events.flatten(using _.clusterTime.flatMap(_.asInstant)).maxOption + val lastEventTimestamp = events.flatten(using _.clusterTime.flatMap(_.asInstant)).maxOption val (toDelete, toIndex) = events.partition(_.isDelete) toIndex .flatten(using _.fullDocument) @@ -148,13 +148,13 @@ object ForumRepo: event.operationType == DELETE || event.fullDocument.exists(_.isErased) object F: - val text = "text" - val topicId = "topicId" - val troll = "troll" - val userId = "userId" + val text = "text" + val topicId = "topicId" + val troll = "troll" + val userId = "userId" val createdAt = "createdAt" val updatedAt = "updatedAt" - val erasedAt = "erasedAt" + val erasedAt = "erasedAt" object Topic: val name = "name" diff --git a/modules/ingestor/src/main/scala/mongo.game.scala b/modules/ingestor-core/src/main/scala/mongo.game.scala similarity index 76% rename from modules/ingestor/src/main/scala/mongo.game.scala rename to modules/ingestor-core/src/main/scala/mongo.game.scala index 203af199..84debcba 100644 --- a/modules/ingestor/src/main/scala/mongo.game.scala +++ b/modules/ingestor-core/src/main/scala/mongo.game.scala @@ -25,7 +25,7 @@ import Repo.{ *, given } object GameRepo: private val interestedOperations = List(UPDATE, DELETE).map(_.getValue) - private val eventFilter = Filter.in("operationType", interestedOperations) + private val eventFilter = Filter.in("operationType", interestedOperations) private val interestedEventFields = List( @@ -41,7 +41,7 @@ object GameRepo: val gameFilter: Filter = // Filter games that finished // https://github.com/lichess-org/scalachess/blob/18edf46a50445048fdc2ee5a83752e5b3884f490/core/src/main/scala/Status.scala#L18-L27 - val statusFilter = Filter.gte("s", 30) + val statusFilter = Filter.gte("s", 30) val noImportFilter = Filter.ne("so", 7) // us fields is the list of player ids, if it's missing then it's // an all anonymous (or anonymous vs stockfish) game @@ -52,7 +52,7 @@ object GameRepo: val changeFilter: Filter = // Filter games that finished // https://github.com/lichess-org/scalachess/blob/18edf46a50445048fdc2ee5a83752e5b3884f490/core/src/main/scala/Status.scala#L18-L27 - val statusFilter = Filter.gte("fullDocument.s", 30) + val statusFilter = Filter.gte("fullDocument.s", 30) val noImportFilter = Filter.ne("fullDocument.so", 7) // us fields is the list of player ids, if it's missing then it's // an all anonymous (or anonymous vs stockfish) game @@ -75,7 +75,7 @@ object GameRepo: def watch(since: Option[Instant]): fs2.Stream[IO, Result[GameSource]] = changes(since) .map: events => - val lastEventTimestamp = events.lastOption.flatMap(_.clusterTime).flatMap(_.asInstant) + val lastEventTimestamp = events.lastOption.flatMap(_.clusterTime).flatMap(_.asInstant) val (toDelete, toIndex) = events.partition(_.operationType == DELETE) Result( toIndex.flatten(using _.fullDocument.map(_.toSource)), @@ -117,44 +117,44 @@ object GameRepo: type PlayerId = String case class DbGame( - id: String, // _id - players: List[PlayerId], // us - winnerId: Option[PlayerId], // wid - createdAt: Instant, // ca - movedAt: Instant, // ua - ply: Int, // t - analysed: Option[Boolean], // an - whitePlayer: Option[DbPlayer], // p0 - blackPlayer: Option[DbPlayer], // p1 - playerIds: String, // is - binaryPieces: Option[Array[Byte]], // ps - huffmanPgn: Option[Array[Byte]], // hp - status: Int, // s - encodedClock: Option[Array[Byte]], // c - moveTimes: Option[Array[Byte]], // mt + id: String, // _id + players: List[PlayerId], // us + winnerId: Option[PlayerId], // wid + createdAt: Instant, // ca + movedAt: Instant, // ua + ply: Int, // t + analysed: Option[Boolean], // an + whitePlayer: Option[DbPlayer], // p0 + blackPlayer: Option[DbPlayer], // p1 + playerIds: String, // is + binaryPieces: Option[Array[Byte]], // ps + huffmanPgn: Option[Array[Byte]], // hp + status: Int, // s + encodedClock: Option[Array[Byte]], // c + moveTimes: Option[Array[Byte]], // mt encodedWhiteClock: Option[Array[Byte]], // cw encodedBlackClock: Option[Array[Byte]], // cb - rated: Option[Boolean], // ra - variant: Option[Int], // v - source: Option[Int], // so - winnerColor: Option[Boolean] // w + rated: Option[Boolean], // ra + variant: Option[Int], // v + source: Option[Int], // so + winnerColor: Option[Boolean] // w ): def clockConfig: Option[Config] = encodedClock.flatMap(ClockDecoder.read) - def clockInit: Option[Int] = clockConfig.map(_.limitSeconds.value) - def clockInc: Option[Int] = clockConfig.map(_.incrementSeconds.value) - def whiteId: Option[PlayerId] = players.headOption - def blackId: Option[PlayerId] = players.lift(1) - def variantOrDefault: Variant = Variant.idOrDefault(variant.map(Variant.Id.apply)) - def speed: Speed = Speed(clockConfig) - def loser: Option[PlayerId] = players.find(_.some != winnerId) - def aiLevel: Option[Int] = whitePlayer.flatMap(_.aiLevel).orElse(blackPlayer.flatMap(_.aiLevel)) + def clockInit: Option[Int] = clockConfig.map(_.limitSeconds.value) + def clockInc: Option[Int] = clockConfig.map(_.incrementSeconds.value) + def whiteId: Option[PlayerId] = players.headOption + def blackId: Option[PlayerId] = players.lift(1) + def variantOrDefault: Variant = Variant.idOrDefault(variant.map(Variant.Id.apply)) + def speed: Speed = Speed(clockConfig) + def loser: Option[PlayerId] = players.find(_.some != winnerId) + def aiLevel: Option[Int] = whitePlayer.flatMap(_.aiLevel).orElse(blackPlayer.flatMap(_.aiLevel)) // https://github.com/lichess-org/lila/blob/65e6dd88e99cfa0068bc790a4518a6edb3513f54/modules/core/src/main/game/Game.scala#L261 private def averageUsersRating = List(whitePlayer.flatMap(_.rating), blackPlayer.flatMap(_.rating)).flatten match case a :: b :: Nil => Some((a + b) / 2) - case a :: Nil => Some((a + 1500) / 2) - case _ => None + case a :: Nil => Some((a + 1500) / 2) + case _ => None // https://github.com/lichess-org/lila/blob/02ac57c4584b89a0df8f343f34074c0135c2d2b4/modules/core/src/main/game/Game.scala#L90-L97 def durationSeconds: Option[Int] = @@ -207,20 +207,20 @@ object DbGame: variant.match case Standard | FromPosition => speed match - case Speed.UltraBullet => 0 - case Speed.Bullet => 1 - case Speed.Blitz => 2 - case Speed.Rapid => 6 - case Speed.Classical => 3 + case Speed.UltraBullet => 0 + case Speed.Bullet => 1 + case Speed.Blitz => 2 + case Speed.Rapid => 6 + case Speed.Classical => 3 case Speed.Correspondence => 4 - case Crazyhouse => 18 - case Chess960 => 11 + case Crazyhouse => 18 + case Chess960 => 11 case KingOfTheHill => 12 - case ThreeCheck => 15 - case Antichess => 13 - case Atomic => 14 - case Horde => 16 - case RacingKings => 17 + case ThreeCheck => 15 + case Antichess => 13 + case Atomic => 14 + case Horde => 16 + case RacingKings => 17 case class DbPlayer( rating: Option[Int], @@ -246,4 +246,4 @@ object ClockDecoder: def read(ba: Array[Byte]): Option[Clock.Config] = ba.take(2).map(toInt) match case Array(b1, b2) => Clock.Config(readClockLimit(b1), Clock.IncrementSeconds(b2)).some - case _ => None + case _ => None diff --git a/modules/ingestor/src/main/scala/mongo.study.scala b/modules/ingestor-core/src/main/scala/mongo.study.scala similarity index 73% rename from modules/ingestor/src/main/scala/mongo.study.scala rename to modules/ingestor-core/src/main/scala/mongo.study.scala index 19a76df6..48b8cd4f 100644 --- a/modules/ingestor/src/main/scala/mongo.study.scala +++ b/modules/ingestor-core/src/main/scala/mongo.study.scala @@ -15,9 +15,21 @@ import Repo.* object StudyRepo: - private val interestedfields = List("_id", F.name, F.members, F.ownerId, F.visibility, F.topics, F.likes) - - private val indexDocProjection = Projection.include(interestedfields) + private val interestedfields = + List( + "_id", + F.name, + F.members, + F.ownerId, + F.visibility, + F.topics, + F.likes, + F.rank, + F.createdAt, + F.updatedAt + ) + + private val indexDocProjection = Projection.include(interestedfields) private val deleteDocProjection = Projection.include(F.oplogId) def apply( @@ -88,30 +100,37 @@ object StudyRepo: .map((since, until) => since -> until.get) extension (docs: List[Document]) - private def toSources: IO[List[StudySourceWithId]] = + private def toSources: IO[List[SourceWithId[StudySource]]] = val studyIds = docs.flatMap(_.id).distinct chapters .byStudyIds(studyIds) .flatMap: chapters => docs.traverseFilter(_.toSource(chapters)) - type StudySourceWithId = (String, StudySource) extension (doc: Document) - private def toSource(chapters: Map[String, StudyData]): IO[Option[StudySourceWithId]] = + private def toSource(chapters: Map[String, StudyData]): IO[Option[SourceWithId[StudySource]]] = doc.id .flatMap: id => ( doc.getName, doc.getOwnerId, - doc.getMembers.some, doc.getChapterNames(chapters), - doc.getChapterTexts(chapters), - doc.getLikes.some, - doc.getPublic.some, - doc.getTopics.some - ) - .mapN(StudySource.apply) - .map(id -> _) + doc.getChapterTexts(chapters) + ).mapN: (name, ownerId, chapterNames, chapterTexts) => + StudySource( + name, + ownerId, + doc.getMembers, + chapterNames, + chapterTexts, + doc.getLikes, + doc.getPublic, + doc.getTopics, + doc.getRank, + doc.getCreatedAt, + doc.getUpdatedAt + ) + .map(id -> _) .pure[IO] .flatTap: source => def reason = @@ -122,24 +141,28 @@ object StudyRepo: + doc.getChapterTexts(chapters).fold("missing doc.chapterTexts; ")(_ => "") info"failed to convert document to source: $doc because $reason".whenA(source.isEmpty) - private def getName = doc.getString(F.name) + private def getName = doc.getString(F.name) private def getOwnerId = doc.getString(F.ownerId) private def getMembers = doc.getDocument(F.members).fold(Nil)(_.toMap.keys.toList) - private def getTopics = doc.getList(F.topics).map(_.flatMap(_.asString)).getOrElse(Nil) - private def getLikes = doc.getInt(F.likes).getOrElse(0) + private def getTopics = doc.getList(F.topics).map(_.flatMap(_.asString)).getOrElse(Nil) + private def getLikes = doc.getInt(F.likes).getOrElse(0) private def getChapterTexts(chapters: Map[String, StudyData]) = chapters.get(doc.id.getOrElse("")).map(_.chapterTexts) private def getChapterNames(chapters: Map[String, StudyData]) = chapters.get(doc.id.getOrElse("")).map(_.chapterNames) private def getPublic = doc.getString(F.visibility).map(_ == "public").getOrElse(true) + private def getRank = doc.get(F.rank).flatMap(_.asInstant).map(SearchDateTime.fromInstant) + private def getCreatedAt = doc.get(F.createdAt).flatMap(_.asInstant).map(SearchDateTime.fromInstant) + private def getUpdatedAt = doc.get(F.updatedAt).flatMap(_.asInstant).map(SearchDateTime.fromInstant) object F: - val name = "name" - val likes = "likes" - val members = "members" - val ownerId = "ownerId" + val name = "name" + val likes = "likes" + val members = "members" + val ownerId = "ownerId" val visibility = "visibility" - val topics = "topics" - val createdAt = "createdAt" - val updatedAt = "updatedAt" - val oplogId = "o._id" + val topics = "topics" + val createdAt = "createdAt" + val updatedAt = "updatedAt" + val oplogId = "o._id" + val rank = "rank" diff --git a/modules/ingestor/src/main/scala/mongo.team.scala b/modules/ingestor-core/src/main/scala/mongo.team.scala similarity index 88% rename from modules/ingestor/src/main/scala/mongo.team.scala rename to modules/ingestor-core/src/main/scala/mongo.team.scala index 0adafd23..47afdf77 100644 --- a/modules/ingestor/src/main/scala/mongo.team.scala +++ b/modules/ingestor-core/src/main/scala/mongo.team.scala @@ -20,10 +20,10 @@ import Repo.{ *, given } object TeamRepo: private val interestedOperations = List(DELETE, INSERT, UPDATE, REPLACE).map(_.getValue) - private val eventFilter = Filter.in("operationType", interestedOperations) + private val eventFilter = Filter.in("operationType", interestedOperations) private val interestedFields = List("_id", F.name, F.description, F.nbMembers, F.name, F.enabled) - private val postProjection = Projection.include(interestedFields) + private val postProjection = Projection.include(interestedFields) private val interestedEventFields = List("operationType", "clusterTime", "documentKey._id") ++ interestedFields.map("fullDocument." + _) @@ -42,7 +42,7 @@ object TeamRepo: def watch(since: Option[Instant]) = // skip the first event if we're starting from a specific timestamp // since the event at that timestamp is already indexed - val skip = since.fold(0)(_ => 1) + val skip = since.fold(0)(_ => 1) val builder = teams.watch(aggregate) since .fold(builder)(x => builder.startAtOperationTime(x.asBsonTimestamp)) @@ -54,7 +54,7 @@ object TeamRepo: .groupWithin(config.batchSize, config.timeWindows.second) .map(_.toList.distincByDocId) .map: docs => - val lastEventTimestamp = docs.lastOption.flatMap(_.clusterTime).flatMap(_.asInstant) + val lastEventTimestamp = docs.lastOption.flatMap(_.clusterTime).flatMap(_.asInstant) val (toDelete, toIndex) = docs.partition(_.isDelete) Result( toIndex.flatten(using _.fullDocument).toSources, @@ -103,10 +103,10 @@ object TeamRepo: event.fullDocument.fold(false)(x => !x.isEnabled) object F: - val name = "name" + val name = "name" val description = "description" - val nbMembers = "nbMembers" - val enabled = "enabled" - val createdAt = "createdAt" - val updatedAt = "updatedAt" - val erasedAt = "erasedAt" + val nbMembers = "nbMembers" + val enabled = "enabled" + val createdAt = "createdAt" + val updatedAt = "updatedAt" + val erasedAt = "erasedAt" diff --git a/modules/ingestor-core/src/main/scala/mongo.ublog.scala b/modules/ingestor-core/src/main/scala/mongo.ublog.scala new file mode 100644 index 00000000..0eb79b83 --- /dev/null +++ b/modules/ingestor-core/src/main/scala/mongo.ublog.scala @@ -0,0 +1,132 @@ +package lila.search +package ingestor + +import cats.effect.IO +import cats.syntax.all.* +import com.mongodb.client.model.changestream.FullDocument +import com.mongodb.client.model.changestream.OperationType.* +import mongo4cats.bson.Document +import mongo4cats.database.MongoDatabase +import mongo4cats.models.collection.ChangeStreamDocument +import mongo4cats.operations.{ Aggregate, Filter, Projection } +import org.typelevel.log4cats.syntax.* +import org.typelevel.log4cats.{ Logger, LoggerFactory } + +import java.time.Instant +import scala.concurrent.duration.* + +import Repo.{ *, given } + +object UblogRepo: + + private val interestedOperations = List(DELETE, INSERT, REPLACE, UPDATE).map(_.getValue) + + private val interestedFields = + List( + _id, + F.markdown, + F.title, + F.intro, + F.topics, + F.blog, + F.live, + F.livedAt, + F.likes, + F.language, + F.quality + ) + private val postProjection = Projection.include(interestedFields) + + private val interestedEventFields = + List("operationType", "clusterTime", "documentKey._id") ++ interestedFields.map("fullDocument." + _) + private val eventProjection = Projection.include(interestedEventFields) + + private def aggregate() = + Aggregate + .matchBy(Filter.in("operationType", interestedOperations)) + .combinedWith(Aggregate.project(eventProjection)) + + def apply(mongo: MongoDatabase[IO], config: IngestorConfig.Ublog)(using + LoggerFactory[IO] + ): IO[Repo[UblogSource]] = + given Logger[IO] = LoggerFactory[IO].getLogger + mongo.getCollection("ublog_post").map(apply(config)) + + def apply(config: IngestorConfig.Ublog)( + posts: MongoCollection + )(using Logger[IO]): Repo[UblogSource] = new: + + def fetch(since: Instant, until: Instant) = + val filter = range(F.livedAt)(since, until.some) + fs2.Stream.eval(info"Fetching blog posts from $since to $until") *> + posts + .find(filter) + .projection(postProjection) + .boundedStream(config.batchSize) + .chunkN(config.batchSize) + .map(_.toList) + .metered(1.second) + .map: docs => + val (toDelete, toIndex) = docs.partition(!_.isLive) + Result(toIndex.toSources, toDelete.flatten(using _.id.map(Id.apply)), none) + + def watch(since: Option[Instant]): fs2.Stream[IO, Result[UblogSource]] = + val builder = posts.watch(aggregate()) + // skip the first event if we're starting from a specific timestamp + // since the event at that timestamp is already indexed + val skip = since.fold(0)(_ => 1) + since + .fold(builder)(x => builder.startAtOperationTime(x.asBsonTimestamp)) + .fullDocument(FullDocument.UPDATE_LOOKUP) // this is required for update event + .batchSize(config.batchSize) + .boundedStream(config.batchSize) + .drop(skip) + .evalTap(x => debug"Ublog event: $x") + .groupWithin(config.batchSize, config.timeWindows.second) + .map(_.toList.distincByDocId) + .map: docs => + val lastEventTimestamp = docs.flatten(using _.clusterTime.flatMap(_.asInstant)).maxOption + val (toDelete, toIndex) = docs.partition(_.isDelete) + Result( + toIndex.flatten(using _.fullDocument).toSources, + toDelete.flatten(using _.docId.map(Id.apply)), + lastEventTimestamp + ) + + extension (docs: List[Document]) + private def toSources: List[(String, UblogSource)] = + docs.flatten(using doc => (doc.id, doc.toSource).mapN(_ -> _)) + + extension (doc: Document) + private def toSource: Option[UblogSource] = + for + title <- doc.getString(F.title) + intro <- doc.getString(F.intro) + body <- doc.getString(F.markdown) + author <- doc.getString(F.blog).map(_.split(":")(1)) + language <- doc.getString(F.language) + likes <- doc.getAs[Int](F.likes) + topics <- doc.getAs[List[String]](F.topics).map(_.mkString(" ").replaceAll("Chess", "")) + text = s"$title\n$topics\n$author\n$intro\n$body" + date <- doc.getNested(F.livedAt).flatMap(_.asInstant).map(_.toEpochMilli) + quality = doc.getNestedAs[Int](F.quality) + yield UblogSource(text, language, likes, date, quality) + + private def isLive: Boolean = + doc.getBoolean("live").contains(true) && !doc.getNestedAs[Int](F.quality).exists(_ == 0) + + extension (event: ChangeStreamDocument[Document]) + private def isDelete: Boolean = + event.operationType == DELETE || event.fullDocument.exists(!_.isLive) + + object F: + val markdown = "markdown" + val title = "title" + val intro = "intro" + val blog = "blog" + val language = "language" + val likes = "likes" + val live = "live" + val livedAt = "lived.at" + val quality = "automod.quality" + val topics = "topics" diff --git a/modules/ingestor/src/main/scala/app.resources.scala b/modules/ingestor-core/src/main/scala/resources.scala similarity index 69% rename from modules/ingestor/src/main/scala/app.resources.scala rename to modules/ingestor-core/src/main/scala/resources.scala index 1799c342..bc656983 100644 --- a/modules/ingestor/src/main/scala/app.resources.scala +++ b/modules/ingestor-core/src/main/scala/resources.scala @@ -7,7 +7,8 @@ import com.mongodb.ReadPreference import mongo4cats.client.MongoClient import mongo4cats.database.MongoDatabase import org.http4s.ember.client.EmberClientBuilder -import org.typelevel.otel4s.metrics.Meter +import org.http4s.otel4s.middleware.metrics.OtelMetrics +import org.typelevel.otel4s.metrics.MeterProvider class AppResources( val lichess: MongoDatabase[IO], @@ -19,17 +20,23 @@ class AppResources( object AppResources: - def instance(conf: AppConfig)(using Meter[IO]): Resource[IO, AppResources] = + def instance(conf: AppConfig)(using MeterProvider[IO]): Resource[IO, AppResources] = ( makeMongoClient(conf.mongo), makeStudyMongoClient(conf.mongo), makeStudyOplogClient(conf.mongo), makeElasticClient(conf.elastic), - KVStore.apply().toResource + KVStore(conf.kvStorePath).toResource ).parMapN(AppResources.apply) - private def makeElasticClient(conf: ElasticConfig)(using Meter[IO]): Resource[IO, ESClient[IO]] = - EmberClientBuilder.default[IO].build.evalMap(ESClient(conf.uri)) + private def makeElasticClient(conf: ElasticConfig)(using MeterProvider[IO]): Resource[IO, ESClient[IO]] = + val metrics = OtelMetrics + .clientMetricsOps[IO]() + .map(org.http4s.client.middleware.Metrics[IO](_, _.uri.renderString.some)) + + (metrics.toResource, EmberClientBuilder.default[IO].build) + .mapN(_.apply(_)) + .map(ESClient(conf.uri)) private def makeMongoClient(conf: MongoConfig) = MongoClient diff --git a/modules/ingestor/src/main/smithy/model.smithy b/modules/ingestor-core/src/main/smithy/model.smithy similarity index 86% rename from modules/ingestor/src/main/smithy/model.smithy rename to modules/ingestor-core/src/main/smithy/model.smithy index 26ae628f..50f94abd 100644 --- a/modules/ingestor/src/main/smithy/model.smithy +++ b/modules/ingestor-core/src/main/smithy/model.smithy @@ -89,7 +89,6 @@ structure StudySource { members: PlayerIds @required chapterNames: String - @required chapterTexts: String @default @@ -98,6 +97,11 @@ structure StudySource { likes: Integer @required public: Boolean + rank: DateTime + @jsonName("createdAt_date") + createdAt: DateTime + @jsonName("updatedAt_date") + updatedAt: DateTime } structure TeamSource { @@ -111,3 +115,15 @@ structure TeamSource { @jsonName("nbm") nbMembers: Integer } + +structure UblogSource { + @required + text: String + @required + language: String + @required + likes: Integer + @required + date: Long + quality: Integer +} diff --git a/modules/ingestor/src/test/scala/HasDocIdTest.scala b/modules/ingestor-core/src/test/scala/HasDocIdTest.scala similarity index 91% rename from modules/ingestor/src/test/scala/HasDocIdTest.scala rename to modules/ingestor-core/src/test/scala/HasDocIdTest.scala index 33afa1dd..6bef44cd 100644 --- a/modules/ingestor/src/test/scala/HasDocIdTest.scala +++ b/modules/ingestor-core/src/test/scala/HasDocIdTest.scala @@ -14,7 +14,7 @@ object HasDocIdTest extends SimpleIOSuite with Checkers: given HasDocId[Change]: extension (a: Change) def docId: Option[String] = a.docId - given Show[Change] = Show.fromToString + given Show[Change] = Show.fromToString given Arbitrary[Change] = Arbitrary: for value <- Gen.posNum[Int] @@ -23,7 +23,7 @@ object HasDocIdTest extends SimpleIOSuite with Checkers: test("distincByDocId is empty when input is empty"): val changes = List.empty[Change] - val result = changes.distincByDocId + val result = changes.distincByDocId IO(expect(List.empty[Option[String]] == result)) test("distincByDocId is empty when all docIds are none"): @@ -41,6 +41,6 @@ object HasDocIdTest extends SimpleIOSuite with Checkers: test("distincByDocId == reverse.distincBy.reverse"): forall: (changes: List[Change]) => - val result = changes.distincByDocId + val result = changes.distincByDocId val doubleReversed = changes.reverse.filter(_.docId.isDefined).distinctBy(_.docId).reverse expect(result == doubleReversed) diff --git a/modules/ingestor/src/main/scala/app.scala b/modules/ingestor/src/main/scala/app.scala deleted file mode 100644 index c2c47130..00000000 --- a/modules/ingestor/src/main/scala/app.scala +++ /dev/null @@ -1,44 +0,0 @@ -package lila.search -package ingestor - -import cats.effect.* -import cats.syntax.all.* -import org.typelevel.log4cats.slf4j.Slf4jFactory -import org.typelevel.log4cats.{ Logger, LoggerFactory } -import org.typelevel.otel4s.experimental.metrics.* -import org.typelevel.otel4s.instrumentation.ce.IORuntimeMetrics -import org.typelevel.otel4s.metrics.{ Meter, MeterProvider } -import org.typelevel.otel4s.sdk.exporter.prometheus.autoconfigure.PrometheusMetricExporterAutoConfigure -import org.typelevel.otel4s.sdk.metrics.SdkMetrics - -object App extends IOApp.Simple: - - given LoggerFactory[IO] = Slf4jFactory.create[IO] - given Logger[IO] = LoggerFactory[IO].getLogger - - override def run: IO[Unit] = app.useForever - - def app: Resource[IO, Unit] = - for - given Meter[IO] <- mkMeter - _ <- RuntimeMetrics.register[IO] - config <- AppConfig.load.toResource - _ <- Logger[IO].info(s"Starting lila-search ingestor with config: ${config.toString}").toResource - _ <- Logger[IO].info(s"BuildInfo: ${BuildInfo.toString}").toResource - res <- AppResources.instance(config) - _ <- IngestorApp(res, config).run() - yield () - - def mkMeter = SdkMetrics - .autoConfigured[IO](_.addExporterConfigurer(PrometheusMetricExporterAutoConfigure[IO])) - .flatMap: sdk => - given meterProvider: MeterProvider[IO] = sdk.meterProvider - IORuntimeMetrics.register[IO](runtime.metrics, IORuntimeMetrics.Config.default) *> - meterProvider.get("lila-search-ingestor").toResource - -class IngestorApp(res: AppResources, config: AppConfig)(using Logger[IO], LoggerFactory[IO]): - def run(): Resource[IO, Unit] = - Ingestors(res.lichess, res.study, res.studyLocal, res.store, res.elastic, config.ingestor) - .flatMap(_.run()) - .toResource - .evalTap(_ => Logger[IO].info("Ingestor started")) diff --git a/project/BuildPlugin.scala b/project/BuildPlugin.scala index c125a0c9..b702e78b 100644 --- a/project/BuildPlugin.scala +++ b/project/BuildPlugin.scala @@ -1,7 +1,7 @@ -import sbt._, Keys._ -import sbt.ScriptedPlugin.autoImport._ -import sbtrelease.ReleasePlugin, ReleasePlugin.autoImport._, ReleaseTransformations._, ReleaseKeys._ -import sbt.ScriptedPlugin.autoImport._ +import sbt.*, Keys.* +import sbt.ScriptedPlugin.autoImport.* +import sbtrelease.ReleasePlugin, ReleasePlugin.autoImport.*, ReleaseTransformations.*, ReleaseKeys.* +import sbt.ScriptedPlugin.autoImport.* import sbt.plugins.{ JvmPlugin, SbtPlugin } diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 31cfa4a9..dcfaada7 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -5,74 +5,76 @@ object Dependencies { val lilaMaven = "lila-maven" at "https://raw.githubusercontent.com/lichess-org/lila-maven/master" val jitpack = "jitpack".at("https://jitpack.io") - val ourResolvers = Seq(lilaMaven, jitpack) + val ourResolvers = Seq(lilaMaven, jitpack, Resolver.sonatypeCentralSnapshots) object V { - val catsEffect = "3.6.1" - val chess = "17.8.0" - val ciris = "3.8.0" - val decline = "2.5.0" - val elastic4s = "9.0.0-RC2" - val fs2 = "3.12.0" - val http4s = "0.23.30" - val iron = "2.5.0" + val catsEffect = "3.6.3" + val catsMtl = "1.6.0" + val ciris = "3.11.1" + val chess = "17.12.3" + val decline = "2.5.0" + val elastic4s = "9.1.1" + val fs2 = "3.12.2" + val http4s = "0.23.33" val mongo4cats = "0.7.13" - val otel4s = "0.12.0" + val otel4s = "0.14.0" + val otel4sHttp4s = "0.15.0" } - def http4s(artifact: String) = "org.http4s" %% s"http4s-$artifact" % V.http4s + def http4s(artifact: String) = "org.http4s" %% s"http4s-$artifact" % V.http4s def smithy4s(artifact: String) = "com.disneystreaming.smithy4s" %% s"smithy4s-$artifact" % smithy4sVersion val chess = "com.github.lichess-org.scalachess" %% "scalachess" % V.chess - val catsCore = "org.typelevel" %% "cats-core" % "2.13.0" + val catsCore = "org.typelevel" %% "cats-core" % "2.13.0" val catsEffect = "org.typelevel" %% "cats-effect" % V.catsEffect + val catsMtl = "org.typelevel" %% "cats-mtl" % V.catsMtl - val fs2 = "co.fs2" %% "fs2-core" % V.fs2 - val fs2IO = "co.fs2" %% "fs2-io" % V.fs2 + val fs2 = "co.fs2" %% "fs2-core" % V.fs2 + val fs2IO = "co.fs2" %% "fs2-io" % V.fs2 - val cirisCore = "is.cir" %% "ciris" % V.ciris - val cirisHtt4s = "is.cir" %% "ciris-http4s" % V.ciris - val iron = "io.github.iltotore" %% "iron" % V.iron - val ironCiris = "io.github.iltotore" %% "iron-ciris" % V.iron + val cirisCore = "is.cir" %% "ciris" % V.ciris + val cirisHtt4s = "is.cir" %% "ciris-http4s" % V.ciris - val http4sServer = http4s("ember-server") - val http4sClient = http4s("client") + val http4sServer = http4s("ember-server") + val http4sClient = http4s("client") val http4sEmberClient = http4s("ember-client") - lazy val smithy4sCore = smithy4s("core") - lazy val smithy4sHttp4s = smithy4s("http4s") + lazy val smithy4sCore = smithy4s("core") + lazy val smithy4sHttp4s = smithy4s("http4s") lazy val smithy4sHttp4sSwagger = smithy4s("http4s-swagger") - lazy val smithy4sJson = smithy4s("json") + lazy val smithy4sJson = smithy4s("json") - val jsoniterCore = "com.github.plokhotnyuk.jsoniter-scala" %% "jsoniter-scala-core" % "2.36.2" - val jsoniterMacro = "com.github.plokhotnyuk.jsoniter-scala" %% "jsoniter-scala-macros" % "2.36.2" + val jsoniterCore = "com.github.plokhotnyuk.jsoniter-scala" %% "jsoniter-scala-core" % "2.37.10" + val jsoniterMacro = "com.github.plokhotnyuk.jsoniter-scala" %% "jsoniter-scala-macros" % "2.37.10" - val playWS = "com.typesafe.play" %% "play-ahc-ws-standalone" % "2.2.11" + val playWS = "com.typesafe.play" %% "play-ahc-ws-standalone" % "2.2.13" val elastic4sHttp4sClient = "nl.gn0s1s" %% "elastic4s-client-http4s" % V.elastic4s val mongo4catsCore = "io.github.kirill5k" %% "mongo4cats-core" % V.mongo4cats val mongo4catsCirce = "io.github.kirill5k" %% "mongo4cats-circe" % V.mongo4cats - val circe = "io.circe" %% "circe-core" % "0.14.13" + val circe = "io.circe" %% "circe-core" % "0.14.15" - val otel4sCore = "org.typelevel" %% "otel4s-core" % V.otel4s + val log4Cats = "org.typelevel" %% "log4cats-slf4j" % "2.7.1" + val logback = "ch.qos.logback" % "logback-classic" % "1.5.20" + val ducktape = "io.github.arainko" %% "ducktape" % "0.2.10" + + val otel4sCore = "org.typelevel" %% "otel4s-core" % V.otel4s val otel4sPrometheusExporter = "org.typelevel" %% "otel4s-sdk-exporter-prometheus" % V.otel4s val otel4sSdk = "org.typelevel" %% "otel4s-sdk" % V.otel4s - val otel4sInstrumentationMetrics = "org.typelevel" %% "otel4s-instrumentation-metrics" % V.otel4s - val otel4sMetrics = "org.typelevel" %% "otel4s-experimental-metrics" % "0.6.0" - - val log4Cats = "org.typelevel" %% "log4cats-slf4j" % "2.7.1" - val logback = "ch.qos.logback" % "logback-classic" % "1.5.18" + val otel4sInstrumentationMetrics = "org.typelevel" %% "otel4s-instrumentation-metrics" % V.otel4s + val otel4sMetrics = "org.typelevel" %% "otel4s-experimental-metrics" % "0.8.0" - val ducktape = "io.github.arainko" %% "ducktape" % "0.2.8" + val otel4sHttp4sCore = "org.http4s" %% "http4s-otel4s-middleware-core" % V.otel4sHttp4s + val otel4sHttp4sMetrics = "org.http4s" %% "http4s-otel4s-middleware-metrics" % V.otel4sHttp4s val declineCore = "com.monovore" %% "decline" % V.decline val declineCatsEffect = "com.monovore" %% "decline-effect" % V.decline - val testContainers = "com.dimafeng" %% "testcontainers-scala-core" % "0.43.0" % Test - val weaver = "org.typelevel" %% "weaver-cats" % "0.9.0" % Test - val weaverScalaCheck = "org.typelevel" %% "weaver-scalacheck" % "0.9.0" % Test - val catsEffectTestKit = "org.typelevel" %% "cats-effect-testkit" % V.catsEffect % Test - val scalacheck = "org.scalacheck" %% "scalacheck" % "1.17.0" % Test + val testContainers = "com.dimafeng" %% "testcontainers-scala-core" % "0.43.6" % Test + val weaver = "org.typelevel" %% "weaver-cats" % "0.10.1" % Test + val weaverScalaCheck = "org.typelevel" %% "weaver-scalacheck" % "0.10.1" % Test + val catsEffectTestKit = "org.typelevel" %% "cats-effect-testkit" % V.catsEffect % Test + val scalacheck = "org.scalacheck" %% "scalacheck" % "1.17.0" % Test } diff --git a/project/build.properties b/project/build.properties index 61c9b1cb..01a16ed1 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.11.1 +sbt.version=1.11.7 diff --git a/project/plugins.sbt b/project/plugins.sbt index cddf18f7..fbc6e8c0 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,15 +1,15 @@ -addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.14.3") +addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.14.4") -addSbtPlugin("com.disneystreaming.smithy4s" % "smithy4s-sbt-codegen" % "0.18.37") +addSbtPlugin("com.disneystreaming.smithy4s" % "smithy4s-sbt-codegen" % "0.18.43") addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.13.1") addSbtPlugin("com.github.sbt" % "sbt-git" % "2.1.0") -addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.11.1") +addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.11.4") addSbtPlugin("com.github.sbt" % "sbt-release" % "1.4.0") -addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.4") +addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.6") addSbtPlugin("org.typelevel" % "sbt-tpolecat" % "0.5.2") diff --git a/version.sbt b/version.sbt index ae4e84d1..08831ae1 100644 --- a/version.sbt +++ b/version.sbt @@ -1 +1 @@ -ThisBuild / version := "3.1.9" +ThisBuild / version := "3.3.1-SNAPSHOT"