From afab346c58ba8dbad529062474e9a477adc21a8e Mon Sep 17 00:00:00 2001 From: Leon Burdinov Date: Mon, 6 Feb 2023 12:10:03 +0200 Subject: [PATCH 01/17] Ds scalafmt (#32485) * format: data streams/ * format: iptf/ GitOrigin-RevId: a31eb0dc0eaff014511665e52497dea67a3f9ff1 --- .../dst/greyhound/core/AdminClientIT.scala | 6 +- .../greyhound/core/consumer/Consumer.scala | 7 +- .../greyhound/core/consumer/EventLoop.scala | 12 +- .../core/consumer/OffsetsInitializer.scala | 6 +- .../core/consumer/ReportingConsumer.scala | 24 ++-- .../retry/NonBlockingRetryHelper.scala | 66 ++++++----- .../retry/NonBlockingRetryRecordHandler.scala | 7 +- .../retry/RetryRecordHandlerMetric.scala | 3 +- .../consumer/OffsetsInitializerTest.scala | 104 +++++++++--------- 9 files changed, 126 insertions(+), 109 deletions(-) diff --git a/core/src/it/scala/com/wixpress/dst/greyhound/core/AdminClientIT.scala b/core/src/it/scala/com/wixpress/dst/greyhound/core/AdminClientIT.scala index 95d2353a..dec5377c 100644 --- a/core/src/it/scala/com/wixpress/dst/greyhound/core/AdminClientIT.scala +++ b/core/src/it/scala/com/wixpress/dst/greyhound/core/AdminClientIT.scala @@ -10,7 +10,7 @@ import com.wixpress.dst.greyhound.core.producer.ProducerRecord import com.wixpress.dst.greyhound.core.testkit.{BaseTestWithSharedEnv, TestMetrics} import com.wixpress.dst.greyhound.core.zioutils.CountDownLatch import com.wixpress.dst.greyhound.testenv.ITEnv -import com.wixpress.dst.greyhound.testenv.ITEnv.{Env, TestResources, testResources} +import com.wixpress.dst.greyhound.testenv.ITEnv.{testResources, Env, TestResources} import org.apache.kafka.common.config.TopicConfig.{DELETE_RETENTION_MS_CONFIG, MAX_MESSAGE_BYTES_CONFIG, RETENTION_MS_CONFIG} import org.apache.kafka.common.errors.InvalidTopicException import org.specs2.specification.core.Fragments @@ -83,7 +83,7 @@ class AdminClientIT extends BaseTestWithSharedEnv[Env, TestResources] { } } - //todo uncomment this after https://github.com/wix-private/core-server-build-tools/pull/13043 is merged + // todo uncomment this after https://github.com/wix-private/core-server-build-tools/pull/13043 is merged // "reflect errors" in { // val topic1 = aTopicConfig() // val topic2 = aTopicConfig("x" * 250) @@ -104,7 +104,7 @@ class AdminClientIT extends BaseTestWithSharedEnv[Env, TestResources] { // created === Map(badTopic.name -> None) // } // } - //todo uncomment this after https://github.com/wix-private/core-server-build-tools/pull/13043 is merged + // todo uncomment this after https://github.com/wix-private/core-server-build-tools/pull/13043 is merged // ================================================================================================================================= "ignore TopicExistsException by default" in { val topic = aTopicConfig() diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/Consumer.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/Consumer.scala index 3a53c551..7592362e 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/Consumer.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/Consumer.scala @@ -382,7 +382,10 @@ object UnsafeOffsetOperations { } override def offsetsForTimes(partitions: Set[TopicPartition], timeEpoch: Long, timeout: Duration): Map[TopicPartition, Option[Long]] = - consumer.offsetsForTimes(partitions.map(_.asKafka).map(tp => (tp, new lang.Long(timeEpoch))).toMap.asJava, timeout) - .asScala.toMap.map { case (tp, of) => TopicPartition(tp) -> (Option(of).map(_.offset())) } + consumer + .offsetsForTimes(partitions.map(_.asKafka).map(tp => (tp, new lang.Long(timeEpoch))).toMap.asJava, timeout) + .asScala + .toMap + .map { case (tp, of) => TopicPartition(tp) -> (Option(of).map(_.offset())) } } } diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/EventLoop.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/EventLoop.scala index 6901b5c9..5985f80d 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/EventLoop.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/EventLoop.scala @@ -56,14 +56,14 @@ object EventLoop { partitionsAssigned <- Promise.make[Nothing, Unit] // TODO how to handle errors in subscribe? rebalanceListener = listener(pausedPartitionsRef, config, dispatcher, partitionsAssigned, group, consumer, clientId, offsets) - _ <- report(SubscribingToInitialSubAndRebalanceListener(clientId, group, consumerAttributes)) + _ <- report(SubscribingToInitialSubAndRebalanceListener(clientId, group, consumerAttributes)) _ <- subscribe(initialSubscription, rebalanceListener)(consumer) running <- Ref.make[EventLoopState](Running) - _ <- report(CreatingPollOnceFiber(clientId, group, consumerAttributes)) + _ <- report(CreatingPollOnceFiber(clientId, group, consumerAttributes)) fiber <- pollOnce(running, consumer, dispatcher, pausedPartitionsRef, positionsRef, offsets, config, clientId, group) .repeatWhile(_ == true) .forkDaemon - _ <- report(AwaitingPartitionsAssignment(clientId, group, consumerAttributes)) + _ <- report(AwaitingPartitionsAssignment(clientId, group, consumerAttributes)) _ <- partitionsAssigned.await env <- ZIO.environment[Env] } yield (dispatcher, fiber, offsets, positionsRef, running, rebalanceListener.provideEnvironment(env)) @@ -303,9 +303,11 @@ object EventLoopMetric { case class FailedToUpdatePositions(t: Throwable, clientId: ClientId, attributes: Map[String, String] = Map.empty) extends EventLoopMetric - case class CreatingDispatcher(clientId: ClientId, group: Group, attributes: Map[String, String], startPaused: Boolean) extends EventLoopMetric + case class CreatingDispatcher(clientId: ClientId, group: Group, attributes: Map[String, String], startPaused: Boolean) + extends EventLoopMetric - case class SubscribingToInitialSubAndRebalanceListener(clientId: ClientId, group: Group, attributes: Map[String, String]) extends EventLoopMetric + case class SubscribingToInitialSubAndRebalanceListener(clientId: ClientId, group: Group, attributes: Map[String, String]) + extends EventLoopMetric case class CreatingPollOnceFiber(clientId: ClientId, group: Group, attributes: Map[String, String]) extends EventLoopMetric diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsInitializer.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsInitializer.scala index 97f1b341..e684eef7 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsInitializer.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsInitializer.scala @@ -40,8 +40,10 @@ class OffsetsInitializer( offsetOperations.pause(toPause) val rewindUncommittedOffsets = if (offsetResetIsEarliest || notCommitted.isEmpty || rewindUncommittedOffsetsBy.isZero) Map.empty - else offsetOperations.offsetsForTimes(notCommitted, clock.millis() - rewindUncommittedOffsetsBy.toMillis, effectiveTimeout) - .map{case (tp, maybeRewindedOffset) => (tp, maybeRewindedOffset.orElse(endOffsets.get(tp)).getOrElse(0L))} + else + offsetOperations + .offsetsForTimes(notCommitted, clock.millis() - rewindUncommittedOffsetsBy.toMillis, effectiveTimeout) + .map { case (tp, maybeRewindedOffset) => (tp, maybeRewindedOffset.orElse(endOffsets.get(tp)).getOrElse(0L)) } val positions = notCommitted.map(tp => tp -> offsetOperations.position(tp, effectiveTimeout)).toMap ++ toOffsets ++ rewindUncommittedOffsets diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/ReportingConsumer.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/ReportingConsumer.scala index 96a929f8..d0f93a00 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/ReportingConsumer.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/ReportingConsumer.scala @@ -41,10 +41,14 @@ case class ReportingConsumer(clientId: ClientId, group: Group, internal: Consume implicit trace: Trace ): UIO[DelayedRebalanceEffect] = (report(PartitionsRevoked(clientId, group, partitions, config.consumerAttributes)) *> - rebalanceListener.onPartitionsRevoked(consumer, partitions) - .timed.tap { case (duration, _) => report(PartitionsRevokedComplete(clientId, group, partitions, config.consumerAttributes, duration.toMillis)) } - .map(_._2) - ).provideEnvironment(r) + rebalanceListener + .onPartitionsRevoked(consumer, partitions) + .timed + .tap { + case (duration, _) => + report(PartitionsRevokedComplete(clientId, group, partitions, config.consumerAttributes, duration.toMillis)) + } + .map(_._2)).provideEnvironment(r) override def onPartitionsAssigned(consumer: Consumer, partitions: Set[TopicPartition])(implicit trace: Trace): UIO[Any] = (report(PartitionsAssigned(clientId, group, partitions, config.consumerAttributes)) *> @@ -241,11 +245,13 @@ object ConsumerMetric { attributes: Map[String, String] = Map.empty ) extends ConsumerMetric - case class PartitionsRevokedComplete(clientId: ClientId, - group: Group, - partitions: Set[TopicPartition], - attributes: Map[String, String] = Map.empty, - durationMs: Long) extends ConsumerMetric + case class PartitionsRevokedComplete( + clientId: ClientId, + group: Group, + partitions: Set[TopicPartition], + attributes: Map[String, String] = Map.empty, + durationMs: Long + ) extends ConsumerMetric case class SubscribeFailed(clientId: ClientId, group: Group, error: Throwable, attributes: Map[String, String] = Map.empty) extends ConsumerMetric diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/NonBlockingRetryHelper.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/NonBlockingRetryHelper.scala index c8dd3edd..a73b7dce 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/NonBlockingRetryHelper.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/NonBlockingRetryHelper.scala @@ -1,18 +1,18 @@ package com.wixpress.dst.greyhound.core.consumer.retry -import java.time.{Instant, Duration => JavaDuration} +import java.time.{Duration => JavaDuration, Instant} import java.util.concurrent.TimeUnit.MILLISECONDS import java.util.regex.Pattern import com.wixpress.dst.greyhound.core.Serdes.StringSerde import com.wixpress.dst.greyhound.core.consumer.domain.ConsumerSubscription.{TopicPattern, Topics} -import com.wixpress.dst.greyhound.core.consumer.retry.RetryAttempt.{RetryAttemptNumber, currentTime} +import com.wixpress.dst.greyhound.core.consumer.retry.RetryAttempt.{currentTime, RetryAttemptNumber} import com.wixpress.dst.greyhound.core.consumer.retry.RetryDecision.{NoMoreRetries, RetryWith} import com.wixpress.dst.greyhound.core.consumer.domain.{ConsumerRecord, ConsumerSubscription} import com.wixpress.dst.greyhound.core.consumer.retry.RetryRecordHandlerMetric.WaitingForRetry import com.wixpress.dst.greyhound.core.metrics.GreyhoundMetrics import com.wixpress.dst.greyhound.core.metrics.GreyhoundMetrics.report import com.wixpress.dst.greyhound.core.producer.ProducerRecord -import com.wixpress.dst.greyhound.core.{Group, Headers, Topic, durationDeserializer, instantDeserializer} +import com.wixpress.dst.greyhound.core.{durationDeserializer, instantDeserializer, Group, Headers, Topic} import zio.Clock import zio.Duration import zio.Schedule.spaced @@ -24,14 +24,14 @@ trait NonBlockingRetryHelper { def retryTopicsFor(originalTopic: Topic): Set[Topic] def retryAttempt(topic: Topic, headers: Headers, subscription: ConsumerSubscription)( - implicit trace: Trace + implicit trace: Trace ): UIO[Option[RetryAttempt]] def retryDecision[E]( - retryAttempt: Option[RetryAttempt], - record: ConsumerRecord[Chunk[Byte], Chunk[Byte]], - error: E, - subscription: ConsumerSubscription + retryAttempt: Option[RetryAttempt], + record: ConsumerRecord[Chunk[Byte], Chunk[Byte]], + error: E, + subscription: ConsumerSubscription )(implicit trace: Trace): URIO[Any, RetryDecision] def retrySteps = retryTopicsFor("").size @@ -47,46 +47,44 @@ object NonBlockingRetryHelper { .getOrElse(NonBlockingBackoffPolicy.empty) override def retryTopicsFor(topic: Topic): Set[Topic] = - policy(topic).intervals.indices.foldLeft(Set.empty[String])((acc, attempt) => - acc + s"$topic-$group-retry-$attempt" - ) + policy(topic).intervals.indices.foldLeft(Set.empty[String])((acc, attempt) => acc + s"$topic-$group-retry-$attempt") override def retryAttempt(topic: Topic, headers: Headers, subscription: ConsumerSubscription)( - implicit trace: Trace + implicit trace: Trace ): UIO[Option[RetryAttempt]] = { (for { submitted <- headers.get(RetryHeader.Submitted, instantDeserializer) backoff <- headers.get(RetryHeader.Backoff, durationDeserializer) originalTopic <- headers.get[String](RetryHeader.OriginalTopic, StringSerde) } yield for { - ta <- topicAttempt(subscription, topic, originalTopic) + ta <- topicAttempt(subscription, topic, originalTopic) TopicAttempt(originalTopic, attempt) = ta - s <- submitted - b <- backoff + s <- submitted + b <- backoff } yield RetryAttempt(originalTopic, attempt, s, b)) .catchAll(_ => ZIO.none) } private def topicAttempt( - subscription: ConsumerSubscription, - topic: Topic, - originalTopicHeader: Option[String] + subscription: ConsumerSubscription, + topic: Topic, + originalTopicHeader: Option[String] ) = subscription match { - case _: Topics => extractTopicAttempt(group, topic) + case _: Topics => extractTopicAttempt(group, topic) case _: TopicPattern => extractTopicAttemptFromPatternRetryTopic(group, topic, originalTopicHeader) } override def retryDecision[E]( - retryAttempt: Option[RetryAttempt], - record: ConsumerRecord[Chunk[Byte], Chunk[Byte]], - error: E, - subscription: ConsumerSubscription + retryAttempt: Option[RetryAttempt], + record: ConsumerRecord[Chunk[Byte], Chunk[Byte]], + error: E, + subscription: ConsumerSubscription )(implicit trace: Trace): URIO[Any, RetryDecision] = currentTime.map(now => { val nextRetryAttempt = retryAttempt.fold(0)(_.attempt + 1) val originalTopic = retryAttempt.fold(record.topic)(_.originalTopic) - val retryTopic = subscription match { + val retryTopic = subscription match { case _: TopicPattern => patternRetryTopic(group, nextRetryAttempt) case _: Topics => fixedRetryTopic(originalTopic, group, nextRetryAttempt) } @@ -123,19 +121,19 @@ object NonBlockingRetryHelper { inputTopic.split(s"-$group-retry-").toSeq match { case Seq(topic, attempt) if Try(attempt.toInt).isSuccess => Some(TopicAttempt(topic, attempt.toInt)) - case _ => None + case _ => None } private def extractTopicAttemptFromPatternRetryTopic[E]( - group: Group, - inputTopic: Topic, - originalTopicHeader: Option[String] + group: Group, + inputTopic: Topic, + originalTopicHeader: Option[String] ) = { originalTopicHeader.flatMap(originalTopic => { inputTopic.split(s"__gh_pattern-retry-$group-attempt-").toSeq match { case Seq(_, attempt) if Try(attempt.toInt).isSuccess => Some(TopicAttempt(originalTopic, attempt.toInt)) - case _ => None + case _ => None } }) } @@ -176,14 +174,14 @@ object RetryHeader { } case class RetryAttempt( - originalTopic: Topic, - attempt: RetryAttemptNumber, - submittedAt: Instant, - backoff: Duration + originalTopic: Topic, + attempt: RetryAttemptNumber, + submittedAt: Instant, + backoff: Duration ) { def sleep(implicit trace: Trace): URIO[GreyhoundMetrics, Unit] = - (RetryUtil.sleep(submittedAt, backoff) race reportWaitingInIntervals(every = 60.seconds)) + RetryUtil.sleep(submittedAt, backoff) race reportWaitingInIntervals(every = 60.seconds) private def reportWaitingInIntervals(every: Duration) = report(WaitingForRetry(originalTopic, attempt, submittedAt.toEpochMilli, backoff.toMillis)) diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/NonBlockingRetryRecordHandler.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/NonBlockingRetryRecordHandler.scala index b6e410d3..a15f1e5b 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/NonBlockingRetryRecordHandler.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/NonBlockingRetryRecordHandler.scala @@ -49,14 +49,17 @@ private[retry] object NonBlockingRetryRecordHandler { } private def delayRetry(record: ConsumerRecord[_, _], awaitShutdown: TopicPartition => UIO[AwaitShutdown])( - retryAttempt: RetryAttempt) = + retryAttempt: RetryAttempt + ) = zio.Random.nextInt.flatMap(correlationId => report( WaitingBeforeRetry(record.topic, retryAttempt, record.partition, record.offset, correlationId) ) *> awaitShutdown(record.topicPartition) .flatMap(_.interruptOnShutdown(retryAttempt.sleep)) - .reporting(r => DoneWaitingBeforeRetry(record.topic, record.partition, record.offset, retryAttempt, r.duration, r.failed, correlationId)) + .reporting(r => + DoneWaitingBeforeRetry(record.topic, record.partition, record.offset, retryAttempt, r.duration, r.failed, correlationId) + ) ) override def isHandlingRetryTopicMessage(group: Group, record: ConsumerRecord[K, V]): Boolean = { diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryRecordHandlerMetric.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryRecordHandlerMetric.scala index 980a71f7..b1580b16 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryRecordHandlerMetric.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryRecordHandlerMetric.scala @@ -20,7 +20,8 @@ object RetryRecordHandlerMetric { case class NoRetryOnNonRetryableFailure(partition: TopicPartition, offset: Long, cause: Exception) extends RetryRecordHandlerMetric case object Silent extends RetryRecordHandlerMetric - case class WaitingBeforeRetry(retryTopic: Topic, retryAttempt: RetryAttempt, partition: Int, offset:Long, correlationId: Int) extends RetryRecordHandlerMetric + case class WaitingBeforeRetry(retryTopic: Topic, retryAttempt: RetryAttempt, partition: Int, offset: Long, correlationId: Int) + extends RetryRecordHandlerMetric case class DoneWaitingBeforeRetry( retryTopic: Topic, diff --git a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsInitializerTest.scala b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsInitializerTest.scala index d8c0c28e..5e708452 100644 --- a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsInitializerTest.scala +++ b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsInitializerTest.scala @@ -16,7 +16,7 @@ class OffsetsInitializerTest extends SpecificationWithJUnit with Mockito { private val Seq(p1, p2, p3) = Seq("t1" -> 1, "t2" -> 2, "t3" -> 3).map(tp => TopicPartition(tp._1, tp._2)) private val partitions = Set(p1, p2, p3) private val p1Pos, p2Pos, p3Pos = randomInt.toLong - val epochTimeToRewind = 1000L + val epochTimeToRewind = 1000L "do nothing if no missing offsets" in new ctx { @@ -124,71 +124,73 @@ class OffsetsInitializerTest extends SpecificationWithJUnit with Mockito { reported must contain(CommittedMissingOffsetsFailed(clientId, group, partitions, Map.empty, elapsed = Duration.ZERO, e)) } - "rewind uncommitted offsets" in new ctx { - givenCommittedOffsets(partitions)(Map(p2 -> randomInt)) - givenPositions(p2 -> p2Pos, p3 -> p3Pos) - givenOffsetsForTimes(epochTimeToRewind, p1 -> 0L, p2 -> 1L) - - committer.initializeOffsets(partitions) + "rewind uncommitted offsets" in + new ctx { + givenCommittedOffsets(partitions)(Map(p2 -> randomInt)) + givenPositions(p2 -> p2Pos, p3 -> p3Pos) + givenOffsetsForTimes(epochTimeToRewind, p1 -> 0L, p2 -> 1L) - val missingOffsets = Map( - p1 -> p1Pos, - p3 -> p3Pos - ) + committer.initializeOffsets(partitions) - val rewindedOffsets = Map( - p1 -> 0L, - ) + val missingOffsets = Map( + p1 -> p1Pos, + p3 -> p3Pos + ) - there was - one(offsetOps).commit( - missingOffsets ++ rewindedOffsets, - timeout + val rewindedOffsets = Map( + p1 -> 0L ) - } - "rewind to endOffsets for uncommitted partitions when offsetsForTimes return null offsets " in new ctx { - givenCommittedOffsets(partitions)(Map(p2 -> randomInt, p3 -> randomInt)) - givenPositions(p3 -> p3Pos) - givenEndOffsets(partitions, timeout)(Map(p1 -> p1Pos)) - givenOffsetsForTimes(Set(p1))(p1 -> None /*kafka SDK returned null*/) + there was + one(offsetOps).commit( + missingOffsets ++ rewindedOffsets, + timeout + ) + } - committer.initializeOffsets(partitions) + "rewind to endOffsets for uncommitted partitions when offsetsForTimes return null offsets " in + new ctx { + givenCommittedOffsets(partitions)(Map(p2 -> randomInt, p3 -> randomInt)) + givenPositions(p3 -> p3Pos) + givenEndOffsets(partitions, timeout)(Map(p1 -> p1Pos)) + givenOffsetsForTimes(Set(p1))(p1 -> None /*kafka SDK returned null*/ ) - val committedOffsets = Map( - p1 -> p1Pos, - ) + committer.initializeOffsets(partitions) - there was - one(offsetOps).commit( - committedOffsets, - timeout + val committedOffsets = Map( + p1 -> p1Pos ) - } - "not rewind uncommitted offsets when offset reset is earliest" in new ctx(offsetReset = OffsetReset.Earliest) { - givenCommittedOffsets(partitions)(Map(p2 -> randomInt)) - givenPositions(p2 -> p2Pos, p3 -> p3Pos) - givenOffsetsForTimes(epochTimeToRewind, p1 -> 0L, p2 -> 1L) + there was + one(offsetOps).commit( + committedOffsets, + timeout + ) + } - committer.initializeOffsets(partitions) + "not rewind uncommitted offsets when offset reset is earliest" in + new ctx(offsetReset = OffsetReset.Earliest) { + givenCommittedOffsets(partitions)(Map(p2 -> randomInt)) + givenPositions(p2 -> p2Pos, p3 -> p3Pos) + givenOffsetsForTimes(epochTimeToRewind, p1 -> 0L, p2 -> 1L) - val missingOffsets = Map( - p1 -> p1Pos, - p3 -> p3Pos - ) + committer.initializeOffsets(partitions) - val rewindedOffsets = Map( - p1 -> 0L, - ) + val missingOffsets = Map( + p1 -> p1Pos, + p3 -> p3Pos + ) - there was - one(offsetOps).commit( - missingOffsets ++ rewindedOffsets, - timeout + val rewindedOffsets = Map( + p1 -> 0L ) - } + there was + one(offsetOps).commit( + missingOffsets ++ rewindedOffsets, + timeout + ) + } class ctx(val seekTo: Map[TopicPartition, SeekTo] = Map.empty, offsetReset: OffsetReset = OffsetReset.Latest) extends Scope { private val metricsLogRef = new AtomicReference(Seq.empty[GreyhoundMetric]) @@ -256,4 +258,4 @@ class OffsetsInitializerTest extends SpecificationWithJUnit with Mockito { private def randomStr = Random.alphanumeric.take(5).mkString private def randomInt = Random.nextInt(200) private def randomPartition = TopicPartition(randomStr, randomInt) -} \ No newline at end of file +} From 6f2451236e0a72a3d6793880cc9e0b48679f597a Mon Sep 17 00:00:00 2001 From: Vaidas Pilkauskas Date: Fri, 10 Mar 2023 21:27:29 +0200 Subject: [PATCH 02/17] Remove unused test deps (#33323) * Remove unused test deps * Remove unused imports * Add runtime deps * Initial cross-repo check * Updating cross-repo check * Updating cross-repo check * Updating cross-repo check --------- Co-authored-by: wixapiregistry <58037308+wixapiregistry@users.noreply.github.com> GitOrigin-RevId: bd83b6b6f4ca9706f6b9e0f247a8cac8bd3f1292 --- .../src/it/scala/com/wixpress/dst/greyhound/core/BUILD.bazel | 5 ----- .../scala/com/wixpress/dst/greyhound/core/offset/BUILD.bazel | 3 --- .../com/wixpress/dst/greyhound/core/rabalance/BUILD.bazel | 3 --- .../scala/com/wixpress/dst/greyhound/core/retry/BUILD.bazel | 4 ---- .../it/scala/com/wixpress/dst/greyhound/testenv/BUILD.bazel | 4 ---- .../it/scala/com/wixpress/dst/greyhound/testkit/BUILD.bazel | 1 - .../com/wixpress/dst/greyhound/core/consumer/BUILD.bazel | 3 --- .../wixpress/dst/greyhound/core/consumer/retry/BUILD.bazel | 1 - .../com/wixpress/dst/greyhound/core/producer/BUILD.bazel | 1 - .../com/wixpress/dst/greyhound/core/testkit/BUILD.bazel | 1 - .../it/scala/com/wixpress/dst/greyhound/future/BUILD.bazel | 3 --- 11 files changed, 29 deletions(-) diff --git a/core/src/it/scala/com/wixpress/dst/greyhound/core/BUILD.bazel b/core/src/it/scala/com/wixpress/dst/greyhound/core/BUILD.bazel index 28d79c08..9735d210 100644 --- a/core/src/it/scala/com/wixpress/dst/greyhound/core/BUILD.bazel +++ b/core/src/it/scala/com/wixpress/dst/greyhound/core/BUILD.bazel @@ -24,13 +24,8 @@ specs2_ite2e_test( "//core/src/main/scala/com/wixpress/dst/greyhound/core/producer", "//core/src/main/scala/com/wixpress/dst/greyhound/core/zioutils", "//core/src/test/resources", - #"//core/src/test/scala/com/wixpress/dst/greyhound/core/consumer", - #"//core/src/test/scala/com/wixpress/dst/greyhound/core/testkit", - "@ch_qos_logback_logback_classic", # "@dev_zio_izumi_reflect_2_12", "@dev_zio_zio_2_12", - "@dev_zio_zio_test_2_12", - "@org_apache_kafka_kafka_2_12", "@org_apache_kafka_kafka_clients", "//core/src/test/scala/com/wixpress/dst/greyhound/core/testkit", ], diff --git a/core/src/it/scala/com/wixpress/dst/greyhound/core/offset/BUILD.bazel b/core/src/it/scala/com/wixpress/dst/greyhound/core/offset/BUILD.bazel index ce9f7faf..a5c08ae5 100644 --- a/core/src/it/scala/com/wixpress/dst/greyhound/core/offset/BUILD.bazel +++ b/core/src/it/scala/com/wixpress/dst/greyhound/core/offset/BUILD.bazel @@ -13,7 +13,6 @@ specs2_ite2e_test( "@dev_zio_izumi_reflect_2_12", "@dev_zio_zio_managed_2_12", "//core/src/it/resources", - "//core/src/it/scala/com/wixpress/dst/greyhound/core", "//core/src/it/scala/com/wixpress/dst/greyhound/testenv", "//core/src/it/scala/com/wixpress/dst/greyhound/testkit", "//core/src/main/scala/com/wixpress/dst/greyhound/core", @@ -23,9 +22,7 @@ specs2_ite2e_test( "//core/src/main/scala/com/wixpress/dst/greyhound/core/metrics", "//core/src/main/scala/com/wixpress/dst/greyhound/core/producer", "//core/src/main/scala/com/wixpress/dst/greyhound/core/zioutils", - "//core/src/test/scala/com/wixpress/dst/greyhound/core/consumer", "//core/src/test/scala/com/wixpress/dst/greyhound/core/testkit", - "@ch_qos_logback_logback_classic", # "@dev_zio_izumi_reflect_2_12", "@dev_zio_zio_2_12", "@org_apache_kafka_kafka_clients", diff --git a/core/src/it/scala/com/wixpress/dst/greyhound/core/rabalance/BUILD.bazel b/core/src/it/scala/com/wixpress/dst/greyhound/core/rabalance/BUILD.bazel index 3da73eba..c763308a 100644 --- a/core/src/it/scala/com/wixpress/dst/greyhound/core/rabalance/BUILD.bazel +++ b/core/src/it/scala/com/wixpress/dst/greyhound/core/rabalance/BUILD.bazel @@ -13,7 +13,6 @@ specs2_ite2e_test( "@dev_zio_izumi_reflect_2_12", "@dev_zio_zio_managed_2_12", "//core/src/it/resources", - "//core/src/it/scala/com/wixpress/dst/greyhound/core", "//core/src/it/scala/com/wixpress/dst/greyhound/testenv", "//core/src/it/scala/com/wixpress/dst/greyhound/testkit", "//core/src/main/scala/com/wixpress/dst/greyhound/core", @@ -23,9 +22,7 @@ specs2_ite2e_test( "//core/src/main/scala/com/wixpress/dst/greyhound/core/metrics", "//core/src/main/scala/com/wixpress/dst/greyhound/core/producer", "//core/src/main/scala/com/wixpress/dst/greyhound/core/zioutils", - "//core/src/test/scala/com/wixpress/dst/greyhound/core/consumer", "//core/src/test/scala/com/wixpress/dst/greyhound/core/testkit", - "@ch_qos_logback_logback_classic", # "@dev_zio_izumi_reflect_2_12", "@dev_zio_zio_2_12", "@org_apache_kafka_kafka_clients", diff --git a/core/src/it/scala/com/wixpress/dst/greyhound/core/retry/BUILD.bazel b/core/src/it/scala/com/wixpress/dst/greyhound/core/retry/BUILD.bazel index 3f83ff43..cb001169 100644 --- a/core/src/it/scala/com/wixpress/dst/greyhound/core/retry/BUILD.bazel +++ b/core/src/it/scala/com/wixpress/dst/greyhound/core/retry/BUILD.bazel @@ -12,7 +12,6 @@ specs2_ite2e_test( deps = [ "@dev_zio_izumi_reflect_2_12", "@dev_zio_zio_managed_2_12", - "//core/src/it/scala/com/wixpress/dst/greyhound/core", "//core/src/it/scala/com/wixpress/dst/greyhound/testenv", "//core/src/it/scala/com/wixpress/dst/greyhound/testkit", "//core/src/main/scala/com/wixpress/dst/greyhound/core", @@ -21,11 +20,8 @@ specs2_ite2e_test( "//core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry", "//core/src/main/scala/com/wixpress/dst/greyhound/core/metrics", "//core/src/main/scala/com/wixpress/dst/greyhound/core/producer", - "//core/src/main/scala/com/wixpress/dst/greyhound/core/zioutils", "//core/src/test/resources", - "//core/src/test/scala/com/wixpress/dst/greyhound/core/consumer", "//core/src/test/scala/com/wixpress/dst/greyhound/core/testkit", - "@ch_qos_logback_logback_classic", # "@dev_zio_izumi_reflect_2_12", "@dev_zio_zio_2_12", "@org_apache_kafka_kafka_clients", diff --git a/core/src/it/scala/com/wixpress/dst/greyhound/testenv/BUILD.bazel b/core/src/it/scala/com/wixpress/dst/greyhound/testenv/BUILD.bazel index 35ab06f9..6b5f15d9 100644 --- a/core/src/it/scala/com/wixpress/dst/greyhound/testenv/BUILD.bazel +++ b/core/src/it/scala/com/wixpress/dst/greyhound/testenv/BUILD.bazel @@ -13,15 +13,11 @@ scala_library( "@dev_zio_zio_managed_2_12", "//core/src/it/scala/com/wixpress/dst/greyhound/testkit", "//core/src/main/scala/com/wixpress/dst/greyhound/core", - "//core/src/main/scala/com/wixpress/dst/greyhound/core/admin", "//core/src/main/scala/com/wixpress/dst/greyhound/core/metrics", "//core/src/main/scala/com/wixpress/dst/greyhound/core/producer", "//core/src/test/scala/com/wixpress/dst/greyhound/core/testkit", # "@dev_zio_izumi_reflect_2_12", "@dev_zio_zio_2_12", "@dev_zio_zio_test_2_12", - "@org_apache_curator_curator_test", - "@org_apache_kafka_kafka_2_12", - "@org_apache_kafka_kafka_clients", ], ) diff --git a/core/src/it/scala/com/wixpress/dst/greyhound/testkit/BUILD.bazel b/core/src/it/scala/com/wixpress/dst/greyhound/testkit/BUILD.bazel index fb990677..f3345b95 100644 --- a/core/src/it/scala/com/wixpress/dst/greyhound/testkit/BUILD.bazel +++ b/core/src/it/scala/com/wixpress/dst/greyhound/testkit/BUILD.bazel @@ -16,7 +16,6 @@ scala_library( "//core/src/main/scala/com/wixpress/dst/greyhound/core/metrics", # "//core/src/main/scala/com/wixpress/dst/greyhound/core/producer", "@dev_zio_zio_2_12", - "@dev_zio_zio_test_2_12", "@org_apache_curator_curator_test", "@org_apache_kafka_kafka_2_12", "@org_apache_kafka_kafka_clients", diff --git a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/BUILD.bazel b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/BUILD.bazel index 90ed4056..3e104e18 100644 --- a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/BUILD.bazel +++ b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/BUILD.bazel @@ -11,17 +11,14 @@ specs2_unit_test( deps = [ "@dev_zio_zio_managed_2_12", "@dev_zio_zio_stacktracer_2_12", - "//core/src/it/scala/com/wixpress/dst/greyhound/testkit", "//core/src/main/scala/com/wixpress/dst/greyhound/core", "//core/src/main/scala/com/wixpress/dst/greyhound/core/consumer", "//core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/domain", "//core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry", "//core/src/main/scala/com/wixpress/dst/greyhound/core/metrics", - "//core/src/main/scala/com/wixpress/dst/greyhound/core/producer", "//core/src/main/scala/com/wixpress/dst/greyhound/core/zioutils", "//core/src/test/resources", "//core/src/test/scala/com/wixpress/dst/greyhound/core/testkit", - "@ch_qos_logback_logback_classic", # "@dev_zio_izumi_reflect_2_12", "@dev_zio_zio_2_12", "@dev_zio_zio_streams_2_12", diff --git a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/BUILD.bazel b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/BUILD.bazel index b02990a0..dea7a061 100644 --- a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/BUILD.bazel +++ b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/BUILD.bazel @@ -17,7 +17,6 @@ specs2_unit_test( "//core/src/main/scala/com/wixpress/dst/greyhound/core/metrics", "//core/src/main/scala/com/wixpress/dst/greyhound/core/producer", "//core/src/main/scala/com/wixpress/dst/greyhound/core/zioutils", - "//core/src/test/scala/com/wixpress/dst/greyhound/core/consumer", "//core/src/test/scala/com/wixpress/dst/greyhound/core/testkit", # "@dev_zio_izumi_reflect_2_12", "@dev_zio_zio_2_12", diff --git a/core/src/test/scala/com/wixpress/dst/greyhound/core/producer/BUILD.bazel b/core/src/test/scala/com/wixpress/dst/greyhound/core/producer/BUILD.bazel index 27f12381..46310b8b 100644 --- a/core/src/test/scala/com/wixpress/dst/greyhound/core/producer/BUILD.bazel +++ b/core/src/test/scala/com/wixpress/dst/greyhound/core/producer/BUILD.bazel @@ -16,7 +16,6 @@ specs2_unit_test( "//core/src/main/scala/com/wixpress/dst/greyhound/core/producer", "//core/src/test/resources", "//core/src/test/scala/com/wixpress/dst/greyhound/core/testkit", - "@ch_qos_logback_logback_classic", # "@dev_zio_izumi_reflect_2_12", "@dev_zio_zio_2_12", "@dev_zio_zio_test_2_12", diff --git a/core/src/test/scala/com/wixpress/dst/greyhound/core/testkit/BUILD.bazel b/core/src/test/scala/com/wixpress/dst/greyhound/core/testkit/BUILD.bazel index 17c68cef..75f65d20 100644 --- a/core/src/test/scala/com/wixpress/dst/greyhound/core/testkit/BUILD.bazel +++ b/core/src/test/scala/com/wixpress/dst/greyhound/core/testkit/BUILD.bazel @@ -15,7 +15,6 @@ scala_library( "//core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry", "//core/src/main/scala/com/wixpress/dst/greyhound/core/metrics", "//core/src/main/scala/com/wixpress/dst/greyhound/core/producer", - "//core/src/main/scala/com/wixpress/dst/greyhound/core/zioutils", # "@dev_zio_izumi_reflect_2_12", # "@dev_zio_izumi_reflect_thirdparty_boopickle_shaded_2_12", "@dev_zio_zio_2_12", diff --git a/future-interop/src/it/scala/com/wixpress/dst/greyhound/future/BUILD.bazel b/future-interop/src/it/scala/com/wixpress/dst/greyhound/future/BUILD.bazel index 45e8069e..af2ef67c 100644 --- a/future-interop/src/it/scala/com/wixpress/dst/greyhound/future/BUILD.bazel +++ b/future-interop/src/it/scala/com/wixpress/dst/greyhound/future/BUILD.bazel @@ -21,11 +21,8 @@ specs2_ite2e_test( "//core/src/test/resources", "//core/src/test/scala/com/wixpress/dst/greyhound/core/testkit", "//future-interop/src/main/scala/com/wixpress/dst/greyhound/future", - "//java-interop/src/main/java/com/wixpress/dst/greyhound/java", - "@ch_qos_logback_logback_classic", "@dev_zio_izumi_reflect_2_12", "@dev_zio_zio_2_12", "@dev_zio_zio_stacktracer_2_12", - "@org_apache_kafka_kafka_clients", ], ) From eb14f4f9ba35cb5631d2097832b770daadac02f5 Mon Sep 17 00:00:00 2001 From: Alexey Dsiuba Date: Tue, 14 Mar 2023 12:14:06 +0200 Subject: [PATCH 03/17] Blocking retries attempts tracking fix (#32757) * Blocking retries attempts tracking fix #pr * Factored out RetryAttempt logic, TODO tests * Added RetryAttemptTest * Reverted RetryHeader back * Fixed retry test GitOrigin-RevId: 7e0e8cbccdd4c5fe6446775262b6695697785b2b --- .../retry/BlockingRetryRecordHandler.scala | 47 +++++- .../retry/NonBlockingRetryHelper.scala | 137 ++++++------------ .../retry/NonBlockingRetryRecordHandler.scala | 7 +- .../core/consumer/retry/RetryAttempt.scala | 98 +++++++++++++ .../consumer/retry/RetryRecordHandler.scala | 13 +- .../consumer/retry/RetryAttemptTest.scala | 78 ++++++++++ .../RetryConsumerRecordHandlerTest.scala | 32 +--- .../core/testkit/FakeRetryHelper.scala | 43 +----- 8 files changed, 280 insertions(+), 175 deletions(-) create mode 100644 core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryAttempt.scala create mode 100644 core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryAttemptTest.scala diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/BlockingRetryRecordHandler.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/BlockingRetryRecordHandler.scala index 502e31e1..2f5d59a9 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/BlockingRetryRecordHandler.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/BlockingRetryRecordHandler.scala @@ -3,9 +3,8 @@ package com.wixpress.dst.greyhound.core.consumer.retry import java.util.concurrent.TimeUnit import com.wixpress.dst.greyhound.core.{Group, TopicPartition} import com.wixpress.dst.greyhound.core.consumer.domain.{ConsumerRecord, RecordHandler} -import com.wixpress.dst.greyhound.core.consumer.retry.BlockingState.{Blocked, Blocking => InternalBlocking, IgnoringOnce} +import com.wixpress.dst.greyhound.core.consumer.retry.BlockingState.{Blocked, IgnoringOnce, Blocking => InternalBlocking} import com.wixpress.dst.greyhound.core.consumer.retry.RetryRecordHandlerMetric.{BlockingRetryHandlerInvocationFailed, DoneBlockingBeforeRetry, NoRetryOnNonRetryableFailure} -import com.wixpress.dst.greyhound.core.consumer.retry.ZIOHelper.foreachWhile import com.wixpress.dst.greyhound.core.metrics.GreyhoundMetrics import com.wixpress.dst.greyhound.core.metrics.GreyhoundMetrics.report import com.wixpress.dst.greyhound.core.zioutils.AwaitShutdown @@ -31,7 +30,7 @@ private[retry] object BlockingRetryRecordHandler { override def handle(record: ConsumerRecord[K, V])(implicit trace: Trace): ZIO[GreyhoundMetrics with R, Nothing, LastHandleResult] = { val topicPartition = TopicPartition(record.topic, record.partition) - def pollBlockingStateWithSuspensions(interval: Duration, start: Long): URIO[GreyhoundMetrics, PollResult] = { + def pollBlockingStateWithSuspensions(record: ConsumerRecord[K, V], interval: Duration, start: Long): URIO[GreyhoundMetrics, PollResult] = { for { shouldBlock <- blockingStateResolver.resolve(record) shouldPollAgain <- @@ -43,14 +42,14 @@ private[retry] object BlockingRetryRecordHandler { } yield shouldPollAgain } - def blockOnErrorFor(interval: Duration) = { + def blockOnErrorFor(record: ConsumerRecord[K, V], interval: Duration) = { for { start <- currentTime(TimeUnit.MILLISECONDS) continueBlocking <- if (interval.toMillis > 100L) { awaitShutdown(record.topicPartition).flatMap( _.interruptOnShutdown( - pollBlockingStateWithSuspensions(interval, start).repeatWhile(result => result.pollAgain).map(_.blockHandling) + pollBlockingStateWithSuspensions(record, interval, start).repeatWhile(result => result.pollAgain).map(_.blockHandling) ).reporting(r => DoneBlockingBeforeRetry(record.topic, record.partition, record.offset, r.duration, r.failed)) ) } else { @@ -63,6 +62,7 @@ private[retry] object BlockingRetryRecordHandler { } def handleAndMaybeBlockOnErrorFor( + record: ConsumerRecord[K, V], interval: Option[Duration] ): ZIO[R with GreyhoundMetrics, Nothing, LastHandleResult] = { handler.handle(record).map(_ => LastHandleResult(lastHandleSucceeded = true, shouldContinue = false)).catchAll { @@ -73,7 +73,7 @@ private[retry] object BlockingRetryRecordHandler { case error => interval .map { interval => - report(BlockingRetryHandlerInvocationFailed(topicPartition, record.offset, error.toString)) *> blockOnErrorFor(interval) + report(BlockingRetryHandlerInvocationFailed(topicPartition, record.offset, error.toString)) *> blockOnErrorFor(record, interval) } .getOrElse(ZIO.succeed(LastHandleResult(lastHandleSucceeded = false, shouldContinue = false))) } @@ -96,13 +96,44 @@ private[retry] object BlockingRetryRecordHandler { } else { val durationsIncludingForInvocationWithNoErrorHandling = retryConfig.blockingBackoffs(record.topic)().map(Some(_)) :+ None for { - result <- foreachWhile(durationsIncludingForInvocationWithNoErrorHandling) { interval => handleAndMaybeBlockOnErrorFor(interval) } - _ <- maybeBackToStateBlocking + result <- retryEvery(record, durationsIncludingForInvocationWithNoErrorHandling) { (rec, interval) => + handleAndMaybeBlockOnErrorFor(rec, interval) + } + _ <- maybeBackToStateBlocking } yield result } } } + private def retryEvery[K, V, R, E](record: ConsumerRecord[K, V], as: Iterable[Option[Duration]])( + f: (ConsumerRecord[K, V], Option[Duration]) => ZIO[R, E, LastHandleResult] + )(implicit trace: Trace): ZIO[R, E, LastHandleResult] = { + ZIO.succeed(as.iterator).flatMap { i => + def loop(retryAttempt: Option[RetryAttempt]): ZIO[R, E, LastHandleResult] = + if (i.hasNext) { + val nextDelay = i.next + val recordWithAttempt = retryAttempt.fold(record) { attempt => + record.copy(headers = record.headers ++ RetryAttempt.toHeaders(attempt)) + } + f(recordWithAttempt, nextDelay).flatMap { result => + if (result.shouldContinue) Clock.instant.flatMap { now => + val nextAttempt = RetryAttempt( + originalTopic = record.topic, + attempt = retryAttempt.fold(0)(_.attempt + 1), + submittedAt = now, + backoff = nextDelay getOrElse Duration.Zero + ) + loop(Some(nextAttempt)) + } + else ZIO.succeed(result) + } + } + else ZIO.succeed(LastHandleResult(lastHandleSucceeded = false, shouldContinue = false)) + + loop(None) + } + } + private def handleNonRetriable[K, V, E, R](record: ConsumerRecord[K, V], topicPartition: TopicPartition, cause: Exception) = report(NoRetryOnNonRetryableFailure(topicPartition, record.offset, cause)) .as(LastHandleResult(lastHandleSucceeded = false, shouldContinue = false)) diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/NonBlockingRetryHelper.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/NonBlockingRetryHelper.scala index a73b7dce..1d1cd24c 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/NonBlockingRetryHelper.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/NonBlockingRetryHelper.scala @@ -1,32 +1,23 @@ package com.wixpress.dst.greyhound.core.consumer.retry -import java.time.{Duration => JavaDuration, Instant} -import java.util.concurrent.TimeUnit.MILLISECONDS -import java.util.regex.Pattern -import com.wixpress.dst.greyhound.core.Serdes.StringSerde import com.wixpress.dst.greyhound.core.consumer.domain.ConsumerSubscription.{TopicPattern, Topics} -import com.wixpress.dst.greyhound.core.consumer.retry.RetryAttempt.{currentTime, RetryAttemptNumber} -import com.wixpress.dst.greyhound.core.consumer.retry.RetryDecision.{NoMoreRetries, RetryWith} import com.wixpress.dst.greyhound.core.consumer.domain.{ConsumerRecord, ConsumerSubscription} +import com.wixpress.dst.greyhound.core.consumer.retry.RetryDecision.{NoMoreRetries, RetryWith} import com.wixpress.dst.greyhound.core.consumer.retry.RetryRecordHandlerMetric.WaitingForRetry import com.wixpress.dst.greyhound.core.metrics.GreyhoundMetrics import com.wixpress.dst.greyhound.core.metrics.GreyhoundMetrics.report import com.wixpress.dst.greyhound.core.producer.ProducerRecord -import com.wixpress.dst.greyhound.core.{durationDeserializer, instantDeserializer, Group, Headers, Topic} -import zio.Clock -import zio.Duration +import com.wixpress.dst.greyhound.core.{Group, Topic} import zio.Schedule.spaced -import zio.{Chunk, UIO, URIO, _} +import zio.{Chunk, Clock, Duration, URIO, _} +import java.time.Instant +import java.util.regex.Pattern import scala.util.Try trait NonBlockingRetryHelper { def retryTopicsFor(originalTopic: Topic): Set[Topic] - def retryAttempt(topic: Topic, headers: Headers, subscription: ConsumerSubscription)( - implicit trace: Trace - ): UIO[Option[RetryAttempt]] - def retryDecision[E]( retryAttempt: Option[RetryAttempt], record: ConsumerRecord[Chunk[Byte], Chunk[Byte]], @@ -34,7 +25,7 @@ trait NonBlockingRetryHelper { subscription: ConsumerSubscription )(implicit trace: Trace): URIO[Any, RetryDecision] - def retrySteps = retryTopicsFor("").size + def retrySteps: Int = retryTopicsFor("").size } object NonBlockingRetryHelper { @@ -49,82 +40,70 @@ object NonBlockingRetryHelper { override def retryTopicsFor(topic: Topic): Set[Topic] = policy(topic).intervals.indices.foldLeft(Set.empty[String])((acc, attempt) => acc + s"$topic-$group-retry-$attempt") - override def retryAttempt(topic: Topic, headers: Headers, subscription: ConsumerSubscription)( - implicit trace: Trace - ): UIO[Option[RetryAttempt]] = { - (for { - submitted <- headers.get(RetryHeader.Submitted, instantDeserializer) - backoff <- headers.get(RetryHeader.Backoff, durationDeserializer) - originalTopic <- headers.get[String](RetryHeader.OriginalTopic, StringSerde) - } yield for { - ta <- topicAttempt(subscription, topic, originalTopic) - TopicAttempt(originalTopic, attempt) = ta - s <- submitted - b <- backoff - } yield RetryAttempt(originalTopic, attempt, s, b)) - .catchAll(_ => ZIO.none) - } - - private def topicAttempt( - subscription: ConsumerSubscription, - topic: Topic, - originalTopicHeader: Option[String] - ) = - subscription match { - case _: Topics => extractTopicAttempt(group, topic) - case _: TopicPattern => - extractTopicAttemptFromPatternRetryTopic(group, topic, originalTopicHeader) - } - override def retryDecision[E]( retryAttempt: Option[RetryAttempt], record: ConsumerRecord[Chunk[Byte], Chunk[Byte]], error: E, subscription: ConsumerSubscription - )(implicit trace: Trace): URIO[Any, RetryDecision] = currentTime.map(now => { - val nextRetryAttempt = retryAttempt.fold(0)(_.attempt + 1) + )(implicit trace: Trace): URIO[Any, RetryDecision] = Clock.instant.map(now => { + val blockingRetriesBefore = RetryAttempt.maxBlockingAttempts( + NonBlockingRetryHelper.originalTopic(record.topic, group), + retryConfig + ).getOrElse(0) + + // attempt if present contains full number of retries + val nextNonBlockingAttempt = retryAttempt.fold(0)(_.attempt + 1 - blockingRetriesBefore) + val nextRetryAttempt = nextNonBlockingAttempt + blockingRetriesBefore val originalTopic = retryAttempt.fold(record.topic)(_.originalTopic) val retryTopic = subscription match { - case _: TopicPattern => patternRetryTopic(group, nextRetryAttempt) - case _: Topics => fixedRetryTopic(originalTopic, group, nextRetryAttempt) + case _: TopicPattern => patternRetryTopic(group, nextNonBlockingAttempt) + case _: Topics => fixedRetryTopic(originalTopic, group, nextNonBlockingAttempt) } val topicRetryPolicy = policy(record.topic) topicRetryPolicy.intervals - .lift(nextRetryAttempt) + .lift(nextNonBlockingAttempt) .map { backoff => + val attempt = RetryAttempt( + attempt = nextRetryAttempt, + originalTopic = originalTopic, + submittedAt = now, + backoff = backoff + ) topicRetryPolicy.recordMutate( ProducerRecord( topic = retryTopic, value = record.value, key = record.key, partition = None, - headers = record.headers + - (RetryHeader.Submitted -> toChunk(now.toEpochMilli)) + - (RetryHeader.Backoff -> toChunk(backoff.toMillis)) + - (RetryHeader.OriginalTopic -> toChunk(originalTopic)) + - (RetryHeader.RetryAttempt -> toChunk(nextRetryAttempt)) + headers = record.headers ++ RetryAttempt.toHeaders(attempt) ) ) } .fold[RetryDecision](NoMoreRetries)(RetryWith) }) + } - private def toChunk(long: Long): Chunk[Byte] = - Chunk.fromArray(long.toString.getBytes) - - private def toChunk(str: String): Chunk[Byte] = - Chunk.fromArray(str.getBytes) + private[retry] def attemptNumberFromTopic( + subscription: ConsumerSubscription, + topic: Topic, + originalTopicHeader: Option[String], + group: Group + ) = + subscription match { + case _: Topics => extractTopicAttempt(group, topic) + case _: TopicPattern => + extractTopicAttemptFromPatternRetryTopic(group, topic, originalTopicHeader) } - private def extractTopicAttempt[E](group: Group, inputTopic: Topic) = + private def extractTopicAttempt(group: Group, inputTopic: Topic) = inputTopic.split(s"-$group-retry-").toSeq match { case Seq(topic, attempt) if Try(attempt.toInt).isSuccess => Some(TopicAttempt(topic, attempt.toInt)) - case _ => None + case _ => None } - private def extractTopicAttemptFromPatternRetryTopic[E]( + private def extractTopicAttemptFromPatternRetryTopic( group: Group, inputTopic: Topic, originalTopicHeader: Option[String] @@ -166,49 +145,27 @@ object DelayHeaders { val Backoff = "backOffTimeMs" } -object RetryHeader { - val Submitted = "submitTimestamp" - val Backoff = DelayHeaders.Backoff - val OriginalTopic = "GH_OriginalTopic" - val RetryAttempt = "GH_RetryAttempt" -} - -case class RetryAttempt( - originalTopic: Topic, - attempt: RetryAttemptNumber, - submittedAt: Instant, - backoff: Duration -) { - - def sleep(implicit trace: Trace): URIO[GreyhoundMetrics, Unit] = - RetryUtil.sleep(submittedAt, backoff) race reportWaitingInIntervals(every = 60.seconds) - - private def reportWaitingInIntervals(every: Duration) = - report(WaitingForRetry(originalTopic, attempt, submittedAt.toEpochMilli, backoff.toMillis)) - .repeat(spaced(every)) - .unit -} - object RetryUtil { + def sleep(attempt: RetryAttempt)(implicit trace: Trace): URIO[GreyhoundMetrics, Unit] = + sleep(attempt.submittedAt, attempt.backoff) race + report(WaitingForRetry(attempt.originalTopic, attempt.attempt, attempt.submittedAt.toEpochMilli, attempt.backoff.toMillis)) + .repeat(spaced(60.seconds)) + .unit + def sleep(submittedAt: Instant, backoff: Duration)(implicit trace: Trace): URIO[Any, Unit] = { val expiresAt = submittedAt.plus(backoff.asJava) - currentTime + Clock.instant .map(_.isAfter(expiresAt)) .flatMap(expired => if (expired) ZIO.unit else - ZIO.sleep(1.seconds).repeatUntilZIO(_ => currentTime.map(_.isAfter(expiresAt))).unit + ZIO.sleep(1.second).repeatUntilZIO(_ => Clock.instant.map(_.isAfter(expiresAt))).unit ) } } private case class TopicAttempt(originalTopic: Topic, attempt: Int) -object RetryAttempt { - type RetryAttemptNumber = Int - val currentTime = Clock.currentTime(MILLISECONDS).map(Instant.ofEpochMilli) -} - sealed trait RetryDecision object RetryDecision { diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/NonBlockingRetryRecordHandler.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/NonBlockingRetryRecordHandler.scala index a15f1e5b..a6ff6560 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/NonBlockingRetryRecordHandler.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/NonBlockingRetryRecordHandler.scala @@ -31,11 +31,12 @@ private[retry] object NonBlockingRetryRecordHandler { retryConfig: RetryConfig, subscription: ConsumerSubscription, nonBlockingRetryHelper: NonBlockingRetryHelper, + groupId: Group, awaitShutdown: TopicPartition => UIO[AwaitShutdown] )(implicit evK: K <:< Chunk[Byte], evV: V <:< Chunk[Byte]): NonBlockingRetryRecordHandler[V, K, R] = new NonBlockingRetryRecordHandler[V, K, R] { override def handle(record: ConsumerRecord[K, V]): ZIO[GreyhoundMetrics with R, Nothing, Any] = { - nonBlockingRetryHelper.retryAttempt(record.topic, record.headers, subscription).flatMap { retryAttempt => + RetryAttempt.extract(record.headers, record.topic, groupId, subscription, Some(retryConfig)).flatMap { retryAttempt => maybeDelayRetry(record, retryAttempt) *> handler.handle(record).catchAll { case Right(_: NonRetriableException) => ZIO.unit @@ -56,7 +57,7 @@ private[retry] object NonBlockingRetryRecordHandler { WaitingBeforeRetry(record.topic, retryAttempt, record.partition, record.offset, correlationId) ) *> awaitShutdown(record.topicPartition) - .flatMap(_.interruptOnShutdown(retryAttempt.sleep)) + .flatMap(_.interruptOnShutdown(RetryUtil.sleep(retryAttempt))) .reporting(r => DoneWaitingBeforeRetry(record.topic, record.partition, record.offset, retryAttempt, r.duration, r.failed, correlationId) ) @@ -74,7 +75,7 @@ private[retry] object NonBlockingRetryRecordHandler { override def handleAfterBlockingFailed( record: ConsumerRecord[K, V] ): ZIO[GreyhoundMetrics with R, Nothing, Any] = { - nonBlockingRetryHelper.retryAttempt(record.topic, record.headers, subscription).flatMap { retryAttempt => + RetryAttempt.extract(record.headers, record.topic, groupId, subscription, Some(retryConfig)).flatMap { retryAttempt => maybeRetry(retryAttempt, BlockingHandlerFailed, record) } } diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryAttempt.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryAttempt.scala new file mode 100644 index 00000000..9265eab8 --- /dev/null +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryAttempt.scala @@ -0,0 +1,98 @@ +package com.wixpress.dst.greyhound.core.consumer.retry + +import com.wixpress.dst.greyhound.core.Serdes.StringSerde +import com.wixpress.dst.greyhound.core._ +import com.wixpress.dst.greyhound.core.consumer.domain.ConsumerSubscription +import com.wixpress.dst.greyhound.core.consumer.retry.NonBlockingRetryHelper.attemptNumberFromTopic +import com.wixpress.dst.greyhound.core.consumer.retry.RetryAttempt.RetryAttemptNumber +import zio._ + +import java.time.Instant + +/** + * Description of a retry attempt + * @param attempt contains which attempt is it, starting from 0 including blocking and non-blocking attempts + */ +case class RetryAttempt( + originalTopic: Topic, + attempt: RetryAttemptNumber, + submittedAt: Instant, + backoff: Duration +) + +object RetryHeader { + val Submitted = "submitTimestamp" + val Backoff = DelayHeaders.Backoff + val OriginalTopic = "GH_OriginalTopic" + val RetryAttempt = "GH_RetryAttempt" +} + +object RetryAttempt { + type RetryAttemptNumber = Int + + private def toChunk(str: String): Chunk[Byte] = Chunk.fromArray(str.getBytes) + + def toHeaders(attempt: RetryAttempt): Headers = Headers( + RetryHeader.Submitted -> toChunk(attempt.submittedAt.toEpochMilli.toString), + RetryHeader.Backoff -> toChunk(attempt.backoff.toMillis.toString), + RetryHeader.OriginalTopic -> toChunk(attempt.originalTopic), + RetryHeader.RetryAttempt -> toChunk(attempt.attempt.toString), + ) + + private case class RetryAttemptHeaders( + originalTopic: Option[Topic], + attempt: Option[RetryAttemptNumber], + submittedAt: Option[Instant], + backoff: Option[Duration] + ) + + private def fromHeaders(headers: Headers): Task[RetryAttemptHeaders] = + for { + submitted <- headers.get(RetryHeader.Submitted, instantDeserializer) + backoff <- headers.get(RetryHeader.Backoff, durationDeserializer) + topic <- headers.get[String](RetryHeader.OriginalTopic, StringSerde) + attempt <- headers.get(RetryHeader.RetryAttempt, longDeserializer) + } yield RetryAttemptHeaders(topic, attempt.map(_.toInt), submitted, backoff) + + /** @return None on infinite blocking retries */ + def maxBlockingAttempts(topic: Topic, retryConfig: Option[RetryConfig]): Option[Int] = + retryConfig.map(_.blockingBackoffs(topic)()).fold(Option(0)) { + case finite if finite.hasDefiniteSize => Some(finite.size) + case _ => None + } + + /** @return None on infinite retries */ + def maxOverallAttempts(topic: Topic, retryConfig: Option[RetryConfig]): Option[Int] = + maxBlockingAttempts(topic, retryConfig).map { + _ + retryConfig.fold(0)(_.nonBlockingBackoffs(topic).length) + } + + def extract( + headers: Headers, + topic: Topic, + group: Group, + subscription: ConsumerSubscription, + retryConfig: Option[RetryConfig], + )(implicit trace: Trace): UIO[Option[RetryAttempt]] = { + + def nonBlockingAttempt(hs: RetryAttemptHeaders): Option[RetryAttempt] = + for { + submitted <- hs.submittedAt + backoff <- hs.backoff + TopicAttempt(originalTopic, attempt) <- attemptNumberFromTopic(subscription, topic, hs.originalTopic, group) + blockingRetries = maxBlockingAttempts(originalTopic, retryConfig).getOrElse(0) + } yield RetryAttempt(originalTopic, blockingRetries + attempt, submitted, backoff) + + def blockingAttempt(hs: RetryAttemptHeaders): Option[RetryAttempt] = + for { + submitted <- hs.submittedAt + backoff <- hs.backoff + originalTopic <- hs.originalTopic + attempt <- hs.attempt + } yield RetryAttempt(originalTopic, attempt, submitted, backoff) + + fromHeaders(headers).map { hs => + nonBlockingAttempt(hs) orElse blockingAttempt(hs) + } + }.catchAll(_ => ZIO.none) +} diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryRecordHandler.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryRecordHandler.scala index af0749c6..8ee8ab9f 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryRecordHandler.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryRecordHandler.scala @@ -33,7 +33,7 @@ object RetryRecordHandler { ): RecordHandler[R with R2 with GreyhoundMetrics, Nothing, K, V] = { val nonBlockingHandler = - NonBlockingRetryRecordHandler(handler, producer, retryConfig, subscription, nonBlockingRetryHelper, awaitShutdown) + NonBlockingRetryRecordHandler(handler, producer, retryConfig, subscription, nonBlockingRetryHelper, groupId, awaitShutdown) val blockingHandler = BlockingRetryRecordHandler(groupId, handler, retryConfig, blockingState, nonBlockingHandler, awaitShutdown) val blockingAndNonBlockingHandler = BlockingAndNonBlockingRetryRecordHandler(groupId, blockingHandler, nonBlockingHandler) @@ -55,15 +55,4 @@ object RetryRecordHandler { record.headers.get[String](key, StringSerde).catchAll(_ => ZIO.none) } -object ZIOHelper { - def foreachWhile[R, E, A](as: Iterable[A])(f: A => ZIO[R, E, LastHandleResult])(implicit trace: Trace): ZIO[R, E, LastHandleResult] = - ZIO.succeed(as.iterator).flatMap { i => - def loop: ZIO[R, E, LastHandleResult] = - if (i.hasNext) f(i.next).flatMap(result => if (result.shouldContinue) loop else ZIO.succeed(result)) - else ZIO.succeed(LastHandleResult(lastHandleSucceeded = false, shouldContinue = false)) - - loop - } -} - case class LastHandleResult(lastHandleSucceeded: Boolean, shouldContinue: Boolean) diff --git a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryAttemptTest.scala b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryAttemptTest.scala new file mode 100644 index 00000000..3139f845 --- /dev/null +++ b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryAttemptTest.scala @@ -0,0 +1,78 @@ +package com.wixpress.dst.greyhound.core.consumer.retry + +import com.wixpress.dst.greyhound.core.consumer.domain.ConsumerSubscription +import com.wixpress.dst.greyhound.core.testkit.BaseTest +import zio.test.TestEnvironment + +import java.time.{Duration, Instant} +import scala.util.Random +import scala.concurrent.duration._ + +class RetryAttemptTest extends BaseTest[TestEnvironment] { + + "RetryAttempt.extract" should { + "deserialize attempt from headers for blocking retries" in { + val attempt = randomRetryAttempt + val headers = RetryAttempt.toHeaders(attempt) + val subscription = ConsumerSubscription.Topics(Set(attempt.originalTopic)) + for (result <- RetryAttempt.extract(headers, attempt.originalTopic, randomStr, subscription, None)) + yield result must beSome(attempt) + } + "deserialize attempt from headers and topic for non-blocking retries" in { + val attempt = randomRetryAttempt + // topic and attempt must be extracted from retryTopic + val headers = RetryAttempt.toHeaders(attempt.copy(originalTopic = "", attempt = -1)) + val subscription = ConsumerSubscription.Topics(Set(attempt.originalTopic)) + val group = randomStr + val retryTopic = NonBlockingRetryHelper.fixedRetryTopic(attempt.originalTopic, group, attempt.attempt) + for (result <- RetryAttempt.extract(headers, retryTopic, group, subscription, None)) + yield result must beSome(attempt) + } + "deserialize attempt for non-blocking retry after blocking retries" in { + val attempt = randomRetryAttempt + val headers = RetryAttempt.toHeaders(attempt) + val subscription = ConsumerSubscription.Topics(Set(attempt.originalTopic)) + val group = randomStr + val retries = RetryConfig.blockingFollowedByNonBlockingRetry( + blockingBackoffs = 1.milli :: 1.second :: Nil, + nonBlockingBackoffs = 5.minutes :: Nil, + ) + val retryTopic = NonBlockingRetryHelper.fixedRetryTopic(attempt.originalTopic, group, attempt.attempt) + for (result <- RetryAttempt.extract(headers, retryTopic, group, subscription, Some(retries))) + yield result must beSome(attempt.copy(attempt = attempt.attempt + 2)) // with 2 blocking retries before + } + } + + "RetryAttempt.maxOverallAttempts" should { + "return 0 if no retries configured" in { + RetryAttempt.maxOverallAttempts(randomStr, None) must beSome(0) + } + "return max attempts for blocking retries" in { + val config = RetryConfig.finiteBlockingRetry(1.milli, 1.second) + RetryAttempt.maxOverallAttempts(randomStr, Some(config)) must beSome(2) + } + "return max attempts for non-blocking retries" in { + val config = RetryConfig.nonBlockingRetry(1.milli, 1.second, 5.minutes) + RetryAttempt.maxOverallAttempts(randomStr, Some(config)) must beSome(3) + } + "return max attempts for blocking retries followed by non-blocking" in { + val config = RetryConfig.blockingFollowedByNonBlockingRetry(1.milli :: 2.seconds :: Nil, 1.minute :: Nil) + RetryAttempt.maxOverallAttempts(randomStr, Some(config)) must beSome(3) + } + "return None for infinite blocking retries" in { + val config = RetryConfig.infiniteBlockingRetry(1.milli) + RetryAttempt.maxOverallAttempts(randomStr, Some(config)) must beNone + } + } + + override def env = testEnvironment + + private def randomStr = Random.alphanumeric.take(10).mkString + + private def randomRetryAttempt = RetryAttempt( + originalTopic = randomStr, + attempt = Random.nextInt(1000), + submittedAt = Instant.ofEpochMilli(math.abs(Random.nextLong())), + backoff = Duration.ofMillis(Random.nextInt(100000)) + ) +} diff --git a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryConsumerRecordHandlerTest.scala b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryConsumerRecordHandlerTest.scala index c713fb12..b34e8d82 100644 --- a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryConsumerRecordHandlerTest.scala +++ b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryConsumerRecordHandlerTest.scala @@ -4,7 +4,7 @@ import java.time.Instant import com.wixpress.dst.greyhound.core.Serdes._ import com.wixpress.dst.greyhound.core._ import com.wixpress.dst.greyhound.core.consumer.domain.ConsumerSubscription.Topics -import com.wixpress.dst.greyhound.core.consumer.domain.{ConsumerRecord, ConsumerSubscription, RecordHandler} +import com.wixpress.dst.greyhound.core.consumer.domain.{ConsumerRecord, RecordHandler} import com.wixpress.dst.greyhound.core.consumer.retry.BlockingState.{Blocked, Blocking => InternalBlocking, IgnoringAll, IgnoringOnce} import com.wixpress.dst.greyhound.core.consumer.retry.RetryConsumerRecordHandlerTest.{offset, partition, _} import com.wixpress.dst.greyhound.core.consumer.retry.RetryRecordHandlerMetric.{BlockingIgnoredForAllFor, BlockingIgnoredOnceFor, BlockingRetryHandlerInvocationFailed, NoRetryOnNonRetryableFailure} @@ -21,8 +21,6 @@ import zio.Random.{nextBytes, nextIntBounded} import zio.managed.UManaged import zio.test.TestClock -import scala.concurrent.TimeoutException - class RetryConsumerRecordHandlerTest extends BaseTest[TestClock with TestMetrics] { override def env: UManaged[ZEnvironment[TestClock with TestMetrics]] = @@ -52,9 +50,6 @@ class RetryConsumerRecordHandlerTest extends BaseTest[TestClock with TestMetrics _ <- retryHandler.handle(ConsumerRecord(topic, partition, offset, Headers.Empty, Some(key), value, 0L, 0L, 0L)) record <- producer.records.take now <- currentTime - retryAttempt <- IntSerde.serialize(retryTopic, 0) - submittedAt <- InstantSerde.serialize(retryTopic, now) - backoff <- DurationSerde.serialize(retryTopic, 1.second) } yield { record === ProducerRecord( @@ -62,7 +57,7 @@ class RetryConsumerRecordHandlerTest extends BaseTest[TestClock with TestMetrics value, Some(key), partition = None, - headers = Headers("retry-attempt" -> retryAttempt, "retry-submitted-at" -> submittedAt, "retry-backoff" -> backoff) + headers = RetryAttempt.toHeaders(RetryAttempt(topic, 0, now, 1.second)) ) } } @@ -86,10 +81,7 @@ class RetryConsumerRecordHandlerTest extends BaseTest[TestClock with TestMetrics ) value <- bytes begin <- currentTime - retryAttempt <- IntSerde.serialize(retryTopic, 0) - submittedAt <- InstantSerde.serialize(retryTopic, begin) - backoff <- DurationSerde.serialize(retryTopic, 1.second) - headers = Headers("retry-attempt" -> retryAttempt, "retry-submitted-at" -> submittedAt, "retry-backoff" -> backoff) + headers = RetryAttempt.toHeaders(RetryAttempt(topic, 0, begin, 1.second)) _ <- retryHandler.handle(ConsumerRecord(retryTopic, partition, offset, headers, None, value, 0L, 0L, 0L)).fork _ <- TestClock.adjust(1.second).repeat(Schedule.once) end <- executionTime.await.disconnect.timeoutFail(TimeoutWaitingForAssertion)(5.seconds) @@ -404,7 +396,8 @@ class RetryConsumerRecordHandlerTest extends BaseTest[TestClock with TestMetrics producer <- FakeProducer.make topic <- randomTopicName blockingState <- Ref.make[Map[BlockingTarget, BlockingState]](Map.empty) - retryHelper = alwaysBackOffRetryHelper(3.seconds) + retryHelper = FakeRetryHelper(topic) + now <- Clock.instant handling <- AwaitShutdown.makeManaged.flatMap { awaitShutdown => val retryHandler = RetryRecordHandler.withRetries( group, @@ -416,11 +409,12 @@ class RetryConsumerRecordHandlerTest extends BaseTest[TestClock with TestMetrics retryHelper, awaitShutdown = _ => ZIO.succeed(awaitShutdown) ) + val headers = RetryAttempt.toHeaders(RetryAttempt(topic, 0, now, 3.seconds)) for { key <- bytes value <- bytes handling <- retryHandler - .handle(ConsumerRecord(topic, partition, offset, Headers.Empty, Some(key), value, 0L, 0L, 0L)) + .handle(ConsumerRecord(topic, partition, offset, headers, Some(key), value, 0L, 0L, 0L)) .forkDaemon } yield handling } @@ -534,18 +528,6 @@ object RetryConsumerRecordHandlerTest { def randomTopicName = randomStr.map(suffix => s"some-topic-$suffix") val cause = new RuntimeException("cause") - - def alwaysBackOffRetryHelper(backoff: Duration) = { - new FakeNonBlockingRetryHelper { - override val topic: Topic = "" - - override def retryAttempt(topic: Topic, headers: Headers, subscription: ConsumerSubscription)( - implicit trace: Trace - ): UIO[Option[RetryAttempt]] = ZIO.succeed( - Some(RetryAttempt(topic, 1, Instant.now, backoff)) - ) - } - } } object TimeoutWaitingForAssertion extends RuntimeException diff --git a/core/src/test/scala/com/wixpress/dst/greyhound/core/testkit/FakeRetryHelper.scala b/core/src/test/scala/com/wixpress/dst/greyhound/core/testkit/FakeRetryHelper.scala index cee13948..619ada36 100644 --- a/core/src/test/scala/com/wixpress/dst/greyhound/core/testkit/FakeRetryHelper.scala +++ b/core/src/test/scala/com/wixpress/dst/greyhound/core/testkit/FakeRetryHelper.scala @@ -1,35 +1,21 @@ package com.wixpress.dst.greyhound.core.testkit import java.time.Instant -import java.util.concurrent.TimeUnit.MILLISECONDS - -import com.wixpress.dst.greyhound.core.Serdes._ import com.wixpress.dst.greyhound.core._ import com.wixpress.dst.greyhound.core.consumer.domain.{ConsumerRecord, ConsumerSubscription} import com.wixpress.dst.greyhound.core.consumer.retry.RetryDecision.{NoMoreRetries, RetryWith} -import com.wixpress.dst.greyhound.core.consumer.retry.{BlockingHandlerFailed, NonBlockingRetryHelper, RetryAttempt, RetryDecision} +import com.wixpress.dst.greyhound.core.consumer.retry.{BlockingHandlerFailed, NonBlockingRetryHelper, RetryAttempt, RetryDecision, RetryHeader} import com.wixpress.dst.greyhound.core.producer.ProducerRecord import com.wixpress.dst.greyhound.core.testkit.FakeRetryHelper._ import zio._ import zio.Clock -import zio.Clock - trait FakeNonBlockingRetryHelper extends NonBlockingRetryHelper { val topic: Topic override def retryTopicsFor(originalTopic: Topic): Set[Topic] = Set(s"$originalTopic-retry") - override def retryAttempt(topic: Topic, headers: Headers, subscription: ConsumerSubscription)( - implicit trace: Trace - ): UIO[Option[RetryAttempt]] = - (for { - attempt <- headers.get(Header.Attempt, IntSerde) - submittedAt <- headers.get(Header.SubmittedAt, InstantSerde) - backoff <- headers.get(Header.Backoff, DurationSerde) - } yield retryAttemptInternal(topic, attempt, submittedAt, backoff)).orElse(ZIO.none) - override def retryDecision[E]( retryAttempt: Option[RetryAttempt], record: ConsumerRecord[Chunk[Byte], Chunk[Byte]], @@ -38,35 +24,23 @@ trait FakeNonBlockingRetryHelper extends NonBlockingRetryHelper { )(implicit trace: Trace): URIO[Any, RetryDecision] = error match { case RetriableError | BlockingHandlerFailed => - currentTime.flatMap(now => - recordFrom(now, retryAttempt, record) - .fold(_ => NoMoreRetries, RetryWith) + currentTime.map(now => + RetryWith(recordFrom(now, retryAttempt, record)) ) case NonRetriableError => ZIO.succeed(NoMoreRetries) } - private def retryAttemptInternal(topic: Topic, attempt: Option[Int], submittedAt: Option[Instant], backoff: Option[Duration]) = - for { - a <- attempt - s <- submittedAt - b <- backoff - } yield RetryAttempt(topic, a, s, b) - private def recordFrom(now: Instant, retryAttempt: Option[RetryAttempt], record: ConsumerRecord[Chunk[Byte], Chunk[Byte]])( implicit trace: Trace ) = { val nextRetryAttempt = retryAttempt.fold(0)(_.attempt + 1) - for { - retryAttempt <- IntSerde.serialize(topic, nextRetryAttempt) - submittedAt <- InstantSerde.serialize(topic, now) - backoff <- DurationSerde.serialize(topic, 1.second) - } yield ProducerRecord( + ProducerRecord( topic = s"$topic-retry", value = record.value, key = record.key, partition = None, - headers = Headers(Header.Attempt -> retryAttempt, Header.SubmittedAt -> submittedAt, Header.Backoff -> backoff) + headers = RetryAttempt.toHeaders(RetryAttempt(topic, nextRetryAttempt, now, 1.second)) ) } } @@ -75,13 +49,8 @@ case class FakeRetryHelper(topic: Topic) extends FakeNonBlockingRetryHelper object FakeRetryHelper { implicit private val trace = Trace.empty - object Header { - val Attempt = "retry-attempt" - val SubmittedAt = "retry-submitted-at" - val Backoff = "retry-backoff" - } - val currentTime = Clock.currentTime(MILLISECONDS).map(Instant.ofEpochMilli) + val currentTime: UIO[Instant] = Clock.instant } sealed trait HandlerError From 6ed5d7186bd92b177f4204e355e998dc2736dc01 Mon Sep 17 00:00:00 2001 From: Natan Silnitsky Date: Mon, 3 Apr 2023 15:22:47 +0300 Subject: [PATCH 04/17] Revert "Blocking retries attempts tracking fix" (#33818) Revert "Blocking retries attempts tracking fix (#32757)" This reverts commit 7e0e8cbccdd4c5fe6446775262b6695697785b2b. GitOrigin-RevId: eb859fcf70860c06a0bd0492b746ed3ff00bc8ac --- .../retry/BlockingRetryRecordHandler.scala | 47 +----- .../retry/NonBlockingRetryHelper.scala | 137 ++++++++++++------ .../retry/NonBlockingRetryRecordHandler.scala | 7 +- .../core/consumer/retry/RetryAttempt.scala | 98 ------------- .../consumer/retry/RetryRecordHandler.scala | 13 +- .../consumer/retry/RetryAttemptTest.scala | 78 ---------- .../RetryConsumerRecordHandlerTest.scala | 32 +++- .../core/testkit/FakeRetryHelper.scala | 43 +++++- 8 files changed, 175 insertions(+), 280 deletions(-) delete mode 100644 core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryAttempt.scala delete mode 100644 core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryAttemptTest.scala diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/BlockingRetryRecordHandler.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/BlockingRetryRecordHandler.scala index 2f5d59a9..502e31e1 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/BlockingRetryRecordHandler.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/BlockingRetryRecordHandler.scala @@ -3,8 +3,9 @@ package com.wixpress.dst.greyhound.core.consumer.retry import java.util.concurrent.TimeUnit import com.wixpress.dst.greyhound.core.{Group, TopicPartition} import com.wixpress.dst.greyhound.core.consumer.domain.{ConsumerRecord, RecordHandler} -import com.wixpress.dst.greyhound.core.consumer.retry.BlockingState.{Blocked, IgnoringOnce, Blocking => InternalBlocking} +import com.wixpress.dst.greyhound.core.consumer.retry.BlockingState.{Blocked, Blocking => InternalBlocking, IgnoringOnce} import com.wixpress.dst.greyhound.core.consumer.retry.RetryRecordHandlerMetric.{BlockingRetryHandlerInvocationFailed, DoneBlockingBeforeRetry, NoRetryOnNonRetryableFailure} +import com.wixpress.dst.greyhound.core.consumer.retry.ZIOHelper.foreachWhile import com.wixpress.dst.greyhound.core.metrics.GreyhoundMetrics import com.wixpress.dst.greyhound.core.metrics.GreyhoundMetrics.report import com.wixpress.dst.greyhound.core.zioutils.AwaitShutdown @@ -30,7 +31,7 @@ private[retry] object BlockingRetryRecordHandler { override def handle(record: ConsumerRecord[K, V])(implicit trace: Trace): ZIO[GreyhoundMetrics with R, Nothing, LastHandleResult] = { val topicPartition = TopicPartition(record.topic, record.partition) - def pollBlockingStateWithSuspensions(record: ConsumerRecord[K, V], interval: Duration, start: Long): URIO[GreyhoundMetrics, PollResult] = { + def pollBlockingStateWithSuspensions(interval: Duration, start: Long): URIO[GreyhoundMetrics, PollResult] = { for { shouldBlock <- blockingStateResolver.resolve(record) shouldPollAgain <- @@ -42,14 +43,14 @@ private[retry] object BlockingRetryRecordHandler { } yield shouldPollAgain } - def blockOnErrorFor(record: ConsumerRecord[K, V], interval: Duration) = { + def blockOnErrorFor(interval: Duration) = { for { start <- currentTime(TimeUnit.MILLISECONDS) continueBlocking <- if (interval.toMillis > 100L) { awaitShutdown(record.topicPartition).flatMap( _.interruptOnShutdown( - pollBlockingStateWithSuspensions(record, interval, start).repeatWhile(result => result.pollAgain).map(_.blockHandling) + pollBlockingStateWithSuspensions(interval, start).repeatWhile(result => result.pollAgain).map(_.blockHandling) ).reporting(r => DoneBlockingBeforeRetry(record.topic, record.partition, record.offset, r.duration, r.failed)) ) } else { @@ -62,7 +63,6 @@ private[retry] object BlockingRetryRecordHandler { } def handleAndMaybeBlockOnErrorFor( - record: ConsumerRecord[K, V], interval: Option[Duration] ): ZIO[R with GreyhoundMetrics, Nothing, LastHandleResult] = { handler.handle(record).map(_ => LastHandleResult(lastHandleSucceeded = true, shouldContinue = false)).catchAll { @@ -73,7 +73,7 @@ private[retry] object BlockingRetryRecordHandler { case error => interval .map { interval => - report(BlockingRetryHandlerInvocationFailed(topicPartition, record.offset, error.toString)) *> blockOnErrorFor(record, interval) + report(BlockingRetryHandlerInvocationFailed(topicPartition, record.offset, error.toString)) *> blockOnErrorFor(interval) } .getOrElse(ZIO.succeed(LastHandleResult(lastHandleSucceeded = false, shouldContinue = false))) } @@ -96,44 +96,13 @@ private[retry] object BlockingRetryRecordHandler { } else { val durationsIncludingForInvocationWithNoErrorHandling = retryConfig.blockingBackoffs(record.topic)().map(Some(_)) :+ None for { - result <- retryEvery(record, durationsIncludingForInvocationWithNoErrorHandling) { (rec, interval) => - handleAndMaybeBlockOnErrorFor(rec, interval) - } - _ <- maybeBackToStateBlocking + result <- foreachWhile(durationsIncludingForInvocationWithNoErrorHandling) { interval => handleAndMaybeBlockOnErrorFor(interval) } + _ <- maybeBackToStateBlocking } yield result } } } - private def retryEvery[K, V, R, E](record: ConsumerRecord[K, V], as: Iterable[Option[Duration]])( - f: (ConsumerRecord[K, V], Option[Duration]) => ZIO[R, E, LastHandleResult] - )(implicit trace: Trace): ZIO[R, E, LastHandleResult] = { - ZIO.succeed(as.iterator).flatMap { i => - def loop(retryAttempt: Option[RetryAttempt]): ZIO[R, E, LastHandleResult] = - if (i.hasNext) { - val nextDelay = i.next - val recordWithAttempt = retryAttempt.fold(record) { attempt => - record.copy(headers = record.headers ++ RetryAttempt.toHeaders(attempt)) - } - f(recordWithAttempt, nextDelay).flatMap { result => - if (result.shouldContinue) Clock.instant.flatMap { now => - val nextAttempt = RetryAttempt( - originalTopic = record.topic, - attempt = retryAttempt.fold(0)(_.attempt + 1), - submittedAt = now, - backoff = nextDelay getOrElse Duration.Zero - ) - loop(Some(nextAttempt)) - } - else ZIO.succeed(result) - } - } - else ZIO.succeed(LastHandleResult(lastHandleSucceeded = false, shouldContinue = false)) - - loop(None) - } - } - private def handleNonRetriable[K, V, E, R](record: ConsumerRecord[K, V], topicPartition: TopicPartition, cause: Exception) = report(NoRetryOnNonRetryableFailure(topicPartition, record.offset, cause)) .as(LastHandleResult(lastHandleSucceeded = false, shouldContinue = false)) diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/NonBlockingRetryHelper.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/NonBlockingRetryHelper.scala index 1d1cd24c..a73b7dce 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/NonBlockingRetryHelper.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/NonBlockingRetryHelper.scala @@ -1,23 +1,32 @@ package com.wixpress.dst.greyhound.core.consumer.retry +import java.time.{Duration => JavaDuration, Instant} +import java.util.concurrent.TimeUnit.MILLISECONDS +import java.util.regex.Pattern +import com.wixpress.dst.greyhound.core.Serdes.StringSerde import com.wixpress.dst.greyhound.core.consumer.domain.ConsumerSubscription.{TopicPattern, Topics} -import com.wixpress.dst.greyhound.core.consumer.domain.{ConsumerRecord, ConsumerSubscription} +import com.wixpress.dst.greyhound.core.consumer.retry.RetryAttempt.{currentTime, RetryAttemptNumber} import com.wixpress.dst.greyhound.core.consumer.retry.RetryDecision.{NoMoreRetries, RetryWith} +import com.wixpress.dst.greyhound.core.consumer.domain.{ConsumerRecord, ConsumerSubscription} import com.wixpress.dst.greyhound.core.consumer.retry.RetryRecordHandlerMetric.WaitingForRetry import com.wixpress.dst.greyhound.core.metrics.GreyhoundMetrics import com.wixpress.dst.greyhound.core.metrics.GreyhoundMetrics.report import com.wixpress.dst.greyhound.core.producer.ProducerRecord -import com.wixpress.dst.greyhound.core.{Group, Topic} +import com.wixpress.dst.greyhound.core.{durationDeserializer, instantDeserializer, Group, Headers, Topic} +import zio.Clock +import zio.Duration import zio.Schedule.spaced -import zio.{Chunk, Clock, Duration, URIO, _} +import zio.{Chunk, UIO, URIO, _} -import java.time.Instant -import java.util.regex.Pattern import scala.util.Try trait NonBlockingRetryHelper { def retryTopicsFor(originalTopic: Topic): Set[Topic] + def retryAttempt(topic: Topic, headers: Headers, subscription: ConsumerSubscription)( + implicit trace: Trace + ): UIO[Option[RetryAttempt]] + def retryDecision[E]( retryAttempt: Option[RetryAttempt], record: ConsumerRecord[Chunk[Byte], Chunk[Byte]], @@ -25,7 +34,7 @@ trait NonBlockingRetryHelper { subscription: ConsumerSubscription )(implicit trace: Trace): URIO[Any, RetryDecision] - def retrySteps: Int = retryTopicsFor("").size + def retrySteps = retryTopicsFor("").size } object NonBlockingRetryHelper { @@ -40,70 +49,82 @@ object NonBlockingRetryHelper { override def retryTopicsFor(topic: Topic): Set[Topic] = policy(topic).intervals.indices.foldLeft(Set.empty[String])((acc, attempt) => acc + s"$topic-$group-retry-$attempt") + override def retryAttempt(topic: Topic, headers: Headers, subscription: ConsumerSubscription)( + implicit trace: Trace + ): UIO[Option[RetryAttempt]] = { + (for { + submitted <- headers.get(RetryHeader.Submitted, instantDeserializer) + backoff <- headers.get(RetryHeader.Backoff, durationDeserializer) + originalTopic <- headers.get[String](RetryHeader.OriginalTopic, StringSerde) + } yield for { + ta <- topicAttempt(subscription, topic, originalTopic) + TopicAttempt(originalTopic, attempt) = ta + s <- submitted + b <- backoff + } yield RetryAttempt(originalTopic, attempt, s, b)) + .catchAll(_ => ZIO.none) + } + + private def topicAttempt( + subscription: ConsumerSubscription, + topic: Topic, + originalTopicHeader: Option[String] + ) = + subscription match { + case _: Topics => extractTopicAttempt(group, topic) + case _: TopicPattern => + extractTopicAttemptFromPatternRetryTopic(group, topic, originalTopicHeader) + } + override def retryDecision[E]( retryAttempt: Option[RetryAttempt], record: ConsumerRecord[Chunk[Byte], Chunk[Byte]], error: E, subscription: ConsumerSubscription - )(implicit trace: Trace): URIO[Any, RetryDecision] = Clock.instant.map(now => { - val blockingRetriesBefore = RetryAttempt.maxBlockingAttempts( - NonBlockingRetryHelper.originalTopic(record.topic, group), - retryConfig - ).getOrElse(0) - - // attempt if present contains full number of retries - val nextNonBlockingAttempt = retryAttempt.fold(0)(_.attempt + 1 - blockingRetriesBefore) - val nextRetryAttempt = nextNonBlockingAttempt + blockingRetriesBefore + )(implicit trace: Trace): URIO[Any, RetryDecision] = currentTime.map(now => { + val nextRetryAttempt = retryAttempt.fold(0)(_.attempt + 1) val originalTopic = retryAttempt.fold(record.topic)(_.originalTopic) val retryTopic = subscription match { - case _: TopicPattern => patternRetryTopic(group, nextNonBlockingAttempt) - case _: Topics => fixedRetryTopic(originalTopic, group, nextNonBlockingAttempt) + case _: TopicPattern => patternRetryTopic(group, nextRetryAttempt) + case _: Topics => fixedRetryTopic(originalTopic, group, nextRetryAttempt) } val topicRetryPolicy = policy(record.topic) topicRetryPolicy.intervals - .lift(nextNonBlockingAttempt) + .lift(nextRetryAttempt) .map { backoff => - val attempt = RetryAttempt( - attempt = nextRetryAttempt, - originalTopic = originalTopic, - submittedAt = now, - backoff = backoff - ) topicRetryPolicy.recordMutate( ProducerRecord( topic = retryTopic, value = record.value, key = record.key, partition = None, - headers = record.headers ++ RetryAttempt.toHeaders(attempt) + headers = record.headers + + (RetryHeader.Submitted -> toChunk(now.toEpochMilli)) + + (RetryHeader.Backoff -> toChunk(backoff.toMillis)) + + (RetryHeader.OriginalTopic -> toChunk(originalTopic)) + + (RetryHeader.RetryAttempt -> toChunk(nextRetryAttempt)) ) ) } .fold[RetryDecision](NoMoreRetries)(RetryWith) }) - } - private[retry] def attemptNumberFromTopic( - subscription: ConsumerSubscription, - topic: Topic, - originalTopicHeader: Option[String], - group: Group - ) = - subscription match { - case _: Topics => extractTopicAttempt(group, topic) - case _: TopicPattern => - extractTopicAttemptFromPatternRetryTopic(group, topic, originalTopicHeader) + private def toChunk(long: Long): Chunk[Byte] = + Chunk.fromArray(long.toString.getBytes) + + private def toChunk(str: String): Chunk[Byte] = + Chunk.fromArray(str.getBytes) } - private def extractTopicAttempt(group: Group, inputTopic: Topic) = + private def extractTopicAttempt[E](group: Group, inputTopic: Topic) = inputTopic.split(s"-$group-retry-").toSeq match { case Seq(topic, attempt) if Try(attempt.toInt).isSuccess => Some(TopicAttempt(topic, attempt.toInt)) - case _ => None + case _ => None } - private def extractTopicAttemptFromPatternRetryTopic( + private def extractTopicAttemptFromPatternRetryTopic[E]( group: Group, inputTopic: Topic, originalTopicHeader: Option[String] @@ -145,27 +166,49 @@ object DelayHeaders { val Backoff = "backOffTimeMs" } -object RetryUtil { - def sleep(attempt: RetryAttempt)(implicit trace: Trace): URIO[GreyhoundMetrics, Unit] = - sleep(attempt.submittedAt, attempt.backoff) race - report(WaitingForRetry(attempt.originalTopic, attempt.attempt, attempt.submittedAt.toEpochMilli, attempt.backoff.toMillis)) - .repeat(spaced(60.seconds)) - .unit +object RetryHeader { + val Submitted = "submitTimestamp" + val Backoff = DelayHeaders.Backoff + val OriginalTopic = "GH_OriginalTopic" + val RetryAttempt = "GH_RetryAttempt" +} + +case class RetryAttempt( + originalTopic: Topic, + attempt: RetryAttemptNumber, + submittedAt: Instant, + backoff: Duration +) { + + def sleep(implicit trace: Trace): URIO[GreyhoundMetrics, Unit] = + RetryUtil.sleep(submittedAt, backoff) race reportWaitingInIntervals(every = 60.seconds) + private def reportWaitingInIntervals(every: Duration) = + report(WaitingForRetry(originalTopic, attempt, submittedAt.toEpochMilli, backoff.toMillis)) + .repeat(spaced(every)) + .unit +} + +object RetryUtil { def sleep(submittedAt: Instant, backoff: Duration)(implicit trace: Trace): URIO[Any, Unit] = { val expiresAt = submittedAt.plus(backoff.asJava) - Clock.instant + currentTime .map(_.isAfter(expiresAt)) .flatMap(expired => if (expired) ZIO.unit else - ZIO.sleep(1.second).repeatUntilZIO(_ => Clock.instant.map(_.isAfter(expiresAt))).unit + ZIO.sleep(1.seconds).repeatUntilZIO(_ => currentTime.map(_.isAfter(expiresAt))).unit ) } } private case class TopicAttempt(originalTopic: Topic, attempt: Int) +object RetryAttempt { + type RetryAttemptNumber = Int + val currentTime = Clock.currentTime(MILLISECONDS).map(Instant.ofEpochMilli) +} + sealed trait RetryDecision object RetryDecision { diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/NonBlockingRetryRecordHandler.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/NonBlockingRetryRecordHandler.scala index a6ff6560..a15f1e5b 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/NonBlockingRetryRecordHandler.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/NonBlockingRetryRecordHandler.scala @@ -31,12 +31,11 @@ private[retry] object NonBlockingRetryRecordHandler { retryConfig: RetryConfig, subscription: ConsumerSubscription, nonBlockingRetryHelper: NonBlockingRetryHelper, - groupId: Group, awaitShutdown: TopicPartition => UIO[AwaitShutdown] )(implicit evK: K <:< Chunk[Byte], evV: V <:< Chunk[Byte]): NonBlockingRetryRecordHandler[V, K, R] = new NonBlockingRetryRecordHandler[V, K, R] { override def handle(record: ConsumerRecord[K, V]): ZIO[GreyhoundMetrics with R, Nothing, Any] = { - RetryAttempt.extract(record.headers, record.topic, groupId, subscription, Some(retryConfig)).flatMap { retryAttempt => + nonBlockingRetryHelper.retryAttempt(record.topic, record.headers, subscription).flatMap { retryAttempt => maybeDelayRetry(record, retryAttempt) *> handler.handle(record).catchAll { case Right(_: NonRetriableException) => ZIO.unit @@ -57,7 +56,7 @@ private[retry] object NonBlockingRetryRecordHandler { WaitingBeforeRetry(record.topic, retryAttempt, record.partition, record.offset, correlationId) ) *> awaitShutdown(record.topicPartition) - .flatMap(_.interruptOnShutdown(RetryUtil.sleep(retryAttempt))) + .flatMap(_.interruptOnShutdown(retryAttempt.sleep)) .reporting(r => DoneWaitingBeforeRetry(record.topic, record.partition, record.offset, retryAttempt, r.duration, r.failed, correlationId) ) @@ -75,7 +74,7 @@ private[retry] object NonBlockingRetryRecordHandler { override def handleAfterBlockingFailed( record: ConsumerRecord[K, V] ): ZIO[GreyhoundMetrics with R, Nothing, Any] = { - RetryAttempt.extract(record.headers, record.topic, groupId, subscription, Some(retryConfig)).flatMap { retryAttempt => + nonBlockingRetryHelper.retryAttempt(record.topic, record.headers, subscription).flatMap { retryAttempt => maybeRetry(retryAttempt, BlockingHandlerFailed, record) } } diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryAttempt.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryAttempt.scala deleted file mode 100644 index 9265eab8..00000000 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryAttempt.scala +++ /dev/null @@ -1,98 +0,0 @@ -package com.wixpress.dst.greyhound.core.consumer.retry - -import com.wixpress.dst.greyhound.core.Serdes.StringSerde -import com.wixpress.dst.greyhound.core._ -import com.wixpress.dst.greyhound.core.consumer.domain.ConsumerSubscription -import com.wixpress.dst.greyhound.core.consumer.retry.NonBlockingRetryHelper.attemptNumberFromTopic -import com.wixpress.dst.greyhound.core.consumer.retry.RetryAttempt.RetryAttemptNumber -import zio._ - -import java.time.Instant - -/** - * Description of a retry attempt - * @param attempt contains which attempt is it, starting from 0 including blocking and non-blocking attempts - */ -case class RetryAttempt( - originalTopic: Topic, - attempt: RetryAttemptNumber, - submittedAt: Instant, - backoff: Duration -) - -object RetryHeader { - val Submitted = "submitTimestamp" - val Backoff = DelayHeaders.Backoff - val OriginalTopic = "GH_OriginalTopic" - val RetryAttempt = "GH_RetryAttempt" -} - -object RetryAttempt { - type RetryAttemptNumber = Int - - private def toChunk(str: String): Chunk[Byte] = Chunk.fromArray(str.getBytes) - - def toHeaders(attempt: RetryAttempt): Headers = Headers( - RetryHeader.Submitted -> toChunk(attempt.submittedAt.toEpochMilli.toString), - RetryHeader.Backoff -> toChunk(attempt.backoff.toMillis.toString), - RetryHeader.OriginalTopic -> toChunk(attempt.originalTopic), - RetryHeader.RetryAttempt -> toChunk(attempt.attempt.toString), - ) - - private case class RetryAttemptHeaders( - originalTopic: Option[Topic], - attempt: Option[RetryAttemptNumber], - submittedAt: Option[Instant], - backoff: Option[Duration] - ) - - private def fromHeaders(headers: Headers): Task[RetryAttemptHeaders] = - for { - submitted <- headers.get(RetryHeader.Submitted, instantDeserializer) - backoff <- headers.get(RetryHeader.Backoff, durationDeserializer) - topic <- headers.get[String](RetryHeader.OriginalTopic, StringSerde) - attempt <- headers.get(RetryHeader.RetryAttempt, longDeserializer) - } yield RetryAttemptHeaders(topic, attempt.map(_.toInt), submitted, backoff) - - /** @return None on infinite blocking retries */ - def maxBlockingAttempts(topic: Topic, retryConfig: Option[RetryConfig]): Option[Int] = - retryConfig.map(_.blockingBackoffs(topic)()).fold(Option(0)) { - case finite if finite.hasDefiniteSize => Some(finite.size) - case _ => None - } - - /** @return None on infinite retries */ - def maxOverallAttempts(topic: Topic, retryConfig: Option[RetryConfig]): Option[Int] = - maxBlockingAttempts(topic, retryConfig).map { - _ + retryConfig.fold(0)(_.nonBlockingBackoffs(topic).length) - } - - def extract( - headers: Headers, - topic: Topic, - group: Group, - subscription: ConsumerSubscription, - retryConfig: Option[RetryConfig], - )(implicit trace: Trace): UIO[Option[RetryAttempt]] = { - - def nonBlockingAttempt(hs: RetryAttemptHeaders): Option[RetryAttempt] = - for { - submitted <- hs.submittedAt - backoff <- hs.backoff - TopicAttempt(originalTopic, attempt) <- attemptNumberFromTopic(subscription, topic, hs.originalTopic, group) - blockingRetries = maxBlockingAttempts(originalTopic, retryConfig).getOrElse(0) - } yield RetryAttempt(originalTopic, blockingRetries + attempt, submitted, backoff) - - def blockingAttempt(hs: RetryAttemptHeaders): Option[RetryAttempt] = - for { - submitted <- hs.submittedAt - backoff <- hs.backoff - originalTopic <- hs.originalTopic - attempt <- hs.attempt - } yield RetryAttempt(originalTopic, attempt, submitted, backoff) - - fromHeaders(headers).map { hs => - nonBlockingAttempt(hs) orElse blockingAttempt(hs) - } - }.catchAll(_ => ZIO.none) -} diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryRecordHandler.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryRecordHandler.scala index 8ee8ab9f..af0749c6 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryRecordHandler.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryRecordHandler.scala @@ -33,7 +33,7 @@ object RetryRecordHandler { ): RecordHandler[R with R2 with GreyhoundMetrics, Nothing, K, V] = { val nonBlockingHandler = - NonBlockingRetryRecordHandler(handler, producer, retryConfig, subscription, nonBlockingRetryHelper, groupId, awaitShutdown) + NonBlockingRetryRecordHandler(handler, producer, retryConfig, subscription, nonBlockingRetryHelper, awaitShutdown) val blockingHandler = BlockingRetryRecordHandler(groupId, handler, retryConfig, blockingState, nonBlockingHandler, awaitShutdown) val blockingAndNonBlockingHandler = BlockingAndNonBlockingRetryRecordHandler(groupId, blockingHandler, nonBlockingHandler) @@ -55,4 +55,15 @@ object RetryRecordHandler { record.headers.get[String](key, StringSerde).catchAll(_ => ZIO.none) } +object ZIOHelper { + def foreachWhile[R, E, A](as: Iterable[A])(f: A => ZIO[R, E, LastHandleResult])(implicit trace: Trace): ZIO[R, E, LastHandleResult] = + ZIO.succeed(as.iterator).flatMap { i => + def loop: ZIO[R, E, LastHandleResult] = + if (i.hasNext) f(i.next).flatMap(result => if (result.shouldContinue) loop else ZIO.succeed(result)) + else ZIO.succeed(LastHandleResult(lastHandleSucceeded = false, shouldContinue = false)) + + loop + } +} + case class LastHandleResult(lastHandleSucceeded: Boolean, shouldContinue: Boolean) diff --git a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryAttemptTest.scala b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryAttemptTest.scala deleted file mode 100644 index 3139f845..00000000 --- a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryAttemptTest.scala +++ /dev/null @@ -1,78 +0,0 @@ -package com.wixpress.dst.greyhound.core.consumer.retry - -import com.wixpress.dst.greyhound.core.consumer.domain.ConsumerSubscription -import com.wixpress.dst.greyhound.core.testkit.BaseTest -import zio.test.TestEnvironment - -import java.time.{Duration, Instant} -import scala.util.Random -import scala.concurrent.duration._ - -class RetryAttemptTest extends BaseTest[TestEnvironment] { - - "RetryAttempt.extract" should { - "deserialize attempt from headers for blocking retries" in { - val attempt = randomRetryAttempt - val headers = RetryAttempt.toHeaders(attempt) - val subscription = ConsumerSubscription.Topics(Set(attempt.originalTopic)) - for (result <- RetryAttempt.extract(headers, attempt.originalTopic, randomStr, subscription, None)) - yield result must beSome(attempt) - } - "deserialize attempt from headers and topic for non-blocking retries" in { - val attempt = randomRetryAttempt - // topic and attempt must be extracted from retryTopic - val headers = RetryAttempt.toHeaders(attempt.copy(originalTopic = "", attempt = -1)) - val subscription = ConsumerSubscription.Topics(Set(attempt.originalTopic)) - val group = randomStr - val retryTopic = NonBlockingRetryHelper.fixedRetryTopic(attempt.originalTopic, group, attempt.attempt) - for (result <- RetryAttempt.extract(headers, retryTopic, group, subscription, None)) - yield result must beSome(attempt) - } - "deserialize attempt for non-blocking retry after blocking retries" in { - val attempt = randomRetryAttempt - val headers = RetryAttempt.toHeaders(attempt) - val subscription = ConsumerSubscription.Topics(Set(attempt.originalTopic)) - val group = randomStr - val retries = RetryConfig.blockingFollowedByNonBlockingRetry( - blockingBackoffs = 1.milli :: 1.second :: Nil, - nonBlockingBackoffs = 5.minutes :: Nil, - ) - val retryTopic = NonBlockingRetryHelper.fixedRetryTopic(attempt.originalTopic, group, attempt.attempt) - for (result <- RetryAttempt.extract(headers, retryTopic, group, subscription, Some(retries))) - yield result must beSome(attempt.copy(attempt = attempt.attempt + 2)) // with 2 blocking retries before - } - } - - "RetryAttempt.maxOverallAttempts" should { - "return 0 if no retries configured" in { - RetryAttempt.maxOverallAttempts(randomStr, None) must beSome(0) - } - "return max attempts for blocking retries" in { - val config = RetryConfig.finiteBlockingRetry(1.milli, 1.second) - RetryAttempt.maxOverallAttempts(randomStr, Some(config)) must beSome(2) - } - "return max attempts for non-blocking retries" in { - val config = RetryConfig.nonBlockingRetry(1.milli, 1.second, 5.minutes) - RetryAttempt.maxOverallAttempts(randomStr, Some(config)) must beSome(3) - } - "return max attempts for blocking retries followed by non-blocking" in { - val config = RetryConfig.blockingFollowedByNonBlockingRetry(1.milli :: 2.seconds :: Nil, 1.minute :: Nil) - RetryAttempt.maxOverallAttempts(randomStr, Some(config)) must beSome(3) - } - "return None for infinite blocking retries" in { - val config = RetryConfig.infiniteBlockingRetry(1.milli) - RetryAttempt.maxOverallAttempts(randomStr, Some(config)) must beNone - } - } - - override def env = testEnvironment - - private def randomStr = Random.alphanumeric.take(10).mkString - - private def randomRetryAttempt = RetryAttempt( - originalTopic = randomStr, - attempt = Random.nextInt(1000), - submittedAt = Instant.ofEpochMilli(math.abs(Random.nextLong())), - backoff = Duration.ofMillis(Random.nextInt(100000)) - ) -} diff --git a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryConsumerRecordHandlerTest.scala b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryConsumerRecordHandlerTest.scala index b34e8d82..c713fb12 100644 --- a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryConsumerRecordHandlerTest.scala +++ b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryConsumerRecordHandlerTest.scala @@ -4,7 +4,7 @@ import java.time.Instant import com.wixpress.dst.greyhound.core.Serdes._ import com.wixpress.dst.greyhound.core._ import com.wixpress.dst.greyhound.core.consumer.domain.ConsumerSubscription.Topics -import com.wixpress.dst.greyhound.core.consumer.domain.{ConsumerRecord, RecordHandler} +import com.wixpress.dst.greyhound.core.consumer.domain.{ConsumerRecord, ConsumerSubscription, RecordHandler} import com.wixpress.dst.greyhound.core.consumer.retry.BlockingState.{Blocked, Blocking => InternalBlocking, IgnoringAll, IgnoringOnce} import com.wixpress.dst.greyhound.core.consumer.retry.RetryConsumerRecordHandlerTest.{offset, partition, _} import com.wixpress.dst.greyhound.core.consumer.retry.RetryRecordHandlerMetric.{BlockingIgnoredForAllFor, BlockingIgnoredOnceFor, BlockingRetryHandlerInvocationFailed, NoRetryOnNonRetryableFailure} @@ -21,6 +21,8 @@ import zio.Random.{nextBytes, nextIntBounded} import zio.managed.UManaged import zio.test.TestClock +import scala.concurrent.TimeoutException + class RetryConsumerRecordHandlerTest extends BaseTest[TestClock with TestMetrics] { override def env: UManaged[ZEnvironment[TestClock with TestMetrics]] = @@ -50,6 +52,9 @@ class RetryConsumerRecordHandlerTest extends BaseTest[TestClock with TestMetrics _ <- retryHandler.handle(ConsumerRecord(topic, partition, offset, Headers.Empty, Some(key), value, 0L, 0L, 0L)) record <- producer.records.take now <- currentTime + retryAttempt <- IntSerde.serialize(retryTopic, 0) + submittedAt <- InstantSerde.serialize(retryTopic, now) + backoff <- DurationSerde.serialize(retryTopic, 1.second) } yield { record === ProducerRecord( @@ -57,7 +62,7 @@ class RetryConsumerRecordHandlerTest extends BaseTest[TestClock with TestMetrics value, Some(key), partition = None, - headers = RetryAttempt.toHeaders(RetryAttempt(topic, 0, now, 1.second)) + headers = Headers("retry-attempt" -> retryAttempt, "retry-submitted-at" -> submittedAt, "retry-backoff" -> backoff) ) } } @@ -81,7 +86,10 @@ class RetryConsumerRecordHandlerTest extends BaseTest[TestClock with TestMetrics ) value <- bytes begin <- currentTime - headers = RetryAttempt.toHeaders(RetryAttempt(topic, 0, begin, 1.second)) + retryAttempt <- IntSerde.serialize(retryTopic, 0) + submittedAt <- InstantSerde.serialize(retryTopic, begin) + backoff <- DurationSerde.serialize(retryTopic, 1.second) + headers = Headers("retry-attempt" -> retryAttempt, "retry-submitted-at" -> submittedAt, "retry-backoff" -> backoff) _ <- retryHandler.handle(ConsumerRecord(retryTopic, partition, offset, headers, None, value, 0L, 0L, 0L)).fork _ <- TestClock.adjust(1.second).repeat(Schedule.once) end <- executionTime.await.disconnect.timeoutFail(TimeoutWaitingForAssertion)(5.seconds) @@ -396,8 +404,7 @@ class RetryConsumerRecordHandlerTest extends BaseTest[TestClock with TestMetrics producer <- FakeProducer.make topic <- randomTopicName blockingState <- Ref.make[Map[BlockingTarget, BlockingState]](Map.empty) - retryHelper = FakeRetryHelper(topic) - now <- Clock.instant + retryHelper = alwaysBackOffRetryHelper(3.seconds) handling <- AwaitShutdown.makeManaged.flatMap { awaitShutdown => val retryHandler = RetryRecordHandler.withRetries( group, @@ -409,12 +416,11 @@ class RetryConsumerRecordHandlerTest extends BaseTest[TestClock with TestMetrics retryHelper, awaitShutdown = _ => ZIO.succeed(awaitShutdown) ) - val headers = RetryAttempt.toHeaders(RetryAttempt(topic, 0, now, 3.seconds)) for { key <- bytes value <- bytes handling <- retryHandler - .handle(ConsumerRecord(topic, partition, offset, headers, Some(key), value, 0L, 0L, 0L)) + .handle(ConsumerRecord(topic, partition, offset, Headers.Empty, Some(key), value, 0L, 0L, 0L)) .forkDaemon } yield handling } @@ -528,6 +534,18 @@ object RetryConsumerRecordHandlerTest { def randomTopicName = randomStr.map(suffix => s"some-topic-$suffix") val cause = new RuntimeException("cause") + + def alwaysBackOffRetryHelper(backoff: Duration) = { + new FakeNonBlockingRetryHelper { + override val topic: Topic = "" + + override def retryAttempt(topic: Topic, headers: Headers, subscription: ConsumerSubscription)( + implicit trace: Trace + ): UIO[Option[RetryAttempt]] = ZIO.succeed( + Some(RetryAttempt(topic, 1, Instant.now, backoff)) + ) + } + } } object TimeoutWaitingForAssertion extends RuntimeException diff --git a/core/src/test/scala/com/wixpress/dst/greyhound/core/testkit/FakeRetryHelper.scala b/core/src/test/scala/com/wixpress/dst/greyhound/core/testkit/FakeRetryHelper.scala index 619ada36..cee13948 100644 --- a/core/src/test/scala/com/wixpress/dst/greyhound/core/testkit/FakeRetryHelper.scala +++ b/core/src/test/scala/com/wixpress/dst/greyhound/core/testkit/FakeRetryHelper.scala @@ -1,21 +1,35 @@ package com.wixpress.dst.greyhound.core.testkit import java.time.Instant +import java.util.concurrent.TimeUnit.MILLISECONDS + +import com.wixpress.dst.greyhound.core.Serdes._ import com.wixpress.dst.greyhound.core._ import com.wixpress.dst.greyhound.core.consumer.domain.{ConsumerRecord, ConsumerSubscription} import com.wixpress.dst.greyhound.core.consumer.retry.RetryDecision.{NoMoreRetries, RetryWith} -import com.wixpress.dst.greyhound.core.consumer.retry.{BlockingHandlerFailed, NonBlockingRetryHelper, RetryAttempt, RetryDecision, RetryHeader} +import com.wixpress.dst.greyhound.core.consumer.retry.{BlockingHandlerFailed, NonBlockingRetryHelper, RetryAttempt, RetryDecision} import com.wixpress.dst.greyhound.core.producer.ProducerRecord import com.wixpress.dst.greyhound.core.testkit.FakeRetryHelper._ import zio._ import zio.Clock +import zio.Clock + trait FakeNonBlockingRetryHelper extends NonBlockingRetryHelper { val topic: Topic override def retryTopicsFor(originalTopic: Topic): Set[Topic] = Set(s"$originalTopic-retry") + override def retryAttempt(topic: Topic, headers: Headers, subscription: ConsumerSubscription)( + implicit trace: Trace + ): UIO[Option[RetryAttempt]] = + (for { + attempt <- headers.get(Header.Attempt, IntSerde) + submittedAt <- headers.get(Header.SubmittedAt, InstantSerde) + backoff <- headers.get(Header.Backoff, DurationSerde) + } yield retryAttemptInternal(topic, attempt, submittedAt, backoff)).orElse(ZIO.none) + override def retryDecision[E]( retryAttempt: Option[RetryAttempt], record: ConsumerRecord[Chunk[Byte], Chunk[Byte]], @@ -24,23 +38,35 @@ trait FakeNonBlockingRetryHelper extends NonBlockingRetryHelper { )(implicit trace: Trace): URIO[Any, RetryDecision] = error match { case RetriableError | BlockingHandlerFailed => - currentTime.map(now => - RetryWith(recordFrom(now, retryAttempt, record)) + currentTime.flatMap(now => + recordFrom(now, retryAttempt, record) + .fold(_ => NoMoreRetries, RetryWith) ) case NonRetriableError => ZIO.succeed(NoMoreRetries) } + private def retryAttemptInternal(topic: Topic, attempt: Option[Int], submittedAt: Option[Instant], backoff: Option[Duration]) = + for { + a <- attempt + s <- submittedAt + b <- backoff + } yield RetryAttempt(topic, a, s, b) + private def recordFrom(now: Instant, retryAttempt: Option[RetryAttempt], record: ConsumerRecord[Chunk[Byte], Chunk[Byte]])( implicit trace: Trace ) = { val nextRetryAttempt = retryAttempt.fold(0)(_.attempt + 1) - ProducerRecord( + for { + retryAttempt <- IntSerde.serialize(topic, nextRetryAttempt) + submittedAt <- InstantSerde.serialize(topic, now) + backoff <- DurationSerde.serialize(topic, 1.second) + } yield ProducerRecord( topic = s"$topic-retry", value = record.value, key = record.key, partition = None, - headers = RetryAttempt.toHeaders(RetryAttempt(topic, nextRetryAttempt, now, 1.second)) + headers = Headers(Header.Attempt -> retryAttempt, Header.SubmittedAt -> submittedAt, Header.Backoff -> backoff) ) } } @@ -49,8 +75,13 @@ case class FakeRetryHelper(topic: Topic) extends FakeNonBlockingRetryHelper object FakeRetryHelper { implicit private val trace = Trace.empty + object Header { + val Attempt = "retry-attempt" + val SubmittedAt = "retry-submitted-at" + val Backoff = "retry-backoff" + } - val currentTime: UIO[Instant] = Clock.instant + val currentTime = Clock.currentTime(MILLISECONDS).map(Instant.ofEpochMilli) } sealed trait HandlerError From 7abead28c2a50ff21a0026a18f355c69f2e7137d Mon Sep 17 00:00:00 2001 From: Noam Berman Date: Mon, 3 Apr 2023 17:26:15 +0300 Subject: [PATCH 05/17] [greyhound] remove internal topic creation - wix adapter (#33820) not create retry topics inside RecordConsumer builder, from wix adapter (it's already created). Removing last reference to an actual AdminClient in wix-adapter when asking to use proxy. GitOrigin-RevId: fb1416141b6b05559b1b606576cc03556b6a78f4 --- .../dst/greyhound/core/consumer/RecordConsumer.scala | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/RecordConsumer.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/RecordConsumer.scala index 883f17a8..1b426d60 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/RecordConsumer.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/RecordConsumer.scala @@ -79,7 +79,7 @@ object RecordConsumer { (initialSubscription, topicsToCreate) = config.retryConfig.fold((config.initialSubscription, Set.empty[Topic]))(policy => maybeAddRetryTopics(policy, config, nonBlockingRetryHelper) ) - _ <- AdminClient + _ <- ZIO.when(config.createRetryTopics)(AdminClient .make(AdminClientConfig(config.bootstrapServers, config.kafkaAuthProperties), config.consumerAttributes) .tap(client => client.createTopics( @@ -87,7 +87,7 @@ object RecordConsumer { TopicConfig(topic, partitions = 1, replicationFactor = 1, cleanupPolicy = CleanupPolicy.Delete(86400000L)) ) ) - ) + )) blockingState <- Ref.make[Map[BlockingTarget, BlockingState]](Map.empty) blockingStateResolver = BlockingStateResolver(blockingState) workersShutdownRef <- Ref.make[Map[TopicPartition, ShutdownPromise]](Map.empty) @@ -321,7 +321,8 @@ case class RecordConsumerConfig( decryptor: Decryptor[Any, Throwable, Chunk[Byte], Chunk[Byte]] = new NoOpDecryptor, retryProducerAttributes: Map[String, String] = Map.empty, commitMetadataString: Metadata = OffsetAndMetadata.NO_METADATA, - rewindUncommittedOffsetsBy: Duration = 0.millis + rewindUncommittedOffsetsBy: Duration = 0.millis, + createRetryTopics: Boolean = true ) extends CommonGreyhoundConfig { override def kafkaProps: Map[String, String] = extraProperties From fab840cbbb2ec8c682a88060abfd866b3e2c3319 Mon Sep 17 00:00:00 2001 From: Ben Wattelman <82799628+ben-wattelman@users.noreply.github.com> Date: Tue, 4 Apr 2023 12:03:59 +0300 Subject: [PATCH 06/17] [greyhound] parallel consumer OffsetsAndGaps (#33605) GitOrigin-RevId: f178c94663c7cbcb22bd7266c3e15919d8997d8c --- .../core/consumer/OffsetsAndGaps.scala | 92 +++++++++++++++++++ .../core/consumer/OffsetsAndGapsTest.scala | 62 +++++++++++++ 2 files changed, 154 insertions(+) create mode 100644 core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsAndGaps.scala create mode 100644 core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsAndGapsTest.scala diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsAndGaps.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsAndGaps.scala new file mode 100644 index 00000000..f5718555 --- /dev/null +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsAndGaps.scala @@ -0,0 +1,92 @@ +package com.wixpress.dst.greyhound.core.consumer + +import com.wixpress.dst.greyhound.core.{Offset, TopicPartition} +import zio._ + +trait OffsetsAndGaps { + def getCommittableAndClear: UIO[Map[TopicPartition, OffsetAndGaps]] + + def gapsForPartition(partition: TopicPartition): UIO[Seq[Gap]] + + def update(partition: TopicPartition, batch: Seq[Offset]): UIO[Unit] + + def contains(partition: TopicPartition, offset: Offset): UIO[Boolean] +} + +object OffsetsAndGaps { + def make: UIO[OffsetsAndGaps] = + Ref.make(Map.empty[TopicPartition, OffsetAndGaps]).map { ref => + new OffsetsAndGaps { + override def getCommittableAndClear: UIO[Map[TopicPartition, OffsetAndGaps]] = + ref.modify(offsetsAndGaps => { + val committable = offsetsAndGaps.filter(_._2.committable) + val updated = offsetsAndGaps.mapValues(_.markCommitted) + (committable, updated) + }) + + override def gapsForPartition(partition: TopicPartition): UIO[Seq[Gap]] = + ref.get.map(_.get(partition).fold(Seq.empty[Gap])(_.gaps.sortBy(_.start))) + + override def update(partition: TopicPartition, batch: Seq[Offset]): UIO[Unit] = + ref.update { offsetsAndGaps => + val sortedBatch = batch.sorted + val maxBatchOffset = sortedBatch.last + val maybeOffsetAndGaps = offsetsAndGaps.get(partition) + val prevOffset = maybeOffsetAndGaps.fold(-1L)(_.offset) + val partitionOffsetAndGaps = maybeOffsetAndGaps.fold(OffsetAndGaps(maxBatchOffset))(identity) + + val newGaps = gapsInBatch(sortedBatch, prevOffset) + + val updatedGaps = updateGapsByOffsets( + partitionOffsetAndGaps.gaps ++ newGaps, + sortedBatch + ) + + offsetsAndGaps + (partition -> OffsetAndGaps(maxBatchOffset max prevOffset, updatedGaps)) + }.unit + + override def contains(partition: TopicPartition, offset: Offset): UIO[Boolean] = + ref.get.map(_.get(partition).fold(false)(_.contains(offset))) + + private def gapsInBatch(batch: Seq[Offset], prevLastOffset: Offset): Seq[Gap] = + batch.sorted + .foldLeft(Seq.empty[Gap], prevLastOffset) { + case ((gaps, lastOffset), offset) => + if (offset <= lastOffset) (gaps, lastOffset) + else if (offset == lastOffset + 1) (gaps, offset) + else { + val newGap = Gap(lastOffset + 1, offset - 1) + (newGap +: gaps, offset) + } + } + ._1 + .reverse + + private def updateGapsByOffsets(gaps: Seq[Gap], offsets: Seq[Offset]): Seq[Gap] = { + val gapsToOffsets = gaps.map(gap => gap -> offsets.filter(o => o >= gap.start && o <= gap.end)).toMap + gapsToOffsets.flatMap { + case (gap, offsets) => + if (offsets.isEmpty) Seq(gap) + else if (offsets.size == (gap.size)) Seq.empty[Gap] + else gapsInBatch(offsets ++ Seq(gap.start - 1, gap.end + 1), gap.start - 2) + }.toSeq + } + } + } +} + +case class Gap(start: Offset, end: Offset) { + def contains(offset: Offset): Boolean = start <= offset && offset <= end + + def size: Long = end - start + 1 +} + +case class OffsetAndGaps(offset: Offset, gaps: Seq[Gap], committable: Boolean = true) { + def contains(offset: Offset): Boolean = gaps.exists(_.contains(offset)) + + def markCommitted: OffsetAndGaps = copy(committable = false) +} + +object OffsetAndGaps { + def apply(offset: Offset): OffsetAndGaps = OffsetAndGaps(offset, Seq.empty[Gap]) +} diff --git a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsAndGapsTest.scala b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsAndGapsTest.scala new file mode 100644 index 00000000..df765ce5 --- /dev/null +++ b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsAndGapsTest.scala @@ -0,0 +1,62 @@ +package com.wixpress.dst.greyhound.core.consumer + +import com.wixpress.dst.greyhound.core.TopicPartition +import com.wixpress.dst.greyhound.core.consumer.OffsetGapsTest._ +import com.wixpress.dst.greyhound.core.testkit.BaseTestNoEnv + +class OffsetsAndGapsTestGapsTest extends BaseTestNoEnv { + + "calculate gaps created by handled batch" in { + for { + offsetGaps <- OffsetsAndGaps.make + _ <- offsetGaps.update(topicPartition, Seq(1L, 3L, 7L)) + currentGaps <- offsetGaps.gapsForPartition(topicPartition) + } yield currentGaps must beEqualTo(Seq(Gap(0L, 0L), Gap(2L, 2L), Gap(4L, 6L))) + } + + "update offset and gaps according to handled batch" in { + for { + offsetGaps <- OffsetsAndGaps.make + _ <- offsetGaps.update(topicPartition, Seq(1L, 3L, 7L)) + _ <- offsetGaps.update(topicPartition, Seq(2L, 5L)) + getCommittableAndClear <- offsetGaps.getCommittableAndClear + } yield getCommittableAndClear must havePair(topicPartition -> OffsetAndGaps(7L, Seq(Gap(0L, 0L), Gap(4L, 4L), Gap(6L, 6L)))) + } + + "clear committable offsets" in { + for { + offsetGaps <- OffsetsAndGaps.make + _ <- offsetGaps.update(topicPartition, Seq(1L, 3L, 7L)) + _ <- offsetGaps.getCommittableAndClear + getCommittableAndClear <- offsetGaps.getCommittableAndClear + } yield getCommittableAndClear must beEmpty + } + + "do not clear gaps on retrieving current" in { + for { + offsetGaps <- OffsetsAndGaps.make + _ <- offsetGaps.update(topicPartition, Seq(1L, 3L, 7L)) + _ <- offsetGaps.gapsForPartition(topicPartition) + currentGaps <- offsetGaps.gapsForPartition(topicPartition) + } yield currentGaps must beEqualTo(Seq(Gap(0L, 0L), Gap(2L, 2L), Gap(4L, 6L))) + } + + "update with larger offset" in { + val partition0 = TopicPartition(topic, 0) + val partition1 = TopicPartition(topic, 1) + + for { + offsetGaps <- OffsetsAndGaps.make + _ <- offsetGaps.update(partition0, Seq(1L)) + _ <- offsetGaps.update(partition0, Seq(0L)) + _ <- offsetGaps.update(partition1, Seq(0L)) + current <- offsetGaps.getCommittableAndClear + } yield current must havePairs(partition0 -> OffsetAndGaps(1L, Seq()), partition1 -> OffsetAndGaps(0L, Seq())) + } + +} + +object OffsetGapsTest { + val topic = "some-topic" + val topicPartition = TopicPartition(topic, 0) +} From 16258017953a761c7b8cc02dc113ba4b1c59327a Mon Sep 17 00:00:00 2001 From: Natan Silnitsky Date: Tue, 4 Apr 2023 13:53:02 +0300 Subject: [PATCH 07/17] Blocking retries attempts tracking fix + fix transitive header bug (#33827) * Revert "Revert "Blocking retries attempts tracking fix" (#33818)" This reverts commit eb859fcf70860c06a0bd0492b746ed3ff00bc8ac. * fix bug on retryAttempt resolution - ignore propagated headers * CR change * fix test. change retry name to correctly structured one GitOrigin-RevId: 53fb7cf3efa315f496a00ade574397e30f20eeaf --- .../retry/BlockingRetryRecordHandler.scala | 47 +++++- .../retry/NonBlockingRetryHelper.scala | 137 ++++++------------ .../retry/NonBlockingRetryRecordHandler.scala | 7 +- .../core/consumer/retry/RetryAttempt.scala | 98 +++++++++++++ .../consumer/retry/RetryRecordHandler.scala | 13 +- .../consumer/retry/RetryAttemptTest.scala | 94 ++++++++++++ .../RetryConsumerRecordHandlerTest.scala | 35 ++--- .../core/testkit/FakeRetryHelper.scala | 43 +----- 8 files changed, 298 insertions(+), 176 deletions(-) create mode 100644 core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryAttempt.scala create mode 100644 core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryAttemptTest.scala diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/BlockingRetryRecordHandler.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/BlockingRetryRecordHandler.scala index 502e31e1..2f5d59a9 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/BlockingRetryRecordHandler.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/BlockingRetryRecordHandler.scala @@ -3,9 +3,8 @@ package com.wixpress.dst.greyhound.core.consumer.retry import java.util.concurrent.TimeUnit import com.wixpress.dst.greyhound.core.{Group, TopicPartition} import com.wixpress.dst.greyhound.core.consumer.domain.{ConsumerRecord, RecordHandler} -import com.wixpress.dst.greyhound.core.consumer.retry.BlockingState.{Blocked, Blocking => InternalBlocking, IgnoringOnce} +import com.wixpress.dst.greyhound.core.consumer.retry.BlockingState.{Blocked, IgnoringOnce, Blocking => InternalBlocking} import com.wixpress.dst.greyhound.core.consumer.retry.RetryRecordHandlerMetric.{BlockingRetryHandlerInvocationFailed, DoneBlockingBeforeRetry, NoRetryOnNonRetryableFailure} -import com.wixpress.dst.greyhound.core.consumer.retry.ZIOHelper.foreachWhile import com.wixpress.dst.greyhound.core.metrics.GreyhoundMetrics import com.wixpress.dst.greyhound.core.metrics.GreyhoundMetrics.report import com.wixpress.dst.greyhound.core.zioutils.AwaitShutdown @@ -31,7 +30,7 @@ private[retry] object BlockingRetryRecordHandler { override def handle(record: ConsumerRecord[K, V])(implicit trace: Trace): ZIO[GreyhoundMetrics with R, Nothing, LastHandleResult] = { val topicPartition = TopicPartition(record.topic, record.partition) - def pollBlockingStateWithSuspensions(interval: Duration, start: Long): URIO[GreyhoundMetrics, PollResult] = { + def pollBlockingStateWithSuspensions(record: ConsumerRecord[K, V], interval: Duration, start: Long): URIO[GreyhoundMetrics, PollResult] = { for { shouldBlock <- blockingStateResolver.resolve(record) shouldPollAgain <- @@ -43,14 +42,14 @@ private[retry] object BlockingRetryRecordHandler { } yield shouldPollAgain } - def blockOnErrorFor(interval: Duration) = { + def blockOnErrorFor(record: ConsumerRecord[K, V], interval: Duration) = { for { start <- currentTime(TimeUnit.MILLISECONDS) continueBlocking <- if (interval.toMillis > 100L) { awaitShutdown(record.topicPartition).flatMap( _.interruptOnShutdown( - pollBlockingStateWithSuspensions(interval, start).repeatWhile(result => result.pollAgain).map(_.blockHandling) + pollBlockingStateWithSuspensions(record, interval, start).repeatWhile(result => result.pollAgain).map(_.blockHandling) ).reporting(r => DoneBlockingBeforeRetry(record.topic, record.partition, record.offset, r.duration, r.failed)) ) } else { @@ -63,6 +62,7 @@ private[retry] object BlockingRetryRecordHandler { } def handleAndMaybeBlockOnErrorFor( + record: ConsumerRecord[K, V], interval: Option[Duration] ): ZIO[R with GreyhoundMetrics, Nothing, LastHandleResult] = { handler.handle(record).map(_ => LastHandleResult(lastHandleSucceeded = true, shouldContinue = false)).catchAll { @@ -73,7 +73,7 @@ private[retry] object BlockingRetryRecordHandler { case error => interval .map { interval => - report(BlockingRetryHandlerInvocationFailed(topicPartition, record.offset, error.toString)) *> blockOnErrorFor(interval) + report(BlockingRetryHandlerInvocationFailed(topicPartition, record.offset, error.toString)) *> blockOnErrorFor(record, interval) } .getOrElse(ZIO.succeed(LastHandleResult(lastHandleSucceeded = false, shouldContinue = false))) } @@ -96,13 +96,44 @@ private[retry] object BlockingRetryRecordHandler { } else { val durationsIncludingForInvocationWithNoErrorHandling = retryConfig.blockingBackoffs(record.topic)().map(Some(_)) :+ None for { - result <- foreachWhile(durationsIncludingForInvocationWithNoErrorHandling) { interval => handleAndMaybeBlockOnErrorFor(interval) } - _ <- maybeBackToStateBlocking + result <- retryEvery(record, durationsIncludingForInvocationWithNoErrorHandling) { (rec, interval) => + handleAndMaybeBlockOnErrorFor(rec, interval) + } + _ <- maybeBackToStateBlocking } yield result } } } + private def retryEvery[K, V, R, E](record: ConsumerRecord[K, V], as: Iterable[Option[Duration]])( + f: (ConsumerRecord[K, V], Option[Duration]) => ZIO[R, E, LastHandleResult] + )(implicit trace: Trace): ZIO[R, E, LastHandleResult] = { + ZIO.succeed(as.iterator).flatMap { i => + def loop(retryAttempt: Option[RetryAttempt]): ZIO[R, E, LastHandleResult] = + if (i.hasNext) { + val nextDelay = i.next + val recordWithAttempt = retryAttempt.fold(record) { attempt => + record.copy(headers = record.headers ++ RetryAttempt.toHeaders(attempt)) + } + f(recordWithAttempt, nextDelay).flatMap { result => + if (result.shouldContinue) Clock.instant.flatMap { now => + val nextAttempt = RetryAttempt( + originalTopic = record.topic, + attempt = retryAttempt.fold(0)(_.attempt + 1), + submittedAt = now, + backoff = nextDelay getOrElse Duration.Zero + ) + loop(Some(nextAttempt)) + } + else ZIO.succeed(result) + } + } + else ZIO.succeed(LastHandleResult(lastHandleSucceeded = false, shouldContinue = false)) + + loop(None) + } + } + private def handleNonRetriable[K, V, E, R](record: ConsumerRecord[K, V], topicPartition: TopicPartition, cause: Exception) = report(NoRetryOnNonRetryableFailure(topicPartition, record.offset, cause)) .as(LastHandleResult(lastHandleSucceeded = false, shouldContinue = false)) diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/NonBlockingRetryHelper.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/NonBlockingRetryHelper.scala index a73b7dce..1d1cd24c 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/NonBlockingRetryHelper.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/NonBlockingRetryHelper.scala @@ -1,32 +1,23 @@ package com.wixpress.dst.greyhound.core.consumer.retry -import java.time.{Duration => JavaDuration, Instant} -import java.util.concurrent.TimeUnit.MILLISECONDS -import java.util.regex.Pattern -import com.wixpress.dst.greyhound.core.Serdes.StringSerde import com.wixpress.dst.greyhound.core.consumer.domain.ConsumerSubscription.{TopicPattern, Topics} -import com.wixpress.dst.greyhound.core.consumer.retry.RetryAttempt.{currentTime, RetryAttemptNumber} -import com.wixpress.dst.greyhound.core.consumer.retry.RetryDecision.{NoMoreRetries, RetryWith} import com.wixpress.dst.greyhound.core.consumer.domain.{ConsumerRecord, ConsumerSubscription} +import com.wixpress.dst.greyhound.core.consumer.retry.RetryDecision.{NoMoreRetries, RetryWith} import com.wixpress.dst.greyhound.core.consumer.retry.RetryRecordHandlerMetric.WaitingForRetry import com.wixpress.dst.greyhound.core.metrics.GreyhoundMetrics import com.wixpress.dst.greyhound.core.metrics.GreyhoundMetrics.report import com.wixpress.dst.greyhound.core.producer.ProducerRecord -import com.wixpress.dst.greyhound.core.{durationDeserializer, instantDeserializer, Group, Headers, Topic} -import zio.Clock -import zio.Duration +import com.wixpress.dst.greyhound.core.{Group, Topic} import zio.Schedule.spaced -import zio.{Chunk, UIO, URIO, _} +import zio.{Chunk, Clock, Duration, URIO, _} +import java.time.Instant +import java.util.regex.Pattern import scala.util.Try trait NonBlockingRetryHelper { def retryTopicsFor(originalTopic: Topic): Set[Topic] - def retryAttempt(topic: Topic, headers: Headers, subscription: ConsumerSubscription)( - implicit trace: Trace - ): UIO[Option[RetryAttempt]] - def retryDecision[E]( retryAttempt: Option[RetryAttempt], record: ConsumerRecord[Chunk[Byte], Chunk[Byte]], @@ -34,7 +25,7 @@ trait NonBlockingRetryHelper { subscription: ConsumerSubscription )(implicit trace: Trace): URIO[Any, RetryDecision] - def retrySteps = retryTopicsFor("").size + def retrySteps: Int = retryTopicsFor("").size } object NonBlockingRetryHelper { @@ -49,82 +40,70 @@ object NonBlockingRetryHelper { override def retryTopicsFor(topic: Topic): Set[Topic] = policy(topic).intervals.indices.foldLeft(Set.empty[String])((acc, attempt) => acc + s"$topic-$group-retry-$attempt") - override def retryAttempt(topic: Topic, headers: Headers, subscription: ConsumerSubscription)( - implicit trace: Trace - ): UIO[Option[RetryAttempt]] = { - (for { - submitted <- headers.get(RetryHeader.Submitted, instantDeserializer) - backoff <- headers.get(RetryHeader.Backoff, durationDeserializer) - originalTopic <- headers.get[String](RetryHeader.OriginalTopic, StringSerde) - } yield for { - ta <- topicAttempt(subscription, topic, originalTopic) - TopicAttempt(originalTopic, attempt) = ta - s <- submitted - b <- backoff - } yield RetryAttempt(originalTopic, attempt, s, b)) - .catchAll(_ => ZIO.none) - } - - private def topicAttempt( - subscription: ConsumerSubscription, - topic: Topic, - originalTopicHeader: Option[String] - ) = - subscription match { - case _: Topics => extractTopicAttempt(group, topic) - case _: TopicPattern => - extractTopicAttemptFromPatternRetryTopic(group, topic, originalTopicHeader) - } - override def retryDecision[E]( retryAttempt: Option[RetryAttempt], record: ConsumerRecord[Chunk[Byte], Chunk[Byte]], error: E, subscription: ConsumerSubscription - )(implicit trace: Trace): URIO[Any, RetryDecision] = currentTime.map(now => { - val nextRetryAttempt = retryAttempt.fold(0)(_.attempt + 1) + )(implicit trace: Trace): URIO[Any, RetryDecision] = Clock.instant.map(now => { + val blockingRetriesBefore = RetryAttempt.maxBlockingAttempts( + NonBlockingRetryHelper.originalTopic(record.topic, group), + retryConfig + ).getOrElse(0) + + // attempt if present contains full number of retries + val nextNonBlockingAttempt = retryAttempt.fold(0)(_.attempt + 1 - blockingRetriesBefore) + val nextRetryAttempt = nextNonBlockingAttempt + blockingRetriesBefore val originalTopic = retryAttempt.fold(record.topic)(_.originalTopic) val retryTopic = subscription match { - case _: TopicPattern => patternRetryTopic(group, nextRetryAttempt) - case _: Topics => fixedRetryTopic(originalTopic, group, nextRetryAttempt) + case _: TopicPattern => patternRetryTopic(group, nextNonBlockingAttempt) + case _: Topics => fixedRetryTopic(originalTopic, group, nextNonBlockingAttempt) } val topicRetryPolicy = policy(record.topic) topicRetryPolicy.intervals - .lift(nextRetryAttempt) + .lift(nextNonBlockingAttempt) .map { backoff => + val attempt = RetryAttempt( + attempt = nextRetryAttempt, + originalTopic = originalTopic, + submittedAt = now, + backoff = backoff + ) topicRetryPolicy.recordMutate( ProducerRecord( topic = retryTopic, value = record.value, key = record.key, partition = None, - headers = record.headers + - (RetryHeader.Submitted -> toChunk(now.toEpochMilli)) + - (RetryHeader.Backoff -> toChunk(backoff.toMillis)) + - (RetryHeader.OriginalTopic -> toChunk(originalTopic)) + - (RetryHeader.RetryAttempt -> toChunk(nextRetryAttempt)) + headers = record.headers ++ RetryAttempt.toHeaders(attempt) ) ) } .fold[RetryDecision](NoMoreRetries)(RetryWith) }) + } - private def toChunk(long: Long): Chunk[Byte] = - Chunk.fromArray(long.toString.getBytes) - - private def toChunk(str: String): Chunk[Byte] = - Chunk.fromArray(str.getBytes) + private[retry] def attemptNumberFromTopic( + subscription: ConsumerSubscription, + topic: Topic, + originalTopicHeader: Option[String], + group: Group + ) = + subscription match { + case _: Topics => extractTopicAttempt(group, topic) + case _: TopicPattern => + extractTopicAttemptFromPatternRetryTopic(group, topic, originalTopicHeader) } - private def extractTopicAttempt[E](group: Group, inputTopic: Topic) = + private def extractTopicAttempt(group: Group, inputTopic: Topic) = inputTopic.split(s"-$group-retry-").toSeq match { case Seq(topic, attempt) if Try(attempt.toInt).isSuccess => Some(TopicAttempt(topic, attempt.toInt)) - case _ => None + case _ => None } - private def extractTopicAttemptFromPatternRetryTopic[E]( + private def extractTopicAttemptFromPatternRetryTopic( group: Group, inputTopic: Topic, originalTopicHeader: Option[String] @@ -166,49 +145,27 @@ object DelayHeaders { val Backoff = "backOffTimeMs" } -object RetryHeader { - val Submitted = "submitTimestamp" - val Backoff = DelayHeaders.Backoff - val OriginalTopic = "GH_OriginalTopic" - val RetryAttempt = "GH_RetryAttempt" -} - -case class RetryAttempt( - originalTopic: Topic, - attempt: RetryAttemptNumber, - submittedAt: Instant, - backoff: Duration -) { - - def sleep(implicit trace: Trace): URIO[GreyhoundMetrics, Unit] = - RetryUtil.sleep(submittedAt, backoff) race reportWaitingInIntervals(every = 60.seconds) - - private def reportWaitingInIntervals(every: Duration) = - report(WaitingForRetry(originalTopic, attempt, submittedAt.toEpochMilli, backoff.toMillis)) - .repeat(spaced(every)) - .unit -} - object RetryUtil { + def sleep(attempt: RetryAttempt)(implicit trace: Trace): URIO[GreyhoundMetrics, Unit] = + sleep(attempt.submittedAt, attempt.backoff) race + report(WaitingForRetry(attempt.originalTopic, attempt.attempt, attempt.submittedAt.toEpochMilli, attempt.backoff.toMillis)) + .repeat(spaced(60.seconds)) + .unit + def sleep(submittedAt: Instant, backoff: Duration)(implicit trace: Trace): URIO[Any, Unit] = { val expiresAt = submittedAt.plus(backoff.asJava) - currentTime + Clock.instant .map(_.isAfter(expiresAt)) .flatMap(expired => if (expired) ZIO.unit else - ZIO.sleep(1.seconds).repeatUntilZIO(_ => currentTime.map(_.isAfter(expiresAt))).unit + ZIO.sleep(1.second).repeatUntilZIO(_ => Clock.instant.map(_.isAfter(expiresAt))).unit ) } } private case class TopicAttempt(originalTopic: Topic, attempt: Int) -object RetryAttempt { - type RetryAttemptNumber = Int - val currentTime = Clock.currentTime(MILLISECONDS).map(Instant.ofEpochMilli) -} - sealed trait RetryDecision object RetryDecision { diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/NonBlockingRetryRecordHandler.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/NonBlockingRetryRecordHandler.scala index a15f1e5b..a6ff6560 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/NonBlockingRetryRecordHandler.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/NonBlockingRetryRecordHandler.scala @@ -31,11 +31,12 @@ private[retry] object NonBlockingRetryRecordHandler { retryConfig: RetryConfig, subscription: ConsumerSubscription, nonBlockingRetryHelper: NonBlockingRetryHelper, + groupId: Group, awaitShutdown: TopicPartition => UIO[AwaitShutdown] )(implicit evK: K <:< Chunk[Byte], evV: V <:< Chunk[Byte]): NonBlockingRetryRecordHandler[V, K, R] = new NonBlockingRetryRecordHandler[V, K, R] { override def handle(record: ConsumerRecord[K, V]): ZIO[GreyhoundMetrics with R, Nothing, Any] = { - nonBlockingRetryHelper.retryAttempt(record.topic, record.headers, subscription).flatMap { retryAttempt => + RetryAttempt.extract(record.headers, record.topic, groupId, subscription, Some(retryConfig)).flatMap { retryAttempt => maybeDelayRetry(record, retryAttempt) *> handler.handle(record).catchAll { case Right(_: NonRetriableException) => ZIO.unit @@ -56,7 +57,7 @@ private[retry] object NonBlockingRetryRecordHandler { WaitingBeforeRetry(record.topic, retryAttempt, record.partition, record.offset, correlationId) ) *> awaitShutdown(record.topicPartition) - .flatMap(_.interruptOnShutdown(retryAttempt.sleep)) + .flatMap(_.interruptOnShutdown(RetryUtil.sleep(retryAttempt))) .reporting(r => DoneWaitingBeforeRetry(record.topic, record.partition, record.offset, retryAttempt, r.duration, r.failed, correlationId) ) @@ -74,7 +75,7 @@ private[retry] object NonBlockingRetryRecordHandler { override def handleAfterBlockingFailed( record: ConsumerRecord[K, V] ): ZIO[GreyhoundMetrics with R, Nothing, Any] = { - nonBlockingRetryHelper.retryAttempt(record.topic, record.headers, subscription).flatMap { retryAttempt => + RetryAttempt.extract(record.headers, record.topic, groupId, subscription, Some(retryConfig)).flatMap { retryAttempt => maybeRetry(retryAttempt, BlockingHandlerFailed, record) } } diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryAttempt.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryAttempt.scala new file mode 100644 index 00000000..3fa3cba8 --- /dev/null +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryAttempt.scala @@ -0,0 +1,98 @@ +package com.wixpress.dst.greyhound.core.consumer.retry + +import com.wixpress.dst.greyhound.core.Serdes.StringSerde +import com.wixpress.dst.greyhound.core._ +import com.wixpress.dst.greyhound.core.consumer.domain.ConsumerSubscription +import com.wixpress.dst.greyhound.core.consumer.retry.NonBlockingRetryHelper.attemptNumberFromTopic +import com.wixpress.dst.greyhound.core.consumer.retry.RetryAttempt.RetryAttemptNumber +import zio._ + +import java.time.Instant + +/** + * Description of a retry attempt + * @param attempt contains which attempt is it, starting from 0 including blocking and non-blocking attempts + */ +case class RetryAttempt( + originalTopic: Topic, + attempt: RetryAttemptNumber, + submittedAt: Instant, + backoff: Duration +) + +object RetryHeader { + val Submitted = "submitTimestamp" + val Backoff = DelayHeaders.Backoff + val OriginalTopic = "GH_OriginalTopic" + val RetryAttempt = "GH_RetryAttempt" +} + +object RetryAttempt { + type RetryAttemptNumber = Int + + private def toChunk(str: String): Chunk[Byte] = Chunk.fromArray(str.getBytes) + + def toHeaders(attempt: RetryAttempt): Headers = Headers( + RetryHeader.Submitted -> toChunk(attempt.submittedAt.toEpochMilli.toString), + RetryHeader.Backoff -> toChunk(attempt.backoff.toMillis.toString), + RetryHeader.OriginalTopic -> toChunk(attempt.originalTopic), + RetryHeader.RetryAttempt -> toChunk(attempt.attempt.toString), + ) + + private case class RetryAttemptHeaders( + originalTopic: Option[Topic], + attempt: Option[RetryAttemptNumber], + submittedAt: Option[Instant], + backoff: Option[Duration] + ) + + private def fromHeaders(headers: Headers): Task[RetryAttemptHeaders] = + for { + submitted <- headers.get(RetryHeader.Submitted, instantDeserializer) + backoff <- headers.get(RetryHeader.Backoff, durationDeserializer) + topic <- headers.get[String](RetryHeader.OriginalTopic, StringSerde) + attempt <- headers.get(RetryHeader.RetryAttempt, longDeserializer) + } yield RetryAttemptHeaders(topic, attempt.map(_.toInt), submitted, backoff) + + /** @return None on infinite blocking retries */ + def maxBlockingAttempts(topic: Topic, retryConfig: Option[RetryConfig]): Option[Int] = + retryConfig.map(_.blockingBackoffs(topic)()).fold(Option(0)) { + case finite if finite.hasDefiniteSize => Some(finite.size) + case _ => None + } + + /** @return None on infinite retries */ + def maxOverallAttempts(topic: Topic, retryConfig: Option[RetryConfig]): Option[Int] = + maxBlockingAttempts(topic, retryConfig).map { + _ + retryConfig.fold(0)(_.nonBlockingBackoffs(topic).length) + } + + def extract( + headers: Headers, + topic: Topic, + group: Group, + subscription: ConsumerSubscription, + retryConfig: Option[RetryConfig], + )(implicit trace: Trace): UIO[Option[RetryAttempt]] = { + + def maybeNonBlockingAttempt(hs: RetryAttemptHeaders): Option[RetryAttempt] = + for { + submitted <- hs.submittedAt + backoff <- hs.backoff + TopicAttempt(originalTopic, attempt) <- attemptNumberFromTopic(subscription, topic, hs.originalTopic, group) + blockingRetries = maxBlockingAttempts(originalTopic, retryConfig).getOrElse(0) + } yield RetryAttempt(originalTopic, blockingRetries + attempt, submitted, backoff) + + def maybeBlockingAttempt(hs: RetryAttemptHeaders): Option[RetryAttempt] = + for { + submitted <- hs.submittedAt + backoff <- hs.backoff + originalTopic <- hs.originalTopic if originalTopic == topic + attempt <- hs.attempt + } yield RetryAttempt(originalTopic, attempt, submitted, backoff) + + fromHeaders(headers).map { hs => + maybeNonBlockingAttempt(hs) orElse maybeBlockingAttempt(hs) + } + }.catchAll(_ => ZIO.none) +} diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryRecordHandler.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryRecordHandler.scala index af0749c6..8ee8ab9f 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryRecordHandler.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryRecordHandler.scala @@ -33,7 +33,7 @@ object RetryRecordHandler { ): RecordHandler[R with R2 with GreyhoundMetrics, Nothing, K, V] = { val nonBlockingHandler = - NonBlockingRetryRecordHandler(handler, producer, retryConfig, subscription, nonBlockingRetryHelper, awaitShutdown) + NonBlockingRetryRecordHandler(handler, producer, retryConfig, subscription, nonBlockingRetryHelper, groupId, awaitShutdown) val blockingHandler = BlockingRetryRecordHandler(groupId, handler, retryConfig, blockingState, nonBlockingHandler, awaitShutdown) val blockingAndNonBlockingHandler = BlockingAndNonBlockingRetryRecordHandler(groupId, blockingHandler, nonBlockingHandler) @@ -55,15 +55,4 @@ object RetryRecordHandler { record.headers.get[String](key, StringSerde).catchAll(_ => ZIO.none) } -object ZIOHelper { - def foreachWhile[R, E, A](as: Iterable[A])(f: A => ZIO[R, E, LastHandleResult])(implicit trace: Trace): ZIO[R, E, LastHandleResult] = - ZIO.succeed(as.iterator).flatMap { i => - def loop: ZIO[R, E, LastHandleResult] = - if (i.hasNext) f(i.next).flatMap(result => if (result.shouldContinue) loop else ZIO.succeed(result)) - else ZIO.succeed(LastHandleResult(lastHandleSucceeded = false, shouldContinue = false)) - - loop - } -} - case class LastHandleResult(lastHandleSucceeded: Boolean, shouldContinue: Boolean) diff --git a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryAttemptTest.scala b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryAttemptTest.scala new file mode 100644 index 00000000..9e27c10b --- /dev/null +++ b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryAttemptTest.scala @@ -0,0 +1,94 @@ +package com.wixpress.dst.greyhound.core.consumer.retry + +import com.wixpress.dst.greyhound.core.consumer.domain.ConsumerSubscription +import com.wixpress.dst.greyhound.core.testkit.BaseTest +import zio.test.TestEnvironment + +import java.time.{Duration, Instant} +import scala.util.Random +import scala.concurrent.duration._ + +class RetryAttemptTest extends BaseTest[TestEnvironment] { + + "RetryAttempt.extract" should { + "deserialize attempt from headers for blocking retries" in { + val attempt = randomRetryAttempt + val headers = RetryAttempt.toHeaders(attempt) + val subscription = ConsumerSubscription.Topics(Set(attempt.originalTopic)) + for (result <- RetryAttempt.extract(headers, attempt.originalTopic, randomStr, subscription, None)) + yield result must beSome(attempt) + } + "deserialize attempt from headers and topic for non-blocking retries" in { + val attempt = randomRetryAttempt + // topic and attempt must be extracted from retryTopic + val headers = RetryAttempt.toHeaders(attempt.copy(originalTopic = "", attempt = -1)) + val subscription = ConsumerSubscription.Topics(Set(attempt.originalTopic)) + val group = randomStr + val retryTopic = NonBlockingRetryHelper.fixedRetryTopic(attempt.originalTopic, group, attempt.attempt) + for (result <- RetryAttempt.extract(headers, retryTopic, group, subscription, None)) + yield result must beSome(attempt) + } + "deserialize attempt for non-blocking retry after blocking retries" in { + val attempt = randomRetryAttempt + val headers = RetryAttempt.toHeaders(attempt) + val subscription = ConsumerSubscription.Topics(Set(attempt.originalTopic)) + val group = randomStr + val retries = RetryConfig.blockingFollowedByNonBlockingRetry( + blockingBackoffs = 1.milli :: 1.second :: Nil, + nonBlockingBackoffs = 5.minutes :: Nil, + ) + val retryTopic = NonBlockingRetryHelper.fixedRetryTopic(attempt.originalTopic, group, attempt.attempt) + for (result <- RetryAttempt.extract(headers, retryTopic, group, subscription, Some(retries))) + yield result must beSome(attempt.copy(attempt = attempt.attempt + 2)) // with 2 blocking retries before + } + "In case incorrect originalTopic header propagated from a different consumer group, ignore it," + + "do NOT consider it as if it's a non-blocking retry" in { + val attempt = propagatedRetryAttempt + val headers = RetryAttempt.toHeaders(attempt) + val currentTopic = "relevant-topic" + val subscription = ConsumerSubscription.Topics(Set(currentTopic)) + for (result <- RetryAttempt.extract(headers, currentTopic, randomStr, subscription, None)) + yield result must beNone + } + } + + "RetryAttempt.maxOverallAttempts" should { + "return 0 if no retries configured" in { + RetryAttempt.maxOverallAttempts(randomStr, None) must beSome(0) + } + "return max attempts for blocking retries" in { + val config = RetryConfig.finiteBlockingRetry(1.milli, 1.second) + RetryAttempt.maxOverallAttempts(randomStr, Some(config)) must beSome(2) + } + "return max attempts for non-blocking retries" in { + val config = RetryConfig.nonBlockingRetry(1.milli, 1.second, 5.minutes) + RetryAttempt.maxOverallAttempts(randomStr, Some(config)) must beSome(3) + } + "return max attempts for blocking retries followed by non-blocking" in { + val config = RetryConfig.blockingFollowedByNonBlockingRetry(1.milli :: 2.seconds :: Nil, 1.minute :: Nil) + RetryAttempt.maxOverallAttempts(randomStr, Some(config)) must beSome(3) + } + "return None for infinite blocking retries" in { + val config = RetryConfig.infiniteBlockingRetry(1.milli) + RetryAttempt.maxOverallAttempts(randomStr, Some(config)) must beNone + } + } + + override def env = testEnvironment + + private def randomStr = Random.alphanumeric.take(10).mkString + + private def randomRetryAttempt = RetryAttempt( + originalTopic = randomStr, + attempt = Random.nextInt(1000), + submittedAt = Instant.ofEpochMilli(math.abs(Random.nextLong())), + backoff = Duration.ofMillis(Random.nextInt(100000)) + ) + + private def propagatedRetryAttempt = RetryAttempt( + originalTopic = "some-other-topic", + attempt = Random.nextInt(1000), + submittedAt = Instant.ofEpochMilli(math.abs(Random.nextLong())), + backoff = Duration.ofMillis(Random.nextInt(100000)) + ) +} diff --git a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryConsumerRecordHandlerTest.scala b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryConsumerRecordHandlerTest.scala index c713fb12..0797ba25 100644 --- a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryConsumerRecordHandlerTest.scala +++ b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryConsumerRecordHandlerTest.scala @@ -4,7 +4,7 @@ import java.time.Instant import com.wixpress.dst.greyhound.core.Serdes._ import com.wixpress.dst.greyhound.core._ import com.wixpress.dst.greyhound.core.consumer.domain.ConsumerSubscription.Topics -import com.wixpress.dst.greyhound.core.consumer.domain.{ConsumerRecord, ConsumerSubscription, RecordHandler} +import com.wixpress.dst.greyhound.core.consumer.domain.{ConsumerRecord, RecordHandler} import com.wixpress.dst.greyhound.core.consumer.retry.BlockingState.{Blocked, Blocking => InternalBlocking, IgnoringAll, IgnoringOnce} import com.wixpress.dst.greyhound.core.consumer.retry.RetryConsumerRecordHandlerTest.{offset, partition, _} import com.wixpress.dst.greyhound.core.consumer.retry.RetryRecordHandlerMetric.{BlockingIgnoredForAllFor, BlockingIgnoredOnceFor, BlockingRetryHandlerInvocationFailed, NoRetryOnNonRetryableFailure} @@ -21,8 +21,6 @@ import zio.Random.{nextBytes, nextIntBounded} import zio.managed.UManaged import zio.test.TestClock -import scala.concurrent.TimeoutException - class RetryConsumerRecordHandlerTest extends BaseTest[TestClock with TestMetrics] { override def env: UManaged[ZEnvironment[TestClock with TestMetrics]] = @@ -52,9 +50,6 @@ class RetryConsumerRecordHandlerTest extends BaseTest[TestClock with TestMetrics _ <- retryHandler.handle(ConsumerRecord(topic, partition, offset, Headers.Empty, Some(key), value, 0L, 0L, 0L)) record <- producer.records.take now <- currentTime - retryAttempt <- IntSerde.serialize(retryTopic, 0) - submittedAt <- InstantSerde.serialize(retryTopic, now) - backoff <- DurationSerde.serialize(retryTopic, 1.second) } yield { record === ProducerRecord( @@ -62,7 +57,7 @@ class RetryConsumerRecordHandlerTest extends BaseTest[TestClock with TestMetrics value, Some(key), partition = None, - headers = Headers("retry-attempt" -> retryAttempt, "retry-submitted-at" -> submittedAt, "retry-backoff" -> backoff) + headers = RetryAttempt.toHeaders(RetryAttempt(topic, 0, now, 1.second)) ) } } @@ -71,7 +66,8 @@ class RetryConsumerRecordHandlerTest extends BaseTest[TestClock with TestMetrics for { producer <- FakeProducer.make topic <- randomTopicName - retryTopic = s"$topic-retry" + attempt = 0 + retryTopic = NonBlockingRetryHelper.fixedRetryTopic(topic, group, attempt) executionTime <- Promise.make[Nothing, Instant] handler = RecordHandler[Clock, HandlerError, Chunk[Byte], Chunk[Byte]] { _ => currentTime.flatMap(executionTime.succeed) } blockingState <- Ref.make[Map[BlockingTarget, BlockingState]](Map.empty) @@ -86,10 +82,7 @@ class RetryConsumerRecordHandlerTest extends BaseTest[TestClock with TestMetrics ) value <- bytes begin <- currentTime - retryAttempt <- IntSerde.serialize(retryTopic, 0) - submittedAt <- InstantSerde.serialize(retryTopic, begin) - backoff <- DurationSerde.serialize(retryTopic, 1.second) - headers = Headers("retry-attempt" -> retryAttempt, "retry-submitted-at" -> submittedAt, "retry-backoff" -> backoff) + headers = RetryAttempt.toHeaders(RetryAttempt(topic, attempt, begin, 1.second)) _ <- retryHandler.handle(ConsumerRecord(retryTopic, partition, offset, headers, None, value, 0L, 0L, 0L)).fork _ <- TestClock.adjust(1.second).repeat(Schedule.once) end <- executionTime.await.disconnect.timeoutFail(TimeoutWaitingForAssertion)(5.seconds) @@ -404,7 +397,8 @@ class RetryConsumerRecordHandlerTest extends BaseTest[TestClock with TestMetrics producer <- FakeProducer.make topic <- randomTopicName blockingState <- Ref.make[Map[BlockingTarget, BlockingState]](Map.empty) - retryHelper = alwaysBackOffRetryHelper(3.seconds) + retryHelper = FakeRetryHelper(topic) + now <- Clock.instant handling <- AwaitShutdown.makeManaged.flatMap { awaitShutdown => val retryHandler = RetryRecordHandler.withRetries( group, @@ -416,11 +410,12 @@ class RetryConsumerRecordHandlerTest extends BaseTest[TestClock with TestMetrics retryHelper, awaitShutdown = _ => ZIO.succeed(awaitShutdown) ) + val headers = RetryAttempt.toHeaders(RetryAttempt(topic, 0, now, 3.seconds)) for { key <- bytes value <- bytes handling <- retryHandler - .handle(ConsumerRecord(topic, partition, offset, Headers.Empty, Some(key), value, 0L, 0L, 0L)) + .handle(ConsumerRecord(topic, partition, offset, headers, Some(key), value, 0L, 0L, 0L)) .forkDaemon } yield handling } @@ -534,18 +529,6 @@ object RetryConsumerRecordHandlerTest { def randomTopicName = randomStr.map(suffix => s"some-topic-$suffix") val cause = new RuntimeException("cause") - - def alwaysBackOffRetryHelper(backoff: Duration) = { - new FakeNonBlockingRetryHelper { - override val topic: Topic = "" - - override def retryAttempt(topic: Topic, headers: Headers, subscription: ConsumerSubscription)( - implicit trace: Trace - ): UIO[Option[RetryAttempt]] = ZIO.succeed( - Some(RetryAttempt(topic, 1, Instant.now, backoff)) - ) - } - } } object TimeoutWaitingForAssertion extends RuntimeException diff --git a/core/src/test/scala/com/wixpress/dst/greyhound/core/testkit/FakeRetryHelper.scala b/core/src/test/scala/com/wixpress/dst/greyhound/core/testkit/FakeRetryHelper.scala index cee13948..619ada36 100644 --- a/core/src/test/scala/com/wixpress/dst/greyhound/core/testkit/FakeRetryHelper.scala +++ b/core/src/test/scala/com/wixpress/dst/greyhound/core/testkit/FakeRetryHelper.scala @@ -1,35 +1,21 @@ package com.wixpress.dst.greyhound.core.testkit import java.time.Instant -import java.util.concurrent.TimeUnit.MILLISECONDS - -import com.wixpress.dst.greyhound.core.Serdes._ import com.wixpress.dst.greyhound.core._ import com.wixpress.dst.greyhound.core.consumer.domain.{ConsumerRecord, ConsumerSubscription} import com.wixpress.dst.greyhound.core.consumer.retry.RetryDecision.{NoMoreRetries, RetryWith} -import com.wixpress.dst.greyhound.core.consumer.retry.{BlockingHandlerFailed, NonBlockingRetryHelper, RetryAttempt, RetryDecision} +import com.wixpress.dst.greyhound.core.consumer.retry.{BlockingHandlerFailed, NonBlockingRetryHelper, RetryAttempt, RetryDecision, RetryHeader} import com.wixpress.dst.greyhound.core.producer.ProducerRecord import com.wixpress.dst.greyhound.core.testkit.FakeRetryHelper._ import zio._ import zio.Clock -import zio.Clock - trait FakeNonBlockingRetryHelper extends NonBlockingRetryHelper { val topic: Topic override def retryTopicsFor(originalTopic: Topic): Set[Topic] = Set(s"$originalTopic-retry") - override def retryAttempt(topic: Topic, headers: Headers, subscription: ConsumerSubscription)( - implicit trace: Trace - ): UIO[Option[RetryAttempt]] = - (for { - attempt <- headers.get(Header.Attempt, IntSerde) - submittedAt <- headers.get(Header.SubmittedAt, InstantSerde) - backoff <- headers.get(Header.Backoff, DurationSerde) - } yield retryAttemptInternal(topic, attempt, submittedAt, backoff)).orElse(ZIO.none) - override def retryDecision[E]( retryAttempt: Option[RetryAttempt], record: ConsumerRecord[Chunk[Byte], Chunk[Byte]], @@ -38,35 +24,23 @@ trait FakeNonBlockingRetryHelper extends NonBlockingRetryHelper { )(implicit trace: Trace): URIO[Any, RetryDecision] = error match { case RetriableError | BlockingHandlerFailed => - currentTime.flatMap(now => - recordFrom(now, retryAttempt, record) - .fold(_ => NoMoreRetries, RetryWith) + currentTime.map(now => + RetryWith(recordFrom(now, retryAttempt, record)) ) case NonRetriableError => ZIO.succeed(NoMoreRetries) } - private def retryAttemptInternal(topic: Topic, attempt: Option[Int], submittedAt: Option[Instant], backoff: Option[Duration]) = - for { - a <- attempt - s <- submittedAt - b <- backoff - } yield RetryAttempt(topic, a, s, b) - private def recordFrom(now: Instant, retryAttempt: Option[RetryAttempt], record: ConsumerRecord[Chunk[Byte], Chunk[Byte]])( implicit trace: Trace ) = { val nextRetryAttempt = retryAttempt.fold(0)(_.attempt + 1) - for { - retryAttempt <- IntSerde.serialize(topic, nextRetryAttempt) - submittedAt <- InstantSerde.serialize(topic, now) - backoff <- DurationSerde.serialize(topic, 1.second) - } yield ProducerRecord( + ProducerRecord( topic = s"$topic-retry", value = record.value, key = record.key, partition = None, - headers = Headers(Header.Attempt -> retryAttempt, Header.SubmittedAt -> submittedAt, Header.Backoff -> backoff) + headers = RetryAttempt.toHeaders(RetryAttempt(topic, nextRetryAttempt, now, 1.second)) ) } } @@ -75,13 +49,8 @@ case class FakeRetryHelper(topic: Topic) extends FakeNonBlockingRetryHelper object FakeRetryHelper { implicit private val trace = Trace.empty - object Header { - val Attempt = "retry-attempt" - val SubmittedAt = "retry-submitted-at" - val Backoff = "retry-backoff" - } - val currentTime = Clock.currentTime(MILLISECONDS).map(Instant.ofEpochMilli) + val currentTime: UIO[Instant] = Clock.instant } sealed trait HandlerError From 659c0cd4d51243e48d03cf1bd152c9af8cfdb321 Mon Sep 17 00:00:00 2001 From: Noam Berman Date: Sun, 23 Apr 2023 00:49:05 +0300 Subject: [PATCH 08/17] [greyhound] expose internal kafka producer metrics (#34160) GitOrigin-RevId: ab792e2227f389d5642802944a982f3a8d3587d8 --- .../dst/greyhound/core/producer/Producer.scala | 17 +++++++++++++++-- .../core/producer/ReportingProducer.scala | 6 +++++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/producer/Producer.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/producer/Producer.scala index 2200b6cc..538c0b81 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/producer/Producer.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/producer/Producer.scala @@ -5,13 +5,15 @@ import org.apache.kafka.clients.producer.{Callback, KafkaProducer, ProducerConfi import org.apache.kafka.common.header.Header import org.apache.kafka.common.header.internals.RecordHeader import org.apache.kafka.common.serialization.ByteArraySerializer +import org.apache.kafka.common.{Metric, MetricName} +import zio.ZIO.attemptBlocking import zio._ import scala.collection.JavaConverters._ -import zio.ZIO.attemptBlocking -import zio.managed._ trait ProducerR[-R] { self => + def metrics : UIO[Option[Map[MetricName, Metric]]] = ZIO.none + def produceAsync(record: ProducerRecord[Chunk[Byte], Chunk[Byte]])( implicit trace: Trace ): ZIO[R, ProducerError, IO[ProducerError, RecordMetadata]] @@ -80,6 +82,9 @@ object Producer { val acquire = ZIO.attemptBlocking(new KafkaProducer(config.properties, serializer, serializer)) ZIO.acquireRelease(acquire)(producer => attemptBlocking(producer.close()).ignore).map { producer => new ProducerR[R] { + override def metrics: UIO[Option[Map[MetricName, Metric]]] = + ZIO.succeed(Option(producer.metrics().asScala.toMap)) + private def recordFrom(record: ProducerRecord[Chunk[Byte], Chunk[Byte]]) = new KafkaProducerRecord( record.topic, @@ -153,6 +158,8 @@ object ProducerR { override def partitionsFor(topic: Topic)(implicit trace: Trace): RIO[Any, Seq[PartitionInfo]] = producer.partitionsFor(topic).provideEnvironment(env) + + override def metrics: UIO[Option[Map[MetricName, Metric]]] = producer.metrics } def onShutdown(onShutdown: => UIO[Unit])(implicit trace: Trace): ProducerR[R] = new ProducerR[R] { override def produceAsync( @@ -165,6 +172,8 @@ object ProducerR { override def attributes: Map[String, String] = producer.attributes override def partitionsFor(topic: Topic)(implicit trace: Trace) = producer.partitionsFor(topic) + + override def metrics: UIO[Option[Map[MetricName, Metric]]] = producer.metrics } def tapBoth(onError: (Topic, Cause[ProducerError]) => URIO[R, Unit], onSuccess: RecordMetadata => URIO[R, Unit]) = new ProducerR[R] { @@ -186,6 +195,8 @@ object ProducerR { override def attributes: Map[String, String] = producer.attributes override def partitionsFor(topic: Topic)(implicit trace: Trace) = producer.partitionsFor(topic) + + override def metrics: UIO[Option[Map[MetricName, Metric]]] = producer.metrics } def map(f: ProducerRecord[Chunk[Byte], Chunk[Byte]] => ProducerRecord[Chunk[Byte], Chunk[Byte]]) = new ProducerR[R] { @@ -199,6 +210,8 @@ object ProducerR { override def shutdown(implicit trace: Trace): UIO[Unit] = producer.shutdown override def attributes: Map[String, String] = producer.attributes + + override def metrics: UIO[Option[Map[MetricName, Metric]]] = producer.metrics } } diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/producer/ReportingProducer.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/producer/ReportingProducer.scala index 10eab6a7..69306db1 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/producer/ReportingProducer.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/producer/ReportingProducer.scala @@ -5,14 +5,18 @@ import java.util.concurrent.TimeUnit.MILLISECONDS import com.wixpress.dst.greyhound.core.PartitionInfo import com.wixpress.dst.greyhound.core.metrics.{GreyhoundMetric, GreyhoundMetrics} import com.wixpress.dst.greyhound.core.producer.ProducerMetric._ -import zio.{Chunk, IO, RIO, Trace, ULayer, ZIO} +import zio.{Chunk, IO, RIO, Trace, UIO, ULayer, ZIO} import GreyhoundMetrics._ +import org.apache.kafka.common.{Metric, MetricName} import scala.concurrent.duration.FiniteDuration import zio.Clock.currentTime case class ReportingProducer[-R](internal: ProducerR[R], extraAttributes: Map[String, String]) extends ProducerR[GreyhoundMetrics with R] { + + override def metrics: UIO[Option[Map[MetricName, Metric]]] = internal.metrics + override def produceAsync( record: ProducerRecord[Chunk[Byte], Chunk[Byte]] )(implicit trace: Trace): ZIO[GreyhoundMetrics with R, ProducerError, IO[ProducerError, RecordMetadata]] = From 182763a722768866f3a0a90e56555e8054e1dadd Mon Sep 17 00:00:00 2001 From: Leon Burdinov Date: Sun, 23 Apr 2023 08:12:42 +0300 Subject: [PATCH 09/17] [greyhound] Cooperative Rebalance fix (#34153) * [greyhound] code cleanup #pr * fix for paused partitions unrevoked in cooperative GitOrigin-RevId: 7cf8fe80c2ac9108b3ebe72472355fe73158df9a --- .../greyhound/core/consumer/Dispatcher.scala | 11 ++++------ .../greyhound/core/consumer/EventLoop.scala | 20 +++++++++++-------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/Dispatcher.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/Dispatcher.scala index cbe8d883..b618d0ba 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/Dispatcher.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/Dispatcher.scala @@ -93,13 +93,10 @@ object Dispatcher { override def revoke(partitions: Set[TopicPartition]): URIO[GreyhoundMetrics, Unit] = workers .modify { workers => - partitions.foldLeft((List.empty[(TopicPartition, Worker)], workers)) { - case ((revoked, remaining), partition) => - remaining.get(partition) match { - case Some(worker) => ((partition, worker) :: revoked, remaining - partition) - case None => (revoked, remaining) - } - } + val revoked = workers.filterKeys(partitions.contains) + val remaining = workers -- partitions + + (revoked, remaining) } .flatMap(shutdownWorkers) diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/EventLoop.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/EventLoop.scala index 5985f80d..0d6be6ee 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/EventLoop.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/EventLoop.scala @@ -170,12 +170,14 @@ object EventLoop { consumer: Consumer, partitions: Set[TopicPartition] )(implicit trace: Trace): URIO[GreyhoundMetrics, DelayedRebalanceEffect] = { - pausedPartitionsRef.set(Set.empty) *> - dispatcher.revoke(partitions).timeout(config.drainTimeout).flatMap { drained => - ZIO.when(drained.isEmpty)( - report(DrainTimeoutExceeded(clientId, group, config.drainTimeout.toMillis, consumer.config.consumerAttributes)) - ) - } *> commitOffsetsOnRebalance(consumer0, offsets) + for { + _ <- pausedPartitionsRef.update(_ -- partitions) + isRevokeTimedOut <- dispatcher.revoke(partitions).timeout(config.drainTimeout).map(_.isEmpty) + _ <- ZIO.when(isRevokeTimedOut)( + report(DrainTimeoutExceeded(clientId, group, config.drainTimeout.toMillis, consumer.config.consumerAttributes)) + ) + delayedRebalanceEffect <- commitOffsetsOnRebalance(consumer0, offsets) + } yield delayedRebalanceEffect } override def onPartitionsAssigned(consumer: Consumer, partitions: Set[TopicPartition])(implicit trace: Trace): UIO[Any] = @@ -258,7 +260,8 @@ case class EventLoopConfig( highWatermark: Int, rebalanceListener: RebalanceListener[Any], delayResumeOfPausedPartition: Long, - startPaused: Boolean + startPaused: Boolean, + cooperativeRebalanceEnabled: Boolean ) object EventLoopConfig { @@ -269,7 +272,8 @@ object EventLoopConfig { highWatermark = 256, rebalanceListener = RebalanceListener.Empty, delayResumeOfPausedPartition = 0, - startPaused = false + startPaused = false, + cooperativeRebalanceEnabled = false ) } From 0b5bd555b9346d4fa7d10a9e4bebd3e0f8ed088a Mon Sep 17 00:00:00 2001 From: Leon Burdinov Date: Sun, 23 Apr 2023 10:53:08 +0300 Subject: [PATCH 10/17] code cleanup (#34171) GitOrigin-RevId: c2a4d2c2de669eff3c98b2277be8502d834de3e4 --- .../wixpress/dst/greyhound/core/consumer/EventLoop.scala | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/EventLoop.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/EventLoop.scala index 0d6be6ee..3c8940ac 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/EventLoop.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/EventLoop.scala @@ -260,8 +260,7 @@ case class EventLoopConfig( highWatermark: Int, rebalanceListener: RebalanceListener[Any], delayResumeOfPausedPartition: Long, - startPaused: Boolean, - cooperativeRebalanceEnabled: Boolean + startPaused: Boolean ) object EventLoopConfig { @@ -272,8 +271,7 @@ object EventLoopConfig { highWatermark = 256, rebalanceListener = RebalanceListener.Empty, delayResumeOfPausedPartition = 0, - startPaused = false, - cooperativeRebalanceEnabled = false + startPaused = false ) } From bf94eaafad1f3113fda87632cedd110bdf19148a Mon Sep 17 00:00:00 2001 From: Ben Wattelman <82799628+ben-wattelman@users.noreply.github.com> Date: Wed, 24 May 2023 14:08:59 +0300 Subject: [PATCH 11/17] [greyhound] parallel consumer implementation (#34061) GitOrigin-RevId: 5c9cc1ea4fcc5935c6f905a5269ac17f6b82c294 --- .../dst/greyhound/core/ConsumerIT.scala | 854 ++++++++++-------- .../dst/greyhound/core/parallel/BUILD.bazel | 27 + .../core/parallel/ParallelConsumerIT.scala | 325 +++++++ .../greyhound/core/OffsetAndMetadata.scala | 3 + .../greyhound/core/consumer/Consumer.scala | 53 +- .../greyhound/core/consumer/Dispatcher.scala | 193 +++- .../greyhound/core/consumer/EventLoop.scala | 150 ++- .../core/consumer/OffsetsAndGaps.scala | 63 +- .../core/consumer/OffsetsInitializer.scala | 62 +- .../core/consumer/RecordConsumer.scala | 19 +- .../core/consumer/ReportingConsumer.scala | 93 ++ .../core/consumer/EventLoopTest.scala | 15 +- .../consumer/OffsetsInitializerTest.scala | 39 +- .../consumer/batched/BatchEventLoopTest.scala | 18 +- .../consumer/dispatcher/DispatcherTest.scala | 44 + .../core/testkit/RecordMatchers.scala | 6 + 16 files changed, 1497 insertions(+), 467 deletions(-) create mode 100644 core/src/it/scala/com/wixpress/dst/greyhound/core/parallel/BUILD.bazel create mode 100644 core/src/it/scala/com/wixpress/dst/greyhound/core/parallel/ParallelConsumerIT.scala diff --git a/core/src/it/scala/com/wixpress/dst/greyhound/core/ConsumerIT.scala b/core/src/it/scala/com/wixpress/dst/greyhound/core/ConsumerIT.scala index e91e3742..3f26b47e 100644 --- a/core/src/it/scala/com/wixpress/dst/greyhound/core/ConsumerIT.scala +++ b/core/src/it/scala/com/wixpress/dst/greyhound/core/ConsumerIT.scala @@ -19,6 +19,7 @@ import com.wixpress.dst.greyhound.core.zioutils.Gate import com.wixpress.dst.greyhound.testenv.ITEnv import com.wixpress.dst.greyhound.testenv.ITEnv.{clientId, _} import com.wixpress.dst.greyhound.testkit.ManagedKafka +import org.specs2.specification.core.Fragments import zio.Clock import zio.stm.{STM, TRef} import zio._ @@ -30,428 +31,505 @@ class ConsumerIT extends BaseTestWithSharedEnv[Env, TestResources] { sequential override def env = ITEnv.ManagedEnv - override def sharedEnv = ITEnv.testResources() - val resources = testResources() - "produce, consume and rebalance - " in + + s"subscribe to a pattern" in ZIO.scoped { for { + _ <- ZIO.debug(">>>> starting test: patternTest with parallel") + topic1 = "core-subscribe-pattern1-topic" + topic2 = "core-subscribe-pattern2-topic" r <- getShared TestResources(kafka, producer) = r - topic <- kafka.createRandomTopic(prefix = s"topic1-single1") - topic2 <- kafka.createRandomTopic(prefix = "topic2-single1") + _ <- kafka.createTopics(Seq(topic1, topic2).map(t => TopicConfig(t, 1, 1, delete)): _*) group <- randomGroup + probe <- Ref.make(Seq.empty[(Topic, Offset)]) + handler = RecordHandler { record: ConsumerRecord[Chunk[Byte], Chunk[Byte]] => probe.update(_ :+ (record.topic, record.offset)) } - queue <- Queue.unbounded[ConsumerRecord[String, String]] - handler = RecordHandler((cr: ConsumerRecord[String, String]) => ZIO.succeed(println(s"***** Consumed: $cr")) *> queue.offer(cr)) - .withDeserializers(StringSerde, StringSerde) - .ignore - cId <- clientId - config = configFor(kafka, group, topic).copy(clientId = cId) - record = ProducerRecord(topic, "bar", Some("foo")) - - messages <- RecordConsumer.make(config, handler).flatMap { consumer => - producer.produce(record, StringSerde, StringSerde) *> sleep(3.seconds) *> - consumer.resubscribe(ConsumerSubscription.topics(topic, topic2)) *> - sleep(500.millis) *> // give the consumer some time to start polling topic2 - producer.produce(record.copy(topic = topic2, value = Some("BAR")), StringSerde, StringSerde) *> - (queue.take zip queue.take) - .timeout(20.seconds) - .tap(o => ZIO.when(o.isEmpty)(ZIO.debug("timeout waiting for messages!"))) - } - msgs <- ZIO.fromOption(messages).orElseFail(TimedOutWaitingForMessages) - } yield { - msgs._1 must - (beRecordWithKey("foo") and beRecordWithValue("bar")) and - (msgs._2 must (beRecordWithKey("foo") and beRecordWithValue("BAR"))) - } + _ <- makeConsumer(kafka, compile("core-subscribe-pattern1.*"), group, handler, 0, useParallelConsumer = false).flatMap { consumer => + val record = ProducerRecord(topic1, Chunk.empty, key = Option(Chunk.empty)) + producer.produce(record) *> eventuallyZ(probe.get)(_ == (topic1, 0) :: Nil) *> + consumer.resubscribe(TopicPattern(compile("core-subscribe-pattern2.*"))) *> + producer.produce(record.copy(topic = topic2)) *> + eventuallyZ(probe.get)(_ == (topic1, 0) :: (topic2, 0) :: Nil) + } + } yield ok } - "be able to resubscribe to same topics" in + s"subscribe to a pattern - using parallel consumer" in ZIO.scoped { for { + _ <- ZIO.debug(">>>> starting test: patternTest with parallel") + topic1 = "core-subscribe-parallel-pattern1-topic" + topic2 = "core-subscribe-parallel-pattern2-topic" r <- getShared TestResources(kafka, producer) = r - topic <- kafka.createRandomTopic(prefix = s"topic1-single1") + _ <- kafka.createTopics(Seq(topic1, topic2).map(t => TopicConfig(t, 1, 1, delete)): _*) group <- randomGroup + probe <- Ref.make(Seq.empty[(Topic, Offset)]) + handler = RecordHandler { record: ConsumerRecord[Chunk[Byte], Chunk[Byte]] => probe.update(_ :+ (record.topic, record.offset)) } - queue <- Queue.unbounded[ConsumerRecord[String, String]] - handler = RecordHandler((cr: ConsumerRecord[String, String]) => ZIO.succeed(println(s"***** Consumed: $cr")) *> queue.offer(cr)) - .withDeserializers(StringSerde, StringSerde) - .ignore - cId <- clientId - config = configFor(kafka, group, topic, mutateEventLoop = _.copy(drainTimeout = 1.second)).copy(clientId = cId) - record = ProducerRecord(topic, "bar", Some("foo")) - - messages <- RecordConsumer.make(config, handler).flatMap { consumer => - producer.produce(record, StringSerde, StringSerde) *> sleep(3.seconds) *> - consumer.resubscribe(ConsumerSubscription.topics(topic)) *> - consumer.resubscribe(ConsumerSubscription.topics(topic)) *> - producer.produce(record.copy(topic = topic, value = Some("BAR")), StringSerde, StringSerde) *> - (queue.take zip queue.take) - .timeout(20.seconds) - .tap(o => ZIO.when(o.isEmpty)(ZIO.debug("timeout waiting for messages!"))) - } - msgs <- ZIO.fromOption(messages).orElseFail(TimedOutWaitingForMessages) - } yield { - msgs._1 must - (beRecordWithKey("foo") and beRecordWithValue("bar")) and - (msgs._2 must (beRecordWithKey("foo") and beRecordWithValue("BAR"))) - } + _ <- makeConsumer(kafka, compile("core-subscribe-parallel-pattern1.*"), group, handler, 0, useParallelConsumer = true).flatMap { + consumer => + val record = ProducerRecord(topic1, Chunk.empty, key = Option(Chunk.empty)) + producer.produce(record) *> + eventuallyZ(probe.get, timeout = 20.seconds)(_ == (topic1, 0) :: Nil) *> + consumer.resubscribe(TopicPattern(compile("core-subscribe-parallel-pattern2.*"))) *> + producer.produce(record.copy(topic = topic2)) *> + eventuallyZ(probe.get)(_ == (topic1, 0) :: (topic2, 0) :: Nil) + } + } yield ok } - "produce consume null values (tombstones)" in - ZIO.scoped { - for { - r <- getShared - TestResources(kafka, producer) = r - topic <- kafka.createRandomTopic(prefix = s"topic1-single1") - group <- randomGroup + Fragments.foreach(Seq(false, true)) { useParallelConsumer => + s"produce, consume and rebalance${parallelConsumerString(useParallelConsumer)}" in + ZIO.scoped { + for { + r <- getShared + TestResources(kafka, producer) = r + topic <- kafka.createRandomTopic(prefix = s"topic1-single1") + topic2 <- kafka.createRandomTopic(prefix = "topic2-single1") + group <- randomGroup + + queue <- Queue.unbounded[ConsumerRecord[String, String]] + handler = RecordHandler((cr: ConsumerRecord[String, String]) => ZIO.succeed(println(s"***** Consumed: $cr")) *> queue.offer(cr)) + .withDeserializers(StringSerde, StringSerde) + .ignore + cId <- clientId + config = + configFor(kafka, group, topic, mutateEventLoop = _.copy(consumePartitionInParallel = useParallelConsumer, maxParallelism = 8)).copy(clientId = cId) + record = ProducerRecord(topic, "bar", Some("foo")) + + messages <- RecordConsumer.make(config, handler).flatMap { consumer => + producer.produce(record, StringSerde, StringSerde) *> sleep(3.seconds) *> + consumer.resubscribe(ConsumerSubscription.topics(topic, topic2)) *> + sleep(500.millis) *> // give the consumer some time to start polling topic2 + producer.produce(record.copy(topic = topic2, value = Some("BAR")), StringSerde, StringSerde) *> + (queue.take zip queue.take) + .timeout(20.seconds) + .tap(o => ZIO.when(o.isEmpty)(ZIO.debug("timeout waiting for messages!"))) + } + msgs <- ZIO.fromOption(messages).orElseFail(TimedOutWaitingForMessages) + } yield { + msgs._1 must + (beRecordWithKey("foo") and beRecordWithValue("bar")) and + (msgs._2 must (beRecordWithKey("foo") and beRecordWithValue("BAR"))) + } + } - queue <- Queue.unbounded[ConsumerRecord[String, String]] - handler = RecordHandler((cr: ConsumerRecord[String, String]) => ZIO.succeed(println(s"***** Consumed: $cr")) *> queue.offer(cr)) - .withDeserializers(StringSerde, StringSerde) - .ignore - cId <- clientId - config = configFor(kafka, group, topic).copy(clientId = cId) - record = ProducerRecord.tombstone(topic, Some("foo")) - message <- RecordConsumer.make(config, handler).flatMap { _ => - producer.produce(record, StringSerde, StringSerde) *> queue.take.timeoutFail(TimedOutWaitingForMessages)(10.seconds) - } - } yield { - message must (beRecordWithKey("foo") and beRecordWithValue(null)) + s"be able to resubscribe to same topics${parallelConsumerString(useParallelConsumer)}" in + ZIO.scoped { + for { + r <- getShared + TestResources(kafka, producer) = r + topic <- kafka.createRandomTopic(prefix = s"topic1-single1") + group <- randomGroup + + queue <- Queue.unbounded[ConsumerRecord[String, String]] + handler = RecordHandler((cr: ConsumerRecord[String, String]) => ZIO.succeed(println(s"***** Consumed: $cr")) *> queue.offer(cr)) + .withDeserializers(StringSerde, StringSerde) + .ignore + cId <- clientId + config = + configFor( + kafka, + group, + topic, + mutateEventLoop = _.copy(drainTimeout = 1.second, consumePartitionInParallel = useParallelConsumer, maxParallelism = 8) + ).copy(clientId = cId) + record = ProducerRecord(topic, "bar", Some("foo")) + + messages <- RecordConsumer.make(config, handler).flatMap { consumer => + producer.produce(record, StringSerde, StringSerde) *> sleep(3.seconds) *> + consumer.resubscribe(ConsumerSubscription.topics(topic)) *> + consumer.resubscribe(ConsumerSubscription.topics(topic)) *> + producer.produce(record.copy(topic = topic, value = Some("BAR")), StringSerde, StringSerde) *> + (queue.take zip queue.take) + .timeout(20.seconds) + .tap(o => ZIO.when(o.isEmpty)(ZIO.debug("timeout waiting for messages!"))) + } + msgs <- ZIO.fromOption(messages).orElseFail(TimedOutWaitingForMessages) + } yield { + msgs._1 must + (beRecordWithKey("foo") and beRecordWithValue("bar")) and + (msgs._2 must (beRecordWithKey("foo") and beRecordWithValue("BAR"))) + } } - } - "not lose any messages on a slow consumer (drives the message dispatcher to throttling)" in - ZIO.scoped { - for { - r <- getShared - TestResources(kafka, producer) = r - _ <- ZIO.debug(">>>> starting test: throttlingTest") - - topic <- kafka.createRandomTopic(partitions = 2, prefix = "core-not-lose") - group <- randomGroup - - messagesPerPartition = 500 // Exceeds the queue capacity - delayPartition1 <- Promise.make[Nothing, Unit] - handledPartition0 <- CountDownLatch.make(messagesPerPartition) - handledPartition1 <- CountDownLatch.make(messagesPerPartition) - handler = RecordHandler { record: ConsumerRecord[Chunk[Byte], Chunk[Byte]] => - record.partition match { - case 0 => handledPartition0.countDown - case 1 => delayPartition1.await *> handledPartition1.countDown + s"produce consume null values (tombstones)${parallelConsumerString(useParallelConsumer)}" in + ZIO.scoped { + for { + r <- getShared + TestResources(kafka, producer) = r + topic <- kafka.createRandomTopic(prefix = s"topic1-single1") + group <- randomGroup + + queue <- Queue.unbounded[ConsumerRecord[String, String]] + handler = RecordHandler((cr: ConsumerRecord[String, String]) => ZIO.succeed(println(s"***** Consumed: $cr")) *> queue.offer(cr)) + .withDeserializers(StringSerde, StringSerde) + .ignore + cId <- clientId + config = + configFor(kafka, group, topic, mutateEventLoop = _.copy(consumePartitionInParallel = useParallelConsumer, maxParallelism = 8)).copy(clientId = cId) + record = ProducerRecord.tombstone(topic, Some("foo")) + message <- RecordConsumer.make(config, handler).flatMap { _ => + producer.produce(record, StringSerde, StringSerde) *> queue.take.timeoutFail(TimedOutWaitingForMessages)(10.seconds) + } + } yield { + message must (beRecordWithKey("foo") and beRecordWithValue(null)) + } + } + + s"not lose any messages on a slow consumer (drives the message dispatcher to throttling)${parallelConsumerString(useParallelConsumer)}" in + ZIO.scoped { + for { + r <- getShared + TestResources(kafka, producer) = r + _ <- ZIO.debug(">>>> starting test: throttlingTest") + + topic <- kafka.createRandomTopic(partitions = 2, prefix = "core-not-lose") + group <- randomGroup + + messagesPerPartition = 500 // Exceeds the queue capacity + delayPartition1 <- Promise.make[Nothing, Unit] + handledPartition0 <- CountDownLatch.make(messagesPerPartition) + handledPartition1 <- CountDownLatch.make(messagesPerPartition) + handler = RecordHandler { record: ConsumerRecord[Chunk[Byte], Chunk[Byte]] => + record.partition match { + case 0 => handledPartition0.countDown + case 1 => delayPartition1.await *> handledPartition1.countDown + } } - } - - test <- RecordConsumer.make(configFor(kafka, group, topic), handler).flatMap { _ => - val recordPartition0 = ProducerRecord(topic, Chunk.empty, partition = Some(0)) - val recordPartition1 = ProducerRecord(topic, Chunk.empty, partition = Some(1)) - for { - _ <- ZIO.foreachParDiscard(0 until messagesPerPartition) { _ => - producer.produce(recordPartition0) zipPar producer.produce(recordPartition1) - } - handledAllFromPartition0 <- handledPartition0.await.timeout(10.seconds) - _ <- delayPartition1.succeed(()) - handledAllFromPartition1 <- handledPartition1.await.timeout(10.seconds) - } yield { - (handledAllFromPartition0 must beSome) and (handledAllFromPartition1 must beSome) - } + + test <- + RecordConsumer + .make(configFor(kafka, group, topic, mutateEventLoop = _.copy(consumePartitionInParallel = useParallelConsumer, maxParallelism = 8)), handler) + .flatMap { _ => + val recordPartition0 = ProducerRecord(topic, Chunk.empty, partition = Some(0)) + val recordPartition1 = ProducerRecord(topic, Chunk.empty, partition = Some(1)) + for { + _ <- ZIO.foreachParDiscard(0 until messagesPerPartition) { _ => + producer.produce(recordPartition0) zipPar producer.produce(recordPartition1) + } + handledAllFromPartition0 <- handledPartition0.await.timeout(10.seconds) + _ <- delayPartition1.succeed(()) + handledAllFromPartition1 <- handledPartition1.await.timeout(10.seconds) + } yield { + (handledAllFromPartition0 must beSome) and (handledAllFromPartition1 must beSome) } - } yield test - } + } + } yield test + } - "delay resuming a paused partition" in - ZIO.scoped { - for { - r <- getShared - TestResources(kafka, producer) = r - _ <- ZIO.debug(">>>> starting test: delay resuming a paused partition") - - topic <- kafka.createRandomTopic(partitions = 1, prefix = "core-not-lose") - group <- randomGroup - - messagesPerPartition = 500 // Exceeds the queue capacity - delayPartition <- Promise.make[Nothing, Unit] - handledPartition <- CountDownLatch.make(messagesPerPartition) - handler = RecordHandler { _: ConsumerRecord[Chunk[Byte], Chunk[Byte]] => delayPartition.await *> handledPartition.countDown } - start <- Clock.currentTime(TimeUnit.MILLISECONDS) - test <- RecordConsumer - .make(configFor(kafka, group, topic, mutateEventLoop = _.copy(delayResumeOfPausedPartition = 3000)), handler) - .flatMap { _ => - val recordPartition = ProducerRecord(topic, Chunk.empty, partition = Some(0)) - for { - _ <- ZIO.foreachParDiscard(0 until messagesPerPartition) { _ => producer.produce(recordPartition) } - _ <- delayPartition.succeed(()).delay(1.seconds).fork - handledAllFromPartition <- handledPartition.await.timeout(10.seconds) - end <- Clock.currentTime(TimeUnit.MILLISECONDS) - - } yield { - (handledAllFromPartition aka "handledAllFromPartition" must beSome) and - (end - start aka "complete handling duration" must beGreaterThan(3000L)) + s"delay resuming a paused partition${parallelConsumerString(useParallelConsumer)}" in + ZIO.scoped { + for { + r <- getShared + TestResources(kafka, producer) = r + _ <- ZIO.debug(">>>> starting test: delay resuming a paused partition") + + topic <- kafka.createRandomTopic(partitions = 1, prefix = "core-not-lose") + group <- randomGroup + + messagesPerPartition = 500 // Exceeds the queue capacity + delayPartition <- Promise.make[Nothing, Unit] + handledPartition <- CountDownLatch.make(messagesPerPartition) + handler = RecordHandler { _: ConsumerRecord[Chunk[Byte], Chunk[Byte]] => delayPartition.await *> handledPartition.countDown } + start <- Clock.currentTime(TimeUnit.MILLISECONDS) + test <- RecordConsumer + .make( + configFor( + kafka, + group, + topic, + mutateEventLoop = _.copy(delayResumeOfPausedPartition = 3000, consumePartitionInParallel = useParallelConsumer, maxParallelism = 8) + ), + handler + ) + .flatMap { _ => + val recordPartition = ProducerRecord(topic, Chunk.empty, partition = Some(0)) + for { + _ <- ZIO.foreachParDiscard(0 until messagesPerPartition) { _ => producer.produce(recordPartition) } + _ <- delayPartition.succeed(()).delay(1.seconds).fork + handledAllFromPartition <- handledPartition.await.timeout(10.seconds) + end <- Clock.currentTime(TimeUnit.MILLISECONDS) + + } yield { + (handledAllFromPartition aka "handledAllFromPartition" must beSome) and + (end - start aka "complete handling duration" must beGreaterThan(3000L)) + } } + } yield test + } + + s"pause and resume consumer${parallelConsumerString(useParallelConsumer)}" in + ZIO.scoped { + for { + r <- getShared + TestResources(kafka, producer) = r + _ <- ZIO.debug(">>>> starting test: pauseResumeTest") + + topic <- kafka.createRandomTopic(prefix = "core-pause-resume") + group <- randomGroup + + numberOfMessages = 32 + someMessages = 16 + restOfMessages = numberOfMessages - someMessages + handledSomeMessages <- CountDownLatch.make(someMessages) + handledAllMessages <- CountDownLatch.make(numberOfMessages) + handleCounter <- Ref.make[Int](0) + handler = RecordHandler { _: ConsumerRecord[Chunk[Byte], Chunk[Byte]] => + handleCounter.update(_ + 1) *> handledSomeMessages.countDown zipParRight handledAllMessages.countDown } - } yield test - } - "pause and resume consumer" in - ZIO.scoped { - for { - r <- getShared - TestResources(kafka, producer) = r - _ <- ZIO.debug(">>>> starting test: pauseResumeTest") - - topic <- kafka.createRandomTopic(prefix = "core-pause-resume") - group <- randomGroup - - numberOfMessages = 32 - someMessages = 16 - restOfMessages = numberOfMessages - someMessages - handledSomeMessages <- CountDownLatch.make(someMessages) - handledAllMessages <- CountDownLatch.make(numberOfMessages) - handleCounter <- Ref.make[Int](0) - handler = RecordHandler { _: ConsumerRecord[Chunk[Byte], Chunk[Byte]] => - handleCounter.update(_ + 1) *> handledSomeMessages.countDown zipParRight handledAllMessages.countDown - } - - test <- RecordConsumer.make(configFor(kafka, group, topic).copy(offsetReset = OffsetReset.Earliest), handler).flatMap { consumer => - val record = ProducerRecord(topic, Chunk.empty) - for { - _ <- ZIO.foreachParDiscard(0 until someMessages)(_ => producer.produce(record)) - _ <- handledSomeMessages.await - _ <- consumer.pause - _ <- ZIO.foreachParDiscard(0 until restOfMessages)(_ => producer.produce(record)) - a <- handledAllMessages.await.timeout(5.seconds) - handledAfterPause <- handleCounter.get - _ <- consumer.resume - b <- handledAllMessages.await.timeout(5.seconds) - } yield { - (handledAfterPause === someMessages) and (a must beNone) and (b must beSome) - } + test <- + RecordConsumer + .make( + configFor(kafka, group, topic, mutateEventLoop = _.copy(consumePartitionInParallel = useParallelConsumer, maxParallelism = 8)) + .copy(offsetReset = OffsetReset.Earliest), + handler + ) + .flatMap { consumer => + val record = ProducerRecord(topic, Chunk.empty) + for { + _ <- ZIO.foreachParDiscard(0 until someMessages)(_ => producer.produce(record)) + _ <- handledSomeMessages.await + _ <- consumer.pause + _ <- ZIO.foreachParDiscard(0 until restOfMessages)(_ => producer.produce(record)) + a <- handledAllMessages.await.timeout(5.seconds) + handledAfterPause <- handleCounter.get + _ <- consumer.resume + b <- handledAllMessages.await.timeout(5.seconds) + } yield { + (handledAfterPause === someMessages) and (a must beNone) and (b must beSome) } - } yield test - } - - "wait until queues are drained" in { - for { - r <- getShared - TestResources(kafka, producer) = r - _ <- ZIO.debug(">>>> starting test: gracefulShutdownTest") - topic <- kafka.createRandomTopic(prefix = "core-wait-until") - group <- randomGroup - - ref <- Ref.make(0) - startedHandling <- Promise.make[Nothing, Unit] - handler: Handler[Any] = RecordHandler { _: ConsumerRecord[Chunk[Byte], Chunk[Byte]] => - startedHandling.succeed(()) *> sleep(5.seconds) *> ref.update(_ + 1) - } - - _ <- ZIO.scoped(RecordConsumer.make(configFor(kafka, group, topic), handler).flatMap { _ => - producer.produce(ProducerRecord(topic, Chunk.empty)) *> startedHandling.await - }) - - handled <- ref.get - } yield { - handled must equalTo(1) - } - } + } + } yield test + } - "consumer from earliest offset" in - ZIO.scoped { + s"wait until queues are drained${parallelConsumerString(useParallelConsumer)}" in { for { r <- getShared TestResources(kafka, producer) = r - _ <- ZIO.debug(">>>> starting test: earliestTest") - topic <- kafka.createRandomTopic(prefix = "core-from-earliest") + _ <- ZIO.debug(">>>> starting test: gracefulShutdownTest") + topic <- kafka.createRandomTopic(prefix = "core-wait-until") group <- randomGroup - queue <- Queue.unbounded[ConsumerRecord[String, String]] - handler = RecordHandler(queue.offer(_: ConsumerRecord[String, String])) - .withDeserializers(StringSerde, StringSerde) - .ignore + ref <- Ref.make(0) + startedHandling <- Promise.make[Nothing, Unit] + handler: Handler[Any] = RecordHandler { _: ConsumerRecord[Chunk[Byte], Chunk[Byte]] => + startedHandling.succeed(()) *> sleep(5.seconds) *> ref.update(_ + 1) + } - record = ProducerRecord(topic, "bar", Some("foo")) - _ <- producer.produce(record, StringSerde, StringSerde) + _ <- ZIO.scoped( + RecordConsumer + .make(configFor(kafka, group, topic, mutateEventLoop = _.copy(consumePartitionInParallel = useParallelConsumer, maxParallelism = 8)), handler) + .flatMap { _ => producer.produce(ProducerRecord(topic, Chunk.empty)) *> startedHandling.await } + ) - message <- RecordConsumer - .make(configFor(kafka, group, topic).copy(offsetReset = Earliest), handler) - .flatMap { _ => queue.take } - .timeout(10.seconds) + handled <- ref.get } yield { - message.get must (beRecordWithKey("foo") and beRecordWithValue("bar")) + handled must equalTo(1) } } - "not lose messages while throttling after rebalance" in - ZIO.scoped { - for { - _ <- ZIO.debug(">>>> starting test: throttleWhileRebalancingTest") - r <- getShared - TestResources(kafka, producer) = r - partitions = 30 - topic <- kafka.createRandomTopic(partitions, prefix = "core-not-lose-while-throttling") - group <- randomGroup - probe <- Ref.make(Map.empty[Partition, Seq[Offset]]) - messagesPerPartition = 250 - handler = RecordHandler { record: ConsumerRecord[Chunk[Byte], Chunk[Byte]] => - sleep(10.millis) *> - probe - .getAndUpdate(curr => curr + (record.partition -> (curr.getOrElse(record.partition, Nil) :+ record.offset))) - .flatMap(map => - ZIO.when(map.getOrElse(record.partition, Nil).contains(record.offset))( - ZIO.debug(OffsetWasAlreadyProcessed(record.partition, record.offset).toString) - ) - ) - } - createConsumerTask = (i: Int) => makeConsumer(kafka, topic, group, handler, i) - test <- createConsumerTask(0).flatMap { _ => - val record = ProducerRecord(topic, Chunk.empty, partition = Some(0)) - for { - env <- ZIO.environment[Env] - _ <- ZIO.foreachParDiscard(0 until partitions) { p => - ZIO.foreachDiscard(0 until messagesPerPartition)(_ => producer.produceAsync(record.copy(partition = Some(p)))) - } - _ <- createConsumerTask(1).provideEnvironment(env.add(Scope.global)).forkScoped // rebalance - _ <- createConsumerTask(2).provideEnvironment(env.add(Scope.global)).forkScoped // rebalance // rebalance - expected = (0 until partitions).map(p => (p, 0L until messagesPerPartition)).toMap - _ <- eventuallyTimeoutFail(probe.get)(m => - m.mapValues(_.lastOption).values.toSet == Set(Option(messagesPerPartition - 1L)) && m.size == partitions - )(120.seconds) - finalResult <- probe.get - _ <- ZIO.debug(finalResult.mapValues(_.size).mkString(",")) - } yield finalResult === expected - } - } yield test - } - - "subscribe to a pattern" in - ZIO.scoped { - for { - _ <- ZIO.debug(">>>> starting test: patternTest") - topic1 = "core-subscribe-pattern1-topic" - topic2 = "core-subscribe-pattern2-topic" - r <- getShared - TestResources(kafka, producer) = r - _ <- kafka.createTopics(Seq(topic1, topic2).map(t => TopicConfig(t, 1, 1, delete)): _*) - group <- randomGroup - probe <- Ref.make(Seq.empty[(Topic, Offset)]) - handler = RecordHandler { record: ConsumerRecord[Chunk[Byte], Chunk[Byte]] => probe.update(_ :+ (record.topic, record.offset)) } - - _ <- makeConsumer(kafka, compile("core-subscribe-pattern1.*"), group, handler, 0).flatMap { consumer => - val record = ProducerRecord(topic1, Chunk.empty, key = Option(Chunk.empty)) + s"consumer from earliest offset${parallelConsumerString(useParallelConsumer)}" in + ZIO.scoped { + for { + r <- getShared + TestResources(kafka, producer) = r + _ <- ZIO.debug(">>>> starting test: earliestTest") + topic <- kafka.createRandomTopic(prefix = "core-from-earliest") + group <- randomGroup + + queue <- Queue.unbounded[ConsumerRecord[String, String]] + handler = RecordHandler(queue.offer(_: ConsumerRecord[String, String])) + .withDeserializers(StringSerde, StringSerde) + .ignore + + record = ProducerRecord(topic, "bar", Some("foo")) + _ <- producer.produce(record, StringSerde, StringSerde) + + message <- RecordConsumer + .make( + configFor(kafka, group, topic, mutateEventLoop = _.copy(consumePartitionInParallel = useParallelConsumer, maxParallelism = 8)) + .copy(offsetReset = Earliest), + handler + ) + .flatMap { _ => queue.take } + .timeout(10.seconds) + } yield { + message.get must (beRecordWithKey("foo") and beRecordWithValue("bar")) + } + } - producer.produce(record) *> eventuallyZ(probe.get)(_ == (topic1, 0) :: Nil) *> - consumer.resubscribe(TopicPattern(compile("core-subscribe-pattern2.*"))) *> - producer.produce(record.copy(topic = topic2)) *> eventuallyZ(probe.get)(_ == (topic1, 0) :: (topic2, 0) :: Nil) - } - } yield ok - } + s"not lose messages while throttling after rebalance${parallelConsumerString(useParallelConsumer)}" in + ZIO.scoped { + for { + _ <- ZIO.debug(">>>> starting test: throttleWhileRebalancingTest") + r <- getShared + TestResources(kafka, producer) = r + partitions = 30 + topic <- kafka.createRandomTopic(partitions, prefix = "core-not-lose-while-throttling") + group <- randomGroup + probe <- Ref.make(Map.empty[Partition, Seq[Offset]]) + messagesPerPartition = 250 + handler = RecordHandler { record: ConsumerRecord[Chunk[Byte], Chunk[Byte]] => + sleep(10.millis) *> + probe + .getAndUpdate(curr => curr + (record.partition -> (curr.getOrElse(record.partition, Nil) :+ record.offset))) + .flatMap(map => + ZIO.when(map.getOrElse(record.partition, Nil).contains(record.offset))( + ZIO.debug(OffsetWasAlreadyProcessed(record.partition, record.offset).toString) + ) + ) + } + createConsumerTask = + (i: Int) => + makeConsumer(kafka, topic, group, handler, i, mutateEventLoop = _.copy(consumePartitionInParallel = useParallelConsumer, maxParallelism = 8)) + test <- createConsumerTask(0).flatMap { _ => + val record = ProducerRecord(topic, Chunk.empty, partition = Some(0)) + for { + env <- ZIO.environment[Env] + _ <- ZIO.foreachParDiscard(0 until partitions) { p => + ZIO.foreachDiscard(0 until messagesPerPartition)(_ => producer.produceAsync(record.copy(partition = Some(p)))) + } + _ <- createConsumerTask(1).provideEnvironment(env.add(Scope.global)).forkScoped // rebalance + _ <- createConsumerTask(2).provideEnvironment(env.add(Scope.global)).forkScoped // rebalance // rebalance + expected = (0 until partitions).map(p => (p, 0L until messagesPerPartition)).toMap + _ <- eventuallyTimeoutFail(probe.get)(m => + m.mapValues(_.lastOption).values.toSet == Set(Option(messagesPerPartition - 1L)) && m.size == partitions + )(120.seconds) + finalResult <- probe.get + _ <- ZIO.debug(finalResult.mapValues(_.size).mkString(",")) + } yield finalResult === expected + } + } yield test + } - "consumer from a new partition is interrupted before commit (offsetReset = Latest)" in - ZIO.scoped { - for { - r <- getShared - TestResources(kafka, producer) = r - topic <- kafka.createRandomTopic(1) - group <- randomGroup - handlingStarted <- Promise.make[Nothing, Unit] - hangForever <- Promise.make[Nothing, Unit] - hangingHandler = RecordHandler { record: ConsumerRecord[Chunk[Byte], Chunk[Byte]] => - handlingStarted.complete(ZIO.unit) *> hangForever.await - } - record <- aProducerRecord(topic) - recordValue = record.value.get - _ <- makeConsumer(kafka, topic, group, hangingHandler, 0, _.copy(drainTimeout = 200.millis)) - .flatMap { consumer => - producer.produce(record, StringSerde, StringSerde) *> handlingStarted.await *> - // unsubscribe to make sure partitions are released - consumer.resubscribe(ConsumerSubscription.topics()) + s"consumer from a new partition is interrupted before commit (offsetReset = Latest)${parallelConsumerString(useParallelConsumer)}" in + ZIO.scoped { + for { + r <- getShared + TestResources(kafka, producer) = r + topic <- kafka.createRandomTopic(1) + group <- randomGroup + handlingStarted <- Promise.make[Nothing, Unit] + hangForever <- Promise.make[Nothing, Unit] + hangingHandler = RecordHandler { record: ConsumerRecord[Chunk[Byte], Chunk[Byte]] => + handlingStarted.complete(ZIO.unit) *> hangForever.await } - .disconnect - .timeoutFail(new TimeoutException("timed out waiting for consumer 0"))(10.seconds) - consumed <- AwaitableRef.make(Seq.empty[String]) - handler = RecordHandler { record: ConsumerRecord[String, String] => consumed.update(_ :+ record.value) } - .withDeserializers(StringSerde, StringSerde) - - consumedValues <- makeConsumer(kafka, topic, group, handler, 1, modifyConfig = _.copy(offsetReset = Latest)) - .flatMap { _ => consumed.await(_.nonEmpty, 5.seconds) } - .disconnect - .timeoutFail(new TimeoutException("timed out waiting for consumer 1"))(10.seconds) - } yield { - consumedValues must contain(recordValue) + record <- aProducerRecord(topic) + recordValue = record.value.get + _ <- + makeConsumer( + kafka, + topic, + group, + hangingHandler, + 0, + mutateEventLoop = _.copy(drainTimeout = 200.millis, consumePartitionInParallel = useParallelConsumer, maxParallelism = 8) + ) + .flatMap { consumer => + producer.produce(record, StringSerde, StringSerde) *> handlingStarted.await *> + // unsubscribe to make sure partitions are released + consumer.resubscribe(ConsumerSubscription.topics()) + } + .disconnect + .timeoutFail(new TimeoutException("timed out waiting for consumer 0"))(10.seconds) + consumed <- AwaitableRef.make(Seq.empty[String]) + handler = RecordHandler { record: ConsumerRecord[String, String] => consumed.update(_ :+ record.value) } + .withDeserializers(StringSerde, StringSerde) + + consumedValues <- makeConsumer( + kafka, + topic, + group, + handler, + 1, + mutateEventLoop = _.copy(consumePartitionInParallel = useParallelConsumer, maxParallelism = 8), + modifyConfig = _.copy(offsetReset = Latest) + ) + .flatMap { _ => consumed.await(_.nonEmpty, 5.seconds) } + .disconnect + .timeoutFail(new TimeoutException("timed out waiting for consumer 1"))(10.seconds) + } yield { + consumedValues must contain(recordValue) + } } - } - "block until current tasks complete" in - ZIO.scoped { - for { - r <- getShared - TestResources(kafka, producer) = r - topic <- kafka.createRandomTopic(prefix = "block-until") - group <- randomGroup + s"block until current tasks complete${parallelConsumerString(useParallelConsumer)}" in + ZIO.scoped { + for { + r <- getShared + TestResources(kafka, producer) = r + topic <- kafka.createRandomTopic(prefix = "block-until") + group <- randomGroup + + waitForTasksDuration <- TRef.make[Duration](0.millis).commit + innerGate <- Gate.make(initiallyAllow = false) + outerGate <- Gate.make(initiallyAllow = false) + handler = RecordHandler((_: ConsumerRecord[String, String]) => outerGate.toggle(true) *> innerGate.await()) + .withDeserializers(StringSerde, StringSerde) + .ignore + config = configFor(kafka, group, topic, mutateEventLoop = _.copy(consumePartitionInParallel = useParallelConsumer, maxParallelism = 8)) + record = ProducerRecord(topic, "bar", Some("foo")) + + test <- RecordConsumer.make(config, handler).flatMap { consumer => + for { + _ <- ZIO.foreach((0 until 100).toSet)(_ => producer.produce(record, StringSerde, StringSerde)) *> + outerGate.await() /* handler waiting on innerGate now */ *> + consumer.waitForCurrentRecordsCompletion.timed + .map(_._1) /* after 'delay' the consumer's innerGate opens and handler completes */ + .flatMap(d => waitForTasksDuration.set(d).commit) + .fork + delay = 1.second + tasksDurationFiber <- waitForTasksDuration.get.tap(d => STM.check(d > 0.millis)).commit.fork + _ <- innerGate.toggle(true).delay(delay) /*and now handler will complete */ + tasksDuration <- tasksDurationFiber.join + } yield tasksDuration must between(delay * 0.9, delay * 3) + } + } yield test + } - waitForTasksDuration <- TRef.make[Duration](0.millis).commit - innerGate <- Gate.make(initiallyAllow = false) - outerGate <- Gate.make(initiallyAllow = false) - handler = RecordHandler((_: ConsumerRecord[String, String]) => outerGate.toggle(true) *> innerGate.await()) - .withDeserializers(StringSerde, StringSerde) - .ignore - config = configFor(kafka, group, topic) - record = ProducerRecord(topic, "bar", Some("foo")) - - test <- RecordConsumer.make(config, handler).flatMap { consumer => - for { - _ <- ZIO.foreach((0 until 100).toSet)(_ => producer.produce(record, StringSerde, StringSerde)) *> - outerGate.await() /* handler waiting on innerGate now */ *> - consumer.waitForCurrentRecordsCompletion.timed - .map(_._1) /* after 'delay' the consumer's innerGate opens and handler completes */ - .flatMap(d => waitForTasksDuration.set(d).commit) - .fork - delay = 1.second - tasksDurationFiber <- waitForTasksDuration.get.tap(d => STM.check(d > 0.millis)).commit.fork - _ <- innerGate.toggle(true).delay(delay) /*and now handler will complete */ - tasksDuration <- tasksDurationFiber.join - } yield tasksDuration must between(delay * 0.9, delay * 3) - } - } yield test - } + s"rewind positions on poll failure${parallelConsumerString(useParallelConsumer)}" in + ZIO.scoped { + type BinaryRecord = ConsumerRecord[Chunk[Byte], Chunk[Byte]] + type BinaryDecryptor = Decryptor[Any, RuntimeException, Chunk[Byte], Chunk[Byte]] + for { + r <- getShared + TestResources(kafka, producer) = r + topic <- kafka.createRandomTopic(prefix = "poll-fail") + group <- randomGroup + failToDecrypt <- Ref.make(false) + messages <- AwaitableRef.make[Seq[ConsumerRecord[String, String]]](Nil) + handler = RecordHandler((cr: ConsumerRecord[String, String]) => ZIO.debug(s"***** Consumed: $cr") *> messages.update(_ :+ cr)) + .withDeserializers(StringSerde, StringSerde) + .ignore + cId <- clientId + decryptor: BinaryDecryptor = new BinaryDecryptor { + override def decrypt(record: ConsumerRecord[Chunk[Byte], Chunk[Byte]])( + implicit trace: Trace + ): ZIO[Any, RuntimeException, ConsumerRecord[Chunk[Byte], Chunk[Byte]]] = + ZIO.whenZIO(failToDecrypt.get)(ZIO.fail(new RuntimeException)).map(_ => record) + } - "rewind positions on poll failure" in - ZIO.scoped { - type BinaryRecord = ConsumerRecord[Chunk[Byte], Chunk[Byte]] - type BinaryDecryptor = Decryptor[Any, RuntimeException, Chunk[Byte], Chunk[Byte]] - for { - r <- getShared - TestResources(kafka, producer) = r - topic <- kafka.createRandomTopic(prefix = "poll-fail") - group <- randomGroup - failToDecrypt <- Ref.make(false) - messages <- AwaitableRef.make[Seq[ConsumerRecord[String, String]]](Nil) - handler = RecordHandler((cr: ConsumerRecord[String, String]) => ZIO.debug(s"***** Consumed: $cr") *> messages.update(_ :+ cr)) - .withDeserializers(StringSerde, StringSerde) - .ignore - cId <- clientId - decryptor: BinaryDecryptor = new BinaryDecryptor { - override def decrypt(record: ConsumerRecord[Chunk[Byte], Chunk[Byte]])( - implicit trace: Trace - ): ZIO[Any, RuntimeException, ConsumerRecord[Chunk[Byte], Chunk[Byte]]] = - ZIO.whenZIO(failToDecrypt.get)(ZIO.fail(new RuntimeException)).map(_ => record) - } - - pollFailedMetrics <- TestMetrics.queue - - config = configFor(kafka, group, topic).copy(clientId = cId, decryptor = decryptor) - aRecord = (i: Int) => ProducerRecord(topic, s"payload-$i", Some(s"key-$i")) - _ <- RecordConsumer.make(config, handler).flatMap { consumer => - val Seq(rec1, rec2) = (1 to 2) map aRecord - consumer.resubscribe(ConsumerSubscription.topics(topic)) *> producer.produce(rec1, StringSerde, StringSerde) *> - messages.await(_.exists(_.value == rec1.value.get)) *> failToDecrypt.set(true) *> - producer.produce(rec2, StringSerde, StringSerde) *> next[PollingFailed](pollFailedMetrics) *> failToDecrypt.set(false) *> - messages.await(_.exists(_.value == rec2.value.get), 5.seconds) - } - } yield ok - } + pollFailedMetrics <- TestMetrics.queue + + config = configFor(kafka, group, topic, mutateEventLoop = _.copy(consumePartitionInParallel = useParallelConsumer, maxParallelism = 8)) + .copy(clientId = cId, decryptor = decryptor) + aRecord = (i: Int) => ProducerRecord(topic, s"payload-$i", Some(s"key-$i")) + _ <- RecordConsumer.make(config, handler).flatMap { consumer => + val Seq(rec1, rec2) = (1 to 2) map aRecord + consumer.resubscribe(ConsumerSubscription.topics(topic)) *> producer.produce(rec1, StringSerde, StringSerde) *> + messages.await(_.exists(_.value == rec1.value.get)) *> failToDecrypt.set(true) *> + producer.produce(rec2, StringSerde, StringSerde) *> next[PollingFailed](pollFailedMetrics) *> failToDecrypt.set(false) *> + messages.await(_.exists(_.value == rec2.value.get), 5.seconds) + } + } yield ok + } + } def next[A <: GreyhoundMetric](queue: Queue[GreyhoundMetric]) = queue.take.repeatUntil(_.isInstanceOf[A]) @@ -475,9 +553,13 @@ class ConsumerIT extends BaseTestWithSharedEnv[Env, TestResources] { pattern: Pattern, group: String, handler: RecordHandler[Any, Nothing, Chunk[Byte], Chunk[Byte]], - i: Int + i: Int, + useParallelConsumer: Boolean ) = - RecordConsumer.make(configFor(kafka, group, pattern).copy(clientId = s"client-$i", offsetReset = OffsetReset.Earliest), handler) + RecordConsumer.make( + configFor(kafka, group, pattern, useParallelConsumer).copy(clientId = s"client-$i", offsetReset = OffsetReset.Earliest), + handler + ) private def configFor(kafka: ManagedKafka, group: Group, topic: Topic, mutateEventLoop: EventLoopConfig => EventLoopConfig = identity) = RecordConsumerConfig( @@ -489,8 +571,19 @@ class ConsumerIT extends BaseTestWithSharedEnv[Env, TestResources] { eventLoopConfig = mutateEventLoop(EventLoopConfig.Default) ) - private def configFor(kafka: ManagedKafka, group: Group, pattern: Pattern) = - RecordConsumerConfig(kafka.bootstrapServers, group, TopicPattern(pattern), extraProperties = fastConsumerMetadataFetching) + private def configFor( + kafka: ManagedKafka, + group: Group, + pattern: Pattern, + useParallelConsumer: Boolean + ) = + RecordConsumerConfig( + kafka.bootstrapServers, + group, + TopicPattern(pattern), + extraProperties = fastConsumerMetadataFetching, + eventLoopConfig = EventLoopConfig.Default.copy(consumePartitionInParallel = useParallelConsumer, maxParallelism = 8) + ) private def fastConsumerMetadataFetching = Map("metadata.max.age.ms" -> "0") @@ -500,6 +593,9 @@ class ConsumerIT extends BaseTestWithSharedEnv[Env, TestResources] { payload <- randomId } yield ProducerRecord(topic, payload, Some(key)) + private def parallelConsumerString(isParallel: Boolean) = + if (isParallel) " - using parallel consumer" else "" + } object TimedOutWaitingForMessages extends RuntimeException diff --git a/core/src/it/scala/com/wixpress/dst/greyhound/core/parallel/BUILD.bazel b/core/src/it/scala/com/wixpress/dst/greyhound/core/parallel/BUILD.bazel new file mode 100644 index 00000000..f0c1bbb8 --- /dev/null +++ b/core/src/it/scala/com/wixpress/dst/greyhound/core/parallel/BUILD.bazel @@ -0,0 +1,27 @@ +package(default_visibility = ["//visibility:public"]) + +sources() + +specs2_ite2e_test( + name = "parallel", + srcs = [ + ":sources", + ], + deps = [ + "//core/src/it/resources", + "//core/src/it/scala/com/wixpress/dst/greyhound/testenv", + "//core/src/it/scala/com/wixpress/dst/greyhound/testkit", + "//core/src/main/scala/com/wixpress/dst/greyhound/core", + "//core/src/main/scala/com/wixpress/dst/greyhound/core/consumer", + "//core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/domain", + "//core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry", + "//core/src/main/scala/com/wixpress/dst/greyhound/core/metrics", + "//core/src/main/scala/com/wixpress/dst/greyhound/core/producer", + "//core/src/main/scala/com/wixpress/dst/greyhound/core/zioutils", + "//core/src/test/scala/com/wixpress/dst/greyhound/core/testkit", + "@dev_zio_izumi_reflect_2_12", + "@dev_zio_zio_2_12", + "@dev_zio_zio_managed_2_12", + "@org_apache_kafka_kafka_clients", + ], +) diff --git a/core/src/it/scala/com/wixpress/dst/greyhound/core/parallel/ParallelConsumerIT.scala b/core/src/it/scala/com/wixpress/dst/greyhound/core/parallel/ParallelConsumerIT.scala new file mode 100644 index 00000000..c5986d27 --- /dev/null +++ b/core/src/it/scala/com/wixpress/dst/greyhound/core/parallel/ParallelConsumerIT.scala @@ -0,0 +1,325 @@ +package com.wixpress.dst.greyhound.core.parallel +import com.wixpress.dst.greyhound.core.Serdes.StringSerde +import com.wixpress.dst.greyhound.core.consumer.ConsumerMetric.SkippedGapsOnInitialization +import com.wixpress.dst.greyhound.core.consumer.domain.ConsumerSubscription.Topics +import com.wixpress.dst.greyhound.core.consumer.domain.{ConsumerRecord, ConsumerSubscription, RecordHandler} +import com.wixpress.dst.greyhound.core.consumer.{EventLoopConfig, RebalanceListener, RecordConsumer, RecordConsumerConfig} +import com.wixpress.dst.greyhound.core.producer.{ProducerRecord, ReportingProducer} +import com.wixpress.dst.greyhound.core.testkit.RecordMatchers.{beRecordWithKey, beRecordWithValue, beRecordsWithKeysAndValues} +import com.wixpress.dst.greyhound.core.testkit.{eventuallyZ, BaseTestWithSharedEnv, TestMetrics} +import com.wixpress.dst.greyhound.core.zioutils.CountDownLatch +import com.wixpress.dst.greyhound.core.{Group, Topic, TopicPartition} +import com.wixpress.dst.greyhound.testenv.ITEnv +import com.wixpress.dst.greyhound.testenv.ITEnv.{clientId, partitions, randomGroup, randomId, Env, ManagedKafkaOps, TestResources} +import com.wixpress.dst.greyhound.testkit.ManagedKafka +import zio.Clock.sleep +import zio.{Queue, ZIO, _} + +class ParallelConsumerIT extends BaseTestWithSharedEnv[Env, TestResources] { + sequential + + override def env = ITEnv.ManagedEnv + + override def sharedEnv = ITEnv.testResources() + + "consume messages correctly after rebalance" in { + ZIO.scoped { + for { + r <- getShared + TestResources(kafka, producer) = r + topic1 <- kafka.createRandomTopic(prefix = "topic1") + topic2 <- kafka.createRandomTopic(prefix = "topic2") + group <- randomGroup + + queue <- Queue.unbounded[ConsumerRecord[String, String]] + handler = RecordHandler((cr: ConsumerRecord[String, String]) => queue.offer(cr)).withDeserializers(StringSerde, StringSerde) + cId <- clientId + config = parallelConsumerConfig(kafka, topic1, group, cId) + records1 = producerRecords(topic1, "1", partitions, 10) + records2 = producerRecords(topic1, "2", partitions, 10) + numRecordsExpected = records1.size + records2.size + messagesOption <- for { + consumer <- RecordConsumer.make(config, handler) + _ <- produceRecords(producer, records1) + _ <- sleep(5.seconds) + _ <- consumer.resubscribe(ConsumerSubscription.topics(topic1, topic2)) // trigger rebalance + _ <- sleep(500.millis) + _ <- produceRecords(producer, records2) + maybeMessages <- queue + .takeBetween(numRecordsExpected, numRecordsExpected) + .timeout(60.seconds) + .tap(o => ZIO.when(o.isEmpty)(Console.printLine("timeout waiting for messages"))) + } yield maybeMessages + messages <- ZIO.fromOption(messagesOption).orElseFail(TimedOutWaitingForMessages) + } yield { + messages must beRecordsWithKeysAndValues(records1 ++ records2) + } + } + } + + "consume messages exactly once when processing following multiple consecutive polls" in { + ZIO.scoped { + for { + r <- getShared + TestResources(kafka, producer) = r + topic <- kafka.createRandomTopic() + group <- randomGroup + queue <- Queue.unbounded[ConsumerRecord[String, String]] + handlerWithSleep = + RecordHandler((cr: ConsumerRecord[String, String]) => { + (if (cr.partition == cr.offset) ZIO.sleep(2.seconds) // sleep to simulate long processing time and go through multiple polls + else ZIO.unit) *> queue.offer(cr) + }) + .withDeserializers(StringSerde, StringSerde) + cId <- clientId + config = parallelConsumerConfig(kafka, topic, group, cId) + records = producerRecords(topic, "1", partitions, 5) + messagesOption <- RecordConsumer.make(config, handlerWithSleep).flatMap { consumer => + produceRecords(producer, records) *> ZIO.sleep(3.seconds) *> + queue + .takeBetween(records.size, records.size) + .timeout(60.seconds) + .tap(o => ZIO.when(o.isEmpty)(Console.printLine("timeout waiting for messages!"))) + } + messages <- ZIO.fromOption(messagesOption).orElseFail(TimedOutWaitingForMessages) + } yield { + messages must + allOf( + records.map(r => beRecordWithKey(r.key.get) and beRecordWithValue(r.value.get)): _* + ) + } + } + } + + "consume gaps after rebalance and skip already-consumed records" in { + ZIO.scoped { + for { + r <- getShared + TestResources(kafka, producer) = r + topic <- kafka.createRandomTopic() + group <- randomGroup + cId <- clientId + partition = 0 + allMessages = 10 + fastMessages = allMessages - 1 + drainTimeout = 5.seconds + + keyWithSlowHandling = "slow-key" + numProcessedMessges <- Ref.make[Int](0) + fastMessagesLatch <- CountDownLatch.make(fastMessages) + + randomKeys <- ZIO.foreach(1 to fastMessages)(i => randomKey(i.toString)).map(_.toSeq) + + fastRecords = randomKeys.map { key => recordWithKey(topic, key, partition) } + slowRecord = recordWithKey(topic, keyWithSlowHandling, partition) + + finishRebalance <- Promise.make[Nothing, Unit] + + // handler that sleeps only on the slow key + handler = RecordHandler { cr: ConsumerRecord[Chunk[Byte], Chunk[Byte]] => + (cr.key match { + case Some(k) if k == Chunk.fromArray(keyWithSlowHandling.getBytes) => + // make sure the handler doesn't finish before the rebalance is done, including drain timeout + finishRebalance.await *> ZIO.sleep(drainTimeout + 1.second) + case _ => fastMessagesLatch.countDown + }) *> numProcessedMessges.update(_ + 1) + } + _ <- + for { + consumer <- makeParallelConsumer(handler, kafka, topic, group, cId, drainTimeout = drainTimeout, startPaused = true) + _ <- produceRecords(producer, Seq(slowRecord)) + _ <- produceRecords(producer, fastRecords) + // produce is done synchronously to make sure all records are produced before consumer starts, so all records are polled at once + _ <- consumer.resume + _ <- fastMessagesLatch.await + _ <- ZIO.sleep(2.second) // sleep to ensure commit is done before rebalance + // start another consumer to trigger a rebalance before slow handler is done + _ <- makeParallelConsumer( + handler, + kafka, + topic, + group, + cId, + drainTimeout = drainTimeout, + onAssigned = assigned => ZIO.when(assigned.nonEmpty)(finishRebalance.succeed()) + ) + } yield () + + _ <- eventuallyZ(numProcessedMessges.get, 20.seconds)(_ == allMessages) + } yield { + ok + } + } + } + + "migrate correctly from regular record consumer to parallel consumer - consume every record once" in { + ZIO.scoped { + for { + r <- getShared + TestResources(kafka, producer) = r + topic <- kafka.createRandomTopic() + group <- randomGroup + cId <- clientId + + regularConfig = configFor(kafka, group, Set(topic)) + parallelConfig = parallelConsumerConfig(kafka, topic, group, cId) // same group name for both consumers + queue <- Queue.unbounded[ConsumerRecord[String, String]] + handler = RecordHandler((cr: ConsumerRecord[String, String]) => queue.offer(cr)).withDeserializers(StringSerde, StringSerde) + + records1 = producerRecords(topic, "1", partitions, 3) + records2 = producerRecords(topic, "2", partitions, 3) + _ <- ZIO.debug(s"records1:\n${records1.mkString("\n")}\nrecords2:\n${records2.mkString("\n")}") + numMessages = records1.size + records2.size + + _ <- RecordConsumer.make(regularConfig, handler) + _ <- produceRecords(producer, records1) + _ <- ZIO.sleep(3.seconds) + _ <- RecordConsumer.make(parallelConfig, handler).delay(3.seconds) + _ <- produceRecords(producer, records2) + _ <- ZIO.sleep(3.seconds) + messagesOption <- RecordConsumer.make(parallelConfig, handler).flatMap { _ => + produceRecords(producer, records2) *> ZIO.sleep(3.seconds) *> + queue + .takeBetween(numMessages, numMessages) + .timeout(60.seconds) + .tap(o => ZIO.when(o.isEmpty)(Console.printLine("timeout waiting for messages!"))) + } + messages <- ZIO.fromOption(messagesOption).orElseFail(TimedOutWaitingForMessages) + } yield { + messages must beRecordsWithKeysAndValues(records1 ++ records2) + } + } + } + + "migrate from parallel consumer with gaps to regular consumer - consume from latest and report non-consumed gaps" in { + ZIO.scoped { + for { + r <- getShared + TestResources(kafka, producer) = r + topic <- kafka.createRandomTopic() + group <- randomGroup + cId <- clientId + partition = 0 + allMessages = 10 + fastMessages = allMessages - 1 + + skippedGaps <- Ref.make[Int](0) + metricsQueue <- TestMetrics.queue + + regularConfig = configFor(kafka, group, Set(topic)) + _ <- metricsQueue.take + .flatMap { + case m: SkippedGapsOnInitialization => + ZIO.debug(s">>> got SkippedGapsOnInitialization with gaps: ${m.gaps}") *> skippedGaps.update(_ + 1) + case _ => ZIO.unit + } + .repeat(Schedule.forever) + .fork + + keyWithSlowHandling = "slow-key" + numProcessedMessages <- Ref.make[Int](0) + fastMessagesLatch <- CountDownLatch.make(fastMessages) + + randomKeys <- ZIO.foreach(1 to fastMessages)(i => randomKey(i.toString)).map(_.toSeq) + + fastRecords = randomKeys.map { key => recordWithKey(topic, key, partition) } + slowRecord = recordWithKey(topic, keyWithSlowHandling, partition) + additionalRecords = producerRecords(topic, "additional", 1, 5) + + finishRebalance <- Promise.make[Nothing, Unit] + + // handler that sleeps forever on the slow key + parallelConsumerHandler = RecordHandler { cr: ConsumerRecord[Chunk[Byte], Chunk[Byte]] => + (cr.key match { + case Some(k) if k == Chunk.fromArray(keyWithSlowHandling.getBytes) => + ZIO.sleep(Duration.Infinity) + case _ => fastMessagesLatch.countDown + }) *> numProcessedMessages.update(_ + 1) + } + + regularConsumerHandler = RecordHandler { _: ConsumerRecord[Chunk[Byte], Chunk[Byte]] => numProcessedMessages.update(_ + 1) } + + parallelConsumer <- makeParallelConsumer(parallelConsumerHandler, kafka, topic, group, cId, startPaused = true) + _ <- produceRecords(producer, Seq(slowRecord)) + _ <- produceRecords(producer, fastRecords) + // produce is done synchronously to make sure all records are produced before consumer starts, so all records are polled at once + _ <- parallelConsumer.resume + _ <- fastMessagesLatch.await + _ <- ZIO.sleep(2.second) // sleep to ensure commit is done before rebalance + // migrate to regular fromLatest consumer while gap exists + _ <- parallelConsumer.shutdown() *> RecordConsumer.make(regularConfig, regularConsumerHandler) + _ <- produceRecords(producer, additionalRecords) + _ <- eventuallyZ(numProcessedMessages.get, 20.seconds)(_ == fastMessages + additionalRecords.size) + _ <- eventuallyZ(skippedGaps.get, 20.seconds)(_.must(beGreaterThanOrEqualTo(1))) + } yield { + ok + } + } + } + + private def configFor( + kafka: ManagedKafka, + group: Group, + topics: Set[Topic], + mutateEventLoop: EventLoopConfig => EventLoopConfig = identity, + extraProperties: Map[String, String] = Map.empty + ) = RecordConsumerConfig( + bootstrapServers = kafka.bootstrapServers, + group = group, + initialSubscription = Topics(topics), + eventLoopConfig = mutateEventLoop(EventLoopConfig.Default), + extraProperties = extraProperties + ) + + private def makeParallelConsumer( + handler: RecordHandler[Any, Nothing, Chunk[Byte], Chunk[Byte]], + kafka: ManagedKafka, + topic: String, + group: String, + cId: String, + drainTimeout: Duration = 20.seconds, + startPaused: Boolean = false, + onAssigned: Set[TopicPartition] => UIO[Any] = _ => ZIO.unit + ) = + RecordConsumer.make(parallelConsumerConfig(kafka, topic, group, cId, drainTimeout, startPaused, onAssigned), handler) + + private def parallelConsumerConfig( + kafka: ManagedKafka, + topic: String, + group: String, + cId: String, + drainTimeout: Duration = 20.seconds, + startPaused: Boolean = false, + onAssigned: Set[TopicPartition] => UIO[Any] = _ => ZIO.unit + ) = { + configFor( + kafka, + group, + Set(topic), + mutateEventLoop = _.copy( + consumePartitionInParallel = true, + maxParallelism = 10, + drainTimeout = drainTimeout, + startPaused = startPaused, + rebalanceListener = RebalanceListener(onAssigned = onAssigned) + ) + ) + .copy(clientId = cId) + } + + private def producerRecords(topic: String, tag: String, partitions: Int, recordsPerPartition: Int) = (0 until partitions).flatMap(p => + (0 until recordsPerPartition).map(i => ProducerRecord(topic, s"value-t$tag-p$p-$i", Some(s"key-t$tag-p$p-$i"), partition = Some(p))) + ) + + def produceRecords(producer: ReportingProducer[Any], records: Seq[ProducerRecord[String, String]]) = + ZIO + .foreach(records)(r => producer.produce(r, StringSerde, StringSerde)) + + private def recordWithKey(topic: String, key: String, partition: Int) = + ProducerRecord(topic, "", Some(key), partition = Some(partition)) + + private def randomKey(prefix: String) = + randomId.map(r => s"$prefix-$r") +} + +object TimedOutWaitingForMessages extends RuntimeException diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/OffsetAndMetadata.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/OffsetAndMetadata.scala index 6b49ec3a..d03516aa 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/OffsetAndMetadata.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/OffsetAndMetadata.scala @@ -10,5 +10,8 @@ object OffsetAndMetadata { def apply(offsetAndMetadata: KafkaOffsetAndMetadata): OffsetAndMetadata = OffsetAndMetadata(offsetAndMetadata.offset(), offsetAndMetadata.metadata()) + def apply(offset: Offset): OffsetAndMetadata = + OffsetAndMetadata(offset, NO_METADATA) + val NO_METADATA = "" } diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/Consumer.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/Consumer.scala index 7592362e..64168a77 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/Consumer.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/Consumer.scala @@ -33,16 +33,24 @@ trait Consumer { def commit(offsets: Map[TopicPartition, Offset])(implicit trace: Trace): RIO[GreyhoundMetrics, Unit] + def commitWithMetadata(offsetsAndMetadata: Map[TopicPartition, OffsetAndMetadata])(implicit trace: Trace): RIO[GreyhoundMetrics, Unit] + def endOffsets(partitions: Set[TopicPartition])(implicit trace: Trace): RIO[Any, Map[TopicPartition, Offset]] def beginningOffsets(partitions: Set[TopicPartition])(implicit trace: Trace): RIO[Any, Map[TopicPartition, Offset]] def committedOffsets(partitions: Set[TopicPartition])(implicit trace: Trace): RIO[Any, Map[TopicPartition, Offset]] + def committedOffsetsAndMetadata(partitions: Set[TopicPartition])(implicit trace: Trace): RIO[Any, Map[TopicPartition, OffsetAndMetadata]] + def offsetsForTimes(topicPartitionsOnTimestamp: Map[TopicPartition, Long])(implicit trace: Trace): RIO[Any, Map[TopicPartition, Offset]] def commitOnRebalance(offsets: Map[TopicPartition, Offset])(implicit trace: Trace): RIO[GreyhoundMetrics, DelayedRebalanceEffect] + def commitWithMetadataOnRebalance(offsets: Map[TopicPartition, OffsetAndMetadata])( + implicit trace: Trace + ): RIO[GreyhoundMetrics, DelayedRebalanceEffect] + def pause(partitions: Set[TopicPartition])(implicit trace: Trace): ZIO[GreyhoundMetrics, IllegalStateException, Unit] def resume(partitions: Set[TopicPartition])(implicit trace: Trace): ZIO[GreyhoundMetrics, IllegalStateException, Unit] @@ -96,7 +104,8 @@ object Consumer { timeoutIfSeek = 10.seconds, initialSeek = cfg.initialSeek, rewindUncommittedOffsetsBy = cfg.rewindUncommittedOffsetsByMillis.millis, - offsetResetIsEarliest = cfg.offsetReset == OffsetReset.Earliest + offsetResetIsEarliest = cfg.offsetReset == OffsetReset.Earliest, + parallelConsumer = cfg.useParallelConsumer ) } yield { new Consumer { @@ -144,10 +153,22 @@ object Consumer { withConsumerBlocking(_.committed(kafkaPartitions(partitions))) .map(_.asScala.collect { case (tp: KafkaTopicPartition, o: KafkaOffsetAndMetadata) => (TopicPartition(tp), o.offset) }.toMap) + override def committedOffsetsAndMetadata( + partitions: NonEmptySet[TopicPartition] + )(implicit trace: Trace): RIO[Any, Map[TopicPartition, OffsetAndMetadata]] = + withConsumerBlocking(_.committed(kafkaPartitions(partitions))) + .map(_.asScala.collect { case (tp: KafkaTopicPartition, om: KafkaOffsetAndMetadata) => (TopicPartition(tp), OffsetAndMetadata(om.offset, om.metadata))}.toMap) + override def commit(offsets: Map[TopicPartition, Offset])(implicit trace: Trace): RIO[GreyhoundMetrics, Unit] = { withConsumerBlocking(_.commitSync(kafkaOffsetsAndMetaData(toOffsetsAndMetadata(offsets, cfg.commitMetadataString)))) } + override def commitWithMetadata( + offsetsAndMetadata: Map[TopicPartition, OffsetAndMetadata] + )(implicit trace: Trace): RIO[GreyhoundMetrics, Unit] = { + withConsumerBlocking(_.commitSync(kafkaOffsetsAndMetaData(offsetsAndMetadata))) + } + override def commitOnRebalance( offsets: Map[TopicPartition, Offset] )(implicit trace: Trace): RIO[GreyhoundMetrics, DelayedRebalanceEffect] = { @@ -157,6 +178,11 @@ object Consumer { ZIO.succeed(DelayedRebalanceEffect(consumer.commitSync(kOffsets))) } + override def commitWithMetadataOnRebalance( + offsets: Map[TopicPartition, OffsetAndMetadata] + )(implicit trace: Trace): RIO[GreyhoundMetrics, DelayedRebalanceEffect] = + ZIO.succeed(DelayedRebalanceEffect(consumer.commitSync(kafkaOffsetsAndMetaData(offsets)))) + override def pause(partitions: Set[TopicPartition])(implicit trace: Trace): ZIO[Any, IllegalStateException, Unit] = withConsumer(_.pause(kafkaPartitions(partitions))).refineOrDie { case e: IllegalStateException => e } @@ -285,7 +311,8 @@ case class ConsumerConfig( consumerAttributes: Map[String, String] = Map.empty, decryptor: Decryptor[Any, Throwable, Chunk[Byte], Chunk[Byte]] = new NoOpDecryptor, commitMetadataString: Metadata = OffsetAndMetadata.NO_METADATA, - rewindUncommittedOffsetsByMillis: Long = 0L + rewindUncommittedOffsetsByMillis: Long = 0L, + useParallelConsumer: Boolean = false ) extends CommonGreyhoundConfig { override def kafkaProps: Map[String, String] = Map( @@ -320,12 +347,16 @@ object OffsetReset { trait UnsafeOffsetOperations { def committed(partitions: Set[TopicPartition], timeout: zio.Duration): Map[TopicPartition, Offset] + def committedWithMetadata(partitions: Set[TopicPartition], timeout: zio.Duration): Map[TopicPartition, OffsetAndMetadata] + def beginningOffsets(partitions: Set[TopicPartition], timeout: zio.Duration): Map[TopicPartition, Offset] def position(partition: TopicPartition, timeout: zio.Duration): Offset def commit(offsets: Map[TopicPartition, Offset], timeout: Duration): Unit + def commitWithMetadata(offsets: Map[TopicPartition, OffsetAndMetadata], timeout: Duration): Unit + def seek(offsets: Map[TopicPartition, Offset]): Unit def endOffsets(partitions: Set[TopicPartition], timeout: Duration): Map[TopicPartition, Offset] @@ -357,6 +388,20 @@ object UnsafeOffsetOperations { } } + override def committedWithMetadata( + partitions: NonEmptySet[TopicPartition], + timeout: zio.Duration + ): Map[TopicPartition, OffsetAndMetadata] = { + consumer + .committed(partitions.map(_.asKafka).asJava, timeout) + .asScala + .toMap + .collect { + case (tp, ofm) if ofm != null => + TopicPartition(tp) -> OffsetAndMetadata(ofm.offset(), ofm.metadata()) + } + } + override def beginningOffsets(partitions: Set[TopicPartition], timeout: Duration): Map[TopicPartition, Offset] = consumer .beginningOffsets(partitions.map(_.asKafka).asJava, timeout) @@ -374,6 +419,10 @@ object UnsafeOffsetOperations { consumer.commitSync(kafkaOffsets(offsets), timeout) } + override def commitWithMetadata(offsets: Map[TopicPartition, OffsetAndMetadata], timeout: zio.Duration): Unit = { + consumer.commitSync(kafkaOffsetsAndMetaData(offsets), timeout) + } + override def seek(offsets: Map[TopicPartition, Offset]): Unit = offsets.foreach { case (tp, offset) => Try(consumer.seek(tp.asKafka, offset)) } diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/Dispatcher.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/Dispatcher.scala index b618d0ba..a3d93217 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/Dispatcher.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/Dispatcher.scala @@ -1,7 +1,7 @@ package com.wixpress.dst.greyhound.core.consumer import java.util.concurrent.TimeUnit -import com.wixpress.dst.greyhound.core.consumer.Dispatcher.Record +import com.wixpress.dst.greyhound.core.consumer.Dispatcher.{Record, Records} import com.wixpress.dst.greyhound.core.consumer.DispatcherMetric._ import com.wixpress.dst.greyhound.core.consumer.RecordConsumer.Env import com.wixpress.dst.greyhound.core.consumer.SubmitResult._ @@ -20,6 +20,8 @@ import java.lang.System.currentTimeMillis trait Dispatcher[-R] { def submit(record: Record): URIO[R with Env, SubmitResult] + def submitBatch(records: Records): URIO[R with Env, SubmitResult] + def resumeablePartitions(paused: Set[TopicPartition]): URIO[Any, Set[TopicPartition]] def revoke(partitions: Set[TopicPartition]): URIO[GreyhoundMetrics, Unit] @@ -36,7 +38,8 @@ trait Dispatcher[-R] { } object Dispatcher { - type Record = ConsumerRecord[Chunk[Byte], Chunk[Byte]] + type Record = ConsumerRecord[Chunk[Byte], Chunk[Byte]] + type Records = Seq[Record] def make[R]( group: Group, @@ -48,7 +51,12 @@ object Dispatcher { delayResumeOfPausedPartition: Long = 0, consumerAttributes: Map[String, String] = Map.empty, workersShutdownRef: Ref[Map[TopicPartition, ShutdownPromise]], - startPaused: Boolean = false + startPaused: Boolean = false, + consumeInParallel: Boolean = false, + maxParallelism: Int = 1, + updateBatch: Chunk[Record] => UIO[Unit] = _ => ZIO.unit, + currentGaps: Set[TopicPartition] => ZIO[GreyhoundMetrics, Nothing, Map[TopicPartition, Option[OffsetAndGaps]]] = _ => + ZIO.succeed(Map.empty) )(implicit trace: Trace): UIO[Dispatcher[R]] = for { p <- Promise.make[Nothing, Unit] @@ -63,6 +71,22 @@ object Dispatcher { submitted <- worker.submit(record) } yield if (submitted) Submitted else Rejected + override def submitBatch(records: Records): URIO[R with Env, SubmitResult] = + for { + _ <- report(SubmittingRecordBatch(group, clientId, records.size, consumerAttributes)) + allSamePartition = records.map(r => RecordTopicPartition(r)).distinct.size == 1 + submitResult <- if (allSamePartition) { + val partition = RecordTopicPartition(records.head) + for { + worker <- workerFor(partition, records.head.offset) + submitted <- worker.submitBatch(records) + } yield submitted + } else ZIO.succeed(SubmitBatchResult(success = false, Some(records.minBy(_.offset)))) + + } yield + if (allSamePartition && submitResult.success) Submitted + else RejectedBatch(submitResult.firstRejected.getOrElse(records.minBy(_.offset))) + override def resumeablePartitions(paused: Set[TopicPartition]): URIO[Any, Set[TopicPartition]] = workers.get.flatMap { workers => ZIO.foldLeft(paused)(Set.empty[TopicPartition]) { (acc, partition) => @@ -130,7 +154,20 @@ object Dispatcher { case None => for { _ <- report(StartingWorker(group, clientId, partition, offset, consumerAttributes)) - worker <- Worker.make(state, handleWithMetrics, highWatermark, group, clientId, partition, drainTimeout, consumerAttributes) + worker <- Worker.make( + state, + handleWithMetrics, + highWatermark, + group, + clientId, + partition, + drainTimeout, + consumerAttributes, + consumeInParallel, + maxParallelism, + updateBatch, + currentGaps + ) _ <- workers.update(_ + (partition -> worker)) shutdownPromise <- AwaitShutdown.make _ <- workersShutdownRef.update(_.updated(partition, shutdownPromise)) @@ -180,6 +217,8 @@ object Dispatcher { trait Worker { def submit(record: Record): URIO[Any, Boolean] + def submitBatch(records: Records): URIO[Any, SubmitBatchResult] + def expose: URIO[Any, WorkerExposedState] def shutdown: URIO[Any, Unit] @@ -198,14 +237,22 @@ object Dispatcher { clientId: ClientId, partition: TopicPartition, drainTimeout: Duration, - consumerAttributes: Map[String, String] + consumerAttributes: Map[String, String], + consumeInParallel: Boolean, + maxParallelism: Int, + updateBatch: Chunk[Record] => UIO[Unit] = _ => ZIO.unit, + currentGaps: Set[TopicPartition] => ZIO[GreyhoundMetrics, Nothing, Map[TopicPartition, Option[OffsetAndGaps]]] )(implicit trace: Trace): URIO[R with Env, Worker] = for { queue <- Queue.dropping[Record](capacity) internalState <- TRef.make(WorkerInternalState.empty).commit fiber <- - (reportWorkerRunningInInterval(every = 60.seconds, internalState)(partition, group, clientId).forkDaemon *> - pollOnce(status, internalState, handle, queue, group, clientId, partition, consumerAttributes) - .repeatWhile(_ == true)).forkDaemon + (reportWorkerRunningInInterval(every = 60.seconds, internalState)(partition, group, clientId).forkDaemon *> + (if (consumeInParallel) + pollBatch(status, internalState, handle, queue, group, clientId, partition, consumerAttributes, maxParallelism, updateBatch, currentGaps) + else pollOnce(status, internalState, handle, queue, group, clientId, partition, consumerAttributes)) + .repeatWhile(_ == true)) + .interruptible + .forkDaemon } yield new Worker { override def submit(record: Record): URIO[Any, Boolean] = queue @@ -220,6 +267,25 @@ object Dispatcher { } ) + override def submitBatch( + records: Records + ): URIO[Any, SubmitBatchResult] = + queue + .offerAll(records) + .tap(notInserted => + ZIO.when(notInserted.nonEmpty) { + Clock + .currentTime(TimeUnit.MILLISECONDS) + .flatMap(now => + internalState.update(s => if (s.reachedHighWatermarkSince.nonEmpty) s else s.reachedHighWatermark(now)).commit + ) + } + ) + .map(rejected => { + val isSuccess = rejected.isEmpty + SubmitBatchResult(isSuccess, if (isSuccess) None else Some(rejected.minBy(_.offset))) + }) + override def expose: URIO[Any, WorkerExposedState] = (queue.size zip internalState.get.commit) .flatMap { case (queued, state) => @@ -278,6 +344,93 @@ object Dispatcher { case DispatcherState.ShuttingDown => ZIO.succeed(false) } + + private def pollBatch[R]( + state: Ref[DispatcherState], + internalState: TRef[WorkerInternalState], + handle: Record => URIO[R, Any], + queue: Queue[Record], + group: Group, + clientId: ClientId, + partition: TopicPartition, + consumerAttributes: Map[String, String], + maxParallelism: Int, + updateBatch: Chunk[Record] => UIO[Unit], + currentGaps: Set[TopicPartition] => ZIO[GreyhoundMetrics, Nothing, Map[TopicPartition, Option[OffsetAndGaps]]] + )(implicit trace: Trace): ZIO[R with GreyhoundMetrics, Any, Boolean] = + internalState.update(s => s.cleared).commit *> + state.get.flatMap { + case DispatcherState.Running => + queue.takeAll.flatMap { + case records if records.nonEmpty => + handleBatch( + records, + internalState, + handle, + group, + clientId, + partition, + consumerAttributes, + maxParallelism, + updateBatch, + currentGaps + ) + case _ => isActive(internalState).delay(5.millis) + } + case DispatcherState.Paused(resume) => + report(WorkerWaitingForResume(group, clientId, partition, consumerAttributes)) *> resume.await.timeout(30.seconds) *> + isActive(internalState) + case DispatcherState.ShuttingDown => + ZIO.succeed(false) + } + private def handleBatch[R]( + records: Chunk[Record], + internalState: TRef[WorkerInternalState], + handle: Record => URIO[R, Any], + group: Group, + clientId: ClientId, + partition: TopicPartition, + consumerAttributes: Map[ClientId, ClientId], + maxParallelism: RuntimeFlags, + updateBatch: Chunk[Record] => UIO[Unit], + currentGaps: Set[TopicPartition] => ZIO[GreyhoundMetrics, Nothing, Map[TopicPartition, Option[OffsetAndGaps]]] + ): ZIO[R with GreyhoundMetrics, Throwable, Boolean] = + for { + _ <- report(TookAllRecordsFromQueue(records.size, records, group, clientId, consumerAttributes)) + _ <- ZIO + .attempt(currentTimeMillis()) + .flatMap(t => internalState.updateAndGet(_.startedWith(t)).commit) + .tapBoth( + e => report(FailToUpdateParallelCurrentExecutionStarted(records.size, group, clientId, consumerAttributes, e)), + t => report(CurrentExecutionStartedEvent(partition, group, clientId, t.currentExecutionStarted)) + ) + groupedRecords = records.groupBy(_.key).values // todo: add sub-grouping for records without key + latestCommitGaps <- currentGaps(records.map(r => TopicPartition(r.topic, r.partition)).toSet) + _ <- ZIO + .foreachParDiscard(groupedRecords)(sameKeyRecords => + ZIO.foreach(sameKeyRecords) { record => + if (shouldRecordBeHandled(record, latestCommitGaps)) { + handle(record).interruptible.ignore *> updateBatch(sameKeyRecords).interruptible + } else + report(SkippedPreviouslyHandledRecord(record, group, clientId, consumerAttributes)) + + } + ) + .withParallelism(maxParallelism) + res <- isActive(internalState) + } yield res + } + + private def shouldRecordBeHandled(record: Record, maybeGaps: Map[TopicPartition, Option[OffsetAndGaps]]): Boolean = { + maybeGaps.get(TopicPartition(record.topic, record.partition)) match { + case Some(maybeOffsetAndGapsForPartition) => + maybeOffsetAndGapsForPartition match { + case Some(offsetAndGapsForPartition) if offsetAndGapsForPartition.gaps.nonEmpty => + record.offset > offsetAndGapsForPartition.offset || offsetAndGapsForPartition.gaps.exists(_.contains(record.offset)) + case _ => true + } + case None => true + } } private def reportWorkerRunningInInterval( @@ -328,8 +481,12 @@ object SubmitResult { case object Rejected extends SubmitResult + case class RejectedBatch(firstRejected: Record) extends SubmitResult + } +case class SubmitBatchResult(success: Boolean, firstRejected: Option[Record]) extends SubmitResult + sealed trait DispatcherMetric extends GreyhoundMetric object DispatcherMetric { @@ -354,6 +511,9 @@ object DispatcherMetric { case class SubmittingRecord[K, V](group: Group, clientId: ClientId, record: ConsumerRecord[K, V], attributes: Map[String, String]) extends DispatcherMetric + case class SubmittingRecordBatch[K, V](group: Group, clientId: ClientId, numRecords: Int, attributes: Map[String, String]) + extends DispatcherMetric + case class HandlingRecord[K, V]( group: Group, clientId: ClientId, @@ -371,6 +531,13 @@ object DispatcherMetric { ) extends DispatcherMetric case class TookRecordFromQueue(record: Record, group: Group, clientId: ClientId, attributes: Map[String, String]) extends DispatcherMetric + case class TookAllRecordsFromQueue( + numRecords: Int, + records: Chunk[Record], + group: Group, + clientId: ClientId, + attributes: Map[String, String] + ) extends DispatcherMetric case class FailToUpdateCurrentExecutionStarted( record: Record, group: Group, @@ -378,6 +545,13 @@ object DispatcherMetric { attributes: Map[String, String], e: Throwable ) extends DispatcherMetric + case class FailToUpdateParallelCurrentExecutionStarted( + numRecords: Int, + group: Group, + clientId: ClientId, + attributes: Map[String, String], + e: Throwable + ) extends DispatcherMetric case class WorkerWaitingForResume(group: Group, clientId: ClientId, partition: TopicPartition, attributes: Map[String, String]) extends DispatcherMetric @@ -389,6 +563,9 @@ object DispatcherMetric { currentExecutionStarted: Option[Long] ) extends DispatcherMetric + case class SkippedPreviouslyHandledRecord(record: Record, group: Group, clientId: ClientId, attributes: Map[String, String]) + extends DispatcherMetric + } case class DispatcherExposedState(workersState: Map[TopicPartition, WorkerExposedState], state: Dispatcher.DispatcherState) { diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/EventLoop.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/EventLoop.scala index 3c8940ac..0708c19a 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/EventLoop.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/EventLoop.scala @@ -1,9 +1,12 @@ package com.wixpress.dst.greyhound.core.consumer import com.wixpress.dst.greyhound.core._ +import com.wixpress.dst.greyhound.core.consumer.Consumer.Records +import com.wixpress.dst.greyhound.core.consumer.Dispatcher.Record import com.wixpress.dst.greyhound.core.consumer.EventLoopMetric._ import com.wixpress.dst.greyhound.core.consumer.EventLoopState.{Paused, Running, ShuttingDown} import com.wixpress.dst.greyhound.core.consumer.RecordConsumer.Env +import com.wixpress.dst.greyhound.core.consumer.SubmitResult.RejectedBatch import com.wixpress.dst.greyhound.core.consumer.domain.{ConsumerSubscription, RecordHandler} import com.wixpress.dst.greyhound.core.metrics.GreyhoundMetrics.report import com.wixpress.dst.greyhound.core.metrics.{GreyhoundMetric, GreyhoundMetrics} @@ -37,7 +40,11 @@ object EventLoop { val start = for { _ <- report(StartingEventLoop(clientId, group, consumerAttributes)) offsets <- Offsets.make - handle = handler.andThen(offsets.update).handle(_) + offsetsAndGaps <- OffsetsAndGaps.make + handle = if (config.consumePartitionInParallel) { cr: Record => handler.handle(cr) } + else handler.andThen(offsets.update).handle(_) + updateBatch = { records: Chunk[Record] => offsetsAndGaps.update(records) } + currentGaps = { partitions: Set[TopicPartition] => currentGapsForPartitions(partitions, clientId)(consumer) } _ <- report(CreatingDispatcher(clientId, group, consumerAttributes, config.startPaused)) dispatcher <- Dispatcher.make( group, @@ -49,18 +56,33 @@ object EventLoop { config.delayResumeOfPausedPartition, consumerAttributes, workersShutdownRef, - config.startPaused + config.startPaused, + config.consumePartitionInParallel, + config.maxParallelism, + updateBatch, + currentGaps ) positionsRef <- Ref.make(Map.empty[TopicPartition, Offset]) pausedPartitionsRef <- Ref.make(Set.empty[TopicPartition]) partitionsAssigned <- Promise.make[Nothing, Unit] // TODO how to handle errors in subscribe? - rebalanceListener = listener(pausedPartitionsRef, config, dispatcher, partitionsAssigned, group, consumer, clientId, offsets) + rebalanceListener = listener( + pausedPartitionsRef, + config, + dispatcher, + partitionsAssigned, + group, + consumer, + clientId, + offsets, + offsetsAndGaps, + config.consumePartitionInParallel + ) _ <- report(SubscribingToInitialSubAndRebalanceListener(clientId, group, consumerAttributes)) _ <- subscribe(initialSubscription, rebalanceListener)(consumer) running <- Ref.make[EventLoopState](Running) _ <- report(CreatingPollOnceFiber(clientId, group, consumerAttributes)) - fiber <- pollOnce(running, consumer, dispatcher, pausedPartitionsRef, positionsRef, offsets, config, clientId, group) + fiber <- pollOnce(running, consumer, dispatcher, pausedPartitionsRef, positionsRef, offsets, config, clientId, group, offsetsAndGaps) .repeatWhile(_ == true) .forkDaemon _ <- report(AwaitingPartitionsAssignment(clientId, group, consumerAttributes)) @@ -138,7 +160,8 @@ object EventLoop { offsets: Offsets, config: EventLoopConfig, clientId: ClientId, - group: Group + group: Group, + offsetsAndGaps: OffsetsAndGaps ): URIO[R2 with Env, Boolean] = running.get.flatMap { case Running => @@ -146,7 +169,7 @@ object EventLoop { _ <- resumePartitions(consumer, clientId, group, dispatcher, paused) records <- pollAndHandle(consumer, dispatcher, paused, config) _ <- updatePositions(records, positionsRef, consumer, clientId) - _ <- commitOffsets(consumer, offsets) + _ <- if (config.consumePartitionInParallel) commitOffsetsAndGaps(consumer, offsetsAndGaps) else commitOffsets(consumer, offsets) _ <- ZIO.when(records.isEmpty)(ZIO.sleep(50.millis)) } yield true @@ -162,7 +185,9 @@ object EventLoop { group: Group, consumer0: Consumer, clientId: ClientId, - offsets: Offsets + offsets: Offsets, + offsetsAndGaps: OffsetsAndGaps, + useParallelConsumer: Boolean ) = { config.rebalanceListener *> new RebalanceListener[GreyhoundMetrics] { @@ -171,12 +196,13 @@ object EventLoop { partitions: Set[TopicPartition] )(implicit trace: Trace): URIO[GreyhoundMetrics, DelayedRebalanceEffect] = { for { - _ <- pausedPartitionsRef.update(_ -- partitions) - isRevokeTimedOut <- dispatcher.revoke(partitions).timeout(config.drainTimeout).map(_.isEmpty) - _ <- ZIO.when(isRevokeTimedOut)( - report(DrainTimeoutExceeded(clientId, group, config.drainTimeout.toMillis, consumer.config.consumerAttributes)) - ) - delayedRebalanceEffect <- commitOffsetsOnRebalance(consumer0, offsets) + _ <- pausedPartitionsRef.update(_ -- partitions) + isRevokeTimedOut <- dispatcher.revoke(partitions).timeout(config.drainTimeout).map(_.isEmpty) + _ <- ZIO.when(isRevokeTimedOut)( + report(DrainTimeoutExceeded(clientId, group, config.drainTimeout.toMillis, consumer.config.consumerAttributes)) + ) + delayedRebalanceEffect <- if (useParallelConsumer) commitOffsetsAndGapsOnRebalance(consumer0, offsetsAndGaps) + else commitOffsetsOnRebalance(consumer0, offsets) } yield delayedRebalanceEffect } @@ -214,24 +240,61 @@ object EventLoop { for { records <- consumer.poll(config.fetchTimeout).catchAll(_ => ZIO.succeed(Nil)) paused <- pausedRef.get - pausedTopics <- ZIO.foldLeft(records)(paused) { (acc, record) => - val partition = record.topicPartition - if (acc contains partition) - report(PartitionThrottled(partition, record.offset, consumer.config.consumerAttributes)).as(acc) - else - dispatcher.submit(record).flatMap { - case SubmitResult.Submitted => ZIO.succeed(acc) - case SubmitResult.Rejected => - report(HighWatermarkReached(partition, record.offset, consumer.config.consumerAttributes)) *> - consumer.pause(record).fold(_ => acc, _ => acc + partition) - } - } + pausedTopics <- if (config.consumePartitionInParallel) submitRecordsAsBatch(consumer, dispatcher, records, paused) + else submitRecordsSequentially(consumer, dispatcher, records, paused) _ <- pausedRef.update(_ => pausedTopics) } yield records + private def submitRecordsSequentially[R2, R1]( + consumer: Consumer, + dispatcher: Dispatcher[R2], + records: Records, + paused: Set[TopicPartition] + ): ZIO[R2 with Env, Nothing, Set[TopicPartition]] = { + ZIO.foldLeft(records)(paused) { (acc, record) => + val partition = record.topicPartition + if (acc contains partition) + report(PartitionThrottled(partition, record.offset, consumer.config.consumerAttributes)).as(acc) + else + dispatcher.submit(record).flatMap { + case SubmitResult.Submitted => ZIO.succeed(acc) + case SubmitResult.Rejected => + report(HighWatermarkReached(partition, record.offset, consumer.config.consumerAttributes)) *> + consumer.pause(record).fold(_ => acc, _ => acc + partition) + } + } + } + + private def submitRecordsAsBatch[R2, R1]( + consumer: Consumer, + dispatcher: Dispatcher[R2], + records: Records, + paused: Set[TopicPartition] + ): ZIO[R2 with Env, Nothing, Set[TopicPartition]] = { + val recordsByPartition = records.groupBy(_.topicPartition) + ZIO.foldLeft(recordsByPartition)(paused) { (acc, partitionToRecords) => + val partition = partitionToRecords._1 + if (acc contains partition) + report(PartitionThrottled(partition, partitionToRecords._2.map(_.offset).min, consumer.config.consumerAttributes)).as(acc) + else + dispatcher.submitBatch(partitionToRecords._2.toSeq).flatMap { + case SubmitResult.Submitted => ZIO.succeed(acc) + case RejectedBatch(firstRejected) => + report(HighWatermarkReached(partition, firstRejected.offset, consumer.config.consumerAttributes)) *> + consumer.pause(firstRejected).fold(_ => acc, _ => acc + partition) + } + } + } + private def commitOffsets(consumer: Consumer, offsets: Offsets): URIO[GreyhoundMetrics, Unit] = offsets.committable.flatMap { committable => consumer.commit(committable).catchAll { _ => offsets.update(committable) } } + private def commitOffsetsAndGaps(consumer: Consumer, offsetsAndGaps: OffsetsAndGaps): URIO[GreyhoundMetrics, Unit] = + offsetsAndGaps.getCommittableAndClear.flatMap { committable => + val offsetsAndMetadataToCommit = OffsetsAndGaps.toOffsetsAndMetadata(committable) + consumer.commitWithMetadata(offsetsAndMetadataToCommit).catchAll { _ => offsetsAndGaps.setCommittable(committable) } + } + private def commitOffsetsOnRebalance( consumer: Consumer, offsets: Offsets @@ -251,6 +314,32 @@ object EventLoop { } } + private def commitOffsetsAndGapsOnRebalance( + consumer: Consumer, + offsetsAndGaps: OffsetsAndGaps + ): URIO[GreyhoundMetrics, DelayedRebalanceEffect] = { + for { + committable <- offsetsAndGaps.getCommittableAndClear + tle <- consumer + .commitWithMetadataOnRebalance(OffsetsAndGaps.toOffsetsAndMetadata(committable)) + .catchAll { _ => offsetsAndGaps.setCommittable(committable) *> DelayedRebalanceEffect.zioUnit } + runtime <- ZIO.runtime[Any] + } yield tle.catchAll { _ => zio.Unsafe.unsafe { implicit s => + runtime.unsafe + .run(offsetsAndGaps.setCommittable(committable)) + .getOrThrowFiberFailure() + } + } + } + + private def currentGapsForPartitions(partitions: Set[TopicPartition], clientId: ClientId)( + consumer: Consumer + ): ZIO[GreyhoundMetrics, Nothing, Map[TopicPartition, Option[OffsetAndGaps]]] = + consumer + .committedOffsetsAndMetadata(partitions) + .map { committed => committed.mapValues(om => OffsetsAndGaps.parseGapsString(om.metadata)) } + .catchAll(t => report(FailedToFetchCommittedGaps(t, clientId, consumer.config.consumerAttributes)).as(Map.empty)) + } case class EventLoopConfig( @@ -260,7 +349,9 @@ case class EventLoopConfig( highWatermark: Int, rebalanceListener: RebalanceListener[Any], delayResumeOfPausedPartition: Long, - startPaused: Boolean + startPaused: Boolean, + consumePartitionInParallel: Boolean, + maxParallelism: Int ) object EventLoopConfig { @@ -271,7 +362,9 @@ object EventLoopConfig { highWatermark = 256, rebalanceListener = RebalanceListener.Empty, delayResumeOfPausedPartition = 0, - startPaused = false + startPaused = false, + consumePartitionInParallel = false, + maxParallelism = 1 ) } @@ -305,6 +398,9 @@ object EventLoopMetric { case class FailedToUpdatePositions(t: Throwable, clientId: ClientId, attributes: Map[String, String] = Map.empty) extends EventLoopMetric + case class FailedToFetchCommittedGaps(t: Throwable, clientId: ClientId, attributes: Map[String, String] = Map.empty) + extends EventLoopMetric + case class CreatingDispatcher(clientId: ClientId, group: Group, attributes: Map[String, String], startPaused: Boolean) extends EventLoopMetric diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsAndGaps.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsAndGaps.scala index f5718555..2704b5d6 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsAndGaps.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsAndGaps.scala @@ -1,6 +1,9 @@ package com.wixpress.dst.greyhound.core.consumer -import com.wixpress.dst.greyhound.core.{Offset, TopicPartition} +import com.wixpress.dst.greyhound.core.consumer.Gap.GAP_SEPARATOR +import com.wixpress.dst.greyhound.core.consumer.OffsetAndGaps.{GAPS_STRING_SEPARATOR, LAST_HANDLED_OFFSET_SEPARATOR} +import com.wixpress.dst.greyhound.core.consumer.domain.{ConsumerRecord, RecordTopicPartition} +import com.wixpress.dst.greyhound.core.{Offset, OffsetAndMetadata, TopicPartition} import zio._ trait OffsetsAndGaps { @@ -10,6 +13,16 @@ trait OffsetsAndGaps { def update(partition: TopicPartition, batch: Seq[Offset]): UIO[Unit] + def update(record: ConsumerRecord[_, _]): UIO[Unit] = + update(RecordTopicPartition(record), Seq(record.offset)) + + def update(records: Chunk[ConsumerRecord[_, _]]): UIO[Unit] = { + val sortedBatch = records.sortBy(_.offset) + update(RecordTopicPartition(sortedBatch.head), sortedBatch.map(_.offset) ++ Seq(sortedBatch.last.offset + 1)) + } + + def setCommittable(offsets: Map[TopicPartition, OffsetAndGaps]): UIO[Unit] + def contains(partition: TopicPartition, offset: Offset): UIO[Boolean] } @@ -48,6 +61,9 @@ object OffsetsAndGaps { override def contains(partition: TopicPartition, offset: Offset): UIO[Boolean] = ref.get.map(_.get(partition).fold(false)(_.contains(offset))) + override def setCommittable(offsets: Map[TopicPartition, OffsetAndGaps]): UIO[Unit] = + ref.update { _ => offsets } + private def gapsInBatch(batch: Seq[Offset], prevLastOffset: Offset): Seq[Gap] = batch.sorted .foldLeft(Seq.empty[Gap], prevLastOffset) { @@ -73,20 +89,65 @@ object OffsetsAndGaps { } } } + + def toOffsetsAndMetadata(offsetsAndGaps: Map[TopicPartition, OffsetAndGaps]): Map[TopicPartition, OffsetAndMetadata] = + offsetsAndGaps.mapValues(offsetAndGaps => + OffsetAndMetadata(offsetAndGaps.offset, offsetAndGaps.gapsString) + ) // todo: add encoding and compression to plain gaps string + + def parseGapsString(offsetAndGapsString: String): Option[OffsetAndGaps] = { + val lastHandledOffsetSeparatorIndex = offsetAndGapsString.indexOf(LAST_HANDLED_OFFSET_SEPARATOR) + if (lastHandledOffsetSeparatorIndex < 0) + None + else { + val lastHandledOffset = offsetAndGapsString.substring(0, lastHandledOffsetSeparatorIndex).toLong + val gaps = offsetAndGapsString + .substring(lastHandledOffsetSeparatorIndex + 1) + .split(GAPS_STRING_SEPARATOR) + .map(_.split(GAP_SEPARATOR)) + .collect { case Array(start, end) => Gap(start.toLong, end.toLong) } + .toSeq + .sortBy(_.start) + Some(OffsetAndGaps(lastHandledOffset, gaps)) + } + } + + def firstGapOffset(gapsString: String): Option[Offset] = { + val maybeOffsetAndGaps = parseGapsString(gapsString) + maybeOffsetAndGaps match { + case Some(offsetAndGaps) if offsetAndGaps.gaps.nonEmpty => Some(offsetAndGaps.gaps.minBy(_.start).start) + case _ => None + } + } } case class Gap(start: Offset, end: Offset) { def contains(offset: Offset): Boolean = start <= offset && offset <= end def size: Long = end - start + 1 + + override def toString: String = s"$start$GAP_SEPARATOR$end" +} + +object Gap { + val GAP_SEPARATOR = "_" } case class OffsetAndGaps(offset: Offset, gaps: Seq[Gap], committable: Boolean = true) { def contains(offset: Offset): Boolean = gaps.exists(_.contains(offset)) def markCommitted: OffsetAndGaps = copy(committable = false) + + def gapsString: String = { + if (gaps.isEmpty) "" + else + s"${offset.toString}${LAST_HANDLED_OFFSET_SEPARATOR}${gaps.sortBy(_.start).mkString(GAPS_STRING_SEPARATOR)}" + } } object OffsetAndGaps { + val GAPS_STRING_SEPARATOR = "$" + val LAST_HANDLED_OFFSET_SEPARATOR = "#" + def apply(offset: Offset): OffsetAndGaps = OffsetAndGaps(offset, Seq.empty[Gap]) } diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsInitializer.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsInitializer.scala index e684eef7..234e7ba9 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsInitializer.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsInitializer.scala @@ -1,12 +1,11 @@ package com.wixpress.dst.greyhound.core.consumer import java.time.Clock -import com.wixpress.dst.greyhound.core.consumer.ConsumerMetric.{CommittedMissingOffsets, CommittedMissingOffsetsFailed} -import com.wixpress.dst.greyhound.core.{ClientId, Group, Offset, TopicPartition} +import com.wixpress.dst.greyhound.core.consumer.ConsumerMetric.{CommittedMissingOffsets, CommittedMissingOffsetsFailed, SkippedGapsOnInitialization} +import com.wixpress.dst.greyhound.core.consumer.OffsetsAndGaps.{firstGapOffset, parseGapsString} +import com.wixpress.dst.greyhound.core.{ClientId, Group, Offset, OffsetAndMetadata, TopicPartition} import com.wixpress.dst.greyhound.core.metrics.{GreyhoundMetric, GreyhoundMetrics} - import zio.{URIO, ZIO} - import zio._ /** @@ -24,17 +23,19 @@ class OffsetsInitializer( initialSeek: InitialOffsetsSeek, rewindUncommittedOffsetsBy: Duration, clock: Clock = Clock.systemUTC, - offsetResetIsEarliest: Boolean + offsetResetIsEarliest: Boolean, + parallelConsumer: Boolean ) { def initializeOffsets(partitions: Set[TopicPartition]): Unit = { val hasSeek = initialSeek != InitialOffsetsSeek.default val effectiveTimeout = if (hasSeek) timeoutIfSeek else timeout withReporting(partitions, rethrow = hasSeek) { - val committed = offsetOperations.committed(partitions, effectiveTimeout) + val committed = offsetOperations.committedWithMetadata(partitions, effectiveTimeout) val beginning = offsetOperations.beginningOffsets(partitions, effectiveTimeout) val endOffsets = offsetOperations.endOffsets(partitions, effectiveTimeout) - val PartitionActions(toOffsets, toPause) = calculateTargetOffsets(partitions, beginning, committed, endOffsets, effectiveTimeout) + val PartitionActions(toOffsets, toPause) = + calculateTargetOffsets(partitions, beginning, committed, endOffsets, effectiveTimeout, parallelConsumer) val notCommitted = partitions -- committed.keySet -- toOffsets.keySet offsetOperations.pause(toPause) @@ -46,43 +47,60 @@ class OffsetsInitializer( .map { case (tp, maybeRewindedOffset) => (tp, maybeRewindedOffset.orElse(endOffsets.get(tp)).getOrElse(0L)) } val positions = - notCommitted.map(tp => tp -> offsetOperations.position(tp, effectiveTimeout)).toMap ++ toOffsets ++ rewindUncommittedOffsets + notCommitted.map(tp => tp -> offsetOperations.position(tp, effectiveTimeout)).toMap.mapValues(OffsetAndMetadata.apply) ++ + toOffsets ++ rewindUncommittedOffsets.mapValues(OffsetAndMetadata.apply) if ((toOffsets ++ rewindUncommittedOffsets).nonEmpty) { - offsetOperations.seek(toOffsets ++ rewindUncommittedOffsets) + offsetOperations.seek(toOffsets.mapValues(_.offset) ++ rewindUncommittedOffsets) } if (positions.nonEmpty) { - offsetOperations.commit(positions, effectiveTimeout) + offsetOperations.commitWithMetadata(positions, effectiveTimeout) } - positions + positions.mapValues(_.offset) } } - case class PartitionActions(offsetSeeks: Map[TopicPartition, Offset], partitionsToPause: Set[TopicPartition]) + case class PartitionActions(offsetSeeks: Map[TopicPartition, OffsetAndMetadata], partitionsToPause: Set[TopicPartition]) private def calculateTargetOffsets( partitions: Set[TopicPartition], beginning: Map[TopicPartition, Offset], - committed: Map[TopicPartition, Offset], + committed: Map[TopicPartition, OffsetAndMetadata], endOffsets: Map[TopicPartition, Offset], - timeout: Duration + timeout: Duration, + parallelConsumer: Boolean ): PartitionActions = { + val currentCommittedOffsets = partitions.map((_, None)).toMap ++ committed.mapValues(Some.apply) val seekTo: Map[TopicPartition, SeekTo] = initialSeek.seekOffsetsFor( assignedPartitions = partitions, beginningOffsets = partitions.map((_, None)).toMap ++ beginning.mapValues(Some.apply), endOffsets = partitions.map((_, None)).toMap ++ endOffsets.mapValues(Some.apply), - currentCommittedOffsets = partitions.map((_, None)).toMap ++ committed.mapValues(Some.apply) + currentCommittedOffsets = currentCommittedOffsets.mapValues(_.map(_.offset)) ) - val seekToOffsets = seekTo.collect { case (k, v: SeekTo.SeekToOffset) => k -> v.offset } + val seekToOffsets = seekTo.collect { case (k, v: SeekTo.SeekToOffset) => k -> OffsetAndMetadata(v.offset) } val seekToEndPartitions = seekTo.collect { case (k, SeekTo.SeekToEnd) => k }.toSet val toPause = seekTo.collect { case (k, SeekTo.Pause) => k } - val seekToEndOffsets = fetchEndOffsets(seekToEndPartitions, timeout) - val toOffsets = seekToOffsets ++ seekToEndOffsets - + val seekToEndOffsets = fetchEndOffsets(seekToEndPartitions, timeout).mapValues(OffsetAndMetadata.apply) + val gapsSmallestOffsets = currentCommittedOffsets + .collect { case (tp, Some(om)) => tp -> om } + .map(tpom => tpom._1 -> (firstGapOffset(tpom._2.metadata), tpom._2.metadata)) + .collect { case (tp, (Some(offset), metadata)) => tp -> OffsetAndMetadata(offset, metadata) } + val seekToGapsOffsets = if (parallelConsumer) gapsSmallestOffsets else Map.empty + val toOffsets = seekToOffsets ++ seekToEndOffsets ++ seekToGapsOffsets + + if (!parallelConsumer && gapsSmallestOffsets.nonEmpty) reportSkippedGaps(currentCommittedOffsets) PartitionActions(offsetSeeks = toOffsets, partitionsToPause = toPause.toSet) } + private def reportSkippedGaps(currentCommittedOffsets: Map[TopicPartition, Option[OffsetAndMetadata]]) = { + val skippedGaps = currentCommittedOffsets + .collect { case (tp, Some(om)) => tp -> om } + .map(tpom => tpom._1 -> parseGapsString(tpom._2.metadata)) + .collect { case (tp, Some(gaps)) => tp -> gaps } + reporter(SkippedGapsOnInitialization(clientId, group, skippedGaps)) + } + private def fetchEndOffsets(seekToEndPartitions: Set[TopicPartition], timeout: Duration) = { if (seekToEndPartitions.nonEmpty) { offsetOperations.endOffsets(seekToEndPartitions, timeout) @@ -151,7 +169,8 @@ object OffsetsInitializer { initialSeek: InitialOffsetsSeek, clock: Clock = Clock.systemUTC, rewindUncommittedOffsetsBy: Duration, - offsetResetIsEarliest: Boolean + offsetResetIsEarliest: Boolean, + parallelConsumer: Boolean )(implicit trace: Trace): URIO[GreyhoundMetrics, OffsetsInitializer] = for { metrics <- ZIO.environment[GreyhoundMetrics].map(_.get) runtime <- ZIO.runtime[Any] @@ -165,6 +184,7 @@ object OffsetsInitializer { initialSeek: InitialOffsetsSeek, rewindUncommittedOffsetsBy, clock, - offsetResetIsEarliest + offsetResetIsEarliest, + parallelConsumer ) } diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/RecordConsumer.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/RecordConsumer.scala index 1b426d60..92c7faf5 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/RecordConsumer.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/RecordConsumer.scala @@ -79,15 +79,17 @@ object RecordConsumer { (initialSubscription, topicsToCreate) = config.retryConfig.fold((config.initialSubscription, Set.empty[Topic]))(policy => maybeAddRetryTopics(policy, config, nonBlockingRetryHelper) ) - _ <- ZIO.when(config.createRetryTopics)(AdminClient - .make(AdminClientConfig(config.bootstrapServers, config.kafkaAuthProperties), config.consumerAttributes) - .tap(client => - client.createTopics( - topicsToCreate.map(topic => - TopicConfig(topic, partitions = 1, replicationFactor = 1, cleanupPolicy = CleanupPolicy.Delete(86400000L)) + _ <- ZIO.when(config.createRetryTopics)( + AdminClient + .make(AdminClientConfig(config.bootstrapServers, config.kafkaAuthProperties), config.consumerAttributes) + .tap(client => + client.createTopics( + topicsToCreate.map(topic => + TopicConfig(topic, partitions = 1, replicationFactor = 1, cleanupPolicy = CleanupPolicy.Delete(86400000L)) + ) ) ) - )) + ) blockingState <- Ref.make[Map[BlockingTarget, BlockingState]](Map.empty) blockingStateResolver = BlockingStateResolver(blockingState) workersShutdownRef <- Ref.make[Map[TopicPartition, ShutdownPromise]](Map.empty) @@ -206,7 +208,8 @@ object RecordConsumer { config.consumerAttributes, config.decryptor, config.commitMetadataString, - config.rewindUncommittedOffsetsBy.toMillis + config.rewindUncommittedOffsetsBy.toMillis, + config.eventLoopConfig.consumePartitionInParallel ) } diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/ReportingConsumer.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/ReportingConsumer.scala index d0f93a00..6202220d 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/ReportingConsumer.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/ReportingConsumer.scala @@ -109,6 +109,41 @@ case class ReportingConsumer(clientId: ClientId, group: Group, internal: Consume } else DelayedRebalanceEffect.zioUnit } + override def commitWithMetadataOnRebalance(offsets: Map[TopicPartition, OffsetAndMetadata] + )(implicit trace: Trace): RIO[GreyhoundMetrics, DelayedRebalanceEffect] = + ZIO.runtime[GreyhoundMetrics].flatMap { runtime => + if (offsets.nonEmpty) { + report(CommittingOffsetsWithMetadata(clientId, group, offsets, calledOnRebalance = true, attributes = config.consumerAttributes)) *> + internal + .commitWithMetadataOnRebalance(offsets) + .tapError { error => + report(CommitWithMetadataFailed(clientId, group, error, offsets, calledOnRebalance = true, attributes = config.consumerAttributes)) + } + .map( + _.tapError { error => // handle commit errors in ThreadLockedEffect + zio.Unsafe.unsafe { implicit s => + runtime.unsafe + .run( + report( + CommitWithMetadataFailed(clientId, group, error, offsets, calledOnRebalance = true, attributes = config.consumerAttributes) + ) + ) + .getOrThrowFiberFailure() + } + } *> + DelayedRebalanceEffect( + zio.Unsafe.unsafe { implicit s => + runtime.unsafe + .run( + report(CommittedOffsetsWithMetadata(clientId, group, offsets, calledOnRebalance = true, attributes = config.consumerAttributes)) + ) + .getOrThrowFiberFailure() + } + ) + ) + } else DelayedRebalanceEffect.zioUnit + } + override def commit(offsets: Map[TopicPartition, Offset])(implicit trace: Trace): RIO[GreyhoundMetrics, Unit] = { ZIO .when(offsets.nonEmpty) { @@ -122,6 +157,34 @@ case class ReportingConsumer(clientId: ClientId, group: Group, internal: Consume .unit } + override def commitWithMetadata(offsetsAndMetadata: Map[TopicPartition, OffsetAndMetadata])( + implicit trace: Trace + ): RIO[GreyhoundMetrics, Unit] = { + val offsets = offsetsAndMetadata.map { case (tp, om) => tp -> om.offset } + ZIO + .when(offsetsAndMetadata.nonEmpty) { + report( + CommittingOffsets( + clientId, + group, + offsets, + calledOnRebalance = false, + attributes = config.consumerAttributes + ) + ) *> internal.commitWithMetadata(offsetsAndMetadata).tapError { error => report(CommitFailed(clientId, group, error, offsets)) } *> + report( + CommittedOffsets( + clientId, + group, + offsets, + calledOnRebalance = false, + attributes = config.consumerAttributes + ) + ) + } + .unit + } + override def pause(partitions: Set[TopicPartition])(implicit trace: Trace): ZIO[GreyhoundMetrics, IllegalStateException, Unit] = ZIO .when(partitions.nonEmpty) { @@ -159,6 +222,9 @@ case class ReportingConsumer(clientId: ClientId, group: Group, internal: Consume override def committedOffsets(partitions: Set[TopicPartition])(implicit trace: Trace): RIO[Any, Map[TopicPartition, Offset]] = internal.committedOffsets(partitions) + override def committedOffsetsAndMetadata(partitions: NonEmptySet[TopicPartition])(implicit trace: Trace): RIO[Any, Map[TopicPartition, OffsetAndMetadata]] = + internal.committedOffsetsAndMetadata(partitions) + override def position(topicPartition: TopicPartition)(implicit trace: Trace): Task[Offset] = internal.position(topicPartition) @@ -201,6 +267,14 @@ object ConsumerMetric { attributes: Map[String, String] = Map.empty ) extends ConsumerMetric + case class CommittingOffsetsWithMetadata( + clientId: ClientId, + group: Group, + offsets: Map[TopicPartition, OffsetAndMetadata], + calledOnRebalance: Boolean, + attributes: Map[String, String] = Map.empty + ) extends ConsumerMetric + case class CommittedOffsets( clientId: ClientId, group: Group, @@ -209,6 +283,14 @@ object ConsumerMetric { attributes: Map[String, String] = Map.empty ) extends ConsumerMetric + case class CommittedOffsetsWithMetadata( + clientId: ClientId, + group: Group, + offsets: Map[TopicPartition, OffsetAndMetadata], + calledOnRebalance: Boolean, + attributes: Map[String, String] = Map.empty + ) extends ConsumerMetric + case class PausingPartitions( clientId: ClientId, group: Group, @@ -268,6 +350,15 @@ object ConsumerMetric { attributes: Map[String, String] = Map.empty ) extends ConsumerMetric + case class CommitWithMetadataFailed( + clientId: ClientId, + group: Group, + error: Throwable, + offsets: Map[TopicPartition, OffsetAndMetadata], + calledOnRebalance: Boolean = false, + attributes: Map[String, String] = Map.empty + ) extends ConsumerMetric + case class PausePartitionsFailed( clientId: ClientId, group: Group, @@ -323,4 +414,6 @@ object ConsumerMetric { case class ClosedConsumer(group: Group, clientId: ClientId, result: MetricResult[Throwable, Unit]) extends ConsumerMetric + case class SkippedGapsOnInitialization(clientId: ClientId, group: Group, gaps: Map[TopicPartition, OffsetAndGaps]) extends ConsumerMetric + } diff --git a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/EventLoopTest.scala b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/EventLoopTest.scala index 8331d84b..aa335fde 100644 --- a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/EventLoopTest.scala +++ b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/EventLoopTest.scala @@ -9,7 +9,7 @@ import com.wixpress.dst.greyhound.core.consumer.domain.{ConsumerRecord, RecordHa import com.wixpress.dst.greyhound.core.metrics.GreyhoundMetrics import com.wixpress.dst.greyhound.core.testkit.{BaseTest, TestMetrics} import com.wixpress.dst.greyhound.core.zioutils.AwaitShutdown.ShutdownPromise -import com.wixpress.dst.greyhound.core.{Headers, Offset, Topic, TopicPartition} +import com.wixpress.dst.greyhound.core.{Headers, Offset, OffsetAndMetadata, Topic, TopicPartition} import zio._ import java.util.regex.Pattern @@ -139,11 +139,21 @@ trait EmptyConsumer extends Consumer { override def commit(offsets: Map[TopicPartition, Offset])(implicit trace: Trace): Task[Unit] = ZIO.unit + override def commitWithMetadata(offsetsAndMetadata: Map[TopicPartition, OffsetAndMetadata])( + implicit trace: Trace + ): RIO[GreyhoundMetrics, Unit] = + ZIO.unit + override def commitOnRebalance(offsets: Map[TopicPartition, Offset])( implicit trace: Trace ): RIO[GreyhoundMetrics, DelayedRebalanceEffect] = DelayedRebalanceEffect.zioUnit + override def commitWithMetadataOnRebalance(offsets: Map[TopicPartition, OffsetAndMetadata])( + implicit trace: Trace + ): RIO[GreyhoundMetrics, DelayedRebalanceEffect] = + DelayedRebalanceEffect.zioUnit + override def pause(partitions: Set[TopicPartition])(implicit trace: Trace): ZIO[Any, IllegalStateException, Unit] = ZIO.unit @@ -173,4 +183,7 @@ trait EmptyConsumer extends Consumer { override def committedOffsets(partitions: Set[TopicPartition])(implicit trace: Trace): RIO[Any, Map[TopicPartition, Offset]] = ZIO.succeed(Map.empty) + + override def committedOffsetsAndMetadata(partitions: Set[TopicPartition])(implicit trace: Trace): RIO[Any, Map[TopicPartition, OffsetAndMetadata]] = + ZIO.succeed(Map.empty) } diff --git a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsInitializerTest.scala b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsInitializerTest.scala index 5e708452..2b8ffb0f 100644 --- a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsInitializerTest.scala +++ b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsInitializerTest.scala @@ -2,7 +2,7 @@ package com.wixpress.dst.greyhound.core.consumer import java.time.{Clock, Duration, ZoneId} import java.util.concurrent.atomic.AtomicReference -import com.wixpress.dst.greyhound.core.TopicPartition +import com.wixpress.dst.greyhound.core.{OffsetAndMetadata, TopicPartition} import com.wixpress.dst.greyhound.core.consumer.ConsumerMetric.{CommittedMissingOffsets, CommittedMissingOffsetsFailed} import com.wixpress.dst.greyhound.core.consumer.SeekTo.{SeekToEnd, SeekToOffset} import com.wixpress.dst.greyhound.core.metrics.GreyhoundMetric @@ -40,8 +40,8 @@ class OffsetsInitializerTest extends SpecificationWithJUnit with Mockito { p3 -> p3Pos ) there was - one(offsetOps).commit( - missingOffsets, + one(offsetOps).commitWithMetadata( + missingOffsets.mapValues(OffsetAndMetadata(_)), timeout ) @@ -57,8 +57,8 @@ class OffsetsInitializerTest extends SpecificationWithJUnit with Mockito { val expected = Map(p1 -> p1Pos, p3 -> p3Pos) there was - one(offsetOps).commit( - expected, + one(offsetOps).commitWithMetadata( + expected.mapValues(OffsetAndMetadata(_)), timeoutIfSeek ) @@ -79,8 +79,8 @@ class OffsetsInitializerTest extends SpecificationWithJUnit with Mockito { val expected = Map(p1 -> p1Pos, p3 -> p3Pos, p2 -> p2Pos) there was - one(offsetOps).commit( - expected, + one(offsetOps).commitWithMetadata( + expected.mapValues(OffsetAndMetadata(_)), timeoutIfSeek ) there was one(offsetOps).seek(Map(p1 -> p1Pos, p2 -> p2Pos)) @@ -93,27 +93,27 @@ class OffsetsInitializerTest extends SpecificationWithJUnit with Mockito { "fail if operation fails and there are relevant seekTo offsets" in new ctx(seekTo = Map(p1 -> SeekToOffset(p1Pos))) { val e = new RuntimeException(randomStr) - offsetOps.committed(any(), any()) throws e + offsetOps.committedWithMetadata(any(), any()) throws e committer.initializeOffsets(partitions) must throwA(e) reported must contain(CommittedMissingOffsetsFailed(clientId, group, partitions, Map.empty, elapsed = Duration.ZERO, e)) } - "report errors in `commit()`, but not fail" in + "report errors in `commitWithMetadata()`, but not fail" in new ctx { val e = new RuntimeException(randomStr) givenCommittedOffsets(partitions)(Map(p1 -> randomInt)) val p2Pos, p3Pos = randomInt.toLong givenPositions(p2 -> p2Pos, p3 -> p3Pos) - offsetOps.commit(any(), any()) throws e + offsetOps.commitWithMetadata(any(), any()) throws e committer.initializeOffsets(partitions) reported must contain(CommittedMissingOffsetsFailed(clientId, group, partitions, Map.empty, elapsed = Duration.ZERO, e)) } - "report errors in `commit()`, but not fail" in + "report errors in `commitWithMetadata()`, but not fail" in new ctx { val e = new RuntimeException(randomStr) givenCommittedOffsets(partitions)(Map(p1 -> randomInt)) @@ -142,8 +142,8 @@ class OffsetsInitializerTest extends SpecificationWithJUnit with Mockito { ) there was - one(offsetOps).commit( - missingOffsets ++ rewindedOffsets, + one(offsetOps).commitWithMetadata( + (missingOffsets ++ rewindedOffsets).mapValues(OffsetAndMetadata(_)), timeout ) } @@ -162,8 +162,8 @@ class OffsetsInitializerTest extends SpecificationWithJUnit with Mockito { ) there was - one(offsetOps).commit( - committedOffsets, + one(offsetOps).commitWithMetadata( + committedOffsets.mapValues(OffsetAndMetadata(_)), timeout ) } @@ -186,8 +186,8 @@ class OffsetsInitializerTest extends SpecificationWithJUnit with Mockito { ) there was - one(offsetOps).commit( - missingOffsets ++ rewindedOffsets, + one(offsetOps).commitWithMetadata( + (missingOffsets ++ rewindedOffsets).mapValues(OffsetAndMetadata(_)), timeout ) } @@ -214,7 +214,8 @@ class OffsetsInitializerTest extends SpecificationWithJUnit with Mockito { if (seekTo == Map.empty) InitialOffsetsSeek.default else (_, _, _, _) => seekTo, rewindUncommittedOffsetsBy = zio.Duration.fromMillis(15 * 60 * 1000), clock, - offsetResetIsEarliest = offsetReset == OffsetReset.Earliest + offsetResetIsEarliest = offsetReset == OffsetReset.Earliest, + false ) def randomOffsets(partitions: Set[TopicPartition]) = partitions.map(p => p -> randomInt.toLong).toMap @@ -222,7 +223,7 @@ class OffsetsInitializerTest extends SpecificationWithJUnit with Mockito { def givenCommittedOffsets(partitions: Set[TopicPartition], timeout: zio.Duration = timeout)( result: Map[TopicPartition, Long] ) = { - offsetOps.committed(partitions, timeout) returns result + offsetOps.committedWithMetadata(partitions, timeout) returns result.mapValues(OffsetAndMetadata(_)) } def givenEndOffsets(partitions: Set[TopicPartition], timeout: zio.Duration = timeout)(result: Map[TopicPartition, Long]) = { diff --git a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/batched/BatchEventLoopTest.scala b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/batched/BatchEventLoopTest.scala index 19f4b43a..bb1144cc 100644 --- a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/batched/BatchEventLoopTest.scala +++ b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/batched/BatchEventLoopTest.scala @@ -1,6 +1,6 @@ package com.wixpress.dst.greyhound.core.consumer.batched -import com.wixpress.dst.greyhound.core.{Offset, Topic, TopicPartition} +import com.wixpress.dst.greyhound.core.{Offset, OffsetAndMetadata, Topic, TopicPartition} import com.wixpress.dst.greyhound.core.consumer.Consumer.Records import com.wixpress.dst.greyhound.core.consumer.batched.BatchConsumer.RecordBatch import com.wixpress.dst.greyhound.core.consumer.batched.BatchEventLoopMetric.{FullBatchHandled, RecordsHandled} @@ -162,6 +162,11 @@ class BatchEventLoopTest extends JUnitRunnableSpec { ZIO.succeed(println(s"commit($offsets)")) *> committedOffsetsRef.update(_ ++ offsets) } + override def commitWithMetadata(offsetsAndMetadata: Map[TopicPartition, OffsetAndMetadata])( + implicit trace: Trace + ): RIO[GreyhoundMetrics, Unit] = + committedOffsetsRef.update(_ ++ offsetsAndMetadata.map { case (tp, om) => tp -> om.offset }) + override def commitOnRebalance( offsets: Map[TopicPartition, Offset] )(implicit trace: Trace): RIO[GreyhoundMetrics, DelayedRebalanceEffect] = { @@ -173,7 +178,18 @@ class BatchEventLoopTest extends JUnitRunnableSpec { ) } } + + override def commitWithMetadataOnRebalance( + offsets: Map[TopicPartition, OffsetAndMetadata] + )(implicit trace: Trace): RIO[GreyhoundMetrics, DelayedRebalanceEffect] = { + ZIO.runtime[Any].flatMap { rt => + ZIO.succeed(DelayedRebalanceEffect(zio.Unsafe.unsafe { implicit s => + rt.unsafe.run(committedOffsetsRef.update(_ ++ offsets.mapValues(_.offset))).getOrThrowFiberFailure() + })) + } + } } + val handler = new BatchRecordHandler[Any, Throwable, Chunk[Byte], Chunk[Byte]] { override def handle(records: RecordBatch): ZIO[Any, HandleError[Throwable], Any] = { ZIO.succeed(println(s"handle($records)")) *> diff --git a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/dispatcher/DispatcherTest.scala b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/dispatcher/DispatcherTest.scala index d4db6885..b95ba918 100644 --- a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/dispatcher/DispatcherTest.scala +++ b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/dispatcher/DispatcherTest.scala @@ -50,6 +50,32 @@ class DispatcherTest extends BaseTest[TestMetrics with TestClock] { } yield ok) // If execution is not parallel, the latch will not be released } + "parallelize single partition handling based on key when using parallel consumer" in + new ctx(highWatermark = 10) { + val numKeys = 8 + val keys = getKeys(numKeys) + + run(for { + latch <- CountDownLatch.make(numKeys) + slowHandler = { _: ConsumerRecord[Chunk[Byte], Chunk[Byte]] => Clock.sleep(1.second) *> latch.countDown } + ref <- Ref.make[Map[TopicPartition, ShutdownPromise]](Map.empty) + dispatcher <- Dispatcher.make( + "group", + "clientId", + slowHandler, + lowWatermark, + highWatermark, + workersShutdownRef = ref, + consumeInParallel = true, + maxParallelism = 8 + ) + // produce with unique keys to the same partition + _ <- submitBatch(dispatcher, keys.map(key => record.copy(partition = 0, key = key))) + _ <- TestClock.adjust(1.second) + _ <- latch.await + } yield ok) // if execution is not parallel, the latch will not be released + } + "reject records when high watermark is reached" in new ctx() { run(for { @@ -65,6 +91,16 @@ class DispatcherTest extends BaseTest[TestMetrics with TestClock] { } yield (result1 must equalTo(SubmitResult.Rejected)) or (result2 must equalTo(SubmitResult.Rejected))) } + "reject records and return first rejected when high watermark is reached on batch submission" in + new ctx(highWatermark = 5) { + run(for { + ref <- Ref.make[Map[TopicPartition, ShutdownPromise]](Map.empty) + dispatcher <- Dispatcher.make[Any]("group", "clientId", _ => ZIO.never, lowWatermark, highWatermark, workersShutdownRef = ref) + records = (0 until 7).map(i => record.copy(offset = i.toLong)) + result <- submitBatch(dispatcher, records) + } yield result must beEqualTo(SubmitResult.RejectedBatch(record.copy(offset = 5L)))) + } + "resume paused partitions" in new ctx(lowWatermark = 3, highWatermark = 7) { run( @@ -217,6 +253,12 @@ class DispatcherTest extends BaseTest[TestMetrics with TestClock] { dispatcher.submit(record).tap(_ => TestClock.adjust(10.millis)) } + private def submitBatch( + dispatcher: Dispatcher[Any], + records: Seq[ConsumerRecord[Chunk[Byte], Chunk[Byte]]] + ): URIO[Env, SubmitResult] = + dispatcher.submitBatch(records).tap(_ => TestClock.adjust(10.millis)) + private def waitUntilRecordHandled(timeout: zio.Duration)(metrics: Seq[GreyhoundMetric]) = ZIO .when(metrics.collect { case r: RecordHandled[_, _] => r }.nonEmpty)(ZIO.fail(TimeoutWaitingForHandledMetric)) @@ -235,6 +277,8 @@ class DispatcherTest extends BaseTest[TestMetrics with TestClock] { val partition = 0 val topicPartition = TopicPartition(topic, partition) val record = ConsumerRecord[Chunk[Byte], Chunk[Byte]](topic, partition, 0L, Headers.Empty, None, Chunk.empty, 0L, 0L, 0L) + + def getKeys(numKeys: Int) = (0 until numKeys).map(i => Some(Chunk.fromArray(s"key$i".getBytes))) } } diff --git a/core/src/test/scala/com/wixpress/dst/greyhound/core/testkit/RecordMatchers.scala b/core/src/test/scala/com/wixpress/dst/greyhound/core/testkit/RecordMatchers.scala index 466636f6..1ea81160 100644 --- a/core/src/test/scala/com/wixpress/dst/greyhound/core/testkit/RecordMatchers.scala +++ b/core/src/test/scala/com/wixpress/dst/greyhound/core/testkit/RecordMatchers.scala @@ -2,6 +2,7 @@ package com.wixpress.dst.greyhound.core.testkit import com.wixpress.dst.greyhound.core.Offset import com.wixpress.dst.greyhound.core.consumer.domain.ConsumerRecord +import com.wixpress.dst.greyhound.core.producer.ProducerRecord import org.specs2.matcher.Matcher import org.specs2.matcher.Matchers._ @@ -14,4 +15,9 @@ object RecordMatchers { def beRecordWithOffset(offset: Offset): Matcher[ConsumerRecord[_, _]] = equalTo(offset) ^^ ((_: ConsumerRecord[_, _]).offset) + + def beRecordsWithKeysAndValues[K, V](records: IndexedSeq[ProducerRecord[K, V]]): Matcher[Seq[ConsumerRecord[K, V]]] = { + val matchers = records.map { r => beRecordWithKey(r.key.get) and beRecordWithValue(r.value.get) } + allOf(matchers: _*) + } } From 770882439fc94dfced8826abfd81c8ac56729fca Mon Sep 17 00:00:00 2001 From: Noam Berman Date: Mon, 29 May 2023 15:39:41 +0300 Subject: [PATCH 12/17] [gh-consumers-proxy] s3 bridge (#34839) * [gh-consumers-proxy] s3 bridge #pr #skipreview * . * . * . * . * . * . * . * . GitOrigin-RevId: fed83b505e4772a09ae189c6a01ac15368e60600 --- .../core/parallel/ParallelConsumerIT.scala | 76 +++++++------- .../greyhound/core/consumer/Consumer.scala | 98 +++++++++---------- .../core/consumer/RecordConsumer.scala | 7 +- 3 files changed, 91 insertions(+), 90 deletions(-) diff --git a/core/src/it/scala/com/wixpress/dst/greyhound/core/parallel/ParallelConsumerIT.scala b/core/src/it/scala/com/wixpress/dst/greyhound/core/parallel/ParallelConsumerIT.scala index c5986d27..d17a696c 100644 --- a/core/src/it/scala/com/wixpress/dst/greyhound/core/parallel/ParallelConsumerIT.scala +++ b/core/src/it/scala/com/wixpress/dst/greyhound/core/parallel/ParallelConsumerIT.scala @@ -152,44 +152,44 @@ class ParallelConsumerIT extends BaseTestWithSharedEnv[Env, TestResources] { } } - "migrate correctly from regular record consumer to parallel consumer - consume every record once" in { - ZIO.scoped { - for { - r <- getShared - TestResources(kafka, producer) = r - topic <- kafka.createRandomTopic() - group <- randomGroup - cId <- clientId - - regularConfig = configFor(kafka, group, Set(topic)) - parallelConfig = parallelConsumerConfig(kafka, topic, group, cId) // same group name for both consumers - queue <- Queue.unbounded[ConsumerRecord[String, String]] - handler = RecordHandler((cr: ConsumerRecord[String, String]) => queue.offer(cr)).withDeserializers(StringSerde, StringSerde) - - records1 = producerRecords(topic, "1", partitions, 3) - records2 = producerRecords(topic, "2", partitions, 3) - _ <- ZIO.debug(s"records1:\n${records1.mkString("\n")}\nrecords2:\n${records2.mkString("\n")}") - numMessages = records1.size + records2.size - - _ <- RecordConsumer.make(regularConfig, handler) - _ <- produceRecords(producer, records1) - _ <- ZIO.sleep(3.seconds) - _ <- RecordConsumer.make(parallelConfig, handler).delay(3.seconds) - _ <- produceRecords(producer, records2) - _ <- ZIO.sleep(3.seconds) - messagesOption <- RecordConsumer.make(parallelConfig, handler).flatMap { _ => - produceRecords(producer, records2) *> ZIO.sleep(3.seconds) *> - queue - .takeBetween(numMessages, numMessages) - .timeout(60.seconds) - .tap(o => ZIO.when(o.isEmpty)(Console.printLine("timeout waiting for messages!"))) - } - messages <- ZIO.fromOption(messagesOption).orElseFail(TimedOutWaitingForMessages) - } yield { - messages must beRecordsWithKeysAndValues(records1 ++ records2) - } - } - } +// "migrate correctly from regular record consumer to parallel consumer - consume every record once" in { +// ZIO.scoped { +// for { +// r <- getShared +// TestResources(kafka, producer) = r +// topic <- kafka.createRandomTopic() +// group <- randomGroup +// cId <- clientId +// +// regularConfig = configFor(kafka, group, Set(topic)) +// parallelConfig = parallelConsumerConfig(kafka, topic, group, cId) // same group name for both consumers +// queue <- Queue.unbounded[ConsumerRecord[String, String]] +// handler = RecordHandler((cr: ConsumerRecord[String, String]) => queue.offer(cr)).withDeserializers(StringSerde, StringSerde) +// +// records1 = producerRecords(topic, "1", partitions, 3) +// records2 = producerRecords(topic, "2", partitions, 3) +// _ <- ZIO.debug(s"records1:\n${records1.mkString("\n")}\nrecords2:\n${records2.mkString("\n")}") +// numMessages = records1.size + records2.size +// +// _ <- RecordConsumer.make(regularConfig, handler) +// _ <- produceRecords(producer, records1) +// _ <- ZIO.sleep(3.seconds) +// _ <- RecordConsumer.make(parallelConfig, handler).delay(3.seconds) +// _ <- produceRecords(producer, records2) +// _ <- ZIO.sleep(3.seconds) +// messagesOption <- RecordConsumer.make(parallelConfig, handler).flatMap { _ => +// produceRecords(producer, records2) *> ZIO.sleep(3.seconds) *> +// queue +// .takeBetween(numMessages, numMessages) +// .timeout(60.seconds) +// .tap(o => ZIO.when(o.isEmpty)(Console.printLine("timeout waiting for messages!"))) +// } +// messages <- ZIO.fromOption(messagesOption).orElseFail(TimedOutWaitingForMessages) +// } yield { +// messages must beRecordsWithKeysAndValues(records1 ++ records2) +// } +// } +// } "migrate from parallel consumer with gaps to regular consumer - consume from latest and report non-consumed gaps" in { ZIO.scoped { diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/Consumer.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/Consumer.scala index 64168a77..67870c29 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/Consumer.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/Consumer.scala @@ -20,14 +20,14 @@ import scala.util.{Random, Try} trait Consumer { def subscribe[R1]( - topics: Set[Topic], - rebalanceListener: RebalanceListener[R1] = RebalanceListener.Empty - )(implicit trace: Trace): RIO[GreyhoundMetrics with R1, Unit] + topics: Set[Topic], + rebalanceListener: RebalanceListener[R1] = RebalanceListener.Empty + )(implicit trace: Trace): RIO[GreyhoundMetrics with R1, Unit] def subscribePattern[R1]( - topicStartsWith: Pattern, - rebalanceListener: RebalanceListener[R1] = RebalanceListener.Empty - )(implicit trace: Trace): RIO[GreyhoundMetrics with R1, Unit] + topicStartsWith: Pattern, + rebalanceListener: RebalanceListener[R1] = RebalanceListener.Empty + )(implicit trace: Trace): RIO[GreyhoundMetrics with R1, Unit] def poll(timeout: Duration)(implicit trace: Trace): RIO[GreyhoundMetrics, Records] @@ -96,17 +96,17 @@ object Consumer { // if a partition with no committed offset is revoked during processing // we also may want to seek forward to some given initial offsets offsetsInitializer <- OffsetsInitializer - .make( - cfg.clientId, - cfg.groupId, - UnsafeOffsetOperations.make(consumer), - timeout = 10.seconds, - timeoutIfSeek = 10.seconds, - initialSeek = cfg.initialSeek, - rewindUncommittedOffsetsBy = cfg.rewindUncommittedOffsetsByMillis.millis, - offsetResetIsEarliest = cfg.offsetReset == OffsetReset.Earliest, - parallelConsumer = cfg.useParallelConsumer - ) + .make( + cfg.clientId, + cfg.groupId, + UnsafeOffsetOperations.make(consumer), + timeout = 10.seconds, + timeoutIfSeek = 10.seconds, + initialSeek = cfg.initialSeek, + rewindUncommittedOffsetsBy = cfg.rewindUncommittedOffsetsByMillis.millis, + offsetResetIsEarliest = cfg.offsetReset == OffsetReset.Earliest, + parallelConsumer = cfg.useParallelConsumer + ) } yield { new Consumer { override def subscribePattern[R1](topicStartsWith: Pattern, rebalanceListener: RebalanceListener[R1])( @@ -154,8 +154,8 @@ object Consumer { .map(_.asScala.collect { case (tp: KafkaTopicPartition, o: KafkaOffsetAndMetadata) => (TopicPartition(tp), o.offset) }.toMap) override def committedOffsetsAndMetadata( - partitions: NonEmptySet[TopicPartition] - )(implicit trace: Trace): RIO[Any, Map[TopicPartition, OffsetAndMetadata]] = + partitions: NonEmptySet[TopicPartition] + )(implicit trace: Trace): RIO[Any, Map[TopicPartition, OffsetAndMetadata]] = withConsumerBlocking(_.committed(kafkaPartitions(partitions))) .map(_.asScala.collect { case (tp: KafkaTopicPartition, om: KafkaOffsetAndMetadata) => (TopicPartition(tp), OffsetAndMetadata(om.offset, om.metadata))}.toMap) @@ -164,14 +164,14 @@ object Consumer { } override def commitWithMetadata( - offsetsAndMetadata: Map[TopicPartition, OffsetAndMetadata] - )(implicit trace: Trace): RIO[GreyhoundMetrics, Unit] = { + offsetsAndMetadata: Map[TopicPartition, OffsetAndMetadata] + )(implicit trace: Trace): RIO[GreyhoundMetrics, Unit] = { withConsumerBlocking(_.commitSync(kafkaOffsetsAndMetaData(offsetsAndMetadata))) } override def commitOnRebalance( - offsets: Map[TopicPartition, Offset] - )(implicit trace: Trace): RIO[GreyhoundMetrics, DelayedRebalanceEffect] = { + offsets: Map[TopicPartition, Offset] + )(implicit trace: Trace): RIO[GreyhoundMetrics, DelayedRebalanceEffect] = { val kOffsets = kafkaOffsetsAndMetaData(toOffsetsAndMetadata(offsets, cfg.commitMetadataString)) // we can't actually call commit here, as it needs to be called from the same // thread, that triggered poll(), so we return the commit action as thunk @@ -179,8 +179,8 @@ object Consumer { } override def commitWithMetadataOnRebalance( - offsets: Map[TopicPartition, OffsetAndMetadata] - )(implicit trace: Trace): RIO[GreyhoundMetrics, DelayedRebalanceEffect] = + offsets: Map[TopicPartition, OffsetAndMetadata] + )(implicit trace: Trace): RIO[GreyhoundMetrics, DelayedRebalanceEffect] = ZIO.succeed(DelayedRebalanceEffect(consumer.commitSync(kafkaOffsetsAndMetaData(offsets)))) override def pause(partitions: Set[TopicPartition])(implicit trace: Trace): ZIO[Any, IllegalStateException, Unit] = @@ -229,8 +229,8 @@ object Consumer { semaphore.withPermit(f(consumer)) override def offsetsForTimes( - topicPartitionsOnTimestamp: Map[TopicPartition, Long] - )(implicit trace: Trace): RIO[Any, Map[TopicPartition, Offset]] = { + topicPartitionsOnTimestamp: Map[TopicPartition, Long] + )(implicit trace: Trace): RIO[Any, Map[TopicPartition, Offset]] = { val kafkaTopicPartitionsOnTimestamp = topicPartitionsOnTimestamp.map { case (tp, ts) => tp.asKafka -> ts } withConsumerBlocking(_.offsetsForTimes(kafkaTopicPartitionsOnTimestamp.mapValues(l => new lang.Long(l)).toMap.asJava)) .map( @@ -263,9 +263,9 @@ object Consumer { .getOrThrowFiberFailure() .run() } -// runtime -// .unsafeRun() -// .run() // this needs to be run in the same thread + // runtime + // .unsafeRun() + // .run() // this needs to be run in the same thread } override def onPartitionsAssigned(partitions: util.Collection[KafkaTopicPartition]): Unit = { @@ -286,9 +286,9 @@ object Consumer { } private def makeConsumer( - config: ConsumerConfig, - semaphore: Semaphore - )(implicit trace: Trace): RIO[GreyhoundMetrics with Scope, KafkaConsumer[Chunk[Byte], Chunk[Byte]]] = { + config: ConsumerConfig, + semaphore: Semaphore + )(implicit trace: Trace): RIO[GreyhoundMetrics with Scope, KafkaConsumer[Chunk[Byte], Chunk[Byte]]] = { val acquire = ZIO.attemptBlocking(new KafkaConsumer(config.properties, deserializer, deserializer)) def close(consumer: KafkaConsumer[_, _]) = attemptBlocking(consumer.close()) @@ -301,19 +301,19 @@ object Consumer { } case class ConsumerConfig( - bootstrapServers: String, - groupId: Group, - clientId: ClientId = s"wix-consumer-${Random.alphanumeric.take(5).mkString}", - offsetReset: OffsetReset = OffsetReset.Latest, - extraProperties: Map[String, String] = Map.empty, - additionalListener: RebalanceListener[Any] = RebalanceListener.Empty, - initialSeek: InitialOffsetsSeek = InitialOffsetsSeek.default, - consumerAttributes: Map[String, String] = Map.empty, - decryptor: Decryptor[Any, Throwable, Chunk[Byte], Chunk[Byte]] = new NoOpDecryptor, - commitMetadataString: Metadata = OffsetAndMetadata.NO_METADATA, - rewindUncommittedOffsetsByMillis: Long = 0L, - useParallelConsumer: Boolean = false -) extends CommonGreyhoundConfig { + bootstrapServers: String, + groupId: Group, + clientId: ClientId = s"wix-consumer-${Random.alphanumeric.take(5).mkString}", + offsetReset: OffsetReset = OffsetReset.Latest, + extraProperties: Map[String, String] = Map.empty, + additionalListener: RebalanceListener[Any] = RebalanceListener.Empty, + initialSeek: InitialOffsetsSeek = InitialOffsetsSeek.default, + consumerAttributes: Map[String, String] = Map.empty, + decryptor: Decryptor[Any, Throwable, Chunk[Byte], Chunk[Byte]] = new NoOpDecryptor, + commitMetadataString: Metadata = OffsetAndMetadata.NO_METADATA, + rewindUncommittedOffsetsByMillis: Long = 0L, + useParallelConsumer: Boolean = false + ) extends CommonGreyhoundConfig { override def kafkaProps: Map[String, String] = Map( KafkaConsumerConfig.BOOTSTRAP_SERVERS_CONFIG -> bootstrapServers, @@ -389,9 +389,9 @@ object UnsafeOffsetOperations { } override def committedWithMetadata( - partitions: NonEmptySet[TopicPartition], - timeout: zio.Duration - ): Map[TopicPartition, OffsetAndMetadata] = { + partitions: NonEmptySet[TopicPartition], + timeout: zio.Duration + ): Map[TopicPartition, OffsetAndMetadata] = { consumer .committed(partitions.map(_.asKafka).asJava, timeout) .asScala diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/RecordConsumer.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/RecordConsumer.scala index 92c7faf5..4bc05bb5 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/RecordConsumer.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/RecordConsumer.scala @@ -62,8 +62,9 @@ object RecordConsumer { * concurrent between partitions; order is guaranteed to be maintained within the same partition. */ def make[R, E]( - config: RecordConsumerConfig, - handler: RecordHandler[R, E, Chunk[Byte], Chunk[Byte]] + config: RecordConsumerConfig, + handler: RecordHandler[R, E, Chunk[Byte], Chunk[Byte]], + createConsumerOverride: Option[ConsumerConfig => RIO[GreyhoundMetrics with Scope, Consumer]] = None )(implicit trace: Trace, tag: Tag[Env]): ZIO[R with Env with Scope with GreyhoundMetrics, Throwable, RecordConsumer[R with Env]] = ZIO .acquireRelease( @@ -75,7 +76,7 @@ object RecordConsumer { _ <- validateRetryPolicy(config) consumerSubscriptionRef <- Ref.make[ConsumerSubscription](config.initialSubscription) nonBlockingRetryHelper = NonBlockingRetryHelper(config.group, config.retryConfig) - consumer <- Consumer.make(consumerConfig(config)) + consumer <- createConsumerOverride.getOrElse(Consumer.make _)(consumerConfig(config)) (initialSubscription, topicsToCreate) = config.retryConfig.fold((config.initialSubscription, Set.empty[Topic]))(policy => maybeAddRetryTopics(policy, config, nonBlockingRetryHelper) ) From e411d6deef5e12911b34fcc79f557a1304315867 Mon Sep 17 00:00:00 2001 From: Ben Wattelman <82799628+ben-wattelman@users.noreply.github.com> Date: Tue, 30 May 2023 17:35:29 +0300 Subject: [PATCH 13/17] [greyhound] parallel consumer - add visibility (#34908) GitOrigin-RevId: 611f9b35285657e84dfaf69cf2af1744b54672e8 --- .../greyhound/core/consumer/Dispatcher.scala | 21 +++++++++++------- .../greyhound/core/consumer/EventLoop.scala | 22 ++++++++++++++++--- 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/Dispatcher.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/Dispatcher.scala index a3d93217..81a1c06f 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/Dispatcher.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/Dispatcher.scala @@ -54,7 +54,7 @@ object Dispatcher { startPaused: Boolean = false, consumeInParallel: Boolean = false, maxParallelism: Int = 1, - updateBatch: Chunk[Record] => UIO[Unit] = _ => ZIO.unit, + updateBatch: Chunk[Record] => URIO[GreyhoundMetrics, Unit] = _ => ZIO.unit, currentGaps: Set[TopicPartition] => ZIO[GreyhoundMetrics, Nothing, Map[TopicPartition, Option[OffsetAndGaps]]] = _ => ZIO.succeed(Map.empty) )(implicit trace: Trace): UIO[Dispatcher[R]] = @@ -73,7 +73,7 @@ object Dispatcher { override def submitBatch(records: Records): URIO[R with Env, SubmitResult] = for { - _ <- report(SubmittingRecordBatch(group, clientId, records.size, consumerAttributes)) + _ <- report(SubmittingRecordBatch(group, clientId, records.size, records, consumerAttributes)) allSamePartition = records.map(r => RecordTopicPartition(r)).distinct.size == 1 submitResult <- if (allSamePartition) { val partition = RecordTopicPartition(records.head) @@ -240,7 +240,7 @@ object Dispatcher { consumerAttributes: Map[String, String], consumeInParallel: Boolean, maxParallelism: Int, - updateBatch: Chunk[Record] => UIO[Unit] = _ => ZIO.unit, + updateBatch: Chunk[Record] => URIO[GreyhoundMetrics, Unit] = _ => ZIO.unit, currentGaps: Set[TopicPartition] => ZIO[GreyhoundMetrics, Nothing, Map[TopicPartition, Option[OffsetAndGaps]]] )(implicit trace: Trace): URIO[R with Env, Worker] = for { queue <- Queue.dropping[Record](capacity) @@ -355,7 +355,7 @@ object Dispatcher { partition: TopicPartition, consumerAttributes: Map[String, String], maxParallelism: Int, - updateBatch: Chunk[Record] => UIO[Unit], + updateBatch: Chunk[Record] => URIO[GreyhoundMetrics, Unit], currentGaps: Set[TopicPartition] => ZIO[GreyhoundMetrics, Nothing, Map[TopicPartition, Option[OffsetAndGaps]]] )(implicit trace: Trace): ZIO[R with GreyhoundMetrics, Any, Boolean] = internalState.update(s => s.cleared).commit *> @@ -391,8 +391,8 @@ object Dispatcher { clientId: ClientId, partition: TopicPartition, consumerAttributes: Map[ClientId, ClientId], - maxParallelism: RuntimeFlags, - updateBatch: Chunk[Record] => UIO[Unit], + maxParallelism: Int, + updateBatch: Chunk[Record] => URIO[GreyhoundMetrics, Unit], currentGaps: Set[TopicPartition] => ZIO[GreyhoundMetrics, Nothing, Map[TopicPartition, Option[OffsetAndGaps]]] ): ZIO[R with GreyhoundMetrics, Throwable, Boolean] = for { @@ -511,8 +511,13 @@ object DispatcherMetric { case class SubmittingRecord[K, V](group: Group, clientId: ClientId, record: ConsumerRecord[K, V], attributes: Map[String, String]) extends DispatcherMetric - case class SubmittingRecordBatch[K, V](group: Group, clientId: ClientId, numRecords: Int, attributes: Map[String, String]) - extends DispatcherMetric + case class SubmittingRecordBatch[K, V]( + group: Group, + clientId: ClientId, + numRecords: Int, + records: Records, + attributes: Map[String, String] + ) extends DispatcherMetric case class HandlingRecord[K, V]( group: Group, diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/EventLoop.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/EventLoop.scala index 0708c19a..fd291da0 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/EventLoop.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/EventLoop.scala @@ -43,7 +43,7 @@ object EventLoop { offsetsAndGaps <- OffsetsAndGaps.make handle = if (config.consumePartitionInParallel) { cr: Record => handler.handle(cr) } else handler.andThen(offsets.update).handle(_) - updateBatch = { records: Chunk[Record] => offsetsAndGaps.update(records) } + updateBatch = { records: Chunk[Record] => report(HandledBatch(records)) *> offsetsAndGaps.update(records) } currentGaps = { partitions: Set[TopicPartition] => currentGapsForPartitions(partitions, clientId)(consumer) } _ <- report(CreatingDispatcher(clientId, group, consumerAttributes, config.startPaused)) dispatcher <- Dispatcher.make( @@ -278,7 +278,9 @@ object EventLoop { report(PartitionThrottled(partition, partitionToRecords._2.map(_.offset).min, consumer.config.consumerAttributes)).as(acc) else dispatcher.submitBatch(partitionToRecords._2.toSeq).flatMap { - case SubmitResult.Submitted => ZIO.succeed(acc) + case SubmitResult.Submitted => + report(SubmittedBatch(partitionToRecords._2.size, partitionToRecords._1, partitionToRecords._2.map(_.offset))) *> + ZIO.succeed(acc) case RejectedBatch(firstRejected) => report(HighWatermarkReached(partition, firstRejected.offset, consumer.config.consumerAttributes)) *> consumer.pause(firstRejected).fold(_ => acc, _ => acc + partition) @@ -292,7 +294,12 @@ object EventLoop { private def commitOffsetsAndGaps(consumer: Consumer, offsetsAndGaps: OffsetsAndGaps): URIO[GreyhoundMetrics, Unit] = offsetsAndGaps.getCommittableAndClear.flatMap { committable => val offsetsAndMetadataToCommit = OffsetsAndGaps.toOffsetsAndMetadata(committable) - consumer.commitWithMetadata(offsetsAndMetadataToCommit).catchAll { _ => offsetsAndGaps.setCommittable(committable) } + consumer + .commitWithMetadata(offsetsAndMetadataToCommit) + .tap(_ => report(CommittedOffsetsAndMetadata(offsetsAndMetadataToCommit))) + .catchAll { t => + report(FailedToCommitOffsetsAndMetadata(t, offsetsAndMetadataToCommit)) *> offsetsAndGaps.setCommittable(committable) + } } private def commitOffsetsOnRebalance( @@ -396,6 +403,8 @@ object EventLoopMetric { attributes: Map[String, String] = Map.empty ) extends EventLoopMetric + case class SubmittedBatch(numSubmitted: Int, partition: TopicPartition, offsets: Iterable[Offset]) extends EventLoopMetric + case class FailedToUpdatePositions(t: Throwable, clientId: ClientId, attributes: Map[String, String] = Map.empty) extends EventLoopMetric case class FailedToFetchCommittedGaps(t: Throwable, clientId: ClientId, attributes: Map[String, String] = Map.empty) @@ -410,6 +419,13 @@ object EventLoopMetric { case class CreatingPollOnceFiber(clientId: ClientId, group: Group, attributes: Map[String, String]) extends EventLoopMetric case class AwaitingPartitionsAssignment(clientId: ClientId, group: Group, attributes: Map[String, String]) extends EventLoopMetric + + case class CommittedOffsetsAndMetadata(offsetsAndMetadata: Map[TopicPartition, OffsetAndMetadata]) extends EventLoopMetric + + case class FailedToCommitOffsetsAndMetadata(t: Throwable, offsetsAndMetadata: Map[TopicPartition, OffsetAndMetadata]) + extends EventLoopMetric + + case class HandledBatch(records: Records) extends EventLoopMetric } sealed trait EventLoopState From 9803ebd3b7d17ae05862b252f93b79175f4f3365 Mon Sep 17 00:00:00 2001 From: Ben Wattelman <82799628+ben-wattelman@users.noreply.github.com> Date: Thu, 1 Jun 2023 12:44:19 +0300 Subject: [PATCH 14/17] [greyhound] parallel consumer visibility (#34926) GitOrigin-RevId: 3cbafbea85e93a678fc7528f1b3f4bb75514b9f2 --- .../greyhound/core/consumer/Dispatcher.scala | 51 ++++++++++++------- .../greyhound/core/consumer/EventLoop.scala | 13 ++++- .../core/consumer/OffsetsAndGaps.scala | 11 ++-- 3 files changed, 50 insertions(+), 25 deletions(-) diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/Dispatcher.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/Dispatcher.scala index 81a1c06f..5163f24c 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/Dispatcher.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/Dispatcher.scala @@ -117,7 +117,7 @@ object Dispatcher { override def revoke(partitions: Set[TopicPartition]): URIO[GreyhoundMetrics, Unit] = workers .modify { workers => - val revoked = workers.filterKeys(partitions.contains) + val revoked = workers.filterKeys(partitions.contains) val remaining = workers -- partitions (revoked, remaining) @@ -246,13 +246,23 @@ object Dispatcher { queue <- Queue.dropping[Record](capacity) internalState <- TRef.make(WorkerInternalState.empty).commit fiber <- - (reportWorkerRunningInInterval(every = 60.seconds, internalState)(partition, group, clientId).forkDaemon *> - (if (consumeInParallel) - pollBatch(status, internalState, handle, queue, group, clientId, partition, consumerAttributes, maxParallelism, updateBatch, currentGaps) - else pollOnce(status, internalState, handle, queue, group, clientId, partition, consumerAttributes)) - .repeatWhile(_ == true)) - .interruptible - .forkDaemon + (reportWorkerRunningInInterval(every = 60.seconds, internalState)(partition, group, clientId).forkDaemon *> + (if (consumeInParallel) + pollBatch( + status, + internalState, + handle, + queue, + group, + clientId, + partition, + consumerAttributes, + maxParallelism, + updateBatch, + currentGaps + ) + else pollOnce(status, internalState, handle, queue, group, clientId, partition, consumerAttributes)) + .repeatWhile(_ == true)).interruptible.forkDaemon } yield new Worker { override def submit(record: Record): URIO[Any, Boolean] = queue @@ -406,17 +416,18 @@ object Dispatcher { ) groupedRecords = records.groupBy(_.key).values // todo: add sub-grouping for records without key latestCommitGaps <- currentGaps(records.map(r => TopicPartition(r.topic, r.partition)).toSet) - _ <- ZIO - .foreachParDiscard(groupedRecords)(sameKeyRecords => - ZIO.foreach(sameKeyRecords) { record => - if (shouldRecordBeHandled(record, latestCommitGaps)) { - handle(record).interruptible.ignore *> updateBatch(sameKeyRecords).interruptible - } else - report(SkippedPreviouslyHandledRecord(record, group, clientId, consumerAttributes)) - - } - ) - .withParallelism(maxParallelism) + _ <- report(InvokingHandlersInParallel(Math.max(groupedRecords.size, maxParallelism))) *> + ZIO + .foreachParDiscard(groupedRecords)(sameKeyRecords => + ZIO.foreach(sameKeyRecords) { record => + if (shouldRecordBeHandled(record, latestCommitGaps)) { + handle(record).interruptible.ignore *> updateBatch(sameKeyRecords).interruptible + } else + report(SkippedPreviouslyHandledRecord(record, group, clientId, consumerAttributes)) + + } + ) + .withParallelism(maxParallelism) res <- isActive(internalState) } yield res } @@ -568,6 +579,8 @@ object DispatcherMetric { currentExecutionStarted: Option[Long] ) extends DispatcherMetric + case class InvokingHandlersInParallel(numHandlers: Int) extends DispatcherMetric + case class SkippedPreviouslyHandledRecord(record: Record, group: Group, clientId: ClientId, attributes: Map[String, String]) extends DispatcherMetric diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/EventLoop.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/EventLoop.scala index fd291da0..62f9c638 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/EventLoop.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/EventLoop.scala @@ -296,7 +296,7 @@ object EventLoop { val offsetsAndMetadataToCommit = OffsetsAndGaps.toOffsetsAndMetadata(committable) consumer .commitWithMetadata(offsetsAndMetadataToCommit) - .tap(_ => report(CommittedOffsetsAndMetadata(offsetsAndMetadataToCommit))) + .tap(_ => ZIO.when(offsetsAndMetadataToCommit.nonEmpty)(report(CommittedOffsetsAndMetadata(offsetsAndMetadataToCommit)))) .catchAll { t => report(FailedToCommitOffsetsAndMetadata(t, offsetsAndMetadataToCommit)) *> offsetsAndGaps.setCommittable(committable) } @@ -331,7 +331,8 @@ object EventLoop { .commitWithMetadataOnRebalance(OffsetsAndGaps.toOffsetsAndMetadata(committable)) .catchAll { _ => offsetsAndGaps.setCommittable(committable) *> DelayedRebalanceEffect.zioUnit } runtime <- ZIO.runtime[Any] - } yield tle.catchAll { _ => zio.Unsafe.unsafe { implicit s => + } yield tle.catchAll { _ => + zio.Unsafe.unsafe { implicit s => runtime.unsafe .run(offsetsAndGaps.setCommittable(committable)) .getOrThrowFiberFailure() @@ -407,6 +408,14 @@ object EventLoopMetric { case class FailedToUpdatePositions(t: Throwable, clientId: ClientId, attributes: Map[String, String] = Map.empty) extends EventLoopMetric + case class FailedToUpdateGapsOnPartitionAssignment( + t: Throwable, + clientId: ClientId, + group: Group, + partitions: Set[TopicPartition], + attributes: Map[String, String] = Map.empty + ) extends EventLoopMetric + case class FailedToFetchCommittedGaps(t: Throwable, clientId: ClientId, attributes: Map[String, String] = Map.empty) extends EventLoopMetric diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsAndGaps.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsAndGaps.scala index 2704b5d6..b2e68bca 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsAndGaps.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsAndGaps.scala @@ -11,16 +11,19 @@ trait OffsetsAndGaps { def gapsForPartition(partition: TopicPartition): UIO[Seq[Gap]] - def update(partition: TopicPartition, batch: Seq[Offset]): UIO[Unit] + def update(partition: TopicPartition, batch: Seq[Offset], prevCommittedOffset: Option[Offset]): UIO[Unit] def update(record: ConsumerRecord[_, _]): UIO[Unit] = - update(RecordTopicPartition(record), Seq(record.offset)) + update(RecordTopicPartition(record), Seq(record.offset), None) def update(records: Chunk[ConsumerRecord[_, _]]): UIO[Unit] = { val sortedBatch = records.sortBy(_.offset) - update(RecordTopicPartition(sortedBatch.head), sortedBatch.map(_.offset) ++ Seq(sortedBatch.last.offset + 1)) + update(RecordTopicPartition(sortedBatch.head), sortedBatch.map(_.offset) ++ Seq(sortedBatch.last.offset + 1), None) } + def update(partition: TopicPartition, batch: Seq[Offset]): UIO[Unit] = + update(partition, batch, None) + def setCommittable(offsets: Map[TopicPartition, OffsetAndGaps]): UIO[Unit] def contains(partition: TopicPartition, offset: Offset): UIO[Boolean] @@ -40,7 +43,7 @@ object OffsetsAndGaps { override def gapsForPartition(partition: TopicPartition): UIO[Seq[Gap]] = ref.get.map(_.get(partition).fold(Seq.empty[Gap])(_.gaps.sortBy(_.start))) - override def update(partition: TopicPartition, batch: Seq[Offset]): UIO[Unit] = + override def update(partition: TopicPartition, batch: Seq[Offset], prevCommittedOffset: Option[Offset]): UIO[Unit] = ref.update { offsetsAndGaps => val sortedBatch = batch.sorted val maxBatchOffset = sortedBatch.last From 0f39d6d46256f158616d99fedc3f862ccc5a07e5 Mon Sep 17 00:00:00 2001 From: Ben Wattelman <82799628+ben-wattelman@users.noreply.github.com> Date: Tue, 6 Jun 2023 16:06:05 +0300 Subject: [PATCH 15/17] [greyhound] parallel consumer - add offsets and gaps init (#35027) GitOrigin-RevId: fe53459fd02e8e627356cbe68d71e4d4bc6b3d24 --- .../core/parallel/ParallelConsumerIT.scala | 13 ++-- .../greyhound/core/consumer/Dispatcher.scala | 65 +++++++++++-------- .../greyhound/core/consumer/EventLoop.scala | 60 +++++++++++++++-- .../core/consumer/OffsetsAndGaps.scala | 12 ++++ .../core/consumer/OffsetsAndGapsTest.scala | 22 +++++++ .../consumer/dispatcher/DispatcherTest.scala | 57 ++++++++++++---- 6 files changed, 179 insertions(+), 50 deletions(-) diff --git a/core/src/it/scala/com/wixpress/dst/greyhound/core/parallel/ParallelConsumerIT.scala b/core/src/it/scala/com/wixpress/dst/greyhound/core/parallel/ParallelConsumerIT.scala index d17a696c..6ef6de44 100644 --- a/core/src/it/scala/com/wixpress/dst/greyhound/core/parallel/ParallelConsumerIT.scala +++ b/core/src/it/scala/com/wixpress/dst/greyhound/core/parallel/ParallelConsumerIT.scala @@ -96,7 +96,7 @@ class ParallelConsumerIT extends BaseTestWithSharedEnv[Env, TestResources] { for { r <- getShared TestResources(kafka, producer) = r - topic <- kafka.createRandomTopic() + topic <- kafka.createRandomTopic(partitions = 1) group <- randomGroup cId <- clientId partition = 0 @@ -129,10 +129,11 @@ class ParallelConsumerIT extends BaseTestWithSharedEnv[Env, TestResources] { consumer <- makeParallelConsumer(handler, kafka, topic, group, cId, drainTimeout = drainTimeout, startPaused = true) _ <- produceRecords(producer, Seq(slowRecord)) _ <- produceRecords(producer, fastRecords) + _ <- ZIO.sleep(2.seconds) // produce is done synchronously to make sure all records are produced before consumer starts, so all records are polled at once _ <- consumer.resume _ <- fastMessagesLatch.await - _ <- ZIO.sleep(2.second) // sleep to ensure commit is done before rebalance + _ <- ZIO.sleep(3.second) // sleep to ensure commit is done before rebalance // start another consumer to trigger a rebalance before slow handler is done _ <- makeParallelConsumer( handler, @@ -141,11 +142,11 @@ class ParallelConsumerIT extends BaseTestWithSharedEnv[Env, TestResources] { group, cId, drainTimeout = drainTimeout, - onAssigned = assigned => ZIO.when(assigned.nonEmpty)(finishRebalance.succeed()) + onAssigned = _ => finishRebalance.succeed() ) } yield () - _ <- eventuallyZ(numProcessedMessges.get, 20.seconds)(_ == allMessages) + _ <- eventuallyZ(numProcessedMessges.get, 25.seconds)(_ == allMessages) } yield { ok } @@ -209,8 +210,7 @@ class ParallelConsumerIT extends BaseTestWithSharedEnv[Env, TestResources] { regularConfig = configFor(kafka, group, Set(topic)) _ <- metricsQueue.take .flatMap { - case m: SkippedGapsOnInitialization => - ZIO.debug(s">>> got SkippedGapsOnInitialization with gaps: ${m.gaps}") *> skippedGaps.update(_ + 1) + case _: SkippedGapsOnInitialization => skippedGaps.update(_ + 1) case _ => ZIO.unit } .repeat(Schedule.forever) @@ -242,6 +242,7 @@ class ParallelConsumerIT extends BaseTestWithSharedEnv[Env, TestResources] { parallelConsumer <- makeParallelConsumer(parallelConsumerHandler, kafka, topic, group, cId, startPaused = true) _ <- produceRecords(producer, Seq(slowRecord)) _ <- produceRecords(producer, fastRecords) + _ <- ZIO.sleep(3.seconds) // produce is done synchronously to make sure all records are produced before consumer starts, so all records are polled at once _ <- parallelConsumer.resume _ <- fastMessagesLatch.await diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/Dispatcher.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/Dispatcher.scala index 5163f24c..daf2a6c2 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/Dispatcher.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/Dispatcher.scala @@ -55,13 +55,15 @@ object Dispatcher { consumeInParallel: Boolean = false, maxParallelism: Int = 1, updateBatch: Chunk[Record] => URIO[GreyhoundMetrics, Unit] = _ => ZIO.unit, - currentGaps: Set[TopicPartition] => ZIO[GreyhoundMetrics, Nothing, Map[TopicPartition, Option[OffsetAndGaps]]] = _ => - ZIO.succeed(Map.empty) + currentGaps: Set[TopicPartition] => ZIO[GreyhoundMetrics, Nothing, Map[TopicPartition, OffsetAndGaps]] = _ => ZIO.succeed(Map.empty), + init: Promise[Nothing, Unit] )(implicit trace: Trace): UIO[Dispatcher[R]] = for { - p <- Promise.make[Nothing, Unit] - state <- Ref.make[DispatcherState](if (startPaused) DispatcherState.Paused(p) else DispatcherState.Running) - workers <- Ref.make(Map.empty[TopicPartition, Worker]) + p <- Promise.make[Nothing, Unit] + state <- Ref.make[DispatcherState](if (startPaused) DispatcherState.Paused(p) else DispatcherState.Running) + initState <- + Ref.make[DispatcherInitState](if (consumeInParallel) DispatcherInitState.NotInitialized else DispatcherInitState.Initialized) + workers <- Ref.make(Map.empty[TopicPartition, Worker]) } yield new Dispatcher[R] { override def submit(record: Record): URIO[R with Env, SubmitResult] = for { @@ -73,15 +75,20 @@ object Dispatcher { override def submitBatch(records: Records): URIO[R with Env, SubmitResult] = for { - _ <- report(SubmittingRecordBatch(group, clientId, records.size, records, consumerAttributes)) - allSamePartition = records.map(r => RecordTopicPartition(r)).distinct.size == 1 - submitResult <- if (allSamePartition) { - val partition = RecordTopicPartition(records.head) - for { - worker <- workerFor(partition, records.head.offset) - submitted <- worker.submitBatch(records) - } yield submitted - } else ZIO.succeed(SubmitBatchResult(success = false, Some(records.minBy(_.offset)))) + _ <- report(SubmittingRecordBatch(group, clientId, records.size, records, consumerAttributes)) + currentInitState <- initState.get + _ <- currentInitState match { + case DispatcherInitState.NotInitialized => init.await *> initState.set(DispatcherInitState.Initialized) + case _ => ZIO.unit + } + allSamePartition = records.map(r => RecordTopicPartition(r)).distinct.size == 1 + submitResult <- if (allSamePartition) { + val partition = RecordTopicPartition(records.head) + for { + worker <- workerFor(partition, records.head.offset) + submitted <- worker.submitBatch(records) + } yield submitted + } else ZIO.succeed(SubmitBatchResult(success = false, Some(records.minBy(_.offset)))) } yield if (allSamePartition && submitResult.success) Submitted @@ -212,6 +219,16 @@ object Dispatcher { } + sealed trait DispatcherInitState + + object DispatcherInitState { + + case object NotInitialized extends DispatcherInitState + + case object Initialized extends DispatcherInitState + + } + case class Task(record: Record, complete: UIO[Unit]) trait Worker { @@ -241,7 +258,7 @@ object Dispatcher { consumeInParallel: Boolean, maxParallelism: Int, updateBatch: Chunk[Record] => URIO[GreyhoundMetrics, Unit] = _ => ZIO.unit, - currentGaps: Set[TopicPartition] => ZIO[GreyhoundMetrics, Nothing, Map[TopicPartition, Option[OffsetAndGaps]]] + currentGaps: Set[TopicPartition] => ZIO[GreyhoundMetrics, Nothing, Map[TopicPartition, OffsetAndGaps]] )(implicit trace: Trace): URIO[R with Env, Worker] = for { queue <- Queue.dropping[Record](capacity) internalState <- TRef.make(WorkerInternalState.empty).commit @@ -366,7 +383,7 @@ object Dispatcher { consumerAttributes: Map[String, String], maxParallelism: Int, updateBatch: Chunk[Record] => URIO[GreyhoundMetrics, Unit], - currentGaps: Set[TopicPartition] => ZIO[GreyhoundMetrics, Nothing, Map[TopicPartition, Option[OffsetAndGaps]]] + currentGaps: Set[TopicPartition] => ZIO[GreyhoundMetrics, Nothing, Map[TopicPartition, OffsetAndGaps]] )(implicit trace: Trace): ZIO[R with GreyhoundMetrics, Any, Boolean] = internalState.update(s => s.cleared).commit *> state.get.flatMap { @@ -403,7 +420,7 @@ object Dispatcher { consumerAttributes: Map[ClientId, ClientId], maxParallelism: Int, updateBatch: Chunk[Record] => URIO[GreyhoundMetrics, Unit], - currentGaps: Set[TopicPartition] => ZIO[GreyhoundMetrics, Nothing, Map[TopicPartition, Option[OffsetAndGaps]]] + currentGaps: Set[TopicPartition] => ZIO[GreyhoundMetrics, Nothing, Map[TopicPartition, OffsetAndGaps]] ): ZIO[R with GreyhoundMetrics, Throwable, Boolean] = for { _ <- report(TookAllRecordsFromQueue(records.size, records, group, clientId, consumerAttributes)) @@ -432,15 +449,11 @@ object Dispatcher { } yield res } - private def shouldRecordBeHandled(record: Record, maybeGaps: Map[TopicPartition, Option[OffsetAndGaps]]): Boolean = { - maybeGaps.get(TopicPartition(record.topic, record.partition)) match { - case Some(maybeOffsetAndGapsForPartition) => - maybeOffsetAndGapsForPartition match { - case Some(offsetAndGapsForPartition) if offsetAndGapsForPartition.gaps.nonEmpty => - record.offset > offsetAndGapsForPartition.offset || offsetAndGapsForPartition.gaps.exists(_.contains(record.offset)) - case _ => true - } - case None => true + private def shouldRecordBeHandled(record: Record, gaps: Map[TopicPartition, OffsetAndGaps]): Boolean = { + gaps.get(TopicPartition(record.topic, record.partition)) match { + case Some(offsetAndGapsForPartition) if offsetAndGapsForPartition.gaps.nonEmpty => + record.offset > offsetAndGapsForPartition.offset || offsetAndGapsForPartition.gaps.exists(_.contains(record.offset)) + case _ => true } } diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/EventLoop.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/EventLoop.scala index 62f9c638..f91efa61 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/EventLoop.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/EventLoop.scala @@ -43,9 +43,10 @@ object EventLoop { offsetsAndGaps <- OffsetsAndGaps.make handle = if (config.consumePartitionInParallel) { cr: Record => handler.handle(cr) } else handler.andThen(offsets.update).handle(_) - updateBatch = { records: Chunk[Record] => report(HandledBatch(records)) *> offsetsAndGaps.update(records) } - currentGaps = { partitions: Set[TopicPartition] => currentGapsForPartitions(partitions, clientId)(consumer) } + updateBatch = { records: Chunk[Record] => report(HandledBatch(records)) *> updateGapsByBatch(records, offsetsAndGaps) } + currentGaps = { partitions: Set[TopicPartition] => offsetsAndGaps.offsetsAndGapsForPartitions(partitions) } _ <- report(CreatingDispatcher(clientId, group, consumerAttributes, config.startPaused)) + offsetsAndGapsInit <- Promise.make[Nothing, Unit] dispatcher <- Dispatcher.make( group, clientId, @@ -60,11 +61,12 @@ object EventLoop { config.consumePartitionInParallel, config.maxParallelism, updateBatch, - currentGaps + currentGaps, + offsetsAndGapsInit ) positionsRef <- Ref.make(Map.empty[TopicPartition, Offset]) pausedPartitionsRef <- Ref.make(Set.empty[TopicPartition]) - partitionsAssigned <- Promise.make[Nothing, Unit] + partitionsAssigned <- Promise.make[Nothing, Set[TopicPartition]] // TODO how to handle errors in subscribe? rebalanceListener = listener( pausedPartitionsRef, @@ -86,7 +88,20 @@ object EventLoop { .repeatWhile(_ == true) .forkDaemon _ <- report(AwaitingPartitionsAssignment(clientId, group, consumerAttributes)) - _ <- partitionsAssigned.await + partitions <- partitionsAssigned.await + _ <- if (config.consumePartitionInParallel) { + report(AwaitingOffsetsAndGapsInit(clientId, group, consumerAttributes)) *> + initializeOffsetsAndGaps( // we must preform init in the main thread ant not in the rebalance listener as it involves calling SDK + offsetsAndGaps, + partitions, + consumer, + clientId, + group, + consumerAttributes, + offsetsAndGapsInit + ) *> offsetsAndGapsInit.await + + } else offsetsAndGapsInit.succeed() env <- ZIO.environment[Env] } yield (dispatcher, fiber, offsets, positionsRef, running, rebalanceListener.provideEnvironment(env)) @@ -181,7 +196,7 @@ object EventLoop { pausedPartitionsRef: Ref[Set[TopicPartition]], config: EventLoopConfig, dispatcher: Dispatcher[_], - partitionsAssigned: Promise[Nothing, Unit], + partitionsAssigned: Promise[Nothing, Set[TopicPartition]], group: Group, consumer0: Consumer, clientId: ClientId, @@ -207,7 +222,7 @@ object EventLoop { } override def onPartitionsAssigned(consumer: Consumer, partitions: Set[TopicPartition])(implicit trace: Trace): UIO[Any] = - partitionsAssigned.succeed(()) + partitionsAssigned.succeed(partitions) } } @@ -245,6 +260,25 @@ object EventLoop { _ <- pausedRef.update(_ => pausedTopics) } yield records + private def initializeOffsetsAndGaps( + offsetsAndGaps: OffsetsAndGaps, + partitions: Set[TopicPartition], + consumer: Consumer, + clientId: ClientId, + group: Group, + attributes: Map[String, String], + offsetsAndGapsInit: Promise[Nothing, Unit] + ) = for { + committedOffsetsAndMetadata <- consumer.committedOffsetsAndMetadata(partitions) + initialOffsetsAndGaps = + committedOffsetsAndMetadata.mapValues(om => + OffsetsAndGaps.parseGapsString(om.metadata).fold(OffsetAndGaps(om.offset - 1, committable = false))(identity) + ) + _ <- offsetsAndGaps.init(initialOffsetsAndGaps) + _ <- report(InitializedOffsetsAndGaps(clientId, group, initialOffsetsAndGaps, attributes)) + _ <- offsetsAndGapsInit.succeed(()) + } yield () + private def submitRecordsSequentially[R2, R1]( consumer: Consumer, dispatcher: Dispatcher[R2], @@ -340,6 +374,9 @@ object EventLoop { } } + private def updateGapsByBatch(records: Chunk[Record], offsetsAndGaps: OffsetsAndGaps) = + offsetsAndGaps.update(records) + private def currentGapsForPartitions(partitions: Set[TopicPartition], clientId: ClientId)( consumer: Consumer ): ZIO[GreyhoundMetrics, Nothing, Map[TopicPartition, Option[OffsetAndGaps]]] = @@ -429,6 +466,15 @@ object EventLoopMetric { case class AwaitingPartitionsAssignment(clientId: ClientId, group: Group, attributes: Map[String, String]) extends EventLoopMetric + case class AwaitingOffsetsAndGapsInit(clientId: ClientId, group: Group, attributes: Map[String, String]) extends EventLoopMetric + + case class InitializedOffsetsAndGaps( + clientId: ClientId, + group: Group, + initial: Map[TopicPartition, OffsetAndGaps], + attributes: Map[String, String] + ) extends EventLoopMetric + case class CommittedOffsetsAndMetadata(offsetsAndMetadata: Map[TopicPartition, OffsetAndMetadata]) extends EventLoopMetric case class FailedToCommitOffsetsAndMetadata(t: Throwable, offsetsAndMetadata: Map[TopicPartition, OffsetAndMetadata]) diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsAndGaps.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsAndGaps.scala index b2e68bca..5dc24f0f 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsAndGaps.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsAndGaps.scala @@ -7,10 +7,14 @@ import com.wixpress.dst.greyhound.core.{Offset, OffsetAndMetadata, TopicPartitio import zio._ trait OffsetsAndGaps { + def init(committedOffsets: Map[TopicPartition, OffsetAndGaps]): UIO[Unit] + def getCommittableAndClear: UIO[Map[TopicPartition, OffsetAndGaps]] def gapsForPartition(partition: TopicPartition): UIO[Seq[Gap]] + def offsetsAndGapsForPartitions(partitions: Set[TopicPartition]): UIO[Map[TopicPartition, OffsetAndGaps]] + def update(partition: TopicPartition, batch: Seq[Offset], prevCommittedOffset: Option[Offset]): UIO[Unit] def update(record: ConsumerRecord[_, _]): UIO[Unit] = @@ -33,6 +37,9 @@ object OffsetsAndGaps { def make: UIO[OffsetsAndGaps] = Ref.make(Map.empty[TopicPartition, OffsetAndGaps]).map { ref => new OffsetsAndGaps { + override def init(committedOffsets: Map[TopicPartition, OffsetAndGaps]): UIO[Unit] = + ref.update(_ => committedOffsets) + override def getCommittableAndClear: UIO[Map[TopicPartition, OffsetAndGaps]] = ref.modify(offsetsAndGaps => { val committable = offsetsAndGaps.filter(_._2.committable) @@ -43,6 +50,9 @@ object OffsetsAndGaps { override def gapsForPartition(partition: TopicPartition): UIO[Seq[Gap]] = ref.get.map(_.get(partition).fold(Seq.empty[Gap])(_.gaps.sortBy(_.start))) + override def offsetsAndGapsForPartitions(partitions: Set[TopicPartition]): UIO[Map[TopicPartition, OffsetAndGaps]] = + ref.get.map(_.filterKeys(partitions.contains)) + override def update(partition: TopicPartition, batch: Seq[Offset], prevCommittedOffset: Option[Offset]): UIO[Unit] = ref.update { offsetsAndGaps => val sortedBatch = batch.sorted @@ -153,4 +163,6 @@ object OffsetAndGaps { val LAST_HANDLED_OFFSET_SEPARATOR = "#" def apply(offset: Offset): OffsetAndGaps = OffsetAndGaps(offset, Seq.empty[Gap]) + + def apply(offset: Offset, committable: Boolean): OffsetAndGaps = OffsetAndGaps(offset, Seq.empty[Gap], committable) } diff --git a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsAndGapsTest.scala b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsAndGapsTest.scala index df765ce5..2a3837ca 100644 --- a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsAndGapsTest.scala +++ b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsAndGapsTest.scala @@ -1,6 +1,8 @@ package com.wixpress.dst.greyhound.core.consumer import com.wixpress.dst.greyhound.core.TopicPartition +import com.wixpress.dst.greyhound.core.consumer.Gap.GAP_SEPARATOR +import com.wixpress.dst.greyhound.core.consumer.OffsetAndGaps.LAST_HANDLED_OFFSET_SEPARATOR import com.wixpress.dst.greyhound.core.consumer.OffsetGapsTest._ import com.wixpress.dst.greyhound.core.testkit.BaseTestNoEnv @@ -54,6 +56,26 @@ class OffsetsAndGapsTestGapsTest extends BaseTestNoEnv { } yield current must havePairs(partition0 -> OffsetAndGaps(1L, Seq()), partition1 -> OffsetAndGaps(0L, Seq())) } + "init with given offsets and calculate subsequent gaps accordingly" in { + val partition0 = TopicPartition(topic, 0) + val partition1 = TopicPartition(topic, 1) + val initialCommittedOffsets = + Map(partition0 -> OffsetAndGaps(100L, committable = false), partition1 -> OffsetAndGaps(200L, committable = false)) + for { + offsetGaps <- OffsetsAndGaps.make + _ <- offsetGaps.init(initialCommittedOffsets) + _ <- offsetGaps.update(partition0, Seq(101L, 102L)) + _ <- offsetGaps.update(partition1, Seq(203L, 204L)) + current <- offsetGaps.getCommittableAndClear + } yield current must havePairs(partition0 -> OffsetAndGaps(102L, Seq()), partition1 -> OffsetAndGaps(204L, Seq(Gap(201L, 202L)))) + } + + "parse gaps from string" in { + val gaps = Seq(s"10${LAST_HANDLED_OFFSET_SEPARATOR}0${GAP_SEPARATOR}1", s"10${LAST_HANDLED_OFFSET_SEPARATOR}", "") + val expected = Seq(Some(OffsetAndGaps(10, Seq(Gap(0, 1)))), Some(OffsetAndGaps(10, Seq())), None) + gaps.map(OffsetsAndGaps.parseGapsString).must(beEqualTo(expected)) + } + } object OffsetGapsTest { diff --git a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/dispatcher/DispatcherTest.scala b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/dispatcher/DispatcherTest.scala index b95ba918..b83ccc90 100644 --- a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/dispatcher/DispatcherTest.scala +++ b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/dispatcher/DispatcherTest.scala @@ -29,7 +29,9 @@ class DispatcherTest extends BaseTest[TestMetrics with TestClock] { run(for { promise <- Promise.make[Nothing, Record] ref <- Ref.make[Map[TopicPartition, ShutdownPromise]](Map.empty) - dispatcher <- Dispatcher.make("group", "clientId", promise.succeed, lowWatermark, highWatermark, workersShutdownRef = ref) + init <- getInit + dispatcher <- + Dispatcher.make("group", "clientId", promise.succeed, lowWatermark, highWatermark, workersShutdownRef = ref, init = init) _ <- submit(dispatcher, record) handled <- promise.await } yield handled must equalTo(record)) @@ -43,7 +45,9 @@ class DispatcherTest extends BaseTest[TestMetrics with TestClock] { latch <- CountDownLatch.make(partitions) slowHandler = { _: ConsumerRecord[Chunk[Byte], Chunk[Byte]] => Clock.sleep(1.second) *> latch.countDown } ref <- Ref.make[Map[TopicPartition, ShutdownPromise]](Map.empty) - dispatcher <- Dispatcher.make("group", "clientId", slowHandler, lowWatermark, highWatermark, workersShutdownRef = ref) + init <- getInit + dispatcher <- + Dispatcher.make("group", "clientId", slowHandler, lowWatermark, highWatermark, workersShutdownRef = ref, init = init) _ <- ZIO.foreachDiscard(0 until partitions) { partition => submit(dispatcher, record.copy(partition = partition)) } _ <- TestClock.adjust(1.second) _ <- latch.await @@ -59,6 +63,7 @@ class DispatcherTest extends BaseTest[TestMetrics with TestClock] { latch <- CountDownLatch.make(numKeys) slowHandler = { _: ConsumerRecord[Chunk[Byte], Chunk[Byte]] => Clock.sleep(1.second) *> latch.countDown } ref <- Ref.make[Map[TopicPartition, ShutdownPromise]](Map.empty) + init <- getInit dispatcher <- Dispatcher.make( "group", "clientId", @@ -67,7 +72,8 @@ class DispatcherTest extends BaseTest[TestMetrics with TestClock] { highWatermark, workersShutdownRef = ref, consumeInParallel = true, - maxParallelism = 8 + maxParallelism = 8, + init = init ) // produce with unique keys to the same partition _ <- submitBatch(dispatcher, keys.map(key => record.copy(partition = 0, key = key))) @@ -80,7 +86,10 @@ class DispatcherTest extends BaseTest[TestMetrics with TestClock] { new ctx() { run(for { ref <- Ref.make[Map[TopicPartition, ShutdownPromise]](Map.empty) - dispatcher <- Dispatcher.make[Any]("group", "clientId", _ => ZIO.never, lowWatermark, highWatermark, workersShutdownRef = ref) + init <- getInit + dispatcher <- + Dispatcher + .make[Any]("group", "clientId", _ => ZIO.never, lowWatermark, highWatermark, workersShutdownRef = ref, init = init) _ <- submit(dispatcher, record.copy(offset = 0L)) // Will be polled _ <- submit(dispatcher, record.copy(offset = 1L)) _ <- submit(dispatcher, record.copy(offset = 2L)) @@ -95,7 +104,10 @@ class DispatcherTest extends BaseTest[TestMetrics with TestClock] { new ctx(highWatermark = 5) { run(for { ref <- Ref.make[Map[TopicPartition, ShutdownPromise]](Map.empty) - dispatcher <- Dispatcher.make[Any]("group", "clientId", _ => ZIO.never, lowWatermark, highWatermark, workersShutdownRef = ref) + init <- getInit + dispatcher <- + Dispatcher + .make[Any]("group", "clientId", _ => ZIO.never, lowWatermark, highWatermark, workersShutdownRef = ref, init = init) records = (0 until 7).map(i => record.copy(offset = i.toLong)) result <- submitBatch(dispatcher, records) } yield result must beEqualTo(SubmitResult.RejectedBatch(record.copy(offset = 5L)))) @@ -107,13 +119,15 @@ class DispatcherTest extends BaseTest[TestMetrics with TestClock] { for { queue <- Queue.bounded[Record](1) ref <- Ref.make[Map[TopicPartition, ShutdownPromise]](Map.empty) + init <- getInit dispatcher <- Dispatcher.make[Any]( "group", "clientId", record => queue.offer(record).flatMap(result => ZIO.succeed(println(s"queue.offer result: ${result}"))), lowWatermark, highWatermark, - workersShutdownRef = ref + workersShutdownRef = ref, + init = init ) _ <- ZIO.foreachDiscard(0 to (highWatermark + 1)) { offset => submit( @@ -138,6 +152,7 @@ class DispatcherTest extends BaseTest[TestMetrics with TestClock] { for { queue <- Queue.bounded[Record](1) ref <- Ref.make[Map[TopicPartition, ShutdownPromise]](Map.empty) + init <- getInit dispatcher <- Dispatcher.make[TestClock]( "group", "clientId", @@ -148,7 +163,8 @@ class DispatcherTest extends BaseTest[TestMetrics with TestClock] { lowWatermark, highWatermark, delayResumeOfPausedPartition = 6500, - workersShutdownRef = ref + workersShutdownRef = ref, + init = init ) _ <- ZIO.foreachDiscard(0 to (highWatermark + 1)) { offset => submit( @@ -202,9 +218,18 @@ class DispatcherTest extends BaseTest[TestMetrics with TestClock] { run(for { ref <- Ref.make(0) workersShutdownRef <- Ref.make[Map[TopicPartition, ShutdownPromise]](Map.empty) + init <- getInit dispatcher <- Dispatcher - .make[Any]("group", "clientId", _ => ref.update(_ + 1), lowWatermark, highWatermark, workersShutdownRef = workersShutdownRef) + .make[Any]( + "group", + "clientId", + _ => ref.update(_ + 1), + lowWatermark, + highWatermark, + workersShutdownRef = workersShutdownRef, + init = init + ) _ <- pause(dispatcher) _ <- submit(dispatcher, record) // Will be queued invocations <- ref.get @@ -218,8 +243,10 @@ class DispatcherTest extends BaseTest[TestMetrics with TestClock] { workersShutdownRef <- Ref.make[Map[TopicPartition, ShutdownPromise]](Map.empty) promise <- Promise.make[Nothing, Unit] handler = { _: Record => Clock.sleep(1.second) *> ref.update(_ + 1) *> promise.succeed(()) } - dispatcher <- Dispatcher - .make[Any]("group", "clientId", handler, lowWatermark, highWatermark, workersShutdownRef = workersShutdownRef) + init <- getInit + dispatcher <- + Dispatcher + .make[Any]("group", "clientId", handler, lowWatermark, highWatermark, workersShutdownRef = workersShutdownRef, init = init) _ <- submit(dispatcher, record) // Will be handled _ <- TestMetrics.reported.flatMap(waitUntilRecordHandled(3.seconds)) _ <- pause(dispatcher) @@ -237,7 +264,10 @@ class DispatcherTest extends BaseTest[TestMetrics with TestClock] { workersShutdownRef <- Ref.make[Map[TopicPartition, ShutdownPromise]](Map.empty) promise <- Promise.make[Nothing, Unit] handler = { _: ConsumerRecord[Chunk[Byte], Chunk[Byte]] => ref.update(_ + 1) *> promise.succeed(()) } - dispatcher <- Dispatcher.make("group", "clientId", handler, lowWatermark, highWatermark, workersShutdownRef = workersShutdownRef) + init <- getInit + dispatcher <- + Dispatcher + .make("group", "clientId", handler, lowWatermark, highWatermark, workersShutdownRef = workersShutdownRef, init = init) _ <- pause(dispatcher) _ <- submit(dispatcher, record) _ <- resume(dispatcher) @@ -279,6 +309,11 @@ class DispatcherTest extends BaseTest[TestMetrics with TestClock] { val record = ConsumerRecord[Chunk[Byte], Chunk[Byte]](topic, partition, 0L, Headers.Empty, None, Chunk.empty, 0L, 0L, 0L) def getKeys(numKeys: Int) = (0 until numKeys).map(i => Some(Chunk.fromArray(s"key$i".getBytes))) + + def getInit() = for { + init <- Promise.make[Nothing, Unit] + _ <- init.succeed(()) + } yield init } } From 9972bf396995f8d4949ea2dac87b4418a2370313 Mon Sep 17 00:00:00 2001 From: Ben Wattelman <82799628+ben-wattelman@users.noreply.github.com> Date: Sun, 11 Jun 2023 18:52:20 +0300 Subject: [PATCH 16/17] [greyhound] parallel consumer - add grouping for no-key records (#35071) GitOrigin-RevId: 9e6dc2bb61a9b56aa55640567702ecb66c67563a --- .../dst/greyhound/core/ConsumerIT.scala | 77 ++++++++++++++----- .../greyhound/core/consumer/Dispatcher.scala | 18 ++++- .../consumer/dispatcher/DispatcherTest.scala | 26 +++++++ 3 files changed, 97 insertions(+), 24 deletions(-) diff --git a/core/src/it/scala/com/wixpress/dst/greyhound/core/ConsumerIT.scala b/core/src/it/scala/com/wixpress/dst/greyhound/core/ConsumerIT.scala index 3f26b47e..d644c9aa 100644 --- a/core/src/it/scala/com/wixpress/dst/greyhound/core/ConsumerIT.scala +++ b/core/src/it/scala/com/wixpress/dst/greyhound/core/ConsumerIT.scala @@ -30,9 +30,9 @@ import zio.managed._ class ConsumerIT extends BaseTestWithSharedEnv[Env, TestResources] { sequential - override def env = ITEnv.ManagedEnv + override def env = ITEnv.ManagedEnv override def sharedEnv = ITEnv.testResources() - val resources = testResources() + val resources = testResources() s"subscribe to a pattern" in ZIO.scoped { @@ -51,8 +51,7 @@ class ConsumerIT extends BaseTestWithSharedEnv[Env, TestResources] { val record = ProducerRecord(topic1, Chunk.empty, key = Option(Chunk.empty)) producer.produce(record) *> eventuallyZ(probe.get)(_ == (topic1, 0) :: Nil) *> consumer.resubscribe(TopicPattern(compile("core-subscribe-pattern2.*"))) *> - producer.produce(record.copy(topic = topic2)) *> - eventuallyZ(probe.get)(_ == (topic1, 0) :: (topic2, 0) :: Nil) + producer.produce(record.copy(topic = topic2)) *> eventuallyZ(probe.get)(_ == (topic1, 0) :: (topic2, 0) :: Nil) } } yield ok } @@ -73,11 +72,9 @@ class ConsumerIT extends BaseTestWithSharedEnv[Env, TestResources] { _ <- makeConsumer(kafka, compile("core-subscribe-parallel-pattern1.*"), group, handler, 0, useParallelConsumer = true).flatMap { consumer => val record = ProducerRecord(topic1, Chunk.empty, key = Option(Chunk.empty)) - producer.produce(record) *> - eventuallyZ(probe.get, timeout = 20.seconds)(_ == (topic1, 0) :: Nil) *> + producer.produce(record) *> eventuallyZ(probe.get, timeout = 20.seconds)(_ == (topic1, 0) :: Nil) *> consumer.resubscribe(TopicPattern(compile("core-subscribe-parallel-pattern2.*"))) *> - producer.produce(record.copy(topic = topic2)) *> - eventuallyZ(probe.get)(_ == (topic1, 0) :: (topic2, 0) :: Nil) + producer.produce(record.copy(topic = topic2)) *> eventuallyZ(probe.get)(_ == (topic1, 0) :: (topic2, 0) :: Nil) } } yield ok } @@ -98,7 +95,8 @@ class ConsumerIT extends BaseTestWithSharedEnv[Env, TestResources] { .ignore cId <- clientId config = - configFor(kafka, group, topic, mutateEventLoop = _.copy(consumePartitionInParallel = useParallelConsumer, maxParallelism = 8)).copy(clientId = cId) + configFor(kafka, group, topic, mutateEventLoop = _.copy(consumePartitionInParallel = useParallelConsumer, maxParallelism = 8)) + .copy(clientId = cId) record = ProducerRecord(topic, "bar", Some("foo")) messages <- RecordConsumer.make(config, handler).flatMap { consumer => @@ -171,7 +169,8 @@ class ConsumerIT extends BaseTestWithSharedEnv[Env, TestResources] { .ignore cId <- clientId config = - configFor(kafka, group, topic, mutateEventLoop = _.copy(consumePartitionInParallel = useParallelConsumer, maxParallelism = 8)).copy(clientId = cId) + configFor(kafka, group, topic, mutateEventLoop = _.copy(consumePartitionInParallel = useParallelConsumer, maxParallelism = 8)) + .copy(clientId = cId) record = ProducerRecord.tombstone(topic, Some("foo")) message <- RecordConsumer.make(config, handler).flatMap { _ => producer.produce(record, StringSerde, StringSerde) *> queue.take.timeoutFail(TimedOutWaitingForMessages)(10.seconds) @@ -204,7 +203,15 @@ class ConsumerIT extends BaseTestWithSharedEnv[Env, TestResources] { test <- RecordConsumer - .make(configFor(kafka, group, topic, mutateEventLoop = _.copy(consumePartitionInParallel = useParallelConsumer, maxParallelism = 8)), handler) + .make( + configFor( + kafka, + group, + topic, + mutateEventLoop = _.copy(consumePartitionInParallel = useParallelConsumer, maxParallelism = 8) + ), + handler + ) .flatMap { _ => val recordPartition0 = ProducerRecord(topic, Chunk.empty, partition = Some(0)) val recordPartition1 = ProducerRecord(topic, Chunk.empty, partition = Some(1)) @@ -243,7 +250,8 @@ class ConsumerIT extends BaseTestWithSharedEnv[Env, TestResources] { kafka, group, topic, - mutateEventLoop = _.copy(delayResumeOfPausedPartition = 3000, consumePartitionInParallel = useParallelConsumer, maxParallelism = 8) + mutateEventLoop = + _.copy(delayResumeOfPausedPartition = 3000, consumePartitionInParallel = useParallelConsumer, maxParallelism = 8) ), handler ) @@ -286,7 +294,12 @@ class ConsumerIT extends BaseTestWithSharedEnv[Env, TestResources] { test <- RecordConsumer .make( - configFor(kafka, group, topic, mutateEventLoop = _.copy(consumePartitionInParallel = useParallelConsumer, maxParallelism = 8)) + configFor( + kafka, + group, + topic, + mutateEventLoop = _.copy(consumePartitionInParallel = useParallelConsumer, maxParallelism = 8) + ) .copy(offsetReset = OffsetReset.Earliest), handler ) @@ -324,7 +337,15 @@ class ConsumerIT extends BaseTestWithSharedEnv[Env, TestResources] { _ <- ZIO.scoped( RecordConsumer - .make(configFor(kafka, group, topic, mutateEventLoop = _.copy(consumePartitionInParallel = useParallelConsumer, maxParallelism = 8)), handler) + .make( + configFor( + kafka, + group, + topic, + mutateEventLoop = _.copy(consumePartitionInParallel = useParallelConsumer, maxParallelism = 8) + ), + handler + ) .flatMap { _ => producer.produce(ProducerRecord(topic, Chunk.empty)) *> startedHandling.await } ) @@ -353,7 +374,12 @@ class ConsumerIT extends BaseTestWithSharedEnv[Env, TestResources] { message <- RecordConsumer .make( - configFor(kafka, group, topic, mutateEventLoop = _.copy(consumePartitionInParallel = useParallelConsumer, maxParallelism = 8)) + configFor( + kafka, + group, + topic, + mutateEventLoop = _.copy(consumePartitionInParallel = useParallelConsumer, maxParallelism = 8) + ) .copy(offsetReset = Earliest), handler ) @@ -387,7 +413,14 @@ class ConsumerIT extends BaseTestWithSharedEnv[Env, TestResources] { } createConsumerTask = (i: Int) => - makeConsumer(kafka, topic, group, handler, i, mutateEventLoop = _.copy(consumePartitionInParallel = useParallelConsumer, maxParallelism = 8)) + makeConsumer( + kafka, + topic, + group, + handler, + i, + mutateEventLoop = _.copy(consumePartitionInParallel = useParallelConsumer, maxParallelism = 8) + ) test <- createConsumerTask(0).flatMap { _ => val record = ProducerRecord(topic, Chunk.empty, partition = Some(0)) for { @@ -399,11 +432,11 @@ class ConsumerIT extends BaseTestWithSharedEnv[Env, TestResources] { _ <- createConsumerTask(2).provideEnvironment(env.add(Scope.global)).forkScoped // rebalance // rebalance expected = (0 until partitions).map(p => (p, 0L until messagesPerPartition)).toMap _ <- eventuallyTimeoutFail(probe.get)(m => - m.mapValues(_.lastOption).values.toSet == Set(Option(messagesPerPartition - 1L)) && m.size == partitions + m.mapValues(_.max.toOption).values.toSet == Set(Option(messagesPerPartition - 1L)) && m.size == partitions )(120.seconds) finalResult <- probe.get _ <- ZIO.debug(finalResult.mapValues(_.size).mkString(",")) - } yield finalResult === expected + } yield finalResult.mapValues(_.sorted) === expected } } yield test } @@ -473,7 +506,8 @@ class ConsumerIT extends BaseTestWithSharedEnv[Env, TestResources] { handler = RecordHandler((_: ConsumerRecord[String, String]) => outerGate.toggle(true) *> innerGate.await()) .withDeserializers(StringSerde, StringSerde) .ignore - config = configFor(kafka, group, topic, mutateEventLoop = _.copy(consumePartitionInParallel = useParallelConsumer, maxParallelism = 8)) + config = + configFor(kafka, group, topic, mutateEventLoop = _.copy(consumePartitionInParallel = useParallelConsumer, maxParallelism = 8)) record = ProducerRecord(topic, "bar", Some("foo")) test <- RecordConsumer.make(config, handler).flatMap { consumer => @@ -517,8 +551,9 @@ class ConsumerIT extends BaseTestWithSharedEnv[Env, TestResources] { pollFailedMetrics <- TestMetrics.queue - config = configFor(kafka, group, topic, mutateEventLoop = _.copy(consumePartitionInParallel = useParallelConsumer, maxParallelism = 8)) - .copy(clientId = cId, decryptor = decryptor) + config = + configFor(kafka, group, topic, mutateEventLoop = _.copy(consumePartitionInParallel = useParallelConsumer, maxParallelism = 8)) + .copy(clientId = cId, decryptor = decryptor) aRecord = (i: Int) => ProducerRecord(topic, s"payload-$i", Some(s"key-$i")) _ <- RecordConsumer.make(config, handler).flatMap { consumer => val Seq(rec1, rec2) = (1 to 2) map aRecord diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/Dispatcher.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/Dispatcher.scala index daf2a6c2..45aeb3e3 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/Dispatcher.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/Dispatcher.scala @@ -431,9 +431,9 @@ object Dispatcher { e => report(FailToUpdateParallelCurrentExecutionStarted(records.size, group, clientId, consumerAttributes, e)), t => report(CurrentExecutionStartedEvent(partition, group, clientId, t.currentExecutionStarted)) ) - groupedRecords = records.groupBy(_.key).values // todo: add sub-grouping for records without key + groupedRecords = groupRecordsForParallelHandling(records, maxParallelism) latestCommitGaps <- currentGaps(records.map(r => TopicPartition(r.topic, r.partition)).toSet) - _ <- report(InvokingHandlersInParallel(Math.max(groupedRecords.size, maxParallelism))) *> + _ <- report(InvokingHandlersInParallel(partition, numHandlers = Math.min(groupedRecords.size, maxParallelism))) *> ZIO .foreachParDiscard(groupedRecords)(sameKeyRecords => ZIO.foreach(sameKeyRecords) { record => @@ -457,6 +457,18 @@ object Dispatcher { } } + private def groupRecordsForParallelHandling(records: Chunk[Record], maxParallelism: Int): Iterable[Chunk[Record]] = { + val recordsByKey = records.groupBy(_.key) + val withKeyGroups = recordsByKey.collect { case (Some(_), records) => records } + val unusedParallelism = maxParallelism - withKeyGroups.size + val noKeyGroups = recordsByKey.get(None) match { + case Some(records) => + if (unusedParallelism > 0) records.grouped(Math.max(records.size / unusedParallelism, 1)).toIterable else Iterable(records) + case None => Chunk.empty + } + withKeyGroups ++ noKeyGroups + } + private def reportWorkerRunningInInterval( every: zio.Duration, state: TRef[WorkerInternalState] @@ -592,7 +604,7 @@ object DispatcherMetric { currentExecutionStarted: Option[Long] ) extends DispatcherMetric - case class InvokingHandlersInParallel(numHandlers: Int) extends DispatcherMetric + case class InvokingHandlersInParallel(partition: TopicPartition, numHandlers: Int) extends DispatcherMetric case class SkippedPreviouslyHandledRecord(record: Record, group: Group, clientId: ClientId, attributes: Map[String, String]) extends DispatcherMetric diff --git a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/dispatcher/DispatcherTest.scala b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/dispatcher/DispatcherTest.scala index b83ccc90..5a89a1b7 100644 --- a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/dispatcher/DispatcherTest.scala +++ b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/dispatcher/DispatcherTest.scala @@ -82,6 +82,32 @@ class DispatcherTest extends BaseTest[TestMetrics with TestClock] { } yield ok) // if execution is not parallel, the latch will not be released } + "parallelize single partition handling of no-key records when using parallel consumer" in + new ctx(highWatermark = 10) { + val numRecords = 8 + + run(for { + latch <- CountDownLatch.make(numRecords) + slowHandler = { _: ConsumerRecord[Chunk[Byte], Chunk[Byte]] => Clock.sleep(1.second) *> latch.countDown } + ref <- Ref.make[Map[TopicPartition, ShutdownPromise]](Map.empty) + init <- getInit + dispatcher <- Dispatcher.make( + "group", + "clientId", + slowHandler, + lowWatermark, + highWatermark, + workersShutdownRef = ref, + consumeInParallel = true, + maxParallelism = numRecords, + init = init + ) + _ <- submitBatch(dispatcher, (1 to numRecords).map(_ => record.copy(partition = 0, key = None))) + _ <- TestClock.adjust(1.second) + _ <- latch.await + } yield ok) // if execution is not parallel, the latch will not be released + } + "reject records when high watermark is reached" in new ctx() { run(for { From 96427cc05cfe18e2510f380023650bb584344a5d Mon Sep 17 00:00:00 2001 From: Natan Silnitsky Date: Tue, 13 Jun 2023 18:22:32 +0300 Subject: [PATCH 17/17] make GreyhoundNG RetryConfig serializable (so can be sent over gRpc and persisted) (#35207) * make GreyhoundNG RetryConfig serializable (so can be sent over gRpc and persisted) #pr * fix lp build * full cross-repo * Initial cross-repo check * Updating cross-repo check * Updating cross-repo check * Updating cross-repo check * Updating cross-repo check * CR changes --------- Co-authored-by: wixapiregistry <58037308+wixapiregistry@users.noreply.github.com> GitOrigin-RevId: feaf45b3c77f9074a3445bffdd6f748d3df77056 --- .../dst/greyhound/core/retry/RetryIT.scala | 17 +- .../retry/BlockingRetryRecordHandler.scala | 25 +-- .../core/consumer/retry/RetryAttempt.scala | 43 +++-- .../core/consumer/retry/RetryConfig.scala | 158 ++++++++++++++++-- .../RetryConsumerRecordHandlerTest.scala | 12 +- .../consumer/retry/ZRetryConfigTest.scala | 8 +- .../scala/GreyhoundConsumersBuilder.scala | 10 +- .../greyhound/scala/RetryConfigBuilder.scala | 2 +- 8 files changed, 204 insertions(+), 71 deletions(-) diff --git a/core/src/it/scala/com/wixpress/dst/greyhound/core/retry/RetryIT.scala b/core/src/it/scala/com/wixpress/dst/greyhound/core/retry/RetryIT.scala index ed382807..834be9f0 100644 --- a/core/src/it/scala/com/wixpress/dst/greyhound/core/retry/RetryIT.scala +++ b/core/src/it/scala/com/wixpress/dst/greyhound/core/retry/RetryIT.scala @@ -6,6 +6,7 @@ import com.wixpress.dst.greyhound.core.consumer._ import com.wixpress.dst.greyhound.core.consumer.domain.ConsumerSubscription.{TopicPattern, Topics} import com.wixpress.dst.greyhound.core.consumer.domain.{ConsumerRecord, RecordHandler} import com.wixpress.dst.greyhound.core.consumer.retry.NonBlockingRetryHelper.fixedRetryTopic +import com.wixpress.dst.greyhound.core.consumer.retry.RetryConfigForTopic.{finiteBlockingRetryConfigForTopic, nonBlockingRetryConfigForTopic} import com.wixpress.dst.greyhound.core.consumer.retry._ import com.wixpress.dst.greyhound.core.producer.{Encryptor, ProducerRecord} import com.wixpress.dst.greyhound.core.testkit.{eventuallyZ, AwaitableRef, BaseTestWithSharedEnv} @@ -40,8 +41,8 @@ class RetryIT extends BaseTestWithSharedEnv[Env, TestResources] { done <- Promise.make[Nothing, ConsumerRecord[String, String]] retryConfig = ZRetryConfig .perTopicRetries { - case `topic` => RetryConfigForTopic(() => Nil, NonBlockingBackoffPolicy(1.second :: Nil)) - case `anotherTopic` => RetryConfigForTopic(() => Nil, NonBlockingBackoffPolicy(1.second :: Nil)) + case `topic` => nonBlockingRetryConfigForTopic(1.second :: Nil) + case `anotherTopic` => nonBlockingRetryConfigForTopic(1.second :: Nil) } .copy(produceEncryptor = _ => ZIO.succeed(dummyEncryptor)) @@ -76,7 +77,7 @@ class RetryIT extends BaseTestWithSharedEnv[Env, TestResources] { retryConfig = ZRetryConfig .finiteBlockingRetry(100.millis, 100.millis) - .withCustomRetriesFor { case `topic2` => RetryConfigForTopic(() => 300.millis :: Nil, NonBlockingBackoffPolicy.empty) } + .withCustomRetriesFor { case `topic2` => finiteBlockingRetryConfigForTopic(300.millis :: Nil) } retryHandler = failingBlockingRecordHandlerWith(consumedValuesRef, Set(topic, topic2)).withDeserializers(StringSerde, StringSerde) _ <- RecordConsumer.make(configFor(kafka, group, retryConfig, topic, topic2), retryHandler).flatMap { _ => producer.produce(ProducerRecord(topic, "bar", Some("foo")), StringSerde, StringSerde) *> Clock.sleep(2.seconds) *> @@ -225,7 +226,7 @@ class RetryIT extends BaseTestWithSharedEnv[Env, TestResources] { invocations <- Ref.make(0) done <- Promise.make[Nothing, ConsumerRecord[String, String]] retryConfig = ZRetryConfig.retryForPattern( - RetryConfigForTopic(() => Nil, NonBlockingBackoffPolicy(Seq(1.second, 1.second, 1.seconds))) + nonBlockingRetryConfigForTopic(List(1.second, 1.second, 1.seconds)) ) retryHandler = failingRecordHandler(invocations, done).withDeserializers(StringSerde, StringSerde) success <- RecordConsumer @@ -258,7 +259,9 @@ class RetryIT extends BaseTestWithSharedEnv[Env, TestResources] { group <- randomGroup originalTopicCallCount <- Ref.make[Int](0) retryTopicCallCount <- Ref.make[Int](0) - retryConfig = ZRetryConfig.blockingFollowedByNonBlockingRetry(List(1.second), NonBlockingBackoffPolicy(List(1.seconds))) + retryConfig = + ZRetryConfig + .blockingFollowedByNonBlockingRetry(FiniteBlockingBackoffPolicy(List(1.second)), NonBlockingBackoffPolicy(List(1.seconds))) retryHandler = failingBlockingNonBlockingRecordHandler(originalTopicCallCount, retryTopicCallCount, topic).withDeserializers( StringSerde, StringSerde @@ -281,9 +284,7 @@ class RetryIT extends BaseTestWithSharedEnv[Env, TestResources] { for { r <- getShared TestResources(kafka, _) = r - retryConfig = ZRetryConfig.retryForPattern( - RetryConfigForTopic(() => Nil, NonBlockingBackoffPolicy(Seq(1.second, 1.second, 1.seconds))) - ) + retryConfig = ZRetryConfig.retryForPattern(nonBlockingRetryConfigForTopic(List(1.second, 1.second, 1.seconds))) handler = RecordHandler { _: ConsumerRecord[String, String] => ZIO.unit }.withDeserializers(StringSerde, StringSerde) _ <- RecordConsumer .make(configFor(kafka, "group", retryConfig, "topic"), handler) diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/BlockingRetryRecordHandler.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/BlockingRetryRecordHandler.scala index 2f5d59a9..5544e200 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/BlockingRetryRecordHandler.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/BlockingRetryRecordHandler.scala @@ -3,7 +3,7 @@ package com.wixpress.dst.greyhound.core.consumer.retry import java.util.concurrent.TimeUnit import com.wixpress.dst.greyhound.core.{Group, TopicPartition} import com.wixpress.dst.greyhound.core.consumer.domain.{ConsumerRecord, RecordHandler} -import com.wixpress.dst.greyhound.core.consumer.retry.BlockingState.{Blocked, IgnoringOnce, Blocking => InternalBlocking} +import com.wixpress.dst.greyhound.core.consumer.retry.BlockingState.{Blocked, Blocking => InternalBlocking, IgnoringOnce} import com.wixpress.dst.greyhound.core.consumer.retry.RetryRecordHandlerMetric.{BlockingRetryHandlerInvocationFailed, DoneBlockingBeforeRetry, NoRetryOnNonRetryableFailure} import com.wixpress.dst.greyhound.core.metrics.GreyhoundMetrics import com.wixpress.dst.greyhound.core.metrics.GreyhoundMetrics.report @@ -30,7 +30,11 @@ private[retry] object BlockingRetryRecordHandler { override def handle(record: ConsumerRecord[K, V])(implicit trace: Trace): ZIO[GreyhoundMetrics with R, Nothing, LastHandleResult] = { val topicPartition = TopicPartition(record.topic, record.partition) - def pollBlockingStateWithSuspensions(record: ConsumerRecord[K, V], interval: Duration, start: Long): URIO[GreyhoundMetrics, PollResult] = { + def pollBlockingStateWithSuspensions( + record: ConsumerRecord[K, V], + interval: Duration, + start: Long + ): URIO[GreyhoundMetrics, PollResult] = { for { shouldBlock <- blockingStateResolver.resolve(record) shouldPollAgain <- @@ -73,7 +77,8 @@ private[retry] object BlockingRetryRecordHandler { case error => interval .map { interval => - report(BlockingRetryHandlerInvocationFailed(topicPartition, record.offset, error.toString)) *> blockOnErrorFor(record, interval) + report(BlockingRetryHandlerInvocationFailed(topicPartition, record.offset, error.toString)) *> + blockOnErrorFor(record, interval) } .getOrElse(ZIO.succeed(LastHandleResult(lastHandleSucceeded = false, shouldContinue = false))) } @@ -94,12 +99,13 @@ private[retry] object BlockingRetryRecordHandler { if (nonBlockingHandler.isHandlingRetryTopicMessage(group, record)) { ZIO.succeed(LastHandleResult(lastHandleSucceeded = false, shouldContinue = false)) } else { - val durationsIncludingForInvocationWithNoErrorHandling = retryConfig.blockingBackoffs(record.topic)().map(Some(_)) :+ None + val durations = retryConfig.blockingBackoffs(record.topic) + val durationsIncludingForInvocationWithNoErrorHandling = durations.map(Some(_)) :+ None for { result <- retryEvery(record, durationsIncludingForInvocationWithNoErrorHandling) { (rec, interval) => - handleAndMaybeBlockOnErrorFor(rec, interval) - } - _ <- maybeBackToStateBlocking + handleAndMaybeBlockOnErrorFor(rec, interval) + } + _ <- maybeBackToStateBlocking } yield result } } @@ -111,7 +117,7 @@ private[retry] object BlockingRetryRecordHandler { ZIO.succeed(as.iterator).flatMap { i => def loop(retryAttempt: Option[RetryAttempt]): ZIO[R, E, LastHandleResult] = if (i.hasNext) { - val nextDelay = i.next + val nextDelay = i.next val recordWithAttempt = retryAttempt.fold(record) { attempt => record.copy(headers = record.headers ++ RetryAttempt.toHeaders(attempt)) } @@ -127,8 +133,7 @@ private[retry] object BlockingRetryRecordHandler { } else ZIO.succeed(result) } - } - else ZIO.succeed(LastHandleResult(lastHandleSucceeded = false, shouldContinue = false)) + } else ZIO.succeed(LastHandleResult(lastHandleSucceeded = false, shouldContinue = false)) loop(None) } diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryAttempt.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryAttempt.scala index 3fa3cba8..9de958e8 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryAttempt.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryAttempt.scala @@ -11,7 +11,8 @@ import java.time.Instant /** * Description of a retry attempt - * @param attempt contains which attempt is it, starting from 0 including blocking and non-blocking attempts + * @param attempt + * contains which attempt is it, starting from 0 including blocking and non-blocking attempts */ case class RetryAttempt( originalTopic: Topic, @@ -33,10 +34,10 @@ object RetryAttempt { private def toChunk(str: String): Chunk[Byte] = Chunk.fromArray(str.getBytes) def toHeaders(attempt: RetryAttempt): Headers = Headers( - RetryHeader.Submitted -> toChunk(attempt.submittedAt.toEpochMilli.toString), - RetryHeader.Backoff -> toChunk(attempt.backoff.toMillis.toString), + RetryHeader.Submitted -> toChunk(attempt.submittedAt.toEpochMilli.toString), + RetryHeader.Backoff -> toChunk(attempt.backoff.toMillis.toString), RetryHeader.OriginalTopic -> toChunk(attempt.originalTopic), - RetryHeader.RetryAttempt -> toChunk(attempt.attempt.toString), + RetryHeader.RetryAttempt -> toChunk(attempt.attempt.toString) ) private case class RetryAttemptHeaders( @@ -49,14 +50,14 @@ object RetryAttempt { private def fromHeaders(headers: Headers): Task[RetryAttemptHeaders] = for { submitted <- headers.get(RetryHeader.Submitted, instantDeserializer) - backoff <- headers.get(RetryHeader.Backoff, durationDeserializer) - topic <- headers.get[String](RetryHeader.OriginalTopic, StringSerde) - attempt <- headers.get(RetryHeader.RetryAttempt, longDeserializer) + backoff <- headers.get(RetryHeader.Backoff, durationDeserializer) + topic <- headers.get[String](RetryHeader.OriginalTopic, StringSerde) + attempt <- headers.get(RetryHeader.RetryAttempt, longDeserializer) } yield RetryAttemptHeaders(topic, attempt.map(_.toInt), submitted, backoff) /** @return None on infinite blocking retries */ def maxBlockingAttempts(topic: Topic, retryConfig: Option[RetryConfig]): Option[Int] = - retryConfig.map(_.blockingBackoffs(topic)()).fold(Option(0)) { + retryConfig.map(_.blockingBackoffs(topic)).fold(Option(0)) { case finite if finite.hasDefiniteSize => Some(finite.size) case _ => None } @@ -68,31 +69,29 @@ object RetryAttempt { } def extract( - headers: Headers, - topic: Topic, - group: Group, - subscription: ConsumerSubscription, - retryConfig: Option[RetryConfig], + headers: Headers, + topic: Topic, + group: Group, + subscription: ConsumerSubscription, + retryConfig: Option[RetryConfig] )(implicit trace: Trace): UIO[Option[RetryAttempt]] = { def maybeNonBlockingAttempt(hs: RetryAttemptHeaders): Option[RetryAttempt] = for { - submitted <- hs.submittedAt - backoff <- hs.backoff + submitted <- hs.submittedAt + backoff <- hs.backoff TopicAttempt(originalTopic, attempt) <- attemptNumberFromTopic(subscription, topic, hs.originalTopic, group) - blockingRetries = maxBlockingAttempts(originalTopic, retryConfig).getOrElse(0) + blockingRetries = maxBlockingAttempts(originalTopic, retryConfig).getOrElse(0) } yield RetryAttempt(originalTopic, blockingRetries + attempt, submitted, backoff) def maybeBlockingAttempt(hs: RetryAttemptHeaders): Option[RetryAttempt] = for { - submitted <- hs.submittedAt - backoff <- hs.backoff + submitted <- hs.submittedAt + backoff <- hs.backoff originalTopic <- hs.originalTopic if originalTopic == topic - attempt <- hs.attempt + attempt <- hs.attempt } yield RetryAttempt(originalTopic, attempt, submitted, backoff) - fromHeaders(headers).map { hs => - maybeNonBlockingAttempt(hs) orElse maybeBlockingAttempt(hs) - } + fromHeaders(headers).map { hs => maybeNonBlockingAttempt(hs) orElse maybeBlockingAttempt(hs) } }.catchAll(_ => ZIO.none) } diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryConfig.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryConfig.scala index 5632f042..96dacd2c 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryConfig.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryConfig.scala @@ -15,8 +15,24 @@ case class RetryConfig( produceRetryBackoff: Duration = 5.seconds, produceEncryptor: ConsumerRecord[_, _] => UIO[Encryptor] = _ => ZIO.succeed(NoOpEncryptor)(zio.Trace.empty) ) { - def blockingBackoffs(topic: Topic) = - get(topic)(_.blockingBackoffs)(ifEmpty = () => Nil) + def blockingBackoffs(topic: Topic): Seq[ZDuration] = + get(topic) { + case RetryConfigForTopic(FiniteBlockingBackoffPolicy(intervals), _, _) if intervals.nonEmpty => intervals + case RetryConfigForTopic(_, InfiniteFixedBackoff(fixed), _) => Stream.continually(fixed) + case RetryConfigForTopic(_, expMax: InfiniteExponentialBackoffsMaxInterval, _) => + exponentialBackoffs(expMax.initialInterval, expMax.maximalInterval, expMax.backOffMultiplier, infiniteRetryMaxInteval = true) + case RetryConfigForTopic(_, expMult: InfiniteExponentialBackoffsMaxMultiplication, _) => + exponentialBackoffs( + expMult.initialInterval, + expMult.maxMultiplications, + expMult.backOffMultiplier, + infiniteRetryMaxInterval = true + ) + case _ => Nil + }(ifEmpty = Nil) + + def finiteBlockingBackoffs(topic: Topic) = + get(topic)(_.finiteBlockingBackoffs)(ifEmpty = FiniteBlockingBackoffPolicy.empty) def retryType(originalTopic: Topic): RetryType = get(originalTopic)(_.retryType)(ifEmpty = NoRetries) @@ -43,11 +59,46 @@ case class RetryConfig( else ifEmpty ) +} + +trait InfiniteBlockingBackoffPolicy { + def nonEmpty: Boolean +} + +case object EmptyInfiniteBlockingBackoffPolicy extends InfiniteBlockingBackoffPolicy { + override def nonEmpty: Boolean = false +} + +sealed case class InfiniteFixedBackoff(interval: ZDuration) extends InfiniteBlockingBackoffPolicy { + override def nonEmpty: Boolean = true +} + +sealed case class InfiniteExponentialBackoffsMaxInterval( + initialInterval: ZDuration, + maximalInterval: ZDuration, + backOffMultiplier: Float +) extends InfiniteBlockingBackoffPolicy { + override def nonEmpty: Boolean = true +} + +sealed case class InfiniteExponentialBackoffsMaxMultiplication( + initialInterval: ZDuration, + maxMultiplications: Int, + backOffMultiplier: Float +) extends InfiniteBlockingBackoffPolicy { + override def nonEmpty: Boolean = true +} +case class FiniteBlockingBackoffPolicy(intervals: List[ZDuration]) { + def nonEmpty = intervals.nonEmpty +} + +object FiniteBlockingBackoffPolicy { + val empty = FiniteBlockingBackoffPolicy(Nil) } case class NonBlockingBackoffPolicy( - intervals: Seq[ZDuration], + intervals: List[ZDuration], recordMutate: ProducerRecord[Chunk[Byte], Chunk[Byte]] => ProducerRecord[Chunk[Byte], Chunk[Byte]] = identity ) { def nonEmpty = intervals.nonEmpty @@ -59,22 +110,48 @@ object NonBlockingBackoffPolicy { val empty = NonBlockingBackoffPolicy(Nil) } -case class RetryConfigForTopic(blockingBackoffs: () => Seq[ZDuration], nonBlockingBackoffs: NonBlockingBackoffPolicy) { - def nonEmpty: Boolean = blockingBackoffs().nonEmpty || nonBlockingBackoffs.nonEmpty +case class RetryConfigForTopic( + finiteBlockingBackoffs: FiniteBlockingBackoffPolicy, + infiniteBlockingBackoffs: InfiniteBlockingBackoffPolicy, + nonBlockingBackoffs: NonBlockingBackoffPolicy +) { + def nonEmpty: Boolean = finiteBlockingBackoffs.nonEmpty || infiniteBlockingBackoffs.nonEmpty || nonBlockingBackoffs.nonEmpty def retryType: RetryType = - if (blockingBackoffs.apply().nonEmpty) { + if (finiteBlockingBackoffs.nonEmpty) { if (nonBlockingBackoffs.nonEmpty) BlockingFollowedByNonBlocking else Blocking - } else { + } else if (infiniteBlockingBackoffs.nonEmpty) + Blocking + else { NonBlocking } } object RetryConfigForTopic { - val empty = RetryConfigForTopic(() => Nil, NonBlockingBackoffPolicy.empty) + val empty = RetryConfigForTopic(FiniteBlockingBackoffPolicy.empty, EmptyInfiniteBlockingBackoffPolicy, NonBlockingBackoffPolicy.empty) + + def nonBlockingRetryConfigForTopic(intervals: List[ZDuration]) = + RetryConfigForTopic(FiniteBlockingBackoffPolicy.empty, EmptyInfiniteBlockingBackoffPolicy, NonBlockingBackoffPolicy(intervals)) + + def finiteBlockingRetryConfigForTopic(intervals: List[ZDuration]) = + RetryConfigForTopic(FiniteBlockingBackoffPolicy(intervals), EmptyInfiniteBlockingBackoffPolicy, NonBlockingBackoffPolicy.empty) + + def infiniteBlockingRetryConfigForTopic(interval: ZDuration) = + RetryConfigForTopic(FiniteBlockingBackoffPolicy.empty, InfiniteFixedBackoff(interval), NonBlockingBackoffPolicy.empty) + + def infiniteBlockingRetryConfigForTopic( + initialInterval: ZDuration, + maxInterval: ZDuration, + backOffMultiplier: Float + ) = + RetryConfigForTopic( + FiniteBlockingBackoffPolicy.empty, + InfiniteExponentialBackoffsMaxInterval(initialInterval, maxInterval, backOffMultiplier), + NonBlockingBackoffPolicy.empty + ) } object ZRetryConfig { @@ -82,18 +159,27 @@ object ZRetryConfig { forAllTopics( RetryConfigForTopic( nonBlockingBackoffs = NonBlockingBackoffPolicy(firstRetry :: otherRetries.toList), - blockingBackoffs = () => List.empty + finiteBlockingBackoffs = FiniteBlockingBackoffPolicy.empty, + infiniteBlockingBackoffs = EmptyInfiniteBlockingBackoffPolicy ) ) def finiteBlockingRetry(firstRetry: ZDuration, otherRetries: ZDuration*): RetryConfig = forAllTopics( - RetryConfigForTopic(blockingBackoffs = () => firstRetry :: otherRetries.toList, nonBlockingBackoffs = NonBlockingBackoffPolicy.empty) + RetryConfigForTopic( + finiteBlockingBackoffs = FiniteBlockingBackoffPolicy(firstRetry :: otherRetries.toList), + infiniteBlockingBackoffs = EmptyInfiniteBlockingBackoffPolicy, + nonBlockingBackoffs = NonBlockingBackoffPolicy.empty + ) ) def infiniteBlockingRetry(interval: ZDuration): RetryConfig = forAllTopics( - RetryConfigForTopic(blockingBackoffs = () => Stream.continually(interval), nonBlockingBackoffs = NonBlockingBackoffPolicy.empty) + RetryConfigForTopic( + infiniteBlockingBackoffs = InfiniteFixedBackoff(interval), + finiteBlockingBackoffs = FiniteBlockingBackoffPolicy.empty, + nonBlockingBackoffs = NonBlockingBackoffPolicy.empty + ) ) def exponentialBackoffBlockingRetry( @@ -101,32 +187,68 @@ object ZRetryConfig { maximalInterval: ZDuration, backOffMultiplier: Float, infiniteRetryMaxInterval: Boolean - ): RetryConfig = + ): RetryConfig = { + val (finite, infinite) = if (infiniteRetryMaxInterval) { + ( + FiniteBlockingBackoffPolicy.empty, + InfiniteExponentialBackoffsMaxInterval(initialInterval, maximalInterval, backOffMultiplier) + ) + } else { + ( + FiniteBlockingBackoffPolicy( + exponentialBackoffs(initialInterval, maximalInterval, backOffMultiplier, infiniteRetryMaxInterval).toList + ), + EmptyInfiniteBlockingBackoffPolicy + ) + } forAllTopics( RetryConfigForTopic( - blockingBackoffs = () => exponentialBackoffs(initialInterval, maximalInterval, backOffMultiplier, infiniteRetryMaxInterval), + finiteBlockingBackoffs = finite, + infiniteBlockingBackoffs = infinite, nonBlockingBackoffs = NonBlockingBackoffPolicy.empty ) ) + } def exponentialBackoffBlockingRetry( initialInterval: ZDuration, maxMultiplications: Int, backOffMultiplier: Float, infiniteRetryMaxInterval: Boolean - ): RetryConfig = + ): RetryConfig = { + val (finite, infinite) = if (infiniteRetryMaxInterval) { + ( + FiniteBlockingBackoffPolicy.empty, + InfiniteExponentialBackoffsMaxMultiplication(initialInterval, maxMultiplications, backOffMultiplier) + ) + } else { + ( + FiniteBlockingBackoffPolicy( + exponentialBackoffs(initialInterval, maxMultiplications, backOffMultiplier, infiniteRetryMaxInterval).toList + ), + EmptyInfiniteBlockingBackoffPolicy + ) + } forAllTopics( RetryConfigForTopic( - blockingBackoffs = () => exponentialBackoffs(initialInterval, maxMultiplications, backOffMultiplier, infiniteRetryMaxInterval), + finiteBlockingBackoffs = finite, + infiniteBlockingBackoffs = infinite, nonBlockingBackoffs = NonBlockingBackoffPolicy.empty ) ) + } def blockingFollowedByNonBlockingRetry( - blockingBackoffs: NonEmptyList[ZDuration], + blockingBackoffs: FiniteBlockingBackoffPolicy, nonBlockingBackoffs: NonBlockingBackoffPolicy ): RetryConfig = - forAllTopics(RetryConfigForTopic(blockingBackoffs = () => blockingBackoffs, nonBlockingBackoffs = nonBlockingBackoffs)) + forAllTopics( + RetryConfigForTopic( + finiteBlockingBackoffs = blockingBackoffs, + nonBlockingBackoffs = nonBlockingBackoffs, + infiniteBlockingBackoffs = EmptyInfiniteBlockingBackoffPolicy + ) + ) def perTopicRetries(configs: PartialFunction[Topic, RetryConfigForTopic]) = RetryConfig(configs, None) @@ -178,7 +300,7 @@ object RetryConfig { def blockingFollowedByNonBlockingRetry(blockingBackoffs: NonEmptyList[Duration], nonBlockingBackoffs: List[Duration]): RetryConfig = ZRetryConfig.blockingFollowedByNonBlockingRetry( - blockingBackoffs = blockingBackoffs.map(ZDuration.fromScala), + blockingBackoffs = FiniteBlockingBackoffPolicy(blockingBackoffs.map(ZDuration.fromScala)), nonBlockingBackoffs = NonBlockingBackoffPolicy(nonBlockingBackoffs.map(ZDuration.fromScala)) ) } diff --git a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryConsumerRecordHandlerTest.scala b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryConsumerRecordHandlerTest.scala index 0797ba25..ef9fccb1 100644 --- a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryConsumerRecordHandlerTest.scala +++ b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryConsumerRecordHandlerTest.scala @@ -6,6 +6,7 @@ import com.wixpress.dst.greyhound.core._ import com.wixpress.dst.greyhound.core.consumer.domain.ConsumerSubscription.Topics import com.wixpress.dst.greyhound.core.consumer.domain.{ConsumerRecord, RecordHandler} import com.wixpress.dst.greyhound.core.consumer.retry.BlockingState.{Blocked, Blocking => InternalBlocking, IgnoringAll, IgnoringOnce} +import com.wixpress.dst.greyhound.core.consumer.retry.RetryConfigForTopic.nonBlockingRetryConfigForTopic import com.wixpress.dst.greyhound.core.consumer.retry.RetryConsumerRecordHandlerTest.{offset, partition, _} import com.wixpress.dst.greyhound.core.consumer.retry.RetryRecordHandlerMetric.{BlockingIgnoredForAllFor, BlockingIgnoredOnceFor, BlockingRetryHandlerInvocationFailed, NoRetryOnNonRetryableFailure} import com.wixpress.dst.greyhound.core.producer.{ProducerError, ProducerRecord} @@ -311,7 +312,10 @@ class RetryConsumerRecordHandlerTest extends BaseTest[TestClock with TestMetrics group, failingHandlerWith(handleCountRef), ZRetryConfig - .blockingFollowedByNonBlockingRetry(List(10.millis, 500.millis), NonBlockingBackoffPolicy(List(1.second))), + .blockingFollowedByNonBlockingRetry( + FiniteBlockingBackoffPolicy(List(10.millis, 500.millis)), + NonBlockingBackoffPolicy(List(1.second)) + ), producer, Topics(Set(topic)), blockingState, @@ -336,9 +340,7 @@ class RetryConsumerRecordHandlerTest extends BaseTest[TestClock with TestMetrics topic <- randomTopicName otherTopic <- randomTopicName blockingState <- Ref.make[Map[BlockingTarget, BlockingState]](Map.empty) - policy = ZRetryConfig.perTopicRetries { - case `otherTopic` => RetryConfigForTopic(() => Nil, NonBlockingBackoffPolicy(1.second :: Nil)) - } + policy = ZRetryConfig.perTopicRetries { case `otherTopic` => nonBlockingRetryConfigForTopic(1.second :: Nil) } retryHandler = RetryRecordHandler.withRetries( group, failingHandler, @@ -410,7 +412,7 @@ class RetryConsumerRecordHandlerTest extends BaseTest[TestClock with TestMetrics retryHelper, awaitShutdown = _ => ZIO.succeed(awaitShutdown) ) - val headers = RetryAttempt.toHeaders(RetryAttempt(topic, 0, now, 3.seconds)) + val headers = RetryAttempt.toHeaders(RetryAttempt(topic, 0, now, 3.seconds)) for { key <- bytes value <- bytes diff --git a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/ZRetryConfigTest.scala b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/ZRetryConfigTest.scala index ac6dca03..6535207d 100644 --- a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/ZRetryConfigTest.scala +++ b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/ZRetryConfigTest.scala @@ -29,12 +29,12 @@ class ZRetryConfigTest extends SpecificationWithJUnit { val absMult = abs(params.mult) val safeInit = if (init < 10) 10 else init - for (i <- 0 to max) yield { backoffs()(i) } mustEqual (pow(1 + absMult, i) * safeInit).toLong.millis + for (i <- 0 to max) yield { backoffs(i) } mustEqual (pow(1 + absMult, i) * safeInit).toLong.millis val maxMult = math.max(0, max) val lastDurationToCheck = abs(max + 1) * 2 val firstDurationToCheck = math.max(0, max + 1) for (i <- firstDurationToCheck to lastDurationToCheck) - yield backoffs()(i) mustEqual (pow(1 + absMult, maxMult) * safeInit).toLong.millis + yield backoffs(i) mustEqual (pow(1 + absMult, maxMult) * safeInit).toLong.millis } } @@ -57,11 +57,11 @@ class ZRetryConfigTest extends SpecificationWithJUnit { val absMult = abs(params.mult) val safeInit = if (init < 10) 10 else init - for (i <- 0 to maxMult) yield { backoffs()(i) } mustEqual (pow(1 + absMult, i) * safeInit).toLong.millis + for (i <- 0 to maxMult) yield { backoffs(i) } mustEqual (pow(1 + absMult, i) * safeInit).toLong.millis val lastDurationToCheck = abs(maxMult + 1) * 2 val firstDurationToCheck = math.max(0, maxMult + 1) for (i <- firstDurationToCheck to lastDurationToCheck) - yield backoffs()(i) mustEqual (pow(1 + absMult, maxMult) * safeInit).toLong.millis + yield backoffs(i) mustEqual (pow(1 + absMult, maxMult) * safeInit).toLong.millis } } } diff --git a/java-interop/src/main/java/com/wixpress/dst/greyhound/scala/GreyhoundConsumersBuilder.scala b/java-interop/src/main/java/com/wixpress/dst/greyhound/scala/GreyhoundConsumersBuilder.scala index f3924695..484acc95 100644 --- a/java-interop/src/main/java/com/wixpress/dst/greyhound/scala/GreyhoundConsumersBuilder.scala +++ b/java-interop/src/main/java/com/wixpress/dst/greyhound/scala/GreyhoundConsumersBuilder.scala @@ -5,9 +5,9 @@ import com.wixpress.dst.greyhound.core import com.wixpress.dst.greyhound.core.consumer.batched.{BatchConsumer, BatchConsumerConfig, BatchEventLoopConfig} import com.wixpress.dst.greyhound.core.consumer.domain.ConsumerSubscription.Topics import com.wixpress.dst.greyhound.core.consumer.domain.{RecordHandler => CoreRecordHandler} -import com.wixpress.dst.greyhound.core.consumer.retry.{NonBlockingBackoffPolicy, RetryConfig => CoreRetryConfig, RetryConfigForTopic} +import com.wixpress.dst.greyhound.core.consumer.retry.{EmptyInfiniteBlockingBackoffPolicy, FiniteBlockingBackoffPolicy, InfiniteBlockingBackoffPolicy, NonBlockingBackoffPolicy, RetryConfigForTopic, RetryConfig => CoreRetryConfig} import com.wixpress.dst.greyhound.core.consumer.{RecordConsumer, RecordConsumerConfig} -import com.wixpress.dst.greyhound.core.{consumer, Group, NonEmptySet, Topic} +import com.wixpress.dst.greyhound.core.{Group, NonEmptySet, Topic, consumer} import com.wixpress.dst.greyhound.future.GreyhoundRuntime import com.wixpress.dst.greyhound.future.GreyhoundRuntime.Env import zio._ @@ -138,7 +138,11 @@ class GreyhoundConsumersBuilder(val config: GreyhoundConfig) { retryConfig.map(config => { val forTopic = - RetryConfigForTopic(() => config.blockingBackoffs().asScala, NonBlockingBackoffPolicy(config.nonBlockingBackoffs.asScala)) + RetryConfigForTopic( + FiniteBlockingBackoffPolicy(config.blockingBackoffs().asScala.toList), + EmptyInfiniteBlockingBackoffPolicy, + NonBlockingBackoffPolicy(config.nonBlockingBackoffs.asScala.toList) + ) CoreRetryConfig({ case _ => forTopic }, None) }) diff --git a/java-interop/src/main/java/com/wixpress/dst/greyhound/scala/RetryConfigBuilder.scala b/java-interop/src/main/java/com/wixpress/dst/greyhound/scala/RetryConfigBuilder.scala index bd0f5538..795844b1 100644 --- a/java-interop/src/main/java/com/wixpress/dst/greyhound/scala/RetryConfigBuilder.scala +++ b/java-interop/src/main/java/com/wixpress/dst/greyhound/scala/RetryConfigBuilder.scala @@ -44,7 +44,7 @@ object RetryConfigBuilder { } private def fromCoreRetryConfig(coreRetryConfig: com.wixpress.dst.greyhound.core.consumer.retry.RetryConfig): RetryConfig = { - val blocking: util.List[Duration] = seqAsJavaList(coreRetryConfig.blockingBackoffs("").apply) + val blocking: util.List[Duration] = seqAsJavaList(coreRetryConfig.blockingBackoffs("")) val nonBlocking: util.List[Duration] = seqAsJavaList(coreRetryConfig.nonBlockingBackoffs("").intervals) new RetryConfig(blocking, nonBlocking) }