diff --git a/core/src/it/scala/com/wixpress/dst/greyhound/core/AdminClientIT.scala b/core/src/it/scala/com/wixpress/dst/greyhound/core/AdminClientIT.scala index 95d2353a..dec5377c 100644 --- a/core/src/it/scala/com/wixpress/dst/greyhound/core/AdminClientIT.scala +++ b/core/src/it/scala/com/wixpress/dst/greyhound/core/AdminClientIT.scala @@ -10,7 +10,7 @@ import com.wixpress.dst.greyhound.core.producer.ProducerRecord import com.wixpress.dst.greyhound.core.testkit.{BaseTestWithSharedEnv, TestMetrics} import com.wixpress.dst.greyhound.core.zioutils.CountDownLatch import com.wixpress.dst.greyhound.testenv.ITEnv -import com.wixpress.dst.greyhound.testenv.ITEnv.{Env, TestResources, testResources} +import com.wixpress.dst.greyhound.testenv.ITEnv.{testResources, Env, TestResources} import org.apache.kafka.common.config.TopicConfig.{DELETE_RETENTION_MS_CONFIG, MAX_MESSAGE_BYTES_CONFIG, RETENTION_MS_CONFIG} import org.apache.kafka.common.errors.InvalidTopicException import org.specs2.specification.core.Fragments @@ -83,7 +83,7 @@ class AdminClientIT extends BaseTestWithSharedEnv[Env, TestResources] { } } - //todo uncomment this after https://github.com/wix-private/core-server-build-tools/pull/13043 is merged + // todo uncomment this after https://github.com/wix-private/core-server-build-tools/pull/13043 is merged // "reflect errors" in { // val topic1 = aTopicConfig() // val topic2 = aTopicConfig("x" * 250) @@ -104,7 +104,7 @@ class AdminClientIT extends BaseTestWithSharedEnv[Env, TestResources] { // created === Map(badTopic.name -> None) // } // } - //todo uncomment this after https://github.com/wix-private/core-server-build-tools/pull/13043 is merged + // todo uncomment this after https://github.com/wix-private/core-server-build-tools/pull/13043 is merged // ================================================================================================================================= "ignore TopicExistsException by default" in { val topic = aTopicConfig() diff --git a/core/src/it/scala/com/wixpress/dst/greyhound/core/BUILD.bazel b/core/src/it/scala/com/wixpress/dst/greyhound/core/BUILD.bazel index 28d79c08..9735d210 100644 --- a/core/src/it/scala/com/wixpress/dst/greyhound/core/BUILD.bazel +++ b/core/src/it/scala/com/wixpress/dst/greyhound/core/BUILD.bazel @@ -24,13 +24,8 @@ specs2_ite2e_test( "//core/src/main/scala/com/wixpress/dst/greyhound/core/producer", "//core/src/main/scala/com/wixpress/dst/greyhound/core/zioutils", "//core/src/test/resources", - #"//core/src/test/scala/com/wixpress/dst/greyhound/core/consumer", - #"//core/src/test/scala/com/wixpress/dst/greyhound/core/testkit", - "@ch_qos_logback_logback_classic", # "@dev_zio_izumi_reflect_2_12", "@dev_zio_zio_2_12", - "@dev_zio_zio_test_2_12", - "@org_apache_kafka_kafka_2_12", "@org_apache_kafka_kafka_clients", "//core/src/test/scala/com/wixpress/dst/greyhound/core/testkit", ], diff --git a/core/src/it/scala/com/wixpress/dst/greyhound/core/ConsumerIT.scala b/core/src/it/scala/com/wixpress/dst/greyhound/core/ConsumerIT.scala index e91e3742..d644c9aa 100644 --- a/core/src/it/scala/com/wixpress/dst/greyhound/core/ConsumerIT.scala +++ b/core/src/it/scala/com/wixpress/dst/greyhound/core/ConsumerIT.scala @@ -19,6 +19,7 @@ import com.wixpress.dst.greyhound.core.zioutils.Gate import com.wixpress.dst.greyhound.testenv.ITEnv import com.wixpress.dst.greyhound.testenv.ITEnv.{clientId, _} import com.wixpress.dst.greyhound.testkit.ManagedKafka +import org.specs2.specification.core.Fragments import zio.Clock import zio.stm.{STM, TRef} import zio._ @@ -29,429 +30,541 @@ import zio.managed._ class ConsumerIT extends BaseTestWithSharedEnv[Env, TestResources] { sequential - override def env = ITEnv.ManagedEnv - + override def env = ITEnv.ManagedEnv override def sharedEnv = ITEnv.testResources() + val resources = testResources() - val resources = testResources() - "produce, consume and rebalance - " in + s"subscribe to a pattern" in ZIO.scoped { for { + _ <- ZIO.debug(">>>> starting test: patternTest with parallel") + topic1 = "core-subscribe-pattern1-topic" + topic2 = "core-subscribe-pattern2-topic" r <- getShared TestResources(kafka, producer) = r - topic <- kafka.createRandomTopic(prefix = s"topic1-single1") - topic2 <- kafka.createRandomTopic(prefix = "topic2-single1") + _ <- kafka.createTopics(Seq(topic1, topic2).map(t => TopicConfig(t, 1, 1, delete)): _*) group <- randomGroup + probe <- Ref.make(Seq.empty[(Topic, Offset)]) + handler = RecordHandler { record: ConsumerRecord[Chunk[Byte], Chunk[Byte]] => probe.update(_ :+ (record.topic, record.offset)) } - queue <- Queue.unbounded[ConsumerRecord[String, String]] - handler = RecordHandler((cr: ConsumerRecord[String, String]) => ZIO.succeed(println(s"***** Consumed: $cr")) *> queue.offer(cr)) - .withDeserializers(StringSerde, StringSerde) - .ignore - cId <- clientId - config = configFor(kafka, group, topic).copy(clientId = cId) - record = ProducerRecord(topic, "bar", Some("foo")) - - messages <- RecordConsumer.make(config, handler).flatMap { consumer => - producer.produce(record, StringSerde, StringSerde) *> sleep(3.seconds) *> - consumer.resubscribe(ConsumerSubscription.topics(topic, topic2)) *> - sleep(500.millis) *> // give the consumer some time to start polling topic2 - producer.produce(record.copy(topic = topic2, value = Some("BAR")), StringSerde, StringSerde) *> - (queue.take zip queue.take) - .timeout(20.seconds) - .tap(o => ZIO.when(o.isEmpty)(ZIO.debug("timeout waiting for messages!"))) - } - msgs <- ZIO.fromOption(messages).orElseFail(TimedOutWaitingForMessages) - } yield { - msgs._1 must - (beRecordWithKey("foo") and beRecordWithValue("bar")) and - (msgs._2 must (beRecordWithKey("foo") and beRecordWithValue("BAR"))) - } + _ <- makeConsumer(kafka, compile("core-subscribe-pattern1.*"), group, handler, 0, useParallelConsumer = false).flatMap { consumer => + val record = ProducerRecord(topic1, Chunk.empty, key = Option(Chunk.empty)) + producer.produce(record) *> eventuallyZ(probe.get)(_ == (topic1, 0) :: Nil) *> + consumer.resubscribe(TopicPattern(compile("core-subscribe-pattern2.*"))) *> + producer.produce(record.copy(topic = topic2)) *> eventuallyZ(probe.get)(_ == (topic1, 0) :: (topic2, 0) :: Nil) + } + } yield ok } - "be able to resubscribe to same topics" in + s"subscribe to a pattern - using parallel consumer" in ZIO.scoped { for { + _ <- ZIO.debug(">>>> starting test: patternTest with parallel") + topic1 = "core-subscribe-parallel-pattern1-topic" + topic2 = "core-subscribe-parallel-pattern2-topic" r <- getShared TestResources(kafka, producer) = r - topic <- kafka.createRandomTopic(prefix = s"topic1-single1") + _ <- kafka.createTopics(Seq(topic1, topic2).map(t => TopicConfig(t, 1, 1, delete)): _*) group <- randomGroup + probe <- Ref.make(Seq.empty[(Topic, Offset)]) + handler = RecordHandler { record: ConsumerRecord[Chunk[Byte], Chunk[Byte]] => probe.update(_ :+ (record.topic, record.offset)) } - queue <- Queue.unbounded[ConsumerRecord[String, String]] - handler = RecordHandler((cr: ConsumerRecord[String, String]) => ZIO.succeed(println(s"***** Consumed: $cr")) *> queue.offer(cr)) - .withDeserializers(StringSerde, StringSerde) - .ignore - cId <- clientId - config = configFor(kafka, group, topic, mutateEventLoop = _.copy(drainTimeout = 1.second)).copy(clientId = cId) - record = ProducerRecord(topic, "bar", Some("foo")) - - messages <- RecordConsumer.make(config, handler).flatMap { consumer => - producer.produce(record, StringSerde, StringSerde) *> sleep(3.seconds) *> - consumer.resubscribe(ConsumerSubscription.topics(topic)) *> - consumer.resubscribe(ConsumerSubscription.topics(topic)) *> - producer.produce(record.copy(topic = topic, value = Some("BAR")), StringSerde, StringSerde) *> - (queue.take zip queue.take) - .timeout(20.seconds) - .tap(o => ZIO.when(o.isEmpty)(ZIO.debug("timeout waiting for messages!"))) - } - msgs <- ZIO.fromOption(messages).orElseFail(TimedOutWaitingForMessages) - } yield { - msgs._1 must - (beRecordWithKey("foo") and beRecordWithValue("bar")) and - (msgs._2 must (beRecordWithKey("foo") and beRecordWithValue("BAR"))) - } + _ <- makeConsumer(kafka, compile("core-subscribe-parallel-pattern1.*"), group, handler, 0, useParallelConsumer = true).flatMap { + consumer => + val record = ProducerRecord(topic1, Chunk.empty, key = Option(Chunk.empty)) + producer.produce(record) *> eventuallyZ(probe.get, timeout = 20.seconds)(_ == (topic1, 0) :: Nil) *> + consumer.resubscribe(TopicPattern(compile("core-subscribe-parallel-pattern2.*"))) *> + producer.produce(record.copy(topic = topic2)) *> eventuallyZ(probe.get)(_ == (topic1, 0) :: (topic2, 0) :: Nil) + } + } yield ok } - "produce consume null values (tombstones)" in - ZIO.scoped { - for { - r <- getShared - TestResources(kafka, producer) = r - topic <- kafka.createRandomTopic(prefix = s"topic1-single1") - group <- randomGroup + Fragments.foreach(Seq(false, true)) { useParallelConsumer => + s"produce, consume and rebalance${parallelConsumerString(useParallelConsumer)}" in + ZIO.scoped { + for { + r <- getShared + TestResources(kafka, producer) = r + topic <- kafka.createRandomTopic(prefix = s"topic1-single1") + topic2 <- kafka.createRandomTopic(prefix = "topic2-single1") + group <- randomGroup + + queue <- Queue.unbounded[ConsumerRecord[String, String]] + handler = RecordHandler((cr: ConsumerRecord[String, String]) => ZIO.succeed(println(s"***** Consumed: $cr")) *> queue.offer(cr)) + .withDeserializers(StringSerde, StringSerde) + .ignore + cId <- clientId + config = + configFor(kafka, group, topic, mutateEventLoop = _.copy(consumePartitionInParallel = useParallelConsumer, maxParallelism = 8)) + .copy(clientId = cId) + record = ProducerRecord(topic, "bar", Some("foo")) + + messages <- RecordConsumer.make(config, handler).flatMap { consumer => + producer.produce(record, StringSerde, StringSerde) *> sleep(3.seconds) *> + consumer.resubscribe(ConsumerSubscription.topics(topic, topic2)) *> + sleep(500.millis) *> // give the consumer some time to start polling topic2 + producer.produce(record.copy(topic = topic2, value = Some("BAR")), StringSerde, StringSerde) *> + (queue.take zip queue.take) + .timeout(20.seconds) + .tap(o => ZIO.when(o.isEmpty)(ZIO.debug("timeout waiting for messages!"))) + } + msgs <- ZIO.fromOption(messages).orElseFail(TimedOutWaitingForMessages) + } yield { + msgs._1 must + (beRecordWithKey("foo") and beRecordWithValue("bar")) and + (msgs._2 must (beRecordWithKey("foo") and beRecordWithValue("BAR"))) + } + } - queue <- Queue.unbounded[ConsumerRecord[String, String]] - handler = RecordHandler((cr: ConsumerRecord[String, String]) => ZIO.succeed(println(s"***** Consumed: $cr")) *> queue.offer(cr)) - .withDeserializers(StringSerde, StringSerde) - .ignore - cId <- clientId - config = configFor(kafka, group, topic).copy(clientId = cId) - record = ProducerRecord.tombstone(topic, Some("foo")) - message <- RecordConsumer.make(config, handler).flatMap { _ => - producer.produce(record, StringSerde, StringSerde) *> queue.take.timeoutFail(TimedOutWaitingForMessages)(10.seconds) - } - } yield { - message must (beRecordWithKey("foo") and beRecordWithValue(null)) + s"be able to resubscribe to same topics${parallelConsumerString(useParallelConsumer)}" in + ZIO.scoped { + for { + r <- getShared + TestResources(kafka, producer) = r + topic <- kafka.createRandomTopic(prefix = s"topic1-single1") + group <- randomGroup + + queue <- Queue.unbounded[ConsumerRecord[String, String]] + handler = RecordHandler((cr: ConsumerRecord[String, String]) => ZIO.succeed(println(s"***** Consumed: $cr")) *> queue.offer(cr)) + .withDeserializers(StringSerde, StringSerde) + .ignore + cId <- clientId + config = + configFor( + kafka, + group, + topic, + mutateEventLoop = _.copy(drainTimeout = 1.second, consumePartitionInParallel = useParallelConsumer, maxParallelism = 8) + ).copy(clientId = cId) + record = ProducerRecord(topic, "bar", Some("foo")) + + messages <- RecordConsumer.make(config, handler).flatMap { consumer => + producer.produce(record, StringSerde, StringSerde) *> sleep(3.seconds) *> + consumer.resubscribe(ConsumerSubscription.topics(topic)) *> + consumer.resubscribe(ConsumerSubscription.topics(topic)) *> + producer.produce(record.copy(topic = topic, value = Some("BAR")), StringSerde, StringSerde) *> + (queue.take zip queue.take) + .timeout(20.seconds) + .tap(o => ZIO.when(o.isEmpty)(ZIO.debug("timeout waiting for messages!"))) + } + msgs <- ZIO.fromOption(messages).orElseFail(TimedOutWaitingForMessages) + } yield { + msgs._1 must + (beRecordWithKey("foo") and beRecordWithValue("bar")) and + (msgs._2 must (beRecordWithKey("foo") and beRecordWithValue("BAR"))) + } } - } - "not lose any messages on a slow consumer (drives the message dispatcher to throttling)" in - ZIO.scoped { - for { - r <- getShared - TestResources(kafka, producer) = r - _ <- ZIO.debug(">>>> starting test: throttlingTest") - - topic <- kafka.createRandomTopic(partitions = 2, prefix = "core-not-lose") - group <- randomGroup - - messagesPerPartition = 500 // Exceeds the queue capacity - delayPartition1 <- Promise.make[Nothing, Unit] - handledPartition0 <- CountDownLatch.make(messagesPerPartition) - handledPartition1 <- CountDownLatch.make(messagesPerPartition) - handler = RecordHandler { record: ConsumerRecord[Chunk[Byte], Chunk[Byte]] => - record.partition match { - case 0 => handledPartition0.countDown - case 1 => delayPartition1.await *> handledPartition1.countDown + s"produce consume null values (tombstones)${parallelConsumerString(useParallelConsumer)}" in + ZIO.scoped { + for { + r <- getShared + TestResources(kafka, producer) = r + topic <- kafka.createRandomTopic(prefix = s"topic1-single1") + group <- randomGroup + + queue <- Queue.unbounded[ConsumerRecord[String, String]] + handler = RecordHandler((cr: ConsumerRecord[String, String]) => ZIO.succeed(println(s"***** Consumed: $cr")) *> queue.offer(cr)) + .withDeserializers(StringSerde, StringSerde) + .ignore + cId <- clientId + config = + configFor(kafka, group, topic, mutateEventLoop = _.copy(consumePartitionInParallel = useParallelConsumer, maxParallelism = 8)) + .copy(clientId = cId) + record = ProducerRecord.tombstone(topic, Some("foo")) + message <- RecordConsumer.make(config, handler).flatMap { _ => + producer.produce(record, StringSerde, StringSerde) *> queue.take.timeoutFail(TimedOutWaitingForMessages)(10.seconds) + } + } yield { + message must (beRecordWithKey("foo") and beRecordWithValue(null)) + } + } + + s"not lose any messages on a slow consumer (drives the message dispatcher to throttling)${parallelConsumerString(useParallelConsumer)}" in + ZIO.scoped { + for { + r <- getShared + TestResources(kafka, producer) = r + _ <- ZIO.debug(">>>> starting test: throttlingTest") + + topic <- kafka.createRandomTopic(partitions = 2, prefix = "core-not-lose") + group <- randomGroup + + messagesPerPartition = 500 // Exceeds the queue capacity + delayPartition1 <- Promise.make[Nothing, Unit] + handledPartition0 <- CountDownLatch.make(messagesPerPartition) + handledPartition1 <- CountDownLatch.make(messagesPerPartition) + handler = RecordHandler { record: ConsumerRecord[Chunk[Byte], Chunk[Byte]] => + record.partition match { + case 0 => handledPartition0.countDown + case 1 => delayPartition1.await *> handledPartition1.countDown + } } - } - - test <- RecordConsumer.make(configFor(kafka, group, topic), handler).flatMap { _ => - val recordPartition0 = ProducerRecord(topic, Chunk.empty, partition = Some(0)) - val recordPartition1 = ProducerRecord(topic, Chunk.empty, partition = Some(1)) - for { - _ <- ZIO.foreachParDiscard(0 until messagesPerPartition) { _ => - producer.produce(recordPartition0) zipPar producer.produce(recordPartition1) - } - handledAllFromPartition0 <- handledPartition0.await.timeout(10.seconds) - _ <- delayPartition1.succeed(()) - handledAllFromPartition1 <- handledPartition1.await.timeout(10.seconds) - } yield { - (handledAllFromPartition0 must beSome) and (handledAllFromPartition1 must beSome) - } + + test <- + RecordConsumer + .make( + configFor( + kafka, + group, + topic, + mutateEventLoop = _.copy(consumePartitionInParallel = useParallelConsumer, maxParallelism = 8) + ), + handler + ) + .flatMap { _ => + val recordPartition0 = ProducerRecord(topic, Chunk.empty, partition = Some(0)) + val recordPartition1 = ProducerRecord(topic, Chunk.empty, partition = Some(1)) + for { + _ <- ZIO.foreachParDiscard(0 until messagesPerPartition) { _ => + producer.produce(recordPartition0) zipPar producer.produce(recordPartition1) + } + handledAllFromPartition0 <- handledPartition0.await.timeout(10.seconds) + _ <- delayPartition1.succeed(()) + handledAllFromPartition1 <- handledPartition1.await.timeout(10.seconds) + } yield { + (handledAllFromPartition0 must beSome) and (handledAllFromPartition1 must beSome) } - } yield test - } + } + } yield test + } - "delay resuming a paused partition" in - ZIO.scoped { - for { - r <- getShared - TestResources(kafka, producer) = r - _ <- ZIO.debug(">>>> starting test: delay resuming a paused partition") - - topic <- kafka.createRandomTopic(partitions = 1, prefix = "core-not-lose") - group <- randomGroup - - messagesPerPartition = 500 // Exceeds the queue capacity - delayPartition <- Promise.make[Nothing, Unit] - handledPartition <- CountDownLatch.make(messagesPerPartition) - handler = RecordHandler { _: ConsumerRecord[Chunk[Byte], Chunk[Byte]] => delayPartition.await *> handledPartition.countDown } - start <- Clock.currentTime(TimeUnit.MILLISECONDS) - test <- RecordConsumer - .make(configFor(kafka, group, topic, mutateEventLoop = _.copy(delayResumeOfPausedPartition = 3000)), handler) - .flatMap { _ => - val recordPartition = ProducerRecord(topic, Chunk.empty, partition = Some(0)) - for { - _ <- ZIO.foreachParDiscard(0 until messagesPerPartition) { _ => producer.produce(recordPartition) } - _ <- delayPartition.succeed(()).delay(1.seconds).fork - handledAllFromPartition <- handledPartition.await.timeout(10.seconds) - end <- Clock.currentTime(TimeUnit.MILLISECONDS) - - } yield { - (handledAllFromPartition aka "handledAllFromPartition" must beSome) and - (end - start aka "complete handling duration" must beGreaterThan(3000L)) + s"delay resuming a paused partition${parallelConsumerString(useParallelConsumer)}" in + ZIO.scoped { + for { + r <- getShared + TestResources(kafka, producer) = r + _ <- ZIO.debug(">>>> starting test: delay resuming a paused partition") + + topic <- kafka.createRandomTopic(partitions = 1, prefix = "core-not-lose") + group <- randomGroup + + messagesPerPartition = 500 // Exceeds the queue capacity + delayPartition <- Promise.make[Nothing, Unit] + handledPartition <- CountDownLatch.make(messagesPerPartition) + handler = RecordHandler { _: ConsumerRecord[Chunk[Byte], Chunk[Byte]] => delayPartition.await *> handledPartition.countDown } + start <- Clock.currentTime(TimeUnit.MILLISECONDS) + test <- RecordConsumer + .make( + configFor( + kafka, + group, + topic, + mutateEventLoop = + _.copy(delayResumeOfPausedPartition = 3000, consumePartitionInParallel = useParallelConsumer, maxParallelism = 8) + ), + handler + ) + .flatMap { _ => + val recordPartition = ProducerRecord(topic, Chunk.empty, partition = Some(0)) + for { + _ <- ZIO.foreachParDiscard(0 until messagesPerPartition) { _ => producer.produce(recordPartition) } + _ <- delayPartition.succeed(()).delay(1.seconds).fork + handledAllFromPartition <- handledPartition.await.timeout(10.seconds) + end <- Clock.currentTime(TimeUnit.MILLISECONDS) + + } yield { + (handledAllFromPartition aka "handledAllFromPartition" must beSome) and + (end - start aka "complete handling duration" must beGreaterThan(3000L)) + } } + } yield test + } + + s"pause and resume consumer${parallelConsumerString(useParallelConsumer)}" in + ZIO.scoped { + for { + r <- getShared + TestResources(kafka, producer) = r + _ <- ZIO.debug(">>>> starting test: pauseResumeTest") + + topic <- kafka.createRandomTopic(prefix = "core-pause-resume") + group <- randomGroup + + numberOfMessages = 32 + someMessages = 16 + restOfMessages = numberOfMessages - someMessages + handledSomeMessages <- CountDownLatch.make(someMessages) + handledAllMessages <- CountDownLatch.make(numberOfMessages) + handleCounter <- Ref.make[Int](0) + handler = RecordHandler { _: ConsumerRecord[Chunk[Byte], Chunk[Byte]] => + handleCounter.update(_ + 1) *> handledSomeMessages.countDown zipParRight handledAllMessages.countDown } - } yield test - } - "pause and resume consumer" in - ZIO.scoped { - for { - r <- getShared - TestResources(kafka, producer) = r - _ <- ZIO.debug(">>>> starting test: pauseResumeTest") - - topic <- kafka.createRandomTopic(prefix = "core-pause-resume") - group <- randomGroup - - numberOfMessages = 32 - someMessages = 16 - restOfMessages = numberOfMessages - someMessages - handledSomeMessages <- CountDownLatch.make(someMessages) - handledAllMessages <- CountDownLatch.make(numberOfMessages) - handleCounter <- Ref.make[Int](0) - handler = RecordHandler { _: ConsumerRecord[Chunk[Byte], Chunk[Byte]] => - handleCounter.update(_ + 1) *> handledSomeMessages.countDown zipParRight handledAllMessages.countDown - } - - test <- RecordConsumer.make(configFor(kafka, group, topic).copy(offsetReset = OffsetReset.Earliest), handler).flatMap { consumer => - val record = ProducerRecord(topic, Chunk.empty) - for { - _ <- ZIO.foreachParDiscard(0 until someMessages)(_ => producer.produce(record)) - _ <- handledSomeMessages.await - _ <- consumer.pause - _ <- ZIO.foreachParDiscard(0 until restOfMessages)(_ => producer.produce(record)) - a <- handledAllMessages.await.timeout(5.seconds) - handledAfterPause <- handleCounter.get - _ <- consumer.resume - b <- handledAllMessages.await.timeout(5.seconds) - } yield { - (handledAfterPause === someMessages) and (a must beNone) and (b must beSome) - } + test <- + RecordConsumer + .make( + configFor( + kafka, + group, + topic, + mutateEventLoop = _.copy(consumePartitionInParallel = useParallelConsumer, maxParallelism = 8) + ) + .copy(offsetReset = OffsetReset.Earliest), + handler + ) + .flatMap { consumer => + val record = ProducerRecord(topic, Chunk.empty) + for { + _ <- ZIO.foreachParDiscard(0 until someMessages)(_ => producer.produce(record)) + _ <- handledSomeMessages.await + _ <- consumer.pause + _ <- ZIO.foreachParDiscard(0 until restOfMessages)(_ => producer.produce(record)) + a <- handledAllMessages.await.timeout(5.seconds) + handledAfterPause <- handleCounter.get + _ <- consumer.resume + b <- handledAllMessages.await.timeout(5.seconds) + } yield { + (handledAfterPause === someMessages) and (a must beNone) and (b must beSome) } - } yield test - } - - "wait until queues are drained" in { - for { - r <- getShared - TestResources(kafka, producer) = r - _ <- ZIO.debug(">>>> starting test: gracefulShutdownTest") - topic <- kafka.createRandomTopic(prefix = "core-wait-until") - group <- randomGroup - - ref <- Ref.make(0) - startedHandling <- Promise.make[Nothing, Unit] - handler: Handler[Any] = RecordHandler { _: ConsumerRecord[Chunk[Byte], Chunk[Byte]] => - startedHandling.succeed(()) *> sleep(5.seconds) *> ref.update(_ + 1) - } - - _ <- ZIO.scoped(RecordConsumer.make(configFor(kafka, group, topic), handler).flatMap { _ => - producer.produce(ProducerRecord(topic, Chunk.empty)) *> startedHandling.await - }) - - handled <- ref.get - } yield { - handled must equalTo(1) - } - } + } + } yield test + } - "consumer from earliest offset" in - ZIO.scoped { + s"wait until queues are drained${parallelConsumerString(useParallelConsumer)}" in { for { r <- getShared TestResources(kafka, producer) = r - _ <- ZIO.debug(">>>> starting test: earliestTest") - topic <- kafka.createRandomTopic(prefix = "core-from-earliest") + _ <- ZIO.debug(">>>> starting test: gracefulShutdownTest") + topic <- kafka.createRandomTopic(prefix = "core-wait-until") group <- randomGroup - queue <- Queue.unbounded[ConsumerRecord[String, String]] - handler = RecordHandler(queue.offer(_: ConsumerRecord[String, String])) - .withDeserializers(StringSerde, StringSerde) - .ignore - - record = ProducerRecord(topic, "bar", Some("foo")) - _ <- producer.produce(record, StringSerde, StringSerde) - - message <- RecordConsumer - .make(configFor(kafka, group, topic).copy(offsetReset = Earliest), handler) - .flatMap { _ => queue.take } - .timeout(10.seconds) + ref <- Ref.make(0) + startedHandling <- Promise.make[Nothing, Unit] + handler: Handler[Any] = RecordHandler { _: ConsumerRecord[Chunk[Byte], Chunk[Byte]] => + startedHandling.succeed(()) *> sleep(5.seconds) *> ref.update(_ + 1) + } + + _ <- ZIO.scoped( + RecordConsumer + .make( + configFor( + kafka, + group, + topic, + mutateEventLoop = _.copy(consumePartitionInParallel = useParallelConsumer, maxParallelism = 8) + ), + handler + ) + .flatMap { _ => producer.produce(ProducerRecord(topic, Chunk.empty)) *> startedHandling.await } + ) + + handled <- ref.get } yield { - message.get must (beRecordWithKey("foo") and beRecordWithValue("bar")) + handled must equalTo(1) } } - "not lose messages while throttling after rebalance" in - ZIO.scoped { - for { - _ <- ZIO.debug(">>>> starting test: throttleWhileRebalancingTest") - r <- getShared - TestResources(kafka, producer) = r - partitions = 30 - topic <- kafka.createRandomTopic(partitions, prefix = "core-not-lose-while-throttling") - group <- randomGroup - probe <- Ref.make(Map.empty[Partition, Seq[Offset]]) - messagesPerPartition = 250 - handler = RecordHandler { record: ConsumerRecord[Chunk[Byte], Chunk[Byte]] => - sleep(10.millis) *> - probe - .getAndUpdate(curr => curr + (record.partition -> (curr.getOrElse(record.partition, Nil) :+ record.offset))) - .flatMap(map => - ZIO.when(map.getOrElse(record.partition, Nil).contains(record.offset))( - ZIO.debug(OffsetWasAlreadyProcessed(record.partition, record.offset).toString) - ) - ) - } - createConsumerTask = (i: Int) => makeConsumer(kafka, topic, group, handler, i) - test <- createConsumerTask(0).flatMap { _ => - val record = ProducerRecord(topic, Chunk.empty, partition = Some(0)) - for { - env <- ZIO.environment[Env] - _ <- ZIO.foreachParDiscard(0 until partitions) { p => - ZIO.foreachDiscard(0 until messagesPerPartition)(_ => producer.produceAsync(record.copy(partition = Some(p)))) - } - _ <- createConsumerTask(1).provideEnvironment(env.add(Scope.global)).forkScoped // rebalance - _ <- createConsumerTask(2).provideEnvironment(env.add(Scope.global)).forkScoped // rebalance // rebalance - expected = (0 until partitions).map(p => (p, 0L until messagesPerPartition)).toMap - _ <- eventuallyTimeoutFail(probe.get)(m => - m.mapValues(_.lastOption).values.toSet == Set(Option(messagesPerPartition - 1L)) && m.size == partitions - )(120.seconds) - finalResult <- probe.get - _ <- ZIO.debug(finalResult.mapValues(_.size).mkString(",")) - } yield finalResult === expected - } - } yield test - } - - "subscribe to a pattern" in - ZIO.scoped { - for { - _ <- ZIO.debug(">>>> starting test: patternTest") - topic1 = "core-subscribe-pattern1-topic" - topic2 = "core-subscribe-pattern2-topic" - r <- getShared - TestResources(kafka, producer) = r - _ <- kafka.createTopics(Seq(topic1, topic2).map(t => TopicConfig(t, 1, 1, delete)): _*) - group <- randomGroup - probe <- Ref.make(Seq.empty[(Topic, Offset)]) - handler = RecordHandler { record: ConsumerRecord[Chunk[Byte], Chunk[Byte]] => probe.update(_ :+ (record.topic, record.offset)) } - - _ <- makeConsumer(kafka, compile("core-subscribe-pattern1.*"), group, handler, 0).flatMap { consumer => - val record = ProducerRecord(topic1, Chunk.empty, key = Option(Chunk.empty)) + s"consumer from earliest offset${parallelConsumerString(useParallelConsumer)}" in + ZIO.scoped { + for { + r <- getShared + TestResources(kafka, producer) = r + _ <- ZIO.debug(">>>> starting test: earliestTest") + topic <- kafka.createRandomTopic(prefix = "core-from-earliest") + group <- randomGroup + + queue <- Queue.unbounded[ConsumerRecord[String, String]] + handler = RecordHandler(queue.offer(_: ConsumerRecord[String, String])) + .withDeserializers(StringSerde, StringSerde) + .ignore + + record = ProducerRecord(topic, "bar", Some("foo")) + _ <- producer.produce(record, StringSerde, StringSerde) + + message <- RecordConsumer + .make( + configFor( + kafka, + group, + topic, + mutateEventLoop = _.copy(consumePartitionInParallel = useParallelConsumer, maxParallelism = 8) + ) + .copy(offsetReset = Earliest), + handler + ) + .flatMap { _ => queue.take } + .timeout(10.seconds) + } yield { + message.get must (beRecordWithKey("foo") and beRecordWithValue("bar")) + } + } - producer.produce(record) *> eventuallyZ(probe.get)(_ == (topic1, 0) :: Nil) *> - consumer.resubscribe(TopicPattern(compile("core-subscribe-pattern2.*"))) *> - producer.produce(record.copy(topic = topic2)) *> eventuallyZ(probe.get)(_ == (topic1, 0) :: (topic2, 0) :: Nil) - } - } yield ok - } + s"not lose messages while throttling after rebalance${parallelConsumerString(useParallelConsumer)}" in + ZIO.scoped { + for { + _ <- ZIO.debug(">>>> starting test: throttleWhileRebalancingTest") + r <- getShared + TestResources(kafka, producer) = r + partitions = 30 + topic <- kafka.createRandomTopic(partitions, prefix = "core-not-lose-while-throttling") + group <- randomGroup + probe <- Ref.make(Map.empty[Partition, Seq[Offset]]) + messagesPerPartition = 250 + handler = RecordHandler { record: ConsumerRecord[Chunk[Byte], Chunk[Byte]] => + sleep(10.millis) *> + probe + .getAndUpdate(curr => curr + (record.partition -> (curr.getOrElse(record.partition, Nil) :+ record.offset))) + .flatMap(map => + ZIO.when(map.getOrElse(record.partition, Nil).contains(record.offset))( + ZIO.debug(OffsetWasAlreadyProcessed(record.partition, record.offset).toString) + ) + ) + } + createConsumerTask = + (i: Int) => + makeConsumer( + kafka, + topic, + group, + handler, + i, + mutateEventLoop = _.copy(consumePartitionInParallel = useParallelConsumer, maxParallelism = 8) + ) + test <- createConsumerTask(0).flatMap { _ => + val record = ProducerRecord(topic, Chunk.empty, partition = Some(0)) + for { + env <- ZIO.environment[Env] + _ <- ZIO.foreachParDiscard(0 until partitions) { p => + ZIO.foreachDiscard(0 until messagesPerPartition)(_ => producer.produceAsync(record.copy(partition = Some(p)))) + } + _ <- createConsumerTask(1).provideEnvironment(env.add(Scope.global)).forkScoped // rebalance + _ <- createConsumerTask(2).provideEnvironment(env.add(Scope.global)).forkScoped // rebalance // rebalance + expected = (0 until partitions).map(p => (p, 0L until messagesPerPartition)).toMap + _ <- eventuallyTimeoutFail(probe.get)(m => + m.mapValues(_.max.toOption).values.toSet == Set(Option(messagesPerPartition - 1L)) && m.size == partitions + )(120.seconds) + finalResult <- probe.get + _ <- ZIO.debug(finalResult.mapValues(_.size).mkString(",")) + } yield finalResult.mapValues(_.sorted) === expected + } + } yield test + } - "consumer from a new partition is interrupted before commit (offsetReset = Latest)" in - ZIO.scoped { - for { - r <- getShared - TestResources(kafka, producer) = r - topic <- kafka.createRandomTopic(1) - group <- randomGroup - handlingStarted <- Promise.make[Nothing, Unit] - hangForever <- Promise.make[Nothing, Unit] - hangingHandler = RecordHandler { record: ConsumerRecord[Chunk[Byte], Chunk[Byte]] => - handlingStarted.complete(ZIO.unit) *> hangForever.await - } - record <- aProducerRecord(topic) - recordValue = record.value.get - _ <- makeConsumer(kafka, topic, group, hangingHandler, 0, _.copy(drainTimeout = 200.millis)) - .flatMap { consumer => - producer.produce(record, StringSerde, StringSerde) *> handlingStarted.await *> - // unsubscribe to make sure partitions are released - consumer.resubscribe(ConsumerSubscription.topics()) + s"consumer from a new partition is interrupted before commit (offsetReset = Latest)${parallelConsumerString(useParallelConsumer)}" in + ZIO.scoped { + for { + r <- getShared + TestResources(kafka, producer) = r + topic <- kafka.createRandomTopic(1) + group <- randomGroup + handlingStarted <- Promise.make[Nothing, Unit] + hangForever <- Promise.make[Nothing, Unit] + hangingHandler = RecordHandler { record: ConsumerRecord[Chunk[Byte], Chunk[Byte]] => + handlingStarted.complete(ZIO.unit) *> hangForever.await } - .disconnect - .timeoutFail(new TimeoutException("timed out waiting for consumer 0"))(10.seconds) - consumed <- AwaitableRef.make(Seq.empty[String]) - handler = RecordHandler { record: ConsumerRecord[String, String] => consumed.update(_ :+ record.value) } - .withDeserializers(StringSerde, StringSerde) - - consumedValues <- makeConsumer(kafka, topic, group, handler, 1, modifyConfig = _.copy(offsetReset = Latest)) - .flatMap { _ => consumed.await(_.nonEmpty, 5.seconds) } - .disconnect - .timeoutFail(new TimeoutException("timed out waiting for consumer 1"))(10.seconds) - } yield { - consumedValues must contain(recordValue) + record <- aProducerRecord(topic) + recordValue = record.value.get + _ <- + makeConsumer( + kafka, + topic, + group, + hangingHandler, + 0, + mutateEventLoop = _.copy(drainTimeout = 200.millis, consumePartitionInParallel = useParallelConsumer, maxParallelism = 8) + ) + .flatMap { consumer => + producer.produce(record, StringSerde, StringSerde) *> handlingStarted.await *> + // unsubscribe to make sure partitions are released + consumer.resubscribe(ConsumerSubscription.topics()) + } + .disconnect + .timeoutFail(new TimeoutException("timed out waiting for consumer 0"))(10.seconds) + consumed <- AwaitableRef.make(Seq.empty[String]) + handler = RecordHandler { record: ConsumerRecord[String, String] => consumed.update(_ :+ record.value) } + .withDeserializers(StringSerde, StringSerde) + + consumedValues <- makeConsumer( + kafka, + topic, + group, + handler, + 1, + mutateEventLoop = _.copy(consumePartitionInParallel = useParallelConsumer, maxParallelism = 8), + modifyConfig = _.copy(offsetReset = Latest) + ) + .flatMap { _ => consumed.await(_.nonEmpty, 5.seconds) } + .disconnect + .timeoutFail(new TimeoutException("timed out waiting for consumer 1"))(10.seconds) + } yield { + consumedValues must contain(recordValue) + } } - } - "block until current tasks complete" in - ZIO.scoped { - for { - r <- getShared - TestResources(kafka, producer) = r - topic <- kafka.createRandomTopic(prefix = "block-until") - group <- randomGroup + s"block until current tasks complete${parallelConsumerString(useParallelConsumer)}" in + ZIO.scoped { + for { + r <- getShared + TestResources(kafka, producer) = r + topic <- kafka.createRandomTopic(prefix = "block-until") + group <- randomGroup + + waitForTasksDuration <- TRef.make[Duration](0.millis).commit + innerGate <- Gate.make(initiallyAllow = false) + outerGate <- Gate.make(initiallyAllow = false) + handler = RecordHandler((_: ConsumerRecord[String, String]) => outerGate.toggle(true) *> innerGate.await()) + .withDeserializers(StringSerde, StringSerde) + .ignore + config = + configFor(kafka, group, topic, mutateEventLoop = _.copy(consumePartitionInParallel = useParallelConsumer, maxParallelism = 8)) + record = ProducerRecord(topic, "bar", Some("foo")) + + test <- RecordConsumer.make(config, handler).flatMap { consumer => + for { + _ <- ZIO.foreach((0 until 100).toSet)(_ => producer.produce(record, StringSerde, StringSerde)) *> + outerGate.await() /* handler waiting on innerGate now */ *> + consumer.waitForCurrentRecordsCompletion.timed + .map(_._1) /* after 'delay' the consumer's innerGate opens and handler completes */ + .flatMap(d => waitForTasksDuration.set(d).commit) + .fork + delay = 1.second + tasksDurationFiber <- waitForTasksDuration.get.tap(d => STM.check(d > 0.millis)).commit.fork + _ <- innerGate.toggle(true).delay(delay) /*and now handler will complete */ + tasksDuration <- tasksDurationFiber.join + } yield tasksDuration must between(delay * 0.9, delay * 3) + } + } yield test + } - waitForTasksDuration <- TRef.make[Duration](0.millis).commit - innerGate <- Gate.make(initiallyAllow = false) - outerGate <- Gate.make(initiallyAllow = false) - handler = RecordHandler((_: ConsumerRecord[String, String]) => outerGate.toggle(true) *> innerGate.await()) - .withDeserializers(StringSerde, StringSerde) - .ignore - config = configFor(kafka, group, topic) - record = ProducerRecord(topic, "bar", Some("foo")) - - test <- RecordConsumer.make(config, handler).flatMap { consumer => - for { - _ <- ZIO.foreach((0 until 100).toSet)(_ => producer.produce(record, StringSerde, StringSerde)) *> - outerGate.await() /* handler waiting on innerGate now */ *> - consumer.waitForCurrentRecordsCompletion.timed - .map(_._1) /* after 'delay' the consumer's innerGate opens and handler completes */ - .flatMap(d => waitForTasksDuration.set(d).commit) - .fork - delay = 1.second - tasksDurationFiber <- waitForTasksDuration.get.tap(d => STM.check(d > 0.millis)).commit.fork - _ <- innerGate.toggle(true).delay(delay) /*and now handler will complete */ - tasksDuration <- tasksDurationFiber.join - } yield tasksDuration must between(delay * 0.9, delay * 3) - } - } yield test - } + s"rewind positions on poll failure${parallelConsumerString(useParallelConsumer)}" in + ZIO.scoped { + type BinaryRecord = ConsumerRecord[Chunk[Byte], Chunk[Byte]] + type BinaryDecryptor = Decryptor[Any, RuntimeException, Chunk[Byte], Chunk[Byte]] + for { + r <- getShared + TestResources(kafka, producer) = r + topic <- kafka.createRandomTopic(prefix = "poll-fail") + group <- randomGroup + failToDecrypt <- Ref.make(false) + messages <- AwaitableRef.make[Seq[ConsumerRecord[String, String]]](Nil) + handler = RecordHandler((cr: ConsumerRecord[String, String]) => ZIO.debug(s"***** Consumed: $cr") *> messages.update(_ :+ cr)) + .withDeserializers(StringSerde, StringSerde) + .ignore + cId <- clientId + decryptor: BinaryDecryptor = new BinaryDecryptor { + override def decrypt(record: ConsumerRecord[Chunk[Byte], Chunk[Byte]])( + implicit trace: Trace + ): ZIO[Any, RuntimeException, ConsumerRecord[Chunk[Byte], Chunk[Byte]]] = + ZIO.whenZIO(failToDecrypt.get)(ZIO.fail(new RuntimeException)).map(_ => record) + } - "rewind positions on poll failure" in - ZIO.scoped { - type BinaryRecord = ConsumerRecord[Chunk[Byte], Chunk[Byte]] - type BinaryDecryptor = Decryptor[Any, RuntimeException, Chunk[Byte], Chunk[Byte]] - for { - r <- getShared - TestResources(kafka, producer) = r - topic <- kafka.createRandomTopic(prefix = "poll-fail") - group <- randomGroup - failToDecrypt <- Ref.make(false) - messages <- AwaitableRef.make[Seq[ConsumerRecord[String, String]]](Nil) - handler = RecordHandler((cr: ConsumerRecord[String, String]) => ZIO.debug(s"***** Consumed: $cr") *> messages.update(_ :+ cr)) - .withDeserializers(StringSerde, StringSerde) - .ignore - cId <- clientId - decryptor: BinaryDecryptor = new BinaryDecryptor { - override def decrypt(record: ConsumerRecord[Chunk[Byte], Chunk[Byte]])( - implicit trace: Trace - ): ZIO[Any, RuntimeException, ConsumerRecord[Chunk[Byte], Chunk[Byte]]] = - ZIO.whenZIO(failToDecrypt.get)(ZIO.fail(new RuntimeException)).map(_ => record) - } - - pollFailedMetrics <- TestMetrics.queue - - config = configFor(kafka, group, topic).copy(clientId = cId, decryptor = decryptor) - aRecord = (i: Int) => ProducerRecord(topic, s"payload-$i", Some(s"key-$i")) - _ <- RecordConsumer.make(config, handler).flatMap { consumer => - val Seq(rec1, rec2) = (1 to 2) map aRecord - consumer.resubscribe(ConsumerSubscription.topics(topic)) *> producer.produce(rec1, StringSerde, StringSerde) *> - messages.await(_.exists(_.value == rec1.value.get)) *> failToDecrypt.set(true) *> - producer.produce(rec2, StringSerde, StringSerde) *> next[PollingFailed](pollFailedMetrics) *> failToDecrypt.set(false) *> - messages.await(_.exists(_.value == rec2.value.get), 5.seconds) - } - } yield ok - } + pollFailedMetrics <- TestMetrics.queue + + config = + configFor(kafka, group, topic, mutateEventLoop = _.copy(consumePartitionInParallel = useParallelConsumer, maxParallelism = 8)) + .copy(clientId = cId, decryptor = decryptor) + aRecord = (i: Int) => ProducerRecord(topic, s"payload-$i", Some(s"key-$i")) + _ <- RecordConsumer.make(config, handler).flatMap { consumer => + val Seq(rec1, rec2) = (1 to 2) map aRecord + consumer.resubscribe(ConsumerSubscription.topics(topic)) *> producer.produce(rec1, StringSerde, StringSerde) *> + messages.await(_.exists(_.value == rec1.value.get)) *> failToDecrypt.set(true) *> + producer.produce(rec2, StringSerde, StringSerde) *> next[PollingFailed](pollFailedMetrics) *> failToDecrypt.set(false) *> + messages.await(_.exists(_.value == rec2.value.get), 5.seconds) + } + } yield ok + } + } def next[A <: GreyhoundMetric](queue: Queue[GreyhoundMetric]) = queue.take.repeatUntil(_.isInstanceOf[A]) @@ -475,9 +588,13 @@ class ConsumerIT extends BaseTestWithSharedEnv[Env, TestResources] { pattern: Pattern, group: String, handler: RecordHandler[Any, Nothing, Chunk[Byte], Chunk[Byte]], - i: Int + i: Int, + useParallelConsumer: Boolean ) = - RecordConsumer.make(configFor(kafka, group, pattern).copy(clientId = s"client-$i", offsetReset = OffsetReset.Earliest), handler) + RecordConsumer.make( + configFor(kafka, group, pattern, useParallelConsumer).copy(clientId = s"client-$i", offsetReset = OffsetReset.Earliest), + handler + ) private def configFor(kafka: ManagedKafka, group: Group, topic: Topic, mutateEventLoop: EventLoopConfig => EventLoopConfig = identity) = RecordConsumerConfig( @@ -489,8 +606,19 @@ class ConsumerIT extends BaseTestWithSharedEnv[Env, TestResources] { eventLoopConfig = mutateEventLoop(EventLoopConfig.Default) ) - private def configFor(kafka: ManagedKafka, group: Group, pattern: Pattern) = - RecordConsumerConfig(kafka.bootstrapServers, group, TopicPattern(pattern), extraProperties = fastConsumerMetadataFetching) + private def configFor( + kafka: ManagedKafka, + group: Group, + pattern: Pattern, + useParallelConsumer: Boolean + ) = + RecordConsumerConfig( + kafka.bootstrapServers, + group, + TopicPattern(pattern), + extraProperties = fastConsumerMetadataFetching, + eventLoopConfig = EventLoopConfig.Default.copy(consumePartitionInParallel = useParallelConsumer, maxParallelism = 8) + ) private def fastConsumerMetadataFetching = Map("metadata.max.age.ms" -> "0") @@ -500,6 +628,9 @@ class ConsumerIT extends BaseTestWithSharedEnv[Env, TestResources] { payload <- randomId } yield ProducerRecord(topic, payload, Some(key)) + private def parallelConsumerString(isParallel: Boolean) = + if (isParallel) " - using parallel consumer" else "" + } object TimedOutWaitingForMessages extends RuntimeException diff --git a/core/src/it/scala/com/wixpress/dst/greyhound/core/offset/BUILD.bazel b/core/src/it/scala/com/wixpress/dst/greyhound/core/offset/BUILD.bazel index ce9f7faf..a5c08ae5 100644 --- a/core/src/it/scala/com/wixpress/dst/greyhound/core/offset/BUILD.bazel +++ b/core/src/it/scala/com/wixpress/dst/greyhound/core/offset/BUILD.bazel @@ -13,7 +13,6 @@ specs2_ite2e_test( "@dev_zio_izumi_reflect_2_12", "@dev_zio_zio_managed_2_12", "//core/src/it/resources", - "//core/src/it/scala/com/wixpress/dst/greyhound/core", "//core/src/it/scala/com/wixpress/dst/greyhound/testenv", "//core/src/it/scala/com/wixpress/dst/greyhound/testkit", "//core/src/main/scala/com/wixpress/dst/greyhound/core", @@ -23,9 +22,7 @@ specs2_ite2e_test( "//core/src/main/scala/com/wixpress/dst/greyhound/core/metrics", "//core/src/main/scala/com/wixpress/dst/greyhound/core/producer", "//core/src/main/scala/com/wixpress/dst/greyhound/core/zioutils", - "//core/src/test/scala/com/wixpress/dst/greyhound/core/consumer", "//core/src/test/scala/com/wixpress/dst/greyhound/core/testkit", - "@ch_qos_logback_logback_classic", # "@dev_zio_izumi_reflect_2_12", "@dev_zio_zio_2_12", "@org_apache_kafka_kafka_clients", diff --git a/core/src/it/scala/com/wixpress/dst/greyhound/core/parallel/BUILD.bazel b/core/src/it/scala/com/wixpress/dst/greyhound/core/parallel/BUILD.bazel new file mode 100644 index 00000000..f0c1bbb8 --- /dev/null +++ b/core/src/it/scala/com/wixpress/dst/greyhound/core/parallel/BUILD.bazel @@ -0,0 +1,27 @@ +package(default_visibility = ["//visibility:public"]) + +sources() + +specs2_ite2e_test( + name = "parallel", + srcs = [ + ":sources", + ], + deps = [ + "//core/src/it/resources", + "//core/src/it/scala/com/wixpress/dst/greyhound/testenv", + "//core/src/it/scala/com/wixpress/dst/greyhound/testkit", + "//core/src/main/scala/com/wixpress/dst/greyhound/core", + "//core/src/main/scala/com/wixpress/dst/greyhound/core/consumer", + "//core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/domain", + "//core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry", + "//core/src/main/scala/com/wixpress/dst/greyhound/core/metrics", + "//core/src/main/scala/com/wixpress/dst/greyhound/core/producer", + "//core/src/main/scala/com/wixpress/dst/greyhound/core/zioutils", + "//core/src/test/scala/com/wixpress/dst/greyhound/core/testkit", + "@dev_zio_izumi_reflect_2_12", + "@dev_zio_zio_2_12", + "@dev_zio_zio_managed_2_12", + "@org_apache_kafka_kafka_clients", + ], +) diff --git a/core/src/it/scala/com/wixpress/dst/greyhound/core/parallel/ParallelConsumerIT.scala b/core/src/it/scala/com/wixpress/dst/greyhound/core/parallel/ParallelConsumerIT.scala new file mode 100644 index 00000000..6ef6de44 --- /dev/null +++ b/core/src/it/scala/com/wixpress/dst/greyhound/core/parallel/ParallelConsumerIT.scala @@ -0,0 +1,326 @@ +package com.wixpress.dst.greyhound.core.parallel +import com.wixpress.dst.greyhound.core.Serdes.StringSerde +import com.wixpress.dst.greyhound.core.consumer.ConsumerMetric.SkippedGapsOnInitialization +import com.wixpress.dst.greyhound.core.consumer.domain.ConsumerSubscription.Topics +import com.wixpress.dst.greyhound.core.consumer.domain.{ConsumerRecord, ConsumerSubscription, RecordHandler} +import com.wixpress.dst.greyhound.core.consumer.{EventLoopConfig, RebalanceListener, RecordConsumer, RecordConsumerConfig} +import com.wixpress.dst.greyhound.core.producer.{ProducerRecord, ReportingProducer} +import com.wixpress.dst.greyhound.core.testkit.RecordMatchers.{beRecordWithKey, beRecordWithValue, beRecordsWithKeysAndValues} +import com.wixpress.dst.greyhound.core.testkit.{eventuallyZ, BaseTestWithSharedEnv, TestMetrics} +import com.wixpress.dst.greyhound.core.zioutils.CountDownLatch +import com.wixpress.dst.greyhound.core.{Group, Topic, TopicPartition} +import com.wixpress.dst.greyhound.testenv.ITEnv +import com.wixpress.dst.greyhound.testenv.ITEnv.{clientId, partitions, randomGroup, randomId, Env, ManagedKafkaOps, TestResources} +import com.wixpress.dst.greyhound.testkit.ManagedKafka +import zio.Clock.sleep +import zio.{Queue, ZIO, _} + +class ParallelConsumerIT extends BaseTestWithSharedEnv[Env, TestResources] { + sequential + + override def env = ITEnv.ManagedEnv + + override def sharedEnv = ITEnv.testResources() + + "consume messages correctly after rebalance" in { + ZIO.scoped { + for { + r <- getShared + TestResources(kafka, producer) = r + topic1 <- kafka.createRandomTopic(prefix = "topic1") + topic2 <- kafka.createRandomTopic(prefix = "topic2") + group <- randomGroup + + queue <- Queue.unbounded[ConsumerRecord[String, String]] + handler = RecordHandler((cr: ConsumerRecord[String, String]) => queue.offer(cr)).withDeserializers(StringSerde, StringSerde) + cId <- clientId + config = parallelConsumerConfig(kafka, topic1, group, cId) + records1 = producerRecords(topic1, "1", partitions, 10) + records2 = producerRecords(topic1, "2", partitions, 10) + numRecordsExpected = records1.size + records2.size + messagesOption <- for { + consumer <- RecordConsumer.make(config, handler) + _ <- produceRecords(producer, records1) + _ <- sleep(5.seconds) + _ <- consumer.resubscribe(ConsumerSubscription.topics(topic1, topic2)) // trigger rebalance + _ <- sleep(500.millis) + _ <- produceRecords(producer, records2) + maybeMessages <- queue + .takeBetween(numRecordsExpected, numRecordsExpected) + .timeout(60.seconds) + .tap(o => ZIO.when(o.isEmpty)(Console.printLine("timeout waiting for messages"))) + } yield maybeMessages + messages <- ZIO.fromOption(messagesOption).orElseFail(TimedOutWaitingForMessages) + } yield { + messages must beRecordsWithKeysAndValues(records1 ++ records2) + } + } + } + + "consume messages exactly once when processing following multiple consecutive polls" in { + ZIO.scoped { + for { + r <- getShared + TestResources(kafka, producer) = r + topic <- kafka.createRandomTopic() + group <- randomGroup + queue <- Queue.unbounded[ConsumerRecord[String, String]] + handlerWithSleep = + RecordHandler((cr: ConsumerRecord[String, String]) => { + (if (cr.partition == cr.offset) ZIO.sleep(2.seconds) // sleep to simulate long processing time and go through multiple polls + else ZIO.unit) *> queue.offer(cr) + }) + .withDeserializers(StringSerde, StringSerde) + cId <- clientId + config = parallelConsumerConfig(kafka, topic, group, cId) + records = producerRecords(topic, "1", partitions, 5) + messagesOption <- RecordConsumer.make(config, handlerWithSleep).flatMap { consumer => + produceRecords(producer, records) *> ZIO.sleep(3.seconds) *> + queue + .takeBetween(records.size, records.size) + .timeout(60.seconds) + .tap(o => ZIO.when(o.isEmpty)(Console.printLine("timeout waiting for messages!"))) + } + messages <- ZIO.fromOption(messagesOption).orElseFail(TimedOutWaitingForMessages) + } yield { + messages must + allOf( + records.map(r => beRecordWithKey(r.key.get) and beRecordWithValue(r.value.get)): _* + ) + } + } + } + + "consume gaps after rebalance and skip already-consumed records" in { + ZIO.scoped { + for { + r <- getShared + TestResources(kafka, producer) = r + topic <- kafka.createRandomTopic(partitions = 1) + group <- randomGroup + cId <- clientId + partition = 0 + allMessages = 10 + fastMessages = allMessages - 1 + drainTimeout = 5.seconds + + keyWithSlowHandling = "slow-key" + numProcessedMessges <- Ref.make[Int](0) + fastMessagesLatch <- CountDownLatch.make(fastMessages) + + randomKeys <- ZIO.foreach(1 to fastMessages)(i => randomKey(i.toString)).map(_.toSeq) + + fastRecords = randomKeys.map { key => recordWithKey(topic, key, partition) } + slowRecord = recordWithKey(topic, keyWithSlowHandling, partition) + + finishRebalance <- Promise.make[Nothing, Unit] + + // handler that sleeps only on the slow key + handler = RecordHandler { cr: ConsumerRecord[Chunk[Byte], Chunk[Byte]] => + (cr.key match { + case Some(k) if k == Chunk.fromArray(keyWithSlowHandling.getBytes) => + // make sure the handler doesn't finish before the rebalance is done, including drain timeout + finishRebalance.await *> ZIO.sleep(drainTimeout + 1.second) + case _ => fastMessagesLatch.countDown + }) *> numProcessedMessges.update(_ + 1) + } + _ <- + for { + consumer <- makeParallelConsumer(handler, kafka, topic, group, cId, drainTimeout = drainTimeout, startPaused = true) + _ <- produceRecords(producer, Seq(slowRecord)) + _ <- produceRecords(producer, fastRecords) + _ <- ZIO.sleep(2.seconds) + // produce is done synchronously to make sure all records are produced before consumer starts, so all records are polled at once + _ <- consumer.resume + _ <- fastMessagesLatch.await + _ <- ZIO.sleep(3.second) // sleep to ensure commit is done before rebalance + // start another consumer to trigger a rebalance before slow handler is done + _ <- makeParallelConsumer( + handler, + kafka, + topic, + group, + cId, + drainTimeout = drainTimeout, + onAssigned = _ => finishRebalance.succeed() + ) + } yield () + + _ <- eventuallyZ(numProcessedMessges.get, 25.seconds)(_ == allMessages) + } yield { + ok + } + } + } + +// "migrate correctly from regular record consumer to parallel consumer - consume every record once" in { +// ZIO.scoped { +// for { +// r <- getShared +// TestResources(kafka, producer) = r +// topic <- kafka.createRandomTopic() +// group <- randomGroup +// cId <- clientId +// +// regularConfig = configFor(kafka, group, Set(topic)) +// parallelConfig = parallelConsumerConfig(kafka, topic, group, cId) // same group name for both consumers +// queue <- Queue.unbounded[ConsumerRecord[String, String]] +// handler = RecordHandler((cr: ConsumerRecord[String, String]) => queue.offer(cr)).withDeserializers(StringSerde, StringSerde) +// +// records1 = producerRecords(topic, "1", partitions, 3) +// records2 = producerRecords(topic, "2", partitions, 3) +// _ <- ZIO.debug(s"records1:\n${records1.mkString("\n")}\nrecords2:\n${records2.mkString("\n")}") +// numMessages = records1.size + records2.size +// +// _ <- RecordConsumer.make(regularConfig, handler) +// _ <- produceRecords(producer, records1) +// _ <- ZIO.sleep(3.seconds) +// _ <- RecordConsumer.make(parallelConfig, handler).delay(3.seconds) +// _ <- produceRecords(producer, records2) +// _ <- ZIO.sleep(3.seconds) +// messagesOption <- RecordConsumer.make(parallelConfig, handler).flatMap { _ => +// produceRecords(producer, records2) *> ZIO.sleep(3.seconds) *> +// queue +// .takeBetween(numMessages, numMessages) +// .timeout(60.seconds) +// .tap(o => ZIO.when(o.isEmpty)(Console.printLine("timeout waiting for messages!"))) +// } +// messages <- ZIO.fromOption(messagesOption).orElseFail(TimedOutWaitingForMessages) +// } yield { +// messages must beRecordsWithKeysAndValues(records1 ++ records2) +// } +// } +// } + + "migrate from parallel consumer with gaps to regular consumer - consume from latest and report non-consumed gaps" in { + ZIO.scoped { + for { + r <- getShared + TestResources(kafka, producer) = r + topic <- kafka.createRandomTopic() + group <- randomGroup + cId <- clientId + partition = 0 + allMessages = 10 + fastMessages = allMessages - 1 + + skippedGaps <- Ref.make[Int](0) + metricsQueue <- TestMetrics.queue + + regularConfig = configFor(kafka, group, Set(topic)) + _ <- metricsQueue.take + .flatMap { + case _: SkippedGapsOnInitialization => skippedGaps.update(_ + 1) + case _ => ZIO.unit + } + .repeat(Schedule.forever) + .fork + + keyWithSlowHandling = "slow-key" + numProcessedMessages <- Ref.make[Int](0) + fastMessagesLatch <- CountDownLatch.make(fastMessages) + + randomKeys <- ZIO.foreach(1 to fastMessages)(i => randomKey(i.toString)).map(_.toSeq) + + fastRecords = randomKeys.map { key => recordWithKey(topic, key, partition) } + slowRecord = recordWithKey(topic, keyWithSlowHandling, partition) + additionalRecords = producerRecords(topic, "additional", 1, 5) + + finishRebalance <- Promise.make[Nothing, Unit] + + // handler that sleeps forever on the slow key + parallelConsumerHandler = RecordHandler { cr: ConsumerRecord[Chunk[Byte], Chunk[Byte]] => + (cr.key match { + case Some(k) if k == Chunk.fromArray(keyWithSlowHandling.getBytes) => + ZIO.sleep(Duration.Infinity) + case _ => fastMessagesLatch.countDown + }) *> numProcessedMessages.update(_ + 1) + } + + regularConsumerHandler = RecordHandler { _: ConsumerRecord[Chunk[Byte], Chunk[Byte]] => numProcessedMessages.update(_ + 1) } + + parallelConsumer <- makeParallelConsumer(parallelConsumerHandler, kafka, topic, group, cId, startPaused = true) + _ <- produceRecords(producer, Seq(slowRecord)) + _ <- produceRecords(producer, fastRecords) + _ <- ZIO.sleep(3.seconds) + // produce is done synchronously to make sure all records are produced before consumer starts, so all records are polled at once + _ <- parallelConsumer.resume + _ <- fastMessagesLatch.await + _ <- ZIO.sleep(2.second) // sleep to ensure commit is done before rebalance + // migrate to regular fromLatest consumer while gap exists + _ <- parallelConsumer.shutdown() *> RecordConsumer.make(regularConfig, regularConsumerHandler) + _ <- produceRecords(producer, additionalRecords) + _ <- eventuallyZ(numProcessedMessages.get, 20.seconds)(_ == fastMessages + additionalRecords.size) + _ <- eventuallyZ(skippedGaps.get, 20.seconds)(_.must(beGreaterThanOrEqualTo(1))) + } yield { + ok + } + } + } + + private def configFor( + kafka: ManagedKafka, + group: Group, + topics: Set[Topic], + mutateEventLoop: EventLoopConfig => EventLoopConfig = identity, + extraProperties: Map[String, String] = Map.empty + ) = RecordConsumerConfig( + bootstrapServers = kafka.bootstrapServers, + group = group, + initialSubscription = Topics(topics), + eventLoopConfig = mutateEventLoop(EventLoopConfig.Default), + extraProperties = extraProperties + ) + + private def makeParallelConsumer( + handler: RecordHandler[Any, Nothing, Chunk[Byte], Chunk[Byte]], + kafka: ManagedKafka, + topic: String, + group: String, + cId: String, + drainTimeout: Duration = 20.seconds, + startPaused: Boolean = false, + onAssigned: Set[TopicPartition] => UIO[Any] = _ => ZIO.unit + ) = + RecordConsumer.make(parallelConsumerConfig(kafka, topic, group, cId, drainTimeout, startPaused, onAssigned), handler) + + private def parallelConsumerConfig( + kafka: ManagedKafka, + topic: String, + group: String, + cId: String, + drainTimeout: Duration = 20.seconds, + startPaused: Boolean = false, + onAssigned: Set[TopicPartition] => UIO[Any] = _ => ZIO.unit + ) = { + configFor( + kafka, + group, + Set(topic), + mutateEventLoop = _.copy( + consumePartitionInParallel = true, + maxParallelism = 10, + drainTimeout = drainTimeout, + startPaused = startPaused, + rebalanceListener = RebalanceListener(onAssigned = onAssigned) + ) + ) + .copy(clientId = cId) + } + + private def producerRecords(topic: String, tag: String, partitions: Int, recordsPerPartition: Int) = (0 until partitions).flatMap(p => + (0 until recordsPerPartition).map(i => ProducerRecord(topic, s"value-t$tag-p$p-$i", Some(s"key-t$tag-p$p-$i"), partition = Some(p))) + ) + + def produceRecords(producer: ReportingProducer[Any], records: Seq[ProducerRecord[String, String]]) = + ZIO + .foreach(records)(r => producer.produce(r, StringSerde, StringSerde)) + + private def recordWithKey(topic: String, key: String, partition: Int) = + ProducerRecord(topic, "", Some(key), partition = Some(partition)) + + private def randomKey(prefix: String) = + randomId.map(r => s"$prefix-$r") +} + +object TimedOutWaitingForMessages extends RuntimeException diff --git a/core/src/it/scala/com/wixpress/dst/greyhound/core/rabalance/BUILD.bazel b/core/src/it/scala/com/wixpress/dst/greyhound/core/rabalance/BUILD.bazel index 3da73eba..c763308a 100644 --- a/core/src/it/scala/com/wixpress/dst/greyhound/core/rabalance/BUILD.bazel +++ b/core/src/it/scala/com/wixpress/dst/greyhound/core/rabalance/BUILD.bazel @@ -13,7 +13,6 @@ specs2_ite2e_test( "@dev_zio_izumi_reflect_2_12", "@dev_zio_zio_managed_2_12", "//core/src/it/resources", - "//core/src/it/scala/com/wixpress/dst/greyhound/core", "//core/src/it/scala/com/wixpress/dst/greyhound/testenv", "//core/src/it/scala/com/wixpress/dst/greyhound/testkit", "//core/src/main/scala/com/wixpress/dst/greyhound/core", @@ -23,9 +22,7 @@ specs2_ite2e_test( "//core/src/main/scala/com/wixpress/dst/greyhound/core/metrics", "//core/src/main/scala/com/wixpress/dst/greyhound/core/producer", "//core/src/main/scala/com/wixpress/dst/greyhound/core/zioutils", - "//core/src/test/scala/com/wixpress/dst/greyhound/core/consumer", "//core/src/test/scala/com/wixpress/dst/greyhound/core/testkit", - "@ch_qos_logback_logback_classic", # "@dev_zio_izumi_reflect_2_12", "@dev_zio_zio_2_12", "@org_apache_kafka_kafka_clients", diff --git a/core/src/it/scala/com/wixpress/dst/greyhound/core/retry/BUILD.bazel b/core/src/it/scala/com/wixpress/dst/greyhound/core/retry/BUILD.bazel index 3f83ff43..cb001169 100644 --- a/core/src/it/scala/com/wixpress/dst/greyhound/core/retry/BUILD.bazel +++ b/core/src/it/scala/com/wixpress/dst/greyhound/core/retry/BUILD.bazel @@ -12,7 +12,6 @@ specs2_ite2e_test( deps = [ "@dev_zio_izumi_reflect_2_12", "@dev_zio_zio_managed_2_12", - "//core/src/it/scala/com/wixpress/dst/greyhound/core", "//core/src/it/scala/com/wixpress/dst/greyhound/testenv", "//core/src/it/scala/com/wixpress/dst/greyhound/testkit", "//core/src/main/scala/com/wixpress/dst/greyhound/core", @@ -21,11 +20,8 @@ specs2_ite2e_test( "//core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry", "//core/src/main/scala/com/wixpress/dst/greyhound/core/metrics", "//core/src/main/scala/com/wixpress/dst/greyhound/core/producer", - "//core/src/main/scala/com/wixpress/dst/greyhound/core/zioutils", "//core/src/test/resources", - "//core/src/test/scala/com/wixpress/dst/greyhound/core/consumer", "//core/src/test/scala/com/wixpress/dst/greyhound/core/testkit", - "@ch_qos_logback_logback_classic", # "@dev_zio_izumi_reflect_2_12", "@dev_zio_zio_2_12", "@org_apache_kafka_kafka_clients", diff --git a/core/src/it/scala/com/wixpress/dst/greyhound/core/retry/RetryIT.scala b/core/src/it/scala/com/wixpress/dst/greyhound/core/retry/RetryIT.scala index ed382807..834be9f0 100644 --- a/core/src/it/scala/com/wixpress/dst/greyhound/core/retry/RetryIT.scala +++ b/core/src/it/scala/com/wixpress/dst/greyhound/core/retry/RetryIT.scala @@ -6,6 +6,7 @@ import com.wixpress.dst.greyhound.core.consumer._ import com.wixpress.dst.greyhound.core.consumer.domain.ConsumerSubscription.{TopicPattern, Topics} import com.wixpress.dst.greyhound.core.consumer.domain.{ConsumerRecord, RecordHandler} import com.wixpress.dst.greyhound.core.consumer.retry.NonBlockingRetryHelper.fixedRetryTopic +import com.wixpress.dst.greyhound.core.consumer.retry.RetryConfigForTopic.{finiteBlockingRetryConfigForTopic, nonBlockingRetryConfigForTopic} import com.wixpress.dst.greyhound.core.consumer.retry._ import com.wixpress.dst.greyhound.core.producer.{Encryptor, ProducerRecord} import com.wixpress.dst.greyhound.core.testkit.{eventuallyZ, AwaitableRef, BaseTestWithSharedEnv} @@ -40,8 +41,8 @@ class RetryIT extends BaseTestWithSharedEnv[Env, TestResources] { done <- Promise.make[Nothing, ConsumerRecord[String, String]] retryConfig = ZRetryConfig .perTopicRetries { - case `topic` => RetryConfigForTopic(() => Nil, NonBlockingBackoffPolicy(1.second :: Nil)) - case `anotherTopic` => RetryConfigForTopic(() => Nil, NonBlockingBackoffPolicy(1.second :: Nil)) + case `topic` => nonBlockingRetryConfigForTopic(1.second :: Nil) + case `anotherTopic` => nonBlockingRetryConfigForTopic(1.second :: Nil) } .copy(produceEncryptor = _ => ZIO.succeed(dummyEncryptor)) @@ -76,7 +77,7 @@ class RetryIT extends BaseTestWithSharedEnv[Env, TestResources] { retryConfig = ZRetryConfig .finiteBlockingRetry(100.millis, 100.millis) - .withCustomRetriesFor { case `topic2` => RetryConfigForTopic(() => 300.millis :: Nil, NonBlockingBackoffPolicy.empty) } + .withCustomRetriesFor { case `topic2` => finiteBlockingRetryConfigForTopic(300.millis :: Nil) } retryHandler = failingBlockingRecordHandlerWith(consumedValuesRef, Set(topic, topic2)).withDeserializers(StringSerde, StringSerde) _ <- RecordConsumer.make(configFor(kafka, group, retryConfig, topic, topic2), retryHandler).flatMap { _ => producer.produce(ProducerRecord(topic, "bar", Some("foo")), StringSerde, StringSerde) *> Clock.sleep(2.seconds) *> @@ -225,7 +226,7 @@ class RetryIT extends BaseTestWithSharedEnv[Env, TestResources] { invocations <- Ref.make(0) done <- Promise.make[Nothing, ConsumerRecord[String, String]] retryConfig = ZRetryConfig.retryForPattern( - RetryConfigForTopic(() => Nil, NonBlockingBackoffPolicy(Seq(1.second, 1.second, 1.seconds))) + nonBlockingRetryConfigForTopic(List(1.second, 1.second, 1.seconds)) ) retryHandler = failingRecordHandler(invocations, done).withDeserializers(StringSerde, StringSerde) success <- RecordConsumer @@ -258,7 +259,9 @@ class RetryIT extends BaseTestWithSharedEnv[Env, TestResources] { group <- randomGroup originalTopicCallCount <- Ref.make[Int](0) retryTopicCallCount <- Ref.make[Int](0) - retryConfig = ZRetryConfig.blockingFollowedByNonBlockingRetry(List(1.second), NonBlockingBackoffPolicy(List(1.seconds))) + retryConfig = + ZRetryConfig + .blockingFollowedByNonBlockingRetry(FiniteBlockingBackoffPolicy(List(1.second)), NonBlockingBackoffPolicy(List(1.seconds))) retryHandler = failingBlockingNonBlockingRecordHandler(originalTopicCallCount, retryTopicCallCount, topic).withDeserializers( StringSerde, StringSerde @@ -281,9 +284,7 @@ class RetryIT extends BaseTestWithSharedEnv[Env, TestResources] { for { r <- getShared TestResources(kafka, _) = r - retryConfig = ZRetryConfig.retryForPattern( - RetryConfigForTopic(() => Nil, NonBlockingBackoffPolicy(Seq(1.second, 1.second, 1.seconds))) - ) + retryConfig = ZRetryConfig.retryForPattern(nonBlockingRetryConfigForTopic(List(1.second, 1.second, 1.seconds))) handler = RecordHandler { _: ConsumerRecord[String, String] => ZIO.unit }.withDeserializers(StringSerde, StringSerde) _ <- RecordConsumer .make(configFor(kafka, "group", retryConfig, "topic"), handler) diff --git a/core/src/it/scala/com/wixpress/dst/greyhound/testenv/BUILD.bazel b/core/src/it/scala/com/wixpress/dst/greyhound/testenv/BUILD.bazel index 35ab06f9..6b5f15d9 100644 --- a/core/src/it/scala/com/wixpress/dst/greyhound/testenv/BUILD.bazel +++ b/core/src/it/scala/com/wixpress/dst/greyhound/testenv/BUILD.bazel @@ -13,15 +13,11 @@ scala_library( "@dev_zio_zio_managed_2_12", "//core/src/it/scala/com/wixpress/dst/greyhound/testkit", "//core/src/main/scala/com/wixpress/dst/greyhound/core", - "//core/src/main/scala/com/wixpress/dst/greyhound/core/admin", "//core/src/main/scala/com/wixpress/dst/greyhound/core/metrics", "//core/src/main/scala/com/wixpress/dst/greyhound/core/producer", "//core/src/test/scala/com/wixpress/dst/greyhound/core/testkit", # "@dev_zio_izumi_reflect_2_12", "@dev_zio_zio_2_12", "@dev_zio_zio_test_2_12", - "@org_apache_curator_curator_test", - "@org_apache_kafka_kafka_2_12", - "@org_apache_kafka_kafka_clients", ], ) diff --git a/core/src/it/scala/com/wixpress/dst/greyhound/testkit/BUILD.bazel b/core/src/it/scala/com/wixpress/dst/greyhound/testkit/BUILD.bazel index fb990677..f3345b95 100644 --- a/core/src/it/scala/com/wixpress/dst/greyhound/testkit/BUILD.bazel +++ b/core/src/it/scala/com/wixpress/dst/greyhound/testkit/BUILD.bazel @@ -16,7 +16,6 @@ scala_library( "//core/src/main/scala/com/wixpress/dst/greyhound/core/metrics", # "//core/src/main/scala/com/wixpress/dst/greyhound/core/producer", "@dev_zio_zio_2_12", - "@dev_zio_zio_test_2_12", "@org_apache_curator_curator_test", "@org_apache_kafka_kafka_2_12", "@org_apache_kafka_kafka_clients", diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/OffsetAndMetadata.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/OffsetAndMetadata.scala index 6b49ec3a..d03516aa 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/OffsetAndMetadata.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/OffsetAndMetadata.scala @@ -10,5 +10,8 @@ object OffsetAndMetadata { def apply(offsetAndMetadata: KafkaOffsetAndMetadata): OffsetAndMetadata = OffsetAndMetadata(offsetAndMetadata.offset(), offsetAndMetadata.metadata()) + def apply(offset: Offset): OffsetAndMetadata = + OffsetAndMetadata(offset, NO_METADATA) + val NO_METADATA = "" } diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/Consumer.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/Consumer.scala index 3a53c551..67870c29 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/Consumer.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/Consumer.scala @@ -20,29 +20,37 @@ import scala.util.{Random, Try} trait Consumer { def subscribe[R1]( - topics: Set[Topic], - rebalanceListener: RebalanceListener[R1] = RebalanceListener.Empty - )(implicit trace: Trace): RIO[GreyhoundMetrics with R1, Unit] + topics: Set[Topic], + rebalanceListener: RebalanceListener[R1] = RebalanceListener.Empty + )(implicit trace: Trace): RIO[GreyhoundMetrics with R1, Unit] def subscribePattern[R1]( - topicStartsWith: Pattern, - rebalanceListener: RebalanceListener[R1] = RebalanceListener.Empty - )(implicit trace: Trace): RIO[GreyhoundMetrics with R1, Unit] + topicStartsWith: Pattern, + rebalanceListener: RebalanceListener[R1] = RebalanceListener.Empty + )(implicit trace: Trace): RIO[GreyhoundMetrics with R1, Unit] def poll(timeout: Duration)(implicit trace: Trace): RIO[GreyhoundMetrics, Records] def commit(offsets: Map[TopicPartition, Offset])(implicit trace: Trace): RIO[GreyhoundMetrics, Unit] + def commitWithMetadata(offsetsAndMetadata: Map[TopicPartition, OffsetAndMetadata])(implicit trace: Trace): RIO[GreyhoundMetrics, Unit] + def endOffsets(partitions: Set[TopicPartition])(implicit trace: Trace): RIO[Any, Map[TopicPartition, Offset]] def beginningOffsets(partitions: Set[TopicPartition])(implicit trace: Trace): RIO[Any, Map[TopicPartition, Offset]] def committedOffsets(partitions: Set[TopicPartition])(implicit trace: Trace): RIO[Any, Map[TopicPartition, Offset]] + def committedOffsetsAndMetadata(partitions: Set[TopicPartition])(implicit trace: Trace): RIO[Any, Map[TopicPartition, OffsetAndMetadata]] + def offsetsForTimes(topicPartitionsOnTimestamp: Map[TopicPartition, Long])(implicit trace: Trace): RIO[Any, Map[TopicPartition, Offset]] def commitOnRebalance(offsets: Map[TopicPartition, Offset])(implicit trace: Trace): RIO[GreyhoundMetrics, DelayedRebalanceEffect] + def commitWithMetadataOnRebalance(offsets: Map[TopicPartition, OffsetAndMetadata])( + implicit trace: Trace + ): RIO[GreyhoundMetrics, DelayedRebalanceEffect] + def pause(partitions: Set[TopicPartition])(implicit trace: Trace): ZIO[GreyhoundMetrics, IllegalStateException, Unit] def resume(partitions: Set[TopicPartition])(implicit trace: Trace): ZIO[GreyhoundMetrics, IllegalStateException, Unit] @@ -88,16 +96,17 @@ object Consumer { // if a partition with no committed offset is revoked during processing // we also may want to seek forward to some given initial offsets offsetsInitializer <- OffsetsInitializer - .make( - cfg.clientId, - cfg.groupId, - UnsafeOffsetOperations.make(consumer), - timeout = 10.seconds, - timeoutIfSeek = 10.seconds, - initialSeek = cfg.initialSeek, - rewindUncommittedOffsetsBy = cfg.rewindUncommittedOffsetsByMillis.millis, - offsetResetIsEarliest = cfg.offsetReset == OffsetReset.Earliest - ) + .make( + cfg.clientId, + cfg.groupId, + UnsafeOffsetOperations.make(consumer), + timeout = 10.seconds, + timeoutIfSeek = 10.seconds, + initialSeek = cfg.initialSeek, + rewindUncommittedOffsetsBy = cfg.rewindUncommittedOffsetsByMillis.millis, + offsetResetIsEarliest = cfg.offsetReset == OffsetReset.Earliest, + parallelConsumer = cfg.useParallelConsumer + ) } yield { new Consumer { override def subscribePattern[R1](topicStartsWith: Pattern, rebalanceListener: RebalanceListener[R1])( @@ -144,19 +153,36 @@ object Consumer { withConsumerBlocking(_.committed(kafkaPartitions(partitions))) .map(_.asScala.collect { case (tp: KafkaTopicPartition, o: KafkaOffsetAndMetadata) => (TopicPartition(tp), o.offset) }.toMap) + override def committedOffsetsAndMetadata( + partitions: NonEmptySet[TopicPartition] + )(implicit trace: Trace): RIO[Any, Map[TopicPartition, OffsetAndMetadata]] = + withConsumerBlocking(_.committed(kafkaPartitions(partitions))) + .map(_.asScala.collect { case (tp: KafkaTopicPartition, om: KafkaOffsetAndMetadata) => (TopicPartition(tp), OffsetAndMetadata(om.offset, om.metadata))}.toMap) + override def commit(offsets: Map[TopicPartition, Offset])(implicit trace: Trace): RIO[GreyhoundMetrics, Unit] = { withConsumerBlocking(_.commitSync(kafkaOffsetsAndMetaData(toOffsetsAndMetadata(offsets, cfg.commitMetadataString)))) } + override def commitWithMetadata( + offsetsAndMetadata: Map[TopicPartition, OffsetAndMetadata] + )(implicit trace: Trace): RIO[GreyhoundMetrics, Unit] = { + withConsumerBlocking(_.commitSync(kafkaOffsetsAndMetaData(offsetsAndMetadata))) + } + override def commitOnRebalance( - offsets: Map[TopicPartition, Offset] - )(implicit trace: Trace): RIO[GreyhoundMetrics, DelayedRebalanceEffect] = { + offsets: Map[TopicPartition, Offset] + )(implicit trace: Trace): RIO[GreyhoundMetrics, DelayedRebalanceEffect] = { val kOffsets = kafkaOffsetsAndMetaData(toOffsetsAndMetadata(offsets, cfg.commitMetadataString)) // we can't actually call commit here, as it needs to be called from the same // thread, that triggered poll(), so we return the commit action as thunk ZIO.succeed(DelayedRebalanceEffect(consumer.commitSync(kOffsets))) } + override def commitWithMetadataOnRebalance( + offsets: Map[TopicPartition, OffsetAndMetadata] + )(implicit trace: Trace): RIO[GreyhoundMetrics, DelayedRebalanceEffect] = + ZIO.succeed(DelayedRebalanceEffect(consumer.commitSync(kafkaOffsetsAndMetaData(offsets)))) + override def pause(partitions: Set[TopicPartition])(implicit trace: Trace): ZIO[Any, IllegalStateException, Unit] = withConsumer(_.pause(kafkaPartitions(partitions))).refineOrDie { case e: IllegalStateException => e } @@ -203,8 +229,8 @@ object Consumer { semaphore.withPermit(f(consumer)) override def offsetsForTimes( - topicPartitionsOnTimestamp: Map[TopicPartition, Long] - )(implicit trace: Trace): RIO[Any, Map[TopicPartition, Offset]] = { + topicPartitionsOnTimestamp: Map[TopicPartition, Long] + )(implicit trace: Trace): RIO[Any, Map[TopicPartition, Offset]] = { val kafkaTopicPartitionsOnTimestamp = topicPartitionsOnTimestamp.map { case (tp, ts) => tp.asKafka -> ts } withConsumerBlocking(_.offsetsForTimes(kafkaTopicPartitionsOnTimestamp.mapValues(l => new lang.Long(l)).toMap.asJava)) .map( @@ -237,9 +263,9 @@ object Consumer { .getOrThrowFiberFailure() .run() } -// runtime -// .unsafeRun() -// .run() // this needs to be run in the same thread + // runtime + // .unsafeRun() + // .run() // this needs to be run in the same thread } override def onPartitionsAssigned(partitions: util.Collection[KafkaTopicPartition]): Unit = { @@ -260,9 +286,9 @@ object Consumer { } private def makeConsumer( - config: ConsumerConfig, - semaphore: Semaphore - )(implicit trace: Trace): RIO[GreyhoundMetrics with Scope, KafkaConsumer[Chunk[Byte], Chunk[Byte]]] = { + config: ConsumerConfig, + semaphore: Semaphore + )(implicit trace: Trace): RIO[GreyhoundMetrics with Scope, KafkaConsumer[Chunk[Byte], Chunk[Byte]]] = { val acquire = ZIO.attemptBlocking(new KafkaConsumer(config.properties, deserializer, deserializer)) def close(consumer: KafkaConsumer[_, _]) = attemptBlocking(consumer.close()) @@ -275,18 +301,19 @@ object Consumer { } case class ConsumerConfig( - bootstrapServers: String, - groupId: Group, - clientId: ClientId = s"wix-consumer-${Random.alphanumeric.take(5).mkString}", - offsetReset: OffsetReset = OffsetReset.Latest, - extraProperties: Map[String, String] = Map.empty, - additionalListener: RebalanceListener[Any] = RebalanceListener.Empty, - initialSeek: InitialOffsetsSeek = InitialOffsetsSeek.default, - consumerAttributes: Map[String, String] = Map.empty, - decryptor: Decryptor[Any, Throwable, Chunk[Byte], Chunk[Byte]] = new NoOpDecryptor, - commitMetadataString: Metadata = OffsetAndMetadata.NO_METADATA, - rewindUncommittedOffsetsByMillis: Long = 0L -) extends CommonGreyhoundConfig { + bootstrapServers: String, + groupId: Group, + clientId: ClientId = s"wix-consumer-${Random.alphanumeric.take(5).mkString}", + offsetReset: OffsetReset = OffsetReset.Latest, + extraProperties: Map[String, String] = Map.empty, + additionalListener: RebalanceListener[Any] = RebalanceListener.Empty, + initialSeek: InitialOffsetsSeek = InitialOffsetsSeek.default, + consumerAttributes: Map[String, String] = Map.empty, + decryptor: Decryptor[Any, Throwable, Chunk[Byte], Chunk[Byte]] = new NoOpDecryptor, + commitMetadataString: Metadata = OffsetAndMetadata.NO_METADATA, + rewindUncommittedOffsetsByMillis: Long = 0L, + useParallelConsumer: Boolean = false + ) extends CommonGreyhoundConfig { override def kafkaProps: Map[String, String] = Map( KafkaConsumerConfig.BOOTSTRAP_SERVERS_CONFIG -> bootstrapServers, @@ -320,12 +347,16 @@ object OffsetReset { trait UnsafeOffsetOperations { def committed(partitions: Set[TopicPartition], timeout: zio.Duration): Map[TopicPartition, Offset] + def committedWithMetadata(partitions: Set[TopicPartition], timeout: zio.Duration): Map[TopicPartition, OffsetAndMetadata] + def beginningOffsets(partitions: Set[TopicPartition], timeout: zio.Duration): Map[TopicPartition, Offset] def position(partition: TopicPartition, timeout: zio.Duration): Offset def commit(offsets: Map[TopicPartition, Offset], timeout: Duration): Unit + def commitWithMetadata(offsets: Map[TopicPartition, OffsetAndMetadata], timeout: Duration): Unit + def seek(offsets: Map[TopicPartition, Offset]): Unit def endOffsets(partitions: Set[TopicPartition], timeout: Duration): Map[TopicPartition, Offset] @@ -357,6 +388,20 @@ object UnsafeOffsetOperations { } } + override def committedWithMetadata( + partitions: NonEmptySet[TopicPartition], + timeout: zio.Duration + ): Map[TopicPartition, OffsetAndMetadata] = { + consumer + .committed(partitions.map(_.asKafka).asJava, timeout) + .asScala + .toMap + .collect { + case (tp, ofm) if ofm != null => + TopicPartition(tp) -> OffsetAndMetadata(ofm.offset(), ofm.metadata()) + } + } + override def beginningOffsets(partitions: Set[TopicPartition], timeout: Duration): Map[TopicPartition, Offset] = consumer .beginningOffsets(partitions.map(_.asKafka).asJava, timeout) @@ -374,6 +419,10 @@ object UnsafeOffsetOperations { consumer.commitSync(kafkaOffsets(offsets), timeout) } + override def commitWithMetadata(offsets: Map[TopicPartition, OffsetAndMetadata], timeout: zio.Duration): Unit = { + consumer.commitSync(kafkaOffsetsAndMetaData(offsets), timeout) + } + override def seek(offsets: Map[TopicPartition, Offset]): Unit = offsets.foreach { case (tp, offset) => Try(consumer.seek(tp.asKafka, offset)) } @@ -382,7 +431,10 @@ object UnsafeOffsetOperations { } override def offsetsForTimes(partitions: Set[TopicPartition], timeEpoch: Long, timeout: Duration): Map[TopicPartition, Option[Long]] = - consumer.offsetsForTimes(partitions.map(_.asKafka).map(tp => (tp, new lang.Long(timeEpoch))).toMap.asJava, timeout) - .asScala.toMap.map { case (tp, of) => TopicPartition(tp) -> (Option(of).map(_.offset())) } + consumer + .offsetsForTimes(partitions.map(_.asKafka).map(tp => (tp, new lang.Long(timeEpoch))).toMap.asJava, timeout) + .asScala + .toMap + .map { case (tp, of) => TopicPartition(tp) -> (Option(of).map(_.offset())) } } } diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/Dispatcher.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/Dispatcher.scala index cbe8d883..45aeb3e3 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/Dispatcher.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/Dispatcher.scala @@ -1,7 +1,7 @@ package com.wixpress.dst.greyhound.core.consumer import java.util.concurrent.TimeUnit -import com.wixpress.dst.greyhound.core.consumer.Dispatcher.Record +import com.wixpress.dst.greyhound.core.consumer.Dispatcher.{Record, Records} import com.wixpress.dst.greyhound.core.consumer.DispatcherMetric._ import com.wixpress.dst.greyhound.core.consumer.RecordConsumer.Env import com.wixpress.dst.greyhound.core.consumer.SubmitResult._ @@ -20,6 +20,8 @@ import java.lang.System.currentTimeMillis trait Dispatcher[-R] { def submit(record: Record): URIO[R with Env, SubmitResult] + def submitBatch(records: Records): URIO[R with Env, SubmitResult] + def resumeablePartitions(paused: Set[TopicPartition]): URIO[Any, Set[TopicPartition]] def revoke(partitions: Set[TopicPartition]): URIO[GreyhoundMetrics, Unit] @@ -36,7 +38,8 @@ trait Dispatcher[-R] { } object Dispatcher { - type Record = ConsumerRecord[Chunk[Byte], Chunk[Byte]] + type Record = ConsumerRecord[Chunk[Byte], Chunk[Byte]] + type Records = Seq[Record] def make[R]( group: Group, @@ -48,12 +51,19 @@ object Dispatcher { delayResumeOfPausedPartition: Long = 0, consumerAttributes: Map[String, String] = Map.empty, workersShutdownRef: Ref[Map[TopicPartition, ShutdownPromise]], - startPaused: Boolean = false + startPaused: Boolean = false, + consumeInParallel: Boolean = false, + maxParallelism: Int = 1, + updateBatch: Chunk[Record] => URIO[GreyhoundMetrics, Unit] = _ => ZIO.unit, + currentGaps: Set[TopicPartition] => ZIO[GreyhoundMetrics, Nothing, Map[TopicPartition, OffsetAndGaps]] = _ => ZIO.succeed(Map.empty), + init: Promise[Nothing, Unit] )(implicit trace: Trace): UIO[Dispatcher[R]] = for { - p <- Promise.make[Nothing, Unit] - state <- Ref.make[DispatcherState](if (startPaused) DispatcherState.Paused(p) else DispatcherState.Running) - workers <- Ref.make(Map.empty[TopicPartition, Worker]) + p <- Promise.make[Nothing, Unit] + state <- Ref.make[DispatcherState](if (startPaused) DispatcherState.Paused(p) else DispatcherState.Running) + initState <- + Ref.make[DispatcherInitState](if (consumeInParallel) DispatcherInitState.NotInitialized else DispatcherInitState.Initialized) + workers <- Ref.make(Map.empty[TopicPartition, Worker]) } yield new Dispatcher[R] { override def submit(record: Record): URIO[R with Env, SubmitResult] = for { @@ -63,6 +73,27 @@ object Dispatcher { submitted <- worker.submit(record) } yield if (submitted) Submitted else Rejected + override def submitBatch(records: Records): URIO[R with Env, SubmitResult] = + for { + _ <- report(SubmittingRecordBatch(group, clientId, records.size, records, consumerAttributes)) + currentInitState <- initState.get + _ <- currentInitState match { + case DispatcherInitState.NotInitialized => init.await *> initState.set(DispatcherInitState.Initialized) + case _ => ZIO.unit + } + allSamePartition = records.map(r => RecordTopicPartition(r)).distinct.size == 1 + submitResult <- if (allSamePartition) { + val partition = RecordTopicPartition(records.head) + for { + worker <- workerFor(partition, records.head.offset) + submitted <- worker.submitBatch(records) + } yield submitted + } else ZIO.succeed(SubmitBatchResult(success = false, Some(records.minBy(_.offset)))) + + } yield + if (allSamePartition && submitResult.success) Submitted + else RejectedBatch(submitResult.firstRejected.getOrElse(records.minBy(_.offset))) + override def resumeablePartitions(paused: Set[TopicPartition]): URIO[Any, Set[TopicPartition]] = workers.get.flatMap { workers => ZIO.foldLeft(paused)(Set.empty[TopicPartition]) { (acc, partition) => @@ -93,13 +124,10 @@ object Dispatcher { override def revoke(partitions: Set[TopicPartition]): URIO[GreyhoundMetrics, Unit] = workers .modify { workers => - partitions.foldLeft((List.empty[(TopicPartition, Worker)], workers)) { - case ((revoked, remaining), partition) => - remaining.get(partition) match { - case Some(worker) => ((partition, worker) :: revoked, remaining - partition) - case None => (revoked, remaining) - } - } + val revoked = workers.filterKeys(partitions.contains) + val remaining = workers -- partitions + + (revoked, remaining) } .flatMap(shutdownWorkers) @@ -133,7 +161,20 @@ object Dispatcher { case None => for { _ <- report(StartingWorker(group, clientId, partition, offset, consumerAttributes)) - worker <- Worker.make(state, handleWithMetrics, highWatermark, group, clientId, partition, drainTimeout, consumerAttributes) + worker <- Worker.make( + state, + handleWithMetrics, + highWatermark, + group, + clientId, + partition, + drainTimeout, + consumerAttributes, + consumeInParallel, + maxParallelism, + updateBatch, + currentGaps + ) _ <- workers.update(_ + (partition -> worker)) shutdownPromise <- AwaitShutdown.make _ <- workersShutdownRef.update(_.updated(partition, shutdownPromise)) @@ -178,11 +219,23 @@ object Dispatcher { } + sealed trait DispatcherInitState + + object DispatcherInitState { + + case object NotInitialized extends DispatcherInitState + + case object Initialized extends DispatcherInitState + + } + case class Task(record: Record, complete: UIO[Unit]) trait Worker { def submit(record: Record): URIO[Any, Boolean] + def submitBatch(records: Records): URIO[Any, SubmitBatchResult] + def expose: URIO[Any, WorkerExposedState] def shutdown: URIO[Any, Unit] @@ -201,14 +254,32 @@ object Dispatcher { clientId: ClientId, partition: TopicPartition, drainTimeout: Duration, - consumerAttributes: Map[String, String] + consumerAttributes: Map[String, String], + consumeInParallel: Boolean, + maxParallelism: Int, + updateBatch: Chunk[Record] => URIO[GreyhoundMetrics, Unit] = _ => ZIO.unit, + currentGaps: Set[TopicPartition] => ZIO[GreyhoundMetrics, Nothing, Map[TopicPartition, OffsetAndGaps]] )(implicit trace: Trace): URIO[R with Env, Worker] = for { queue <- Queue.dropping[Record](capacity) internalState <- TRef.make(WorkerInternalState.empty).commit fiber <- (reportWorkerRunningInInterval(every = 60.seconds, internalState)(partition, group, clientId).forkDaemon *> - pollOnce(status, internalState, handle, queue, group, clientId, partition, consumerAttributes) - .repeatWhile(_ == true)).forkDaemon + (if (consumeInParallel) + pollBatch( + status, + internalState, + handle, + queue, + group, + clientId, + partition, + consumerAttributes, + maxParallelism, + updateBatch, + currentGaps + ) + else pollOnce(status, internalState, handle, queue, group, clientId, partition, consumerAttributes)) + .repeatWhile(_ == true)).interruptible.forkDaemon } yield new Worker { override def submit(record: Record): URIO[Any, Boolean] = queue @@ -223,6 +294,25 @@ object Dispatcher { } ) + override def submitBatch( + records: Records + ): URIO[Any, SubmitBatchResult] = + queue + .offerAll(records) + .tap(notInserted => + ZIO.when(notInserted.nonEmpty) { + Clock + .currentTime(TimeUnit.MILLISECONDS) + .flatMap(now => + internalState.update(s => if (s.reachedHighWatermarkSince.nonEmpty) s else s.reachedHighWatermark(now)).commit + ) + } + ) + .map(rejected => { + val isSuccess = rejected.isEmpty + SubmitBatchResult(isSuccess, if (isSuccess) None else Some(rejected.minBy(_.offset))) + }) + override def expose: URIO[Any, WorkerExposedState] = (queue.size zip internalState.get.commit) .flatMap { case (queued, state) => @@ -281,6 +371,102 @@ object Dispatcher { case DispatcherState.ShuttingDown => ZIO.succeed(false) } + + private def pollBatch[R]( + state: Ref[DispatcherState], + internalState: TRef[WorkerInternalState], + handle: Record => URIO[R, Any], + queue: Queue[Record], + group: Group, + clientId: ClientId, + partition: TopicPartition, + consumerAttributes: Map[String, String], + maxParallelism: Int, + updateBatch: Chunk[Record] => URIO[GreyhoundMetrics, Unit], + currentGaps: Set[TopicPartition] => ZIO[GreyhoundMetrics, Nothing, Map[TopicPartition, OffsetAndGaps]] + )(implicit trace: Trace): ZIO[R with GreyhoundMetrics, Any, Boolean] = + internalState.update(s => s.cleared).commit *> + state.get.flatMap { + case DispatcherState.Running => + queue.takeAll.flatMap { + case records if records.nonEmpty => + handleBatch( + records, + internalState, + handle, + group, + clientId, + partition, + consumerAttributes, + maxParallelism, + updateBatch, + currentGaps + ) + case _ => isActive(internalState).delay(5.millis) + } + case DispatcherState.Paused(resume) => + report(WorkerWaitingForResume(group, clientId, partition, consumerAttributes)) *> resume.await.timeout(30.seconds) *> + isActive(internalState) + case DispatcherState.ShuttingDown => + ZIO.succeed(false) + } + private def handleBatch[R]( + records: Chunk[Record], + internalState: TRef[WorkerInternalState], + handle: Record => URIO[R, Any], + group: Group, + clientId: ClientId, + partition: TopicPartition, + consumerAttributes: Map[ClientId, ClientId], + maxParallelism: Int, + updateBatch: Chunk[Record] => URIO[GreyhoundMetrics, Unit], + currentGaps: Set[TopicPartition] => ZIO[GreyhoundMetrics, Nothing, Map[TopicPartition, OffsetAndGaps]] + ): ZIO[R with GreyhoundMetrics, Throwable, Boolean] = + for { + _ <- report(TookAllRecordsFromQueue(records.size, records, group, clientId, consumerAttributes)) + _ <- ZIO + .attempt(currentTimeMillis()) + .flatMap(t => internalState.updateAndGet(_.startedWith(t)).commit) + .tapBoth( + e => report(FailToUpdateParallelCurrentExecutionStarted(records.size, group, clientId, consumerAttributes, e)), + t => report(CurrentExecutionStartedEvent(partition, group, clientId, t.currentExecutionStarted)) + ) + groupedRecords = groupRecordsForParallelHandling(records, maxParallelism) + latestCommitGaps <- currentGaps(records.map(r => TopicPartition(r.topic, r.partition)).toSet) + _ <- report(InvokingHandlersInParallel(partition, numHandlers = Math.min(groupedRecords.size, maxParallelism))) *> + ZIO + .foreachParDiscard(groupedRecords)(sameKeyRecords => + ZIO.foreach(sameKeyRecords) { record => + if (shouldRecordBeHandled(record, latestCommitGaps)) { + handle(record).interruptible.ignore *> updateBatch(sameKeyRecords).interruptible + } else + report(SkippedPreviouslyHandledRecord(record, group, clientId, consumerAttributes)) + + } + ) + .withParallelism(maxParallelism) + res <- isActive(internalState) + } yield res + } + + private def shouldRecordBeHandled(record: Record, gaps: Map[TopicPartition, OffsetAndGaps]): Boolean = { + gaps.get(TopicPartition(record.topic, record.partition)) match { + case Some(offsetAndGapsForPartition) if offsetAndGapsForPartition.gaps.nonEmpty => + record.offset > offsetAndGapsForPartition.offset || offsetAndGapsForPartition.gaps.exists(_.contains(record.offset)) + case _ => true + } + } + + private def groupRecordsForParallelHandling(records: Chunk[Record], maxParallelism: Int): Iterable[Chunk[Record]] = { + val recordsByKey = records.groupBy(_.key) + val withKeyGroups = recordsByKey.collect { case (Some(_), records) => records } + val unusedParallelism = maxParallelism - withKeyGroups.size + val noKeyGroups = recordsByKey.get(None) match { + case Some(records) => + if (unusedParallelism > 0) records.grouped(Math.max(records.size / unusedParallelism, 1)).toIterable else Iterable(records) + case None => Chunk.empty + } + withKeyGroups ++ noKeyGroups } private def reportWorkerRunningInInterval( @@ -331,8 +517,12 @@ object SubmitResult { case object Rejected extends SubmitResult + case class RejectedBatch(firstRejected: Record) extends SubmitResult + } +case class SubmitBatchResult(success: Boolean, firstRejected: Option[Record]) extends SubmitResult + sealed trait DispatcherMetric extends GreyhoundMetric object DispatcherMetric { @@ -357,6 +547,14 @@ object DispatcherMetric { case class SubmittingRecord[K, V](group: Group, clientId: ClientId, record: ConsumerRecord[K, V], attributes: Map[String, String]) extends DispatcherMetric + case class SubmittingRecordBatch[K, V]( + group: Group, + clientId: ClientId, + numRecords: Int, + records: Records, + attributes: Map[String, String] + ) extends DispatcherMetric + case class HandlingRecord[K, V]( group: Group, clientId: ClientId, @@ -374,6 +572,13 @@ object DispatcherMetric { ) extends DispatcherMetric case class TookRecordFromQueue(record: Record, group: Group, clientId: ClientId, attributes: Map[String, String]) extends DispatcherMetric + case class TookAllRecordsFromQueue( + numRecords: Int, + records: Chunk[Record], + group: Group, + clientId: ClientId, + attributes: Map[String, String] + ) extends DispatcherMetric case class FailToUpdateCurrentExecutionStarted( record: Record, group: Group, @@ -381,6 +586,13 @@ object DispatcherMetric { attributes: Map[String, String], e: Throwable ) extends DispatcherMetric + case class FailToUpdateParallelCurrentExecutionStarted( + numRecords: Int, + group: Group, + clientId: ClientId, + attributes: Map[String, String], + e: Throwable + ) extends DispatcherMetric case class WorkerWaitingForResume(group: Group, clientId: ClientId, partition: TopicPartition, attributes: Map[String, String]) extends DispatcherMetric @@ -392,6 +604,11 @@ object DispatcherMetric { currentExecutionStarted: Option[Long] ) extends DispatcherMetric + case class InvokingHandlersInParallel(partition: TopicPartition, numHandlers: Int) extends DispatcherMetric + + case class SkippedPreviouslyHandledRecord(record: Record, group: Group, clientId: ClientId, attributes: Map[String, String]) + extends DispatcherMetric + } case class DispatcherExposedState(workersState: Map[TopicPartition, WorkerExposedState], state: Dispatcher.DispatcherState) { diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/EventLoop.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/EventLoop.scala index 6901b5c9..f91efa61 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/EventLoop.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/EventLoop.scala @@ -1,9 +1,12 @@ package com.wixpress.dst.greyhound.core.consumer import com.wixpress.dst.greyhound.core._ +import com.wixpress.dst.greyhound.core.consumer.Consumer.Records +import com.wixpress.dst.greyhound.core.consumer.Dispatcher.Record import com.wixpress.dst.greyhound.core.consumer.EventLoopMetric._ import com.wixpress.dst.greyhound.core.consumer.EventLoopState.{Paused, Running, ShuttingDown} import com.wixpress.dst.greyhound.core.consumer.RecordConsumer.Env +import com.wixpress.dst.greyhound.core.consumer.SubmitResult.RejectedBatch import com.wixpress.dst.greyhound.core.consumer.domain.{ConsumerSubscription, RecordHandler} import com.wixpress.dst.greyhound.core.metrics.GreyhoundMetrics.report import com.wixpress.dst.greyhound.core.metrics.{GreyhoundMetric, GreyhoundMetrics} @@ -37,8 +40,13 @@ object EventLoop { val start = for { _ <- report(StartingEventLoop(clientId, group, consumerAttributes)) offsets <- Offsets.make - handle = handler.andThen(offsets.update).handle(_) + offsetsAndGaps <- OffsetsAndGaps.make + handle = if (config.consumePartitionInParallel) { cr: Record => handler.handle(cr) } + else handler.andThen(offsets.update).handle(_) + updateBatch = { records: Chunk[Record] => report(HandledBatch(records)) *> updateGapsByBatch(records, offsetsAndGaps) } + currentGaps = { partitions: Set[TopicPartition] => offsetsAndGaps.offsetsAndGapsForPartitions(partitions) } _ <- report(CreatingDispatcher(clientId, group, consumerAttributes, config.startPaused)) + offsetsAndGapsInit <- Promise.make[Nothing, Unit] dispatcher <- Dispatcher.make( group, clientId, @@ -49,22 +57,51 @@ object EventLoop { config.delayResumeOfPausedPartition, consumerAttributes, workersShutdownRef, - config.startPaused + config.startPaused, + config.consumePartitionInParallel, + config.maxParallelism, + updateBatch, + currentGaps, + offsetsAndGapsInit ) positionsRef <- Ref.make(Map.empty[TopicPartition, Offset]) pausedPartitionsRef <- Ref.make(Set.empty[TopicPartition]) - partitionsAssigned <- Promise.make[Nothing, Unit] + partitionsAssigned <- Promise.make[Nothing, Set[TopicPartition]] // TODO how to handle errors in subscribe? - rebalanceListener = listener(pausedPartitionsRef, config, dispatcher, partitionsAssigned, group, consumer, clientId, offsets) - _ <- report(SubscribingToInitialSubAndRebalanceListener(clientId, group, consumerAttributes)) + rebalanceListener = listener( + pausedPartitionsRef, + config, + dispatcher, + partitionsAssigned, + group, + consumer, + clientId, + offsets, + offsetsAndGaps, + config.consumePartitionInParallel + ) + _ <- report(SubscribingToInitialSubAndRebalanceListener(clientId, group, consumerAttributes)) _ <- subscribe(initialSubscription, rebalanceListener)(consumer) running <- Ref.make[EventLoopState](Running) - _ <- report(CreatingPollOnceFiber(clientId, group, consumerAttributes)) - fiber <- pollOnce(running, consumer, dispatcher, pausedPartitionsRef, positionsRef, offsets, config, clientId, group) + _ <- report(CreatingPollOnceFiber(clientId, group, consumerAttributes)) + fiber <- pollOnce(running, consumer, dispatcher, pausedPartitionsRef, positionsRef, offsets, config, clientId, group, offsetsAndGaps) .repeatWhile(_ == true) .forkDaemon - _ <- report(AwaitingPartitionsAssignment(clientId, group, consumerAttributes)) - _ <- partitionsAssigned.await + _ <- report(AwaitingPartitionsAssignment(clientId, group, consumerAttributes)) + partitions <- partitionsAssigned.await + _ <- if (config.consumePartitionInParallel) { + report(AwaitingOffsetsAndGapsInit(clientId, group, consumerAttributes)) *> + initializeOffsetsAndGaps( // we must preform init in the main thread ant not in the rebalance listener as it involves calling SDK + offsetsAndGaps, + partitions, + consumer, + clientId, + group, + consumerAttributes, + offsetsAndGapsInit + ) *> offsetsAndGapsInit.await + + } else offsetsAndGapsInit.succeed() env <- ZIO.environment[Env] } yield (dispatcher, fiber, offsets, positionsRef, running, rebalanceListener.provideEnvironment(env)) @@ -138,7 +175,8 @@ object EventLoop { offsets: Offsets, config: EventLoopConfig, clientId: ClientId, - group: Group + group: Group, + offsetsAndGaps: OffsetsAndGaps ): URIO[R2 with Env, Boolean] = running.get.flatMap { case Running => @@ -146,7 +184,7 @@ object EventLoop { _ <- resumePartitions(consumer, clientId, group, dispatcher, paused) records <- pollAndHandle(consumer, dispatcher, paused, config) _ <- updatePositions(records, positionsRef, consumer, clientId) - _ <- commitOffsets(consumer, offsets) + _ <- if (config.consumePartitionInParallel) commitOffsetsAndGaps(consumer, offsetsAndGaps) else commitOffsets(consumer, offsets) _ <- ZIO.when(records.isEmpty)(ZIO.sleep(50.millis)) } yield true @@ -158,11 +196,13 @@ object EventLoop { pausedPartitionsRef: Ref[Set[TopicPartition]], config: EventLoopConfig, dispatcher: Dispatcher[_], - partitionsAssigned: Promise[Nothing, Unit], + partitionsAssigned: Promise[Nothing, Set[TopicPartition]], group: Group, consumer0: Consumer, clientId: ClientId, - offsets: Offsets + offsets: Offsets, + offsetsAndGaps: OffsetsAndGaps, + useParallelConsumer: Boolean ) = { config.rebalanceListener *> new RebalanceListener[GreyhoundMetrics] { @@ -170,16 +210,19 @@ object EventLoop { consumer: Consumer, partitions: Set[TopicPartition] )(implicit trace: Trace): URIO[GreyhoundMetrics, DelayedRebalanceEffect] = { - pausedPartitionsRef.set(Set.empty) *> - dispatcher.revoke(partitions).timeout(config.drainTimeout).flatMap { drained => - ZIO.when(drained.isEmpty)( - report(DrainTimeoutExceeded(clientId, group, config.drainTimeout.toMillis, consumer.config.consumerAttributes)) - ) - } *> commitOffsetsOnRebalance(consumer0, offsets) + for { + _ <- pausedPartitionsRef.update(_ -- partitions) + isRevokeTimedOut <- dispatcher.revoke(partitions).timeout(config.drainTimeout).map(_.isEmpty) + _ <- ZIO.when(isRevokeTimedOut)( + report(DrainTimeoutExceeded(clientId, group, config.drainTimeout.toMillis, consumer.config.consumerAttributes)) + ) + delayedRebalanceEffect <- if (useParallelConsumer) commitOffsetsAndGapsOnRebalance(consumer0, offsetsAndGaps) + else commitOffsetsOnRebalance(consumer0, offsets) + } yield delayedRebalanceEffect } override def onPartitionsAssigned(consumer: Consumer, partitions: Set[TopicPartition])(implicit trace: Trace): UIO[Any] = - partitionsAssigned.succeed(()) + partitionsAssigned.succeed(partitions) } } @@ -212,24 +255,87 @@ object EventLoop { for { records <- consumer.poll(config.fetchTimeout).catchAll(_ => ZIO.succeed(Nil)) paused <- pausedRef.get - pausedTopics <- ZIO.foldLeft(records)(paused) { (acc, record) => - val partition = record.topicPartition - if (acc contains partition) - report(PartitionThrottled(partition, record.offset, consumer.config.consumerAttributes)).as(acc) - else - dispatcher.submit(record).flatMap { - case SubmitResult.Submitted => ZIO.succeed(acc) - case SubmitResult.Rejected => - report(HighWatermarkReached(partition, record.offset, consumer.config.consumerAttributes)) *> - consumer.pause(record).fold(_ => acc, _ => acc + partition) - } - } + pausedTopics <- if (config.consumePartitionInParallel) submitRecordsAsBatch(consumer, dispatcher, records, paused) + else submitRecordsSequentially(consumer, dispatcher, records, paused) _ <- pausedRef.update(_ => pausedTopics) } yield records + private def initializeOffsetsAndGaps( + offsetsAndGaps: OffsetsAndGaps, + partitions: Set[TopicPartition], + consumer: Consumer, + clientId: ClientId, + group: Group, + attributes: Map[String, String], + offsetsAndGapsInit: Promise[Nothing, Unit] + ) = for { + committedOffsetsAndMetadata <- consumer.committedOffsetsAndMetadata(partitions) + initialOffsetsAndGaps = + committedOffsetsAndMetadata.mapValues(om => + OffsetsAndGaps.parseGapsString(om.metadata).fold(OffsetAndGaps(om.offset - 1, committable = false))(identity) + ) + _ <- offsetsAndGaps.init(initialOffsetsAndGaps) + _ <- report(InitializedOffsetsAndGaps(clientId, group, initialOffsetsAndGaps, attributes)) + _ <- offsetsAndGapsInit.succeed(()) + } yield () + + private def submitRecordsSequentially[R2, R1]( + consumer: Consumer, + dispatcher: Dispatcher[R2], + records: Records, + paused: Set[TopicPartition] + ): ZIO[R2 with Env, Nothing, Set[TopicPartition]] = { + ZIO.foldLeft(records)(paused) { (acc, record) => + val partition = record.topicPartition + if (acc contains partition) + report(PartitionThrottled(partition, record.offset, consumer.config.consumerAttributes)).as(acc) + else + dispatcher.submit(record).flatMap { + case SubmitResult.Submitted => ZIO.succeed(acc) + case SubmitResult.Rejected => + report(HighWatermarkReached(partition, record.offset, consumer.config.consumerAttributes)) *> + consumer.pause(record).fold(_ => acc, _ => acc + partition) + } + } + } + + private def submitRecordsAsBatch[R2, R1]( + consumer: Consumer, + dispatcher: Dispatcher[R2], + records: Records, + paused: Set[TopicPartition] + ): ZIO[R2 with Env, Nothing, Set[TopicPartition]] = { + val recordsByPartition = records.groupBy(_.topicPartition) + ZIO.foldLeft(recordsByPartition)(paused) { (acc, partitionToRecords) => + val partition = partitionToRecords._1 + if (acc contains partition) + report(PartitionThrottled(partition, partitionToRecords._2.map(_.offset).min, consumer.config.consumerAttributes)).as(acc) + else + dispatcher.submitBatch(partitionToRecords._2.toSeq).flatMap { + case SubmitResult.Submitted => + report(SubmittedBatch(partitionToRecords._2.size, partitionToRecords._1, partitionToRecords._2.map(_.offset))) *> + ZIO.succeed(acc) + case RejectedBatch(firstRejected) => + report(HighWatermarkReached(partition, firstRejected.offset, consumer.config.consumerAttributes)) *> + consumer.pause(firstRejected).fold(_ => acc, _ => acc + partition) + } + } + } + private def commitOffsets(consumer: Consumer, offsets: Offsets): URIO[GreyhoundMetrics, Unit] = offsets.committable.flatMap { committable => consumer.commit(committable).catchAll { _ => offsets.update(committable) } } + private def commitOffsetsAndGaps(consumer: Consumer, offsetsAndGaps: OffsetsAndGaps): URIO[GreyhoundMetrics, Unit] = + offsetsAndGaps.getCommittableAndClear.flatMap { committable => + val offsetsAndMetadataToCommit = OffsetsAndGaps.toOffsetsAndMetadata(committable) + consumer + .commitWithMetadata(offsetsAndMetadataToCommit) + .tap(_ => ZIO.when(offsetsAndMetadataToCommit.nonEmpty)(report(CommittedOffsetsAndMetadata(offsetsAndMetadataToCommit)))) + .catchAll { t => + report(FailedToCommitOffsetsAndMetadata(t, offsetsAndMetadataToCommit)) *> offsetsAndGaps.setCommittable(committable) + } + } + private def commitOffsetsOnRebalance( consumer: Consumer, offsets: Offsets @@ -249,6 +355,36 @@ object EventLoop { } } + private def commitOffsetsAndGapsOnRebalance( + consumer: Consumer, + offsetsAndGaps: OffsetsAndGaps + ): URIO[GreyhoundMetrics, DelayedRebalanceEffect] = { + for { + committable <- offsetsAndGaps.getCommittableAndClear + tle <- consumer + .commitWithMetadataOnRebalance(OffsetsAndGaps.toOffsetsAndMetadata(committable)) + .catchAll { _ => offsetsAndGaps.setCommittable(committable) *> DelayedRebalanceEffect.zioUnit } + runtime <- ZIO.runtime[Any] + } yield tle.catchAll { _ => + zio.Unsafe.unsafe { implicit s => + runtime.unsafe + .run(offsetsAndGaps.setCommittable(committable)) + .getOrThrowFiberFailure() + } + } + } + + private def updateGapsByBatch(records: Chunk[Record], offsetsAndGaps: OffsetsAndGaps) = + offsetsAndGaps.update(records) + + private def currentGapsForPartitions(partitions: Set[TopicPartition], clientId: ClientId)( + consumer: Consumer + ): ZIO[GreyhoundMetrics, Nothing, Map[TopicPartition, Option[OffsetAndGaps]]] = + consumer + .committedOffsetsAndMetadata(partitions) + .map { committed => committed.mapValues(om => OffsetsAndGaps.parseGapsString(om.metadata)) } + .catchAll(t => report(FailedToFetchCommittedGaps(t, clientId, consumer.config.consumerAttributes)).as(Map.empty)) + } case class EventLoopConfig( @@ -258,7 +394,9 @@ case class EventLoopConfig( highWatermark: Int, rebalanceListener: RebalanceListener[Any], delayResumeOfPausedPartition: Long, - startPaused: Boolean + startPaused: Boolean, + consumePartitionInParallel: Boolean, + maxParallelism: Int ) object EventLoopConfig { @@ -269,7 +407,9 @@ object EventLoopConfig { highWatermark = 256, rebalanceListener = RebalanceListener.Empty, delayResumeOfPausedPartition = 0, - startPaused = false + startPaused = false, + consumePartitionInParallel = false, + maxParallelism = 1 ) } @@ -301,15 +441,46 @@ object EventLoopMetric { attributes: Map[String, String] = Map.empty ) extends EventLoopMetric + case class SubmittedBatch(numSubmitted: Int, partition: TopicPartition, offsets: Iterable[Offset]) extends EventLoopMetric + case class FailedToUpdatePositions(t: Throwable, clientId: ClientId, attributes: Map[String, String] = Map.empty) extends EventLoopMetric - case class CreatingDispatcher(clientId: ClientId, group: Group, attributes: Map[String, String], startPaused: Boolean) extends EventLoopMetric + case class FailedToUpdateGapsOnPartitionAssignment( + t: Throwable, + clientId: ClientId, + group: Group, + partitions: Set[TopicPartition], + attributes: Map[String, String] = Map.empty + ) extends EventLoopMetric + + case class FailedToFetchCommittedGaps(t: Throwable, clientId: ClientId, attributes: Map[String, String] = Map.empty) + extends EventLoopMetric - case class SubscribingToInitialSubAndRebalanceListener(clientId: ClientId, group: Group, attributes: Map[String, String]) extends EventLoopMetric + case class CreatingDispatcher(clientId: ClientId, group: Group, attributes: Map[String, String], startPaused: Boolean) + extends EventLoopMetric + + case class SubscribingToInitialSubAndRebalanceListener(clientId: ClientId, group: Group, attributes: Map[String, String]) + extends EventLoopMetric case class CreatingPollOnceFiber(clientId: ClientId, group: Group, attributes: Map[String, String]) extends EventLoopMetric case class AwaitingPartitionsAssignment(clientId: ClientId, group: Group, attributes: Map[String, String]) extends EventLoopMetric + + case class AwaitingOffsetsAndGapsInit(clientId: ClientId, group: Group, attributes: Map[String, String]) extends EventLoopMetric + + case class InitializedOffsetsAndGaps( + clientId: ClientId, + group: Group, + initial: Map[TopicPartition, OffsetAndGaps], + attributes: Map[String, String] + ) extends EventLoopMetric + + case class CommittedOffsetsAndMetadata(offsetsAndMetadata: Map[TopicPartition, OffsetAndMetadata]) extends EventLoopMetric + + case class FailedToCommitOffsetsAndMetadata(t: Throwable, offsetsAndMetadata: Map[TopicPartition, OffsetAndMetadata]) + extends EventLoopMetric + + case class HandledBatch(records: Records) extends EventLoopMetric } sealed trait EventLoopState diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsAndGaps.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsAndGaps.scala new file mode 100644 index 00000000..5dc24f0f --- /dev/null +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsAndGaps.scala @@ -0,0 +1,168 @@ +package com.wixpress.dst.greyhound.core.consumer + +import com.wixpress.dst.greyhound.core.consumer.Gap.GAP_SEPARATOR +import com.wixpress.dst.greyhound.core.consumer.OffsetAndGaps.{GAPS_STRING_SEPARATOR, LAST_HANDLED_OFFSET_SEPARATOR} +import com.wixpress.dst.greyhound.core.consumer.domain.{ConsumerRecord, RecordTopicPartition} +import com.wixpress.dst.greyhound.core.{Offset, OffsetAndMetadata, TopicPartition} +import zio._ + +trait OffsetsAndGaps { + def init(committedOffsets: Map[TopicPartition, OffsetAndGaps]): UIO[Unit] + + def getCommittableAndClear: UIO[Map[TopicPartition, OffsetAndGaps]] + + def gapsForPartition(partition: TopicPartition): UIO[Seq[Gap]] + + def offsetsAndGapsForPartitions(partitions: Set[TopicPartition]): UIO[Map[TopicPartition, OffsetAndGaps]] + + def update(partition: TopicPartition, batch: Seq[Offset], prevCommittedOffset: Option[Offset]): UIO[Unit] + + def update(record: ConsumerRecord[_, _]): UIO[Unit] = + update(RecordTopicPartition(record), Seq(record.offset), None) + + def update(records: Chunk[ConsumerRecord[_, _]]): UIO[Unit] = { + val sortedBatch = records.sortBy(_.offset) + update(RecordTopicPartition(sortedBatch.head), sortedBatch.map(_.offset) ++ Seq(sortedBatch.last.offset + 1), None) + } + + def update(partition: TopicPartition, batch: Seq[Offset]): UIO[Unit] = + update(partition, batch, None) + + def setCommittable(offsets: Map[TopicPartition, OffsetAndGaps]): UIO[Unit] + + def contains(partition: TopicPartition, offset: Offset): UIO[Boolean] +} + +object OffsetsAndGaps { + def make: UIO[OffsetsAndGaps] = + Ref.make(Map.empty[TopicPartition, OffsetAndGaps]).map { ref => + new OffsetsAndGaps { + override def init(committedOffsets: Map[TopicPartition, OffsetAndGaps]): UIO[Unit] = + ref.update(_ => committedOffsets) + + override def getCommittableAndClear: UIO[Map[TopicPartition, OffsetAndGaps]] = + ref.modify(offsetsAndGaps => { + val committable = offsetsAndGaps.filter(_._2.committable) + val updated = offsetsAndGaps.mapValues(_.markCommitted) + (committable, updated) + }) + + override def gapsForPartition(partition: TopicPartition): UIO[Seq[Gap]] = + ref.get.map(_.get(partition).fold(Seq.empty[Gap])(_.gaps.sortBy(_.start))) + + override def offsetsAndGapsForPartitions(partitions: Set[TopicPartition]): UIO[Map[TopicPartition, OffsetAndGaps]] = + ref.get.map(_.filterKeys(partitions.contains)) + + override def update(partition: TopicPartition, batch: Seq[Offset], prevCommittedOffset: Option[Offset]): UIO[Unit] = + ref.update { offsetsAndGaps => + val sortedBatch = batch.sorted + val maxBatchOffset = sortedBatch.last + val maybeOffsetAndGaps = offsetsAndGaps.get(partition) + val prevOffset = maybeOffsetAndGaps.fold(-1L)(_.offset) + val partitionOffsetAndGaps = maybeOffsetAndGaps.fold(OffsetAndGaps(maxBatchOffset))(identity) + + val newGaps = gapsInBatch(sortedBatch, prevOffset) + + val updatedGaps = updateGapsByOffsets( + partitionOffsetAndGaps.gaps ++ newGaps, + sortedBatch + ) + + offsetsAndGaps + (partition -> OffsetAndGaps(maxBatchOffset max prevOffset, updatedGaps)) + }.unit + + override def contains(partition: TopicPartition, offset: Offset): UIO[Boolean] = + ref.get.map(_.get(partition).fold(false)(_.contains(offset))) + + override def setCommittable(offsets: Map[TopicPartition, OffsetAndGaps]): UIO[Unit] = + ref.update { _ => offsets } + + private def gapsInBatch(batch: Seq[Offset], prevLastOffset: Offset): Seq[Gap] = + batch.sorted + .foldLeft(Seq.empty[Gap], prevLastOffset) { + case ((gaps, lastOffset), offset) => + if (offset <= lastOffset) (gaps, lastOffset) + else if (offset == lastOffset + 1) (gaps, offset) + else { + val newGap = Gap(lastOffset + 1, offset - 1) + (newGap +: gaps, offset) + } + } + ._1 + .reverse + + private def updateGapsByOffsets(gaps: Seq[Gap], offsets: Seq[Offset]): Seq[Gap] = { + val gapsToOffsets = gaps.map(gap => gap -> offsets.filter(o => o >= gap.start && o <= gap.end)).toMap + gapsToOffsets.flatMap { + case (gap, offsets) => + if (offsets.isEmpty) Seq(gap) + else if (offsets.size == (gap.size)) Seq.empty[Gap] + else gapsInBatch(offsets ++ Seq(gap.start - 1, gap.end + 1), gap.start - 2) + }.toSeq + } + } + } + + def toOffsetsAndMetadata(offsetsAndGaps: Map[TopicPartition, OffsetAndGaps]): Map[TopicPartition, OffsetAndMetadata] = + offsetsAndGaps.mapValues(offsetAndGaps => + OffsetAndMetadata(offsetAndGaps.offset, offsetAndGaps.gapsString) + ) // todo: add encoding and compression to plain gaps string + + def parseGapsString(offsetAndGapsString: String): Option[OffsetAndGaps] = { + val lastHandledOffsetSeparatorIndex = offsetAndGapsString.indexOf(LAST_HANDLED_OFFSET_SEPARATOR) + if (lastHandledOffsetSeparatorIndex < 0) + None + else { + val lastHandledOffset = offsetAndGapsString.substring(0, lastHandledOffsetSeparatorIndex).toLong + val gaps = offsetAndGapsString + .substring(lastHandledOffsetSeparatorIndex + 1) + .split(GAPS_STRING_SEPARATOR) + .map(_.split(GAP_SEPARATOR)) + .collect { case Array(start, end) => Gap(start.toLong, end.toLong) } + .toSeq + .sortBy(_.start) + Some(OffsetAndGaps(lastHandledOffset, gaps)) + } + } + + def firstGapOffset(gapsString: String): Option[Offset] = { + val maybeOffsetAndGaps = parseGapsString(gapsString) + maybeOffsetAndGaps match { + case Some(offsetAndGaps) if offsetAndGaps.gaps.nonEmpty => Some(offsetAndGaps.gaps.minBy(_.start).start) + case _ => None + } + } +} + +case class Gap(start: Offset, end: Offset) { + def contains(offset: Offset): Boolean = start <= offset && offset <= end + + def size: Long = end - start + 1 + + override def toString: String = s"$start$GAP_SEPARATOR$end" +} + +object Gap { + val GAP_SEPARATOR = "_" +} + +case class OffsetAndGaps(offset: Offset, gaps: Seq[Gap], committable: Boolean = true) { + def contains(offset: Offset): Boolean = gaps.exists(_.contains(offset)) + + def markCommitted: OffsetAndGaps = copy(committable = false) + + def gapsString: String = { + if (gaps.isEmpty) "" + else + s"${offset.toString}${LAST_HANDLED_OFFSET_SEPARATOR}${gaps.sortBy(_.start).mkString(GAPS_STRING_SEPARATOR)}" + } +} + +object OffsetAndGaps { + val GAPS_STRING_SEPARATOR = "$" + val LAST_HANDLED_OFFSET_SEPARATOR = "#" + + def apply(offset: Offset): OffsetAndGaps = OffsetAndGaps(offset, Seq.empty[Gap]) + + def apply(offset: Offset, committable: Boolean): OffsetAndGaps = OffsetAndGaps(offset, Seq.empty[Gap], committable) +} diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsInitializer.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsInitializer.scala index 97f1b341..234e7ba9 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsInitializer.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsInitializer.scala @@ -1,12 +1,11 @@ package com.wixpress.dst.greyhound.core.consumer import java.time.Clock -import com.wixpress.dst.greyhound.core.consumer.ConsumerMetric.{CommittedMissingOffsets, CommittedMissingOffsetsFailed} -import com.wixpress.dst.greyhound.core.{ClientId, Group, Offset, TopicPartition} +import com.wixpress.dst.greyhound.core.consumer.ConsumerMetric.{CommittedMissingOffsets, CommittedMissingOffsetsFailed, SkippedGapsOnInitialization} +import com.wixpress.dst.greyhound.core.consumer.OffsetsAndGaps.{firstGapOffset, parseGapsString} +import com.wixpress.dst.greyhound.core.{ClientId, Group, Offset, OffsetAndMetadata, TopicPartition} import com.wixpress.dst.greyhound.core.metrics.{GreyhoundMetric, GreyhoundMetrics} - import zio.{URIO, ZIO} - import zio._ /** @@ -24,63 +23,84 @@ class OffsetsInitializer( initialSeek: InitialOffsetsSeek, rewindUncommittedOffsetsBy: Duration, clock: Clock = Clock.systemUTC, - offsetResetIsEarliest: Boolean + offsetResetIsEarliest: Boolean, + parallelConsumer: Boolean ) { def initializeOffsets(partitions: Set[TopicPartition]): Unit = { val hasSeek = initialSeek != InitialOffsetsSeek.default val effectiveTimeout = if (hasSeek) timeoutIfSeek else timeout withReporting(partitions, rethrow = hasSeek) { - val committed = offsetOperations.committed(partitions, effectiveTimeout) + val committed = offsetOperations.committedWithMetadata(partitions, effectiveTimeout) val beginning = offsetOperations.beginningOffsets(partitions, effectiveTimeout) val endOffsets = offsetOperations.endOffsets(partitions, effectiveTimeout) - val PartitionActions(toOffsets, toPause) = calculateTargetOffsets(partitions, beginning, committed, endOffsets, effectiveTimeout) + val PartitionActions(toOffsets, toPause) = + calculateTargetOffsets(partitions, beginning, committed, endOffsets, effectiveTimeout, parallelConsumer) val notCommitted = partitions -- committed.keySet -- toOffsets.keySet offsetOperations.pause(toPause) val rewindUncommittedOffsets = if (offsetResetIsEarliest || notCommitted.isEmpty || rewindUncommittedOffsetsBy.isZero) Map.empty - else offsetOperations.offsetsForTimes(notCommitted, clock.millis() - rewindUncommittedOffsetsBy.toMillis, effectiveTimeout) - .map{case (tp, maybeRewindedOffset) => (tp, maybeRewindedOffset.orElse(endOffsets.get(tp)).getOrElse(0L))} + else + offsetOperations + .offsetsForTimes(notCommitted, clock.millis() - rewindUncommittedOffsetsBy.toMillis, effectiveTimeout) + .map { case (tp, maybeRewindedOffset) => (tp, maybeRewindedOffset.orElse(endOffsets.get(tp)).getOrElse(0L)) } val positions = - notCommitted.map(tp => tp -> offsetOperations.position(tp, effectiveTimeout)).toMap ++ toOffsets ++ rewindUncommittedOffsets + notCommitted.map(tp => tp -> offsetOperations.position(tp, effectiveTimeout)).toMap.mapValues(OffsetAndMetadata.apply) ++ + toOffsets ++ rewindUncommittedOffsets.mapValues(OffsetAndMetadata.apply) if ((toOffsets ++ rewindUncommittedOffsets).nonEmpty) { - offsetOperations.seek(toOffsets ++ rewindUncommittedOffsets) + offsetOperations.seek(toOffsets.mapValues(_.offset) ++ rewindUncommittedOffsets) } if (positions.nonEmpty) { - offsetOperations.commit(positions, effectiveTimeout) + offsetOperations.commitWithMetadata(positions, effectiveTimeout) } - positions + positions.mapValues(_.offset) } } - case class PartitionActions(offsetSeeks: Map[TopicPartition, Offset], partitionsToPause: Set[TopicPartition]) + case class PartitionActions(offsetSeeks: Map[TopicPartition, OffsetAndMetadata], partitionsToPause: Set[TopicPartition]) private def calculateTargetOffsets( partitions: Set[TopicPartition], beginning: Map[TopicPartition, Offset], - committed: Map[TopicPartition, Offset], + committed: Map[TopicPartition, OffsetAndMetadata], endOffsets: Map[TopicPartition, Offset], - timeout: Duration + timeout: Duration, + parallelConsumer: Boolean ): PartitionActions = { + val currentCommittedOffsets = partitions.map((_, None)).toMap ++ committed.mapValues(Some.apply) val seekTo: Map[TopicPartition, SeekTo] = initialSeek.seekOffsetsFor( assignedPartitions = partitions, beginningOffsets = partitions.map((_, None)).toMap ++ beginning.mapValues(Some.apply), endOffsets = partitions.map((_, None)).toMap ++ endOffsets.mapValues(Some.apply), - currentCommittedOffsets = partitions.map((_, None)).toMap ++ committed.mapValues(Some.apply) + currentCommittedOffsets = currentCommittedOffsets.mapValues(_.map(_.offset)) ) - val seekToOffsets = seekTo.collect { case (k, v: SeekTo.SeekToOffset) => k -> v.offset } + val seekToOffsets = seekTo.collect { case (k, v: SeekTo.SeekToOffset) => k -> OffsetAndMetadata(v.offset) } val seekToEndPartitions = seekTo.collect { case (k, SeekTo.SeekToEnd) => k }.toSet val toPause = seekTo.collect { case (k, SeekTo.Pause) => k } - val seekToEndOffsets = fetchEndOffsets(seekToEndPartitions, timeout) - val toOffsets = seekToOffsets ++ seekToEndOffsets - + val seekToEndOffsets = fetchEndOffsets(seekToEndPartitions, timeout).mapValues(OffsetAndMetadata.apply) + val gapsSmallestOffsets = currentCommittedOffsets + .collect { case (tp, Some(om)) => tp -> om } + .map(tpom => tpom._1 -> (firstGapOffset(tpom._2.metadata), tpom._2.metadata)) + .collect { case (tp, (Some(offset), metadata)) => tp -> OffsetAndMetadata(offset, metadata) } + val seekToGapsOffsets = if (parallelConsumer) gapsSmallestOffsets else Map.empty + val toOffsets = seekToOffsets ++ seekToEndOffsets ++ seekToGapsOffsets + + if (!parallelConsumer && gapsSmallestOffsets.nonEmpty) reportSkippedGaps(currentCommittedOffsets) PartitionActions(offsetSeeks = toOffsets, partitionsToPause = toPause.toSet) } + private def reportSkippedGaps(currentCommittedOffsets: Map[TopicPartition, Option[OffsetAndMetadata]]) = { + val skippedGaps = currentCommittedOffsets + .collect { case (tp, Some(om)) => tp -> om } + .map(tpom => tpom._1 -> parseGapsString(tpom._2.metadata)) + .collect { case (tp, Some(gaps)) => tp -> gaps } + reporter(SkippedGapsOnInitialization(clientId, group, skippedGaps)) + } + private def fetchEndOffsets(seekToEndPartitions: Set[TopicPartition], timeout: Duration) = { if (seekToEndPartitions.nonEmpty) { offsetOperations.endOffsets(seekToEndPartitions, timeout) @@ -149,7 +169,8 @@ object OffsetsInitializer { initialSeek: InitialOffsetsSeek, clock: Clock = Clock.systemUTC, rewindUncommittedOffsetsBy: Duration, - offsetResetIsEarliest: Boolean + offsetResetIsEarliest: Boolean, + parallelConsumer: Boolean )(implicit trace: Trace): URIO[GreyhoundMetrics, OffsetsInitializer] = for { metrics <- ZIO.environment[GreyhoundMetrics].map(_.get) runtime <- ZIO.runtime[Any] @@ -163,6 +184,7 @@ object OffsetsInitializer { initialSeek: InitialOffsetsSeek, rewindUncommittedOffsetsBy, clock, - offsetResetIsEarliest + offsetResetIsEarliest, + parallelConsumer ) } diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/RecordConsumer.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/RecordConsumer.scala index 883f17a8..4bc05bb5 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/RecordConsumer.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/RecordConsumer.scala @@ -62,8 +62,9 @@ object RecordConsumer { * concurrent between partitions; order is guaranteed to be maintained within the same partition. */ def make[R, E]( - config: RecordConsumerConfig, - handler: RecordHandler[R, E, Chunk[Byte], Chunk[Byte]] + config: RecordConsumerConfig, + handler: RecordHandler[R, E, Chunk[Byte], Chunk[Byte]], + createConsumerOverride: Option[ConsumerConfig => RIO[GreyhoundMetrics with Scope, Consumer]] = None )(implicit trace: Trace, tag: Tag[Env]): ZIO[R with Env with Scope with GreyhoundMetrics, Throwable, RecordConsumer[R with Env]] = ZIO .acquireRelease( @@ -75,19 +76,21 @@ object RecordConsumer { _ <- validateRetryPolicy(config) consumerSubscriptionRef <- Ref.make[ConsumerSubscription](config.initialSubscription) nonBlockingRetryHelper = NonBlockingRetryHelper(config.group, config.retryConfig) - consumer <- Consumer.make(consumerConfig(config)) + consumer <- createConsumerOverride.getOrElse(Consumer.make _)(consumerConfig(config)) (initialSubscription, topicsToCreate) = config.retryConfig.fold((config.initialSubscription, Set.empty[Topic]))(policy => maybeAddRetryTopics(policy, config, nonBlockingRetryHelper) ) - _ <- AdminClient - .make(AdminClientConfig(config.bootstrapServers, config.kafkaAuthProperties), config.consumerAttributes) - .tap(client => - client.createTopics( - topicsToCreate.map(topic => - TopicConfig(topic, partitions = 1, replicationFactor = 1, cleanupPolicy = CleanupPolicy.Delete(86400000L)) + _ <- ZIO.when(config.createRetryTopics)( + AdminClient + .make(AdminClientConfig(config.bootstrapServers, config.kafkaAuthProperties), config.consumerAttributes) + .tap(client => + client.createTopics( + topicsToCreate.map(topic => + TopicConfig(topic, partitions = 1, replicationFactor = 1, cleanupPolicy = CleanupPolicy.Delete(86400000L)) + ) ) ) - ) + ) blockingState <- Ref.make[Map[BlockingTarget, BlockingState]](Map.empty) blockingStateResolver = BlockingStateResolver(blockingState) workersShutdownRef <- Ref.make[Map[TopicPartition, ShutdownPromise]](Map.empty) @@ -206,7 +209,8 @@ object RecordConsumer { config.consumerAttributes, config.decryptor, config.commitMetadataString, - config.rewindUncommittedOffsetsBy.toMillis + config.rewindUncommittedOffsetsBy.toMillis, + config.eventLoopConfig.consumePartitionInParallel ) } @@ -321,7 +325,8 @@ case class RecordConsumerConfig( decryptor: Decryptor[Any, Throwable, Chunk[Byte], Chunk[Byte]] = new NoOpDecryptor, retryProducerAttributes: Map[String, String] = Map.empty, commitMetadataString: Metadata = OffsetAndMetadata.NO_METADATA, - rewindUncommittedOffsetsBy: Duration = 0.millis + rewindUncommittedOffsetsBy: Duration = 0.millis, + createRetryTopics: Boolean = true ) extends CommonGreyhoundConfig { override def kafkaProps: Map[String, String] = extraProperties diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/ReportingConsumer.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/ReportingConsumer.scala index 96a929f8..6202220d 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/ReportingConsumer.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/ReportingConsumer.scala @@ -41,10 +41,14 @@ case class ReportingConsumer(clientId: ClientId, group: Group, internal: Consume implicit trace: Trace ): UIO[DelayedRebalanceEffect] = (report(PartitionsRevoked(clientId, group, partitions, config.consumerAttributes)) *> - rebalanceListener.onPartitionsRevoked(consumer, partitions) - .timed.tap { case (duration, _) => report(PartitionsRevokedComplete(clientId, group, partitions, config.consumerAttributes, duration.toMillis)) } - .map(_._2) - ).provideEnvironment(r) + rebalanceListener + .onPartitionsRevoked(consumer, partitions) + .timed + .tap { + case (duration, _) => + report(PartitionsRevokedComplete(clientId, group, partitions, config.consumerAttributes, duration.toMillis)) + } + .map(_._2)).provideEnvironment(r) override def onPartitionsAssigned(consumer: Consumer, partitions: Set[TopicPartition])(implicit trace: Trace): UIO[Any] = (report(PartitionsAssigned(clientId, group, partitions, config.consumerAttributes)) *> @@ -105,6 +109,41 @@ case class ReportingConsumer(clientId: ClientId, group: Group, internal: Consume } else DelayedRebalanceEffect.zioUnit } + override def commitWithMetadataOnRebalance(offsets: Map[TopicPartition, OffsetAndMetadata] + )(implicit trace: Trace): RIO[GreyhoundMetrics, DelayedRebalanceEffect] = + ZIO.runtime[GreyhoundMetrics].flatMap { runtime => + if (offsets.nonEmpty) { + report(CommittingOffsetsWithMetadata(clientId, group, offsets, calledOnRebalance = true, attributes = config.consumerAttributes)) *> + internal + .commitWithMetadataOnRebalance(offsets) + .tapError { error => + report(CommitWithMetadataFailed(clientId, group, error, offsets, calledOnRebalance = true, attributes = config.consumerAttributes)) + } + .map( + _.tapError { error => // handle commit errors in ThreadLockedEffect + zio.Unsafe.unsafe { implicit s => + runtime.unsafe + .run( + report( + CommitWithMetadataFailed(clientId, group, error, offsets, calledOnRebalance = true, attributes = config.consumerAttributes) + ) + ) + .getOrThrowFiberFailure() + } + } *> + DelayedRebalanceEffect( + zio.Unsafe.unsafe { implicit s => + runtime.unsafe + .run( + report(CommittedOffsetsWithMetadata(clientId, group, offsets, calledOnRebalance = true, attributes = config.consumerAttributes)) + ) + .getOrThrowFiberFailure() + } + ) + ) + } else DelayedRebalanceEffect.zioUnit + } + override def commit(offsets: Map[TopicPartition, Offset])(implicit trace: Trace): RIO[GreyhoundMetrics, Unit] = { ZIO .when(offsets.nonEmpty) { @@ -118,6 +157,34 @@ case class ReportingConsumer(clientId: ClientId, group: Group, internal: Consume .unit } + override def commitWithMetadata(offsetsAndMetadata: Map[TopicPartition, OffsetAndMetadata])( + implicit trace: Trace + ): RIO[GreyhoundMetrics, Unit] = { + val offsets = offsetsAndMetadata.map { case (tp, om) => tp -> om.offset } + ZIO + .when(offsetsAndMetadata.nonEmpty) { + report( + CommittingOffsets( + clientId, + group, + offsets, + calledOnRebalance = false, + attributes = config.consumerAttributes + ) + ) *> internal.commitWithMetadata(offsetsAndMetadata).tapError { error => report(CommitFailed(clientId, group, error, offsets)) } *> + report( + CommittedOffsets( + clientId, + group, + offsets, + calledOnRebalance = false, + attributes = config.consumerAttributes + ) + ) + } + .unit + } + override def pause(partitions: Set[TopicPartition])(implicit trace: Trace): ZIO[GreyhoundMetrics, IllegalStateException, Unit] = ZIO .when(partitions.nonEmpty) { @@ -155,6 +222,9 @@ case class ReportingConsumer(clientId: ClientId, group: Group, internal: Consume override def committedOffsets(partitions: Set[TopicPartition])(implicit trace: Trace): RIO[Any, Map[TopicPartition, Offset]] = internal.committedOffsets(partitions) + override def committedOffsetsAndMetadata(partitions: NonEmptySet[TopicPartition])(implicit trace: Trace): RIO[Any, Map[TopicPartition, OffsetAndMetadata]] = + internal.committedOffsetsAndMetadata(partitions) + override def position(topicPartition: TopicPartition)(implicit trace: Trace): Task[Offset] = internal.position(topicPartition) @@ -197,6 +267,14 @@ object ConsumerMetric { attributes: Map[String, String] = Map.empty ) extends ConsumerMetric + case class CommittingOffsetsWithMetadata( + clientId: ClientId, + group: Group, + offsets: Map[TopicPartition, OffsetAndMetadata], + calledOnRebalance: Boolean, + attributes: Map[String, String] = Map.empty + ) extends ConsumerMetric + case class CommittedOffsets( clientId: ClientId, group: Group, @@ -205,6 +283,14 @@ object ConsumerMetric { attributes: Map[String, String] = Map.empty ) extends ConsumerMetric + case class CommittedOffsetsWithMetadata( + clientId: ClientId, + group: Group, + offsets: Map[TopicPartition, OffsetAndMetadata], + calledOnRebalance: Boolean, + attributes: Map[String, String] = Map.empty + ) extends ConsumerMetric + case class PausingPartitions( clientId: ClientId, group: Group, @@ -241,11 +327,13 @@ object ConsumerMetric { attributes: Map[String, String] = Map.empty ) extends ConsumerMetric - case class PartitionsRevokedComplete(clientId: ClientId, - group: Group, - partitions: Set[TopicPartition], - attributes: Map[String, String] = Map.empty, - durationMs: Long) extends ConsumerMetric + case class PartitionsRevokedComplete( + clientId: ClientId, + group: Group, + partitions: Set[TopicPartition], + attributes: Map[String, String] = Map.empty, + durationMs: Long + ) extends ConsumerMetric case class SubscribeFailed(clientId: ClientId, group: Group, error: Throwable, attributes: Map[String, String] = Map.empty) extends ConsumerMetric @@ -262,6 +350,15 @@ object ConsumerMetric { attributes: Map[String, String] = Map.empty ) extends ConsumerMetric + case class CommitWithMetadataFailed( + clientId: ClientId, + group: Group, + error: Throwable, + offsets: Map[TopicPartition, OffsetAndMetadata], + calledOnRebalance: Boolean = false, + attributes: Map[String, String] = Map.empty + ) extends ConsumerMetric + case class PausePartitionsFailed( clientId: ClientId, group: Group, @@ -317,4 +414,6 @@ object ConsumerMetric { case class ClosedConsumer(group: Group, clientId: ClientId, result: MetricResult[Throwable, Unit]) extends ConsumerMetric + case class SkippedGapsOnInitialization(clientId: ClientId, group: Group, gaps: Map[TopicPartition, OffsetAndGaps]) extends ConsumerMetric + } diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/BlockingRetryRecordHandler.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/BlockingRetryRecordHandler.scala index 502e31e1..5544e200 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/BlockingRetryRecordHandler.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/BlockingRetryRecordHandler.scala @@ -5,7 +5,6 @@ import com.wixpress.dst.greyhound.core.{Group, TopicPartition} import com.wixpress.dst.greyhound.core.consumer.domain.{ConsumerRecord, RecordHandler} import com.wixpress.dst.greyhound.core.consumer.retry.BlockingState.{Blocked, Blocking => InternalBlocking, IgnoringOnce} import com.wixpress.dst.greyhound.core.consumer.retry.RetryRecordHandlerMetric.{BlockingRetryHandlerInvocationFailed, DoneBlockingBeforeRetry, NoRetryOnNonRetryableFailure} -import com.wixpress.dst.greyhound.core.consumer.retry.ZIOHelper.foreachWhile import com.wixpress.dst.greyhound.core.metrics.GreyhoundMetrics import com.wixpress.dst.greyhound.core.metrics.GreyhoundMetrics.report import com.wixpress.dst.greyhound.core.zioutils.AwaitShutdown @@ -31,7 +30,11 @@ private[retry] object BlockingRetryRecordHandler { override def handle(record: ConsumerRecord[K, V])(implicit trace: Trace): ZIO[GreyhoundMetrics with R, Nothing, LastHandleResult] = { val topicPartition = TopicPartition(record.topic, record.partition) - def pollBlockingStateWithSuspensions(interval: Duration, start: Long): URIO[GreyhoundMetrics, PollResult] = { + def pollBlockingStateWithSuspensions( + record: ConsumerRecord[K, V], + interval: Duration, + start: Long + ): URIO[GreyhoundMetrics, PollResult] = { for { shouldBlock <- blockingStateResolver.resolve(record) shouldPollAgain <- @@ -43,14 +46,14 @@ private[retry] object BlockingRetryRecordHandler { } yield shouldPollAgain } - def blockOnErrorFor(interval: Duration) = { + def blockOnErrorFor(record: ConsumerRecord[K, V], interval: Duration) = { for { start <- currentTime(TimeUnit.MILLISECONDS) continueBlocking <- if (interval.toMillis > 100L) { awaitShutdown(record.topicPartition).flatMap( _.interruptOnShutdown( - pollBlockingStateWithSuspensions(interval, start).repeatWhile(result => result.pollAgain).map(_.blockHandling) + pollBlockingStateWithSuspensions(record, interval, start).repeatWhile(result => result.pollAgain).map(_.blockHandling) ).reporting(r => DoneBlockingBeforeRetry(record.topic, record.partition, record.offset, r.duration, r.failed)) ) } else { @@ -63,6 +66,7 @@ private[retry] object BlockingRetryRecordHandler { } def handleAndMaybeBlockOnErrorFor( + record: ConsumerRecord[K, V], interval: Option[Duration] ): ZIO[R with GreyhoundMetrics, Nothing, LastHandleResult] = { handler.handle(record).map(_ => LastHandleResult(lastHandleSucceeded = true, shouldContinue = false)).catchAll { @@ -73,7 +77,8 @@ private[retry] object BlockingRetryRecordHandler { case error => interval .map { interval => - report(BlockingRetryHandlerInvocationFailed(topicPartition, record.offset, error.toString)) *> blockOnErrorFor(interval) + report(BlockingRetryHandlerInvocationFailed(topicPartition, record.offset, error.toString)) *> + blockOnErrorFor(record, interval) } .getOrElse(ZIO.succeed(LastHandleResult(lastHandleSucceeded = false, shouldContinue = false))) } @@ -94,15 +99,46 @@ private[retry] object BlockingRetryRecordHandler { if (nonBlockingHandler.isHandlingRetryTopicMessage(group, record)) { ZIO.succeed(LastHandleResult(lastHandleSucceeded = false, shouldContinue = false)) } else { - val durationsIncludingForInvocationWithNoErrorHandling = retryConfig.blockingBackoffs(record.topic)().map(Some(_)) :+ None + val durations = retryConfig.blockingBackoffs(record.topic) + val durationsIncludingForInvocationWithNoErrorHandling = durations.map(Some(_)) :+ None for { - result <- foreachWhile(durationsIncludingForInvocationWithNoErrorHandling) { interval => handleAndMaybeBlockOnErrorFor(interval) } + result <- retryEvery(record, durationsIncludingForInvocationWithNoErrorHandling) { (rec, interval) => + handleAndMaybeBlockOnErrorFor(rec, interval) + } _ <- maybeBackToStateBlocking } yield result } } } + private def retryEvery[K, V, R, E](record: ConsumerRecord[K, V], as: Iterable[Option[Duration]])( + f: (ConsumerRecord[K, V], Option[Duration]) => ZIO[R, E, LastHandleResult] + )(implicit trace: Trace): ZIO[R, E, LastHandleResult] = { + ZIO.succeed(as.iterator).flatMap { i => + def loop(retryAttempt: Option[RetryAttempt]): ZIO[R, E, LastHandleResult] = + if (i.hasNext) { + val nextDelay = i.next + val recordWithAttempt = retryAttempt.fold(record) { attempt => + record.copy(headers = record.headers ++ RetryAttempt.toHeaders(attempt)) + } + f(recordWithAttempt, nextDelay).flatMap { result => + if (result.shouldContinue) Clock.instant.flatMap { now => + val nextAttempt = RetryAttempt( + originalTopic = record.topic, + attempt = retryAttempt.fold(0)(_.attempt + 1), + submittedAt = now, + backoff = nextDelay getOrElse Duration.Zero + ) + loop(Some(nextAttempt)) + } + else ZIO.succeed(result) + } + } else ZIO.succeed(LastHandleResult(lastHandleSucceeded = false, shouldContinue = false)) + + loop(None) + } + } + private def handleNonRetriable[K, V, E, R](record: ConsumerRecord[K, V], topicPartition: TopicPartition, cause: Exception) = report(NoRetryOnNonRetryableFailure(topicPartition, record.offset, cause)) .as(LastHandleResult(lastHandleSucceeded = false, shouldContinue = false)) diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/NonBlockingRetryHelper.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/NonBlockingRetryHelper.scala index c8dd3edd..1d1cd24c 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/NonBlockingRetryHelper.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/NonBlockingRetryHelper.scala @@ -1,40 +1,31 @@ package com.wixpress.dst.greyhound.core.consumer.retry -import java.time.{Instant, Duration => JavaDuration} -import java.util.concurrent.TimeUnit.MILLISECONDS -import java.util.regex.Pattern -import com.wixpress.dst.greyhound.core.Serdes.StringSerde import com.wixpress.dst.greyhound.core.consumer.domain.ConsumerSubscription.{TopicPattern, Topics} -import com.wixpress.dst.greyhound.core.consumer.retry.RetryAttempt.{RetryAttemptNumber, currentTime} -import com.wixpress.dst.greyhound.core.consumer.retry.RetryDecision.{NoMoreRetries, RetryWith} import com.wixpress.dst.greyhound.core.consumer.domain.{ConsumerRecord, ConsumerSubscription} +import com.wixpress.dst.greyhound.core.consumer.retry.RetryDecision.{NoMoreRetries, RetryWith} import com.wixpress.dst.greyhound.core.consumer.retry.RetryRecordHandlerMetric.WaitingForRetry import com.wixpress.dst.greyhound.core.metrics.GreyhoundMetrics import com.wixpress.dst.greyhound.core.metrics.GreyhoundMetrics.report import com.wixpress.dst.greyhound.core.producer.ProducerRecord -import com.wixpress.dst.greyhound.core.{Group, Headers, Topic, durationDeserializer, instantDeserializer} -import zio.Clock -import zio.Duration +import com.wixpress.dst.greyhound.core.{Group, Topic} import zio.Schedule.spaced -import zio.{Chunk, UIO, URIO, _} +import zio.{Chunk, Clock, Duration, URIO, _} +import java.time.Instant +import java.util.regex.Pattern import scala.util.Try trait NonBlockingRetryHelper { def retryTopicsFor(originalTopic: Topic): Set[Topic] - def retryAttempt(topic: Topic, headers: Headers, subscription: ConsumerSubscription)( - implicit trace: Trace - ): UIO[Option[RetryAttempt]] - def retryDecision[E]( - retryAttempt: Option[RetryAttempt], - record: ConsumerRecord[Chunk[Byte], Chunk[Byte]], - error: E, - subscription: ConsumerSubscription + retryAttempt: Option[RetryAttempt], + record: ConsumerRecord[Chunk[Byte], Chunk[Byte]], + error: E, + subscription: ConsumerSubscription )(implicit trace: Trace): URIO[Any, RetryDecision] - def retrySteps = retryTopicsFor("").size + def retrySteps: Int = retryTopicsFor("").size } object NonBlockingRetryHelper { @@ -47,95 +38,81 @@ object NonBlockingRetryHelper { .getOrElse(NonBlockingBackoffPolicy.empty) override def retryTopicsFor(topic: Topic): Set[Topic] = - policy(topic).intervals.indices.foldLeft(Set.empty[String])((acc, attempt) => - acc + s"$topic-$group-retry-$attempt" - ) - - override def retryAttempt(topic: Topic, headers: Headers, subscription: ConsumerSubscription)( - implicit trace: Trace - ): UIO[Option[RetryAttempt]] = { - (for { - submitted <- headers.get(RetryHeader.Submitted, instantDeserializer) - backoff <- headers.get(RetryHeader.Backoff, durationDeserializer) - originalTopic <- headers.get[String](RetryHeader.OriginalTopic, StringSerde) - } yield for { - ta <- topicAttempt(subscription, topic, originalTopic) - TopicAttempt(originalTopic, attempt) = ta - s <- submitted - b <- backoff - } yield RetryAttempt(originalTopic, attempt, s, b)) - .catchAll(_ => ZIO.none) - } - - private def topicAttempt( - subscription: ConsumerSubscription, - topic: Topic, - originalTopicHeader: Option[String] - ) = - subscription match { - case _: Topics => extractTopicAttempt(group, topic) - case _: TopicPattern => - extractTopicAttemptFromPatternRetryTopic(group, topic, originalTopicHeader) - } + policy(topic).intervals.indices.foldLeft(Set.empty[String])((acc, attempt) => acc + s"$topic-$group-retry-$attempt") override def retryDecision[E]( - retryAttempt: Option[RetryAttempt], - record: ConsumerRecord[Chunk[Byte], Chunk[Byte]], - error: E, - subscription: ConsumerSubscription - )(implicit trace: Trace): URIO[Any, RetryDecision] = currentTime.map(now => { - val nextRetryAttempt = retryAttempt.fold(0)(_.attempt + 1) + retryAttempt: Option[RetryAttempt], + record: ConsumerRecord[Chunk[Byte], Chunk[Byte]], + error: E, + subscription: ConsumerSubscription + )(implicit trace: Trace): URIO[Any, RetryDecision] = Clock.instant.map(now => { + val blockingRetriesBefore = RetryAttempt.maxBlockingAttempts( + NonBlockingRetryHelper.originalTopic(record.topic, group), + retryConfig + ).getOrElse(0) + + // attempt if present contains full number of retries + val nextNonBlockingAttempt = retryAttempt.fold(0)(_.attempt + 1 - blockingRetriesBefore) + val nextRetryAttempt = nextNonBlockingAttempt + blockingRetriesBefore val originalTopic = retryAttempt.fold(record.topic)(_.originalTopic) - val retryTopic = subscription match { - case _: TopicPattern => patternRetryTopic(group, nextRetryAttempt) - case _: Topics => fixedRetryTopic(originalTopic, group, nextRetryAttempt) + val retryTopic = subscription match { + case _: TopicPattern => patternRetryTopic(group, nextNonBlockingAttempt) + case _: Topics => fixedRetryTopic(originalTopic, group, nextNonBlockingAttempt) } val topicRetryPolicy = policy(record.topic) topicRetryPolicy.intervals - .lift(nextRetryAttempt) + .lift(nextNonBlockingAttempt) .map { backoff => + val attempt = RetryAttempt( + attempt = nextRetryAttempt, + originalTopic = originalTopic, + submittedAt = now, + backoff = backoff + ) topicRetryPolicy.recordMutate( ProducerRecord( topic = retryTopic, value = record.value, key = record.key, partition = None, - headers = record.headers + - (RetryHeader.Submitted -> toChunk(now.toEpochMilli)) + - (RetryHeader.Backoff -> toChunk(backoff.toMillis)) + - (RetryHeader.OriginalTopic -> toChunk(originalTopic)) + - (RetryHeader.RetryAttempt -> toChunk(nextRetryAttempt)) + headers = record.headers ++ RetryAttempt.toHeaders(attempt) ) ) } .fold[RetryDecision](NoMoreRetries)(RetryWith) }) + } - private def toChunk(long: Long): Chunk[Byte] = - Chunk.fromArray(long.toString.getBytes) - - private def toChunk(str: String): Chunk[Byte] = - Chunk.fromArray(str.getBytes) + private[retry] def attemptNumberFromTopic( + subscription: ConsumerSubscription, + topic: Topic, + originalTopicHeader: Option[String], + group: Group + ) = + subscription match { + case _: Topics => extractTopicAttempt(group, topic) + case _: TopicPattern => + extractTopicAttemptFromPatternRetryTopic(group, topic, originalTopicHeader) } - private def extractTopicAttempt[E](group: Group, inputTopic: Topic) = + private def extractTopicAttempt(group: Group, inputTopic: Topic) = inputTopic.split(s"-$group-retry-").toSeq match { case Seq(topic, attempt) if Try(attempt.toInt).isSuccess => Some(TopicAttempt(topic, attempt.toInt)) case _ => None } - private def extractTopicAttemptFromPatternRetryTopic[E]( - group: Group, - inputTopic: Topic, - originalTopicHeader: Option[String] + private def extractTopicAttemptFromPatternRetryTopic( + group: Group, + inputTopic: Topic, + originalTopicHeader: Option[String] ) = { originalTopicHeader.flatMap(originalTopic => { inputTopic.split(s"__gh_pattern-retry-$group-attempt-").toSeq match { case Seq(_, attempt) if Try(attempt.toInt).isSuccess => Some(TopicAttempt(originalTopic, attempt.toInt)) - case _ => None + case _ => None } }) } @@ -168,49 +145,27 @@ object DelayHeaders { val Backoff = "backOffTimeMs" } -object RetryHeader { - val Submitted = "submitTimestamp" - val Backoff = DelayHeaders.Backoff - val OriginalTopic = "GH_OriginalTopic" - val RetryAttempt = "GH_RetryAttempt" -} - -case class RetryAttempt( - originalTopic: Topic, - attempt: RetryAttemptNumber, - submittedAt: Instant, - backoff: Duration -) { - - def sleep(implicit trace: Trace): URIO[GreyhoundMetrics, Unit] = - (RetryUtil.sleep(submittedAt, backoff) race reportWaitingInIntervals(every = 60.seconds)) - - private def reportWaitingInIntervals(every: Duration) = - report(WaitingForRetry(originalTopic, attempt, submittedAt.toEpochMilli, backoff.toMillis)) - .repeat(spaced(every)) - .unit -} - object RetryUtil { + def sleep(attempt: RetryAttempt)(implicit trace: Trace): URIO[GreyhoundMetrics, Unit] = + sleep(attempt.submittedAt, attempt.backoff) race + report(WaitingForRetry(attempt.originalTopic, attempt.attempt, attempt.submittedAt.toEpochMilli, attempt.backoff.toMillis)) + .repeat(spaced(60.seconds)) + .unit + def sleep(submittedAt: Instant, backoff: Duration)(implicit trace: Trace): URIO[Any, Unit] = { val expiresAt = submittedAt.plus(backoff.asJava) - currentTime + Clock.instant .map(_.isAfter(expiresAt)) .flatMap(expired => if (expired) ZIO.unit else - ZIO.sleep(1.seconds).repeatUntilZIO(_ => currentTime.map(_.isAfter(expiresAt))).unit + ZIO.sleep(1.second).repeatUntilZIO(_ => Clock.instant.map(_.isAfter(expiresAt))).unit ) } } private case class TopicAttempt(originalTopic: Topic, attempt: Int) -object RetryAttempt { - type RetryAttemptNumber = Int - val currentTime = Clock.currentTime(MILLISECONDS).map(Instant.ofEpochMilli) -} - sealed trait RetryDecision object RetryDecision { diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/NonBlockingRetryRecordHandler.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/NonBlockingRetryRecordHandler.scala index b6e410d3..a6ff6560 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/NonBlockingRetryRecordHandler.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/NonBlockingRetryRecordHandler.scala @@ -31,11 +31,12 @@ private[retry] object NonBlockingRetryRecordHandler { retryConfig: RetryConfig, subscription: ConsumerSubscription, nonBlockingRetryHelper: NonBlockingRetryHelper, + groupId: Group, awaitShutdown: TopicPartition => UIO[AwaitShutdown] )(implicit evK: K <:< Chunk[Byte], evV: V <:< Chunk[Byte]): NonBlockingRetryRecordHandler[V, K, R] = new NonBlockingRetryRecordHandler[V, K, R] { override def handle(record: ConsumerRecord[K, V]): ZIO[GreyhoundMetrics with R, Nothing, Any] = { - nonBlockingRetryHelper.retryAttempt(record.topic, record.headers, subscription).flatMap { retryAttempt => + RetryAttempt.extract(record.headers, record.topic, groupId, subscription, Some(retryConfig)).flatMap { retryAttempt => maybeDelayRetry(record, retryAttempt) *> handler.handle(record).catchAll { case Right(_: NonRetriableException) => ZIO.unit @@ -49,14 +50,17 @@ private[retry] object NonBlockingRetryRecordHandler { } private def delayRetry(record: ConsumerRecord[_, _], awaitShutdown: TopicPartition => UIO[AwaitShutdown])( - retryAttempt: RetryAttempt) = + retryAttempt: RetryAttempt + ) = zio.Random.nextInt.flatMap(correlationId => report( WaitingBeforeRetry(record.topic, retryAttempt, record.partition, record.offset, correlationId) ) *> awaitShutdown(record.topicPartition) - .flatMap(_.interruptOnShutdown(retryAttempt.sleep)) - .reporting(r => DoneWaitingBeforeRetry(record.topic, record.partition, record.offset, retryAttempt, r.duration, r.failed, correlationId)) + .flatMap(_.interruptOnShutdown(RetryUtil.sleep(retryAttempt))) + .reporting(r => + DoneWaitingBeforeRetry(record.topic, record.partition, record.offset, retryAttempt, r.duration, r.failed, correlationId) + ) ) override def isHandlingRetryTopicMessage(group: Group, record: ConsumerRecord[K, V]): Boolean = { @@ -71,7 +75,7 @@ private[retry] object NonBlockingRetryRecordHandler { override def handleAfterBlockingFailed( record: ConsumerRecord[K, V] ): ZIO[GreyhoundMetrics with R, Nothing, Any] = { - nonBlockingRetryHelper.retryAttempt(record.topic, record.headers, subscription).flatMap { retryAttempt => + RetryAttempt.extract(record.headers, record.topic, groupId, subscription, Some(retryConfig)).flatMap { retryAttempt => maybeRetry(retryAttempt, BlockingHandlerFailed, record) } } diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryAttempt.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryAttempt.scala new file mode 100644 index 00000000..9de958e8 --- /dev/null +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryAttempt.scala @@ -0,0 +1,97 @@ +package com.wixpress.dst.greyhound.core.consumer.retry + +import com.wixpress.dst.greyhound.core.Serdes.StringSerde +import com.wixpress.dst.greyhound.core._ +import com.wixpress.dst.greyhound.core.consumer.domain.ConsumerSubscription +import com.wixpress.dst.greyhound.core.consumer.retry.NonBlockingRetryHelper.attemptNumberFromTopic +import com.wixpress.dst.greyhound.core.consumer.retry.RetryAttempt.RetryAttemptNumber +import zio._ + +import java.time.Instant + +/** + * Description of a retry attempt + * @param attempt + * contains which attempt is it, starting from 0 including blocking and non-blocking attempts + */ +case class RetryAttempt( + originalTopic: Topic, + attempt: RetryAttemptNumber, + submittedAt: Instant, + backoff: Duration +) + +object RetryHeader { + val Submitted = "submitTimestamp" + val Backoff = DelayHeaders.Backoff + val OriginalTopic = "GH_OriginalTopic" + val RetryAttempt = "GH_RetryAttempt" +} + +object RetryAttempt { + type RetryAttemptNumber = Int + + private def toChunk(str: String): Chunk[Byte] = Chunk.fromArray(str.getBytes) + + def toHeaders(attempt: RetryAttempt): Headers = Headers( + RetryHeader.Submitted -> toChunk(attempt.submittedAt.toEpochMilli.toString), + RetryHeader.Backoff -> toChunk(attempt.backoff.toMillis.toString), + RetryHeader.OriginalTopic -> toChunk(attempt.originalTopic), + RetryHeader.RetryAttempt -> toChunk(attempt.attempt.toString) + ) + + private case class RetryAttemptHeaders( + originalTopic: Option[Topic], + attempt: Option[RetryAttemptNumber], + submittedAt: Option[Instant], + backoff: Option[Duration] + ) + + private def fromHeaders(headers: Headers): Task[RetryAttemptHeaders] = + for { + submitted <- headers.get(RetryHeader.Submitted, instantDeserializer) + backoff <- headers.get(RetryHeader.Backoff, durationDeserializer) + topic <- headers.get[String](RetryHeader.OriginalTopic, StringSerde) + attempt <- headers.get(RetryHeader.RetryAttempt, longDeserializer) + } yield RetryAttemptHeaders(topic, attempt.map(_.toInt), submitted, backoff) + + /** @return None on infinite blocking retries */ + def maxBlockingAttempts(topic: Topic, retryConfig: Option[RetryConfig]): Option[Int] = + retryConfig.map(_.blockingBackoffs(topic)).fold(Option(0)) { + case finite if finite.hasDefiniteSize => Some(finite.size) + case _ => None + } + + /** @return None on infinite retries */ + def maxOverallAttempts(topic: Topic, retryConfig: Option[RetryConfig]): Option[Int] = + maxBlockingAttempts(topic, retryConfig).map { + _ + retryConfig.fold(0)(_.nonBlockingBackoffs(topic).length) + } + + def extract( + headers: Headers, + topic: Topic, + group: Group, + subscription: ConsumerSubscription, + retryConfig: Option[RetryConfig] + )(implicit trace: Trace): UIO[Option[RetryAttempt]] = { + + def maybeNonBlockingAttempt(hs: RetryAttemptHeaders): Option[RetryAttempt] = + for { + submitted <- hs.submittedAt + backoff <- hs.backoff + TopicAttempt(originalTopic, attempt) <- attemptNumberFromTopic(subscription, topic, hs.originalTopic, group) + blockingRetries = maxBlockingAttempts(originalTopic, retryConfig).getOrElse(0) + } yield RetryAttempt(originalTopic, blockingRetries + attempt, submitted, backoff) + + def maybeBlockingAttempt(hs: RetryAttemptHeaders): Option[RetryAttempt] = + for { + submitted <- hs.submittedAt + backoff <- hs.backoff + originalTopic <- hs.originalTopic if originalTopic == topic + attempt <- hs.attempt + } yield RetryAttempt(originalTopic, attempt, submitted, backoff) + + fromHeaders(headers).map { hs => maybeNonBlockingAttempt(hs) orElse maybeBlockingAttempt(hs) } + }.catchAll(_ => ZIO.none) +} diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryConfig.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryConfig.scala index 5632f042..96dacd2c 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryConfig.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryConfig.scala @@ -15,8 +15,24 @@ case class RetryConfig( produceRetryBackoff: Duration = 5.seconds, produceEncryptor: ConsumerRecord[_, _] => UIO[Encryptor] = _ => ZIO.succeed(NoOpEncryptor)(zio.Trace.empty) ) { - def blockingBackoffs(topic: Topic) = - get(topic)(_.blockingBackoffs)(ifEmpty = () => Nil) + def blockingBackoffs(topic: Topic): Seq[ZDuration] = + get(topic) { + case RetryConfigForTopic(FiniteBlockingBackoffPolicy(intervals), _, _) if intervals.nonEmpty => intervals + case RetryConfigForTopic(_, InfiniteFixedBackoff(fixed), _) => Stream.continually(fixed) + case RetryConfigForTopic(_, expMax: InfiniteExponentialBackoffsMaxInterval, _) => + exponentialBackoffs(expMax.initialInterval, expMax.maximalInterval, expMax.backOffMultiplier, infiniteRetryMaxInteval = true) + case RetryConfigForTopic(_, expMult: InfiniteExponentialBackoffsMaxMultiplication, _) => + exponentialBackoffs( + expMult.initialInterval, + expMult.maxMultiplications, + expMult.backOffMultiplier, + infiniteRetryMaxInterval = true + ) + case _ => Nil + }(ifEmpty = Nil) + + def finiteBlockingBackoffs(topic: Topic) = + get(topic)(_.finiteBlockingBackoffs)(ifEmpty = FiniteBlockingBackoffPolicy.empty) def retryType(originalTopic: Topic): RetryType = get(originalTopic)(_.retryType)(ifEmpty = NoRetries) @@ -43,11 +59,46 @@ case class RetryConfig( else ifEmpty ) +} + +trait InfiniteBlockingBackoffPolicy { + def nonEmpty: Boolean +} + +case object EmptyInfiniteBlockingBackoffPolicy extends InfiniteBlockingBackoffPolicy { + override def nonEmpty: Boolean = false +} + +sealed case class InfiniteFixedBackoff(interval: ZDuration) extends InfiniteBlockingBackoffPolicy { + override def nonEmpty: Boolean = true +} + +sealed case class InfiniteExponentialBackoffsMaxInterval( + initialInterval: ZDuration, + maximalInterval: ZDuration, + backOffMultiplier: Float +) extends InfiniteBlockingBackoffPolicy { + override def nonEmpty: Boolean = true +} + +sealed case class InfiniteExponentialBackoffsMaxMultiplication( + initialInterval: ZDuration, + maxMultiplications: Int, + backOffMultiplier: Float +) extends InfiniteBlockingBackoffPolicy { + override def nonEmpty: Boolean = true +} +case class FiniteBlockingBackoffPolicy(intervals: List[ZDuration]) { + def nonEmpty = intervals.nonEmpty +} + +object FiniteBlockingBackoffPolicy { + val empty = FiniteBlockingBackoffPolicy(Nil) } case class NonBlockingBackoffPolicy( - intervals: Seq[ZDuration], + intervals: List[ZDuration], recordMutate: ProducerRecord[Chunk[Byte], Chunk[Byte]] => ProducerRecord[Chunk[Byte], Chunk[Byte]] = identity ) { def nonEmpty = intervals.nonEmpty @@ -59,22 +110,48 @@ object NonBlockingBackoffPolicy { val empty = NonBlockingBackoffPolicy(Nil) } -case class RetryConfigForTopic(blockingBackoffs: () => Seq[ZDuration], nonBlockingBackoffs: NonBlockingBackoffPolicy) { - def nonEmpty: Boolean = blockingBackoffs().nonEmpty || nonBlockingBackoffs.nonEmpty +case class RetryConfigForTopic( + finiteBlockingBackoffs: FiniteBlockingBackoffPolicy, + infiniteBlockingBackoffs: InfiniteBlockingBackoffPolicy, + nonBlockingBackoffs: NonBlockingBackoffPolicy +) { + def nonEmpty: Boolean = finiteBlockingBackoffs.nonEmpty || infiniteBlockingBackoffs.nonEmpty || nonBlockingBackoffs.nonEmpty def retryType: RetryType = - if (blockingBackoffs.apply().nonEmpty) { + if (finiteBlockingBackoffs.nonEmpty) { if (nonBlockingBackoffs.nonEmpty) BlockingFollowedByNonBlocking else Blocking - } else { + } else if (infiniteBlockingBackoffs.nonEmpty) + Blocking + else { NonBlocking } } object RetryConfigForTopic { - val empty = RetryConfigForTopic(() => Nil, NonBlockingBackoffPolicy.empty) + val empty = RetryConfigForTopic(FiniteBlockingBackoffPolicy.empty, EmptyInfiniteBlockingBackoffPolicy, NonBlockingBackoffPolicy.empty) + + def nonBlockingRetryConfigForTopic(intervals: List[ZDuration]) = + RetryConfigForTopic(FiniteBlockingBackoffPolicy.empty, EmptyInfiniteBlockingBackoffPolicy, NonBlockingBackoffPolicy(intervals)) + + def finiteBlockingRetryConfigForTopic(intervals: List[ZDuration]) = + RetryConfigForTopic(FiniteBlockingBackoffPolicy(intervals), EmptyInfiniteBlockingBackoffPolicy, NonBlockingBackoffPolicy.empty) + + def infiniteBlockingRetryConfigForTopic(interval: ZDuration) = + RetryConfigForTopic(FiniteBlockingBackoffPolicy.empty, InfiniteFixedBackoff(interval), NonBlockingBackoffPolicy.empty) + + def infiniteBlockingRetryConfigForTopic( + initialInterval: ZDuration, + maxInterval: ZDuration, + backOffMultiplier: Float + ) = + RetryConfigForTopic( + FiniteBlockingBackoffPolicy.empty, + InfiniteExponentialBackoffsMaxInterval(initialInterval, maxInterval, backOffMultiplier), + NonBlockingBackoffPolicy.empty + ) } object ZRetryConfig { @@ -82,18 +159,27 @@ object ZRetryConfig { forAllTopics( RetryConfigForTopic( nonBlockingBackoffs = NonBlockingBackoffPolicy(firstRetry :: otherRetries.toList), - blockingBackoffs = () => List.empty + finiteBlockingBackoffs = FiniteBlockingBackoffPolicy.empty, + infiniteBlockingBackoffs = EmptyInfiniteBlockingBackoffPolicy ) ) def finiteBlockingRetry(firstRetry: ZDuration, otherRetries: ZDuration*): RetryConfig = forAllTopics( - RetryConfigForTopic(blockingBackoffs = () => firstRetry :: otherRetries.toList, nonBlockingBackoffs = NonBlockingBackoffPolicy.empty) + RetryConfigForTopic( + finiteBlockingBackoffs = FiniteBlockingBackoffPolicy(firstRetry :: otherRetries.toList), + infiniteBlockingBackoffs = EmptyInfiniteBlockingBackoffPolicy, + nonBlockingBackoffs = NonBlockingBackoffPolicy.empty + ) ) def infiniteBlockingRetry(interval: ZDuration): RetryConfig = forAllTopics( - RetryConfigForTopic(blockingBackoffs = () => Stream.continually(interval), nonBlockingBackoffs = NonBlockingBackoffPolicy.empty) + RetryConfigForTopic( + infiniteBlockingBackoffs = InfiniteFixedBackoff(interval), + finiteBlockingBackoffs = FiniteBlockingBackoffPolicy.empty, + nonBlockingBackoffs = NonBlockingBackoffPolicy.empty + ) ) def exponentialBackoffBlockingRetry( @@ -101,32 +187,68 @@ object ZRetryConfig { maximalInterval: ZDuration, backOffMultiplier: Float, infiniteRetryMaxInterval: Boolean - ): RetryConfig = + ): RetryConfig = { + val (finite, infinite) = if (infiniteRetryMaxInterval) { + ( + FiniteBlockingBackoffPolicy.empty, + InfiniteExponentialBackoffsMaxInterval(initialInterval, maximalInterval, backOffMultiplier) + ) + } else { + ( + FiniteBlockingBackoffPolicy( + exponentialBackoffs(initialInterval, maximalInterval, backOffMultiplier, infiniteRetryMaxInterval).toList + ), + EmptyInfiniteBlockingBackoffPolicy + ) + } forAllTopics( RetryConfigForTopic( - blockingBackoffs = () => exponentialBackoffs(initialInterval, maximalInterval, backOffMultiplier, infiniteRetryMaxInterval), + finiteBlockingBackoffs = finite, + infiniteBlockingBackoffs = infinite, nonBlockingBackoffs = NonBlockingBackoffPolicy.empty ) ) + } def exponentialBackoffBlockingRetry( initialInterval: ZDuration, maxMultiplications: Int, backOffMultiplier: Float, infiniteRetryMaxInterval: Boolean - ): RetryConfig = + ): RetryConfig = { + val (finite, infinite) = if (infiniteRetryMaxInterval) { + ( + FiniteBlockingBackoffPolicy.empty, + InfiniteExponentialBackoffsMaxMultiplication(initialInterval, maxMultiplications, backOffMultiplier) + ) + } else { + ( + FiniteBlockingBackoffPolicy( + exponentialBackoffs(initialInterval, maxMultiplications, backOffMultiplier, infiniteRetryMaxInterval).toList + ), + EmptyInfiniteBlockingBackoffPolicy + ) + } forAllTopics( RetryConfigForTopic( - blockingBackoffs = () => exponentialBackoffs(initialInterval, maxMultiplications, backOffMultiplier, infiniteRetryMaxInterval), + finiteBlockingBackoffs = finite, + infiniteBlockingBackoffs = infinite, nonBlockingBackoffs = NonBlockingBackoffPolicy.empty ) ) + } def blockingFollowedByNonBlockingRetry( - blockingBackoffs: NonEmptyList[ZDuration], + blockingBackoffs: FiniteBlockingBackoffPolicy, nonBlockingBackoffs: NonBlockingBackoffPolicy ): RetryConfig = - forAllTopics(RetryConfigForTopic(blockingBackoffs = () => blockingBackoffs, nonBlockingBackoffs = nonBlockingBackoffs)) + forAllTopics( + RetryConfigForTopic( + finiteBlockingBackoffs = blockingBackoffs, + nonBlockingBackoffs = nonBlockingBackoffs, + infiniteBlockingBackoffs = EmptyInfiniteBlockingBackoffPolicy + ) + ) def perTopicRetries(configs: PartialFunction[Topic, RetryConfigForTopic]) = RetryConfig(configs, None) @@ -178,7 +300,7 @@ object RetryConfig { def blockingFollowedByNonBlockingRetry(blockingBackoffs: NonEmptyList[Duration], nonBlockingBackoffs: List[Duration]): RetryConfig = ZRetryConfig.blockingFollowedByNonBlockingRetry( - blockingBackoffs = blockingBackoffs.map(ZDuration.fromScala), + blockingBackoffs = FiniteBlockingBackoffPolicy(blockingBackoffs.map(ZDuration.fromScala)), nonBlockingBackoffs = NonBlockingBackoffPolicy(nonBlockingBackoffs.map(ZDuration.fromScala)) ) } diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryRecordHandler.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryRecordHandler.scala index af0749c6..8ee8ab9f 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryRecordHandler.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryRecordHandler.scala @@ -33,7 +33,7 @@ object RetryRecordHandler { ): RecordHandler[R with R2 with GreyhoundMetrics, Nothing, K, V] = { val nonBlockingHandler = - NonBlockingRetryRecordHandler(handler, producer, retryConfig, subscription, nonBlockingRetryHelper, awaitShutdown) + NonBlockingRetryRecordHandler(handler, producer, retryConfig, subscription, nonBlockingRetryHelper, groupId, awaitShutdown) val blockingHandler = BlockingRetryRecordHandler(groupId, handler, retryConfig, blockingState, nonBlockingHandler, awaitShutdown) val blockingAndNonBlockingHandler = BlockingAndNonBlockingRetryRecordHandler(groupId, blockingHandler, nonBlockingHandler) @@ -55,15 +55,4 @@ object RetryRecordHandler { record.headers.get[String](key, StringSerde).catchAll(_ => ZIO.none) } -object ZIOHelper { - def foreachWhile[R, E, A](as: Iterable[A])(f: A => ZIO[R, E, LastHandleResult])(implicit trace: Trace): ZIO[R, E, LastHandleResult] = - ZIO.succeed(as.iterator).flatMap { i => - def loop: ZIO[R, E, LastHandleResult] = - if (i.hasNext) f(i.next).flatMap(result => if (result.shouldContinue) loop else ZIO.succeed(result)) - else ZIO.succeed(LastHandleResult(lastHandleSucceeded = false, shouldContinue = false)) - - loop - } -} - case class LastHandleResult(lastHandleSucceeded: Boolean, shouldContinue: Boolean) diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryRecordHandlerMetric.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryRecordHandlerMetric.scala index 980a71f7..b1580b16 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryRecordHandlerMetric.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryRecordHandlerMetric.scala @@ -20,7 +20,8 @@ object RetryRecordHandlerMetric { case class NoRetryOnNonRetryableFailure(partition: TopicPartition, offset: Long, cause: Exception) extends RetryRecordHandlerMetric case object Silent extends RetryRecordHandlerMetric - case class WaitingBeforeRetry(retryTopic: Topic, retryAttempt: RetryAttempt, partition: Int, offset:Long, correlationId: Int) extends RetryRecordHandlerMetric + case class WaitingBeforeRetry(retryTopic: Topic, retryAttempt: RetryAttempt, partition: Int, offset: Long, correlationId: Int) + extends RetryRecordHandlerMetric case class DoneWaitingBeforeRetry( retryTopic: Topic, diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/producer/Producer.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/producer/Producer.scala index 2200b6cc..538c0b81 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/producer/Producer.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/producer/Producer.scala @@ -5,13 +5,15 @@ import org.apache.kafka.clients.producer.{Callback, KafkaProducer, ProducerConfi import org.apache.kafka.common.header.Header import org.apache.kafka.common.header.internals.RecordHeader import org.apache.kafka.common.serialization.ByteArraySerializer +import org.apache.kafka.common.{Metric, MetricName} +import zio.ZIO.attemptBlocking import zio._ import scala.collection.JavaConverters._ -import zio.ZIO.attemptBlocking -import zio.managed._ trait ProducerR[-R] { self => + def metrics : UIO[Option[Map[MetricName, Metric]]] = ZIO.none + def produceAsync(record: ProducerRecord[Chunk[Byte], Chunk[Byte]])( implicit trace: Trace ): ZIO[R, ProducerError, IO[ProducerError, RecordMetadata]] @@ -80,6 +82,9 @@ object Producer { val acquire = ZIO.attemptBlocking(new KafkaProducer(config.properties, serializer, serializer)) ZIO.acquireRelease(acquire)(producer => attemptBlocking(producer.close()).ignore).map { producer => new ProducerR[R] { + override def metrics: UIO[Option[Map[MetricName, Metric]]] = + ZIO.succeed(Option(producer.metrics().asScala.toMap)) + private def recordFrom(record: ProducerRecord[Chunk[Byte], Chunk[Byte]]) = new KafkaProducerRecord( record.topic, @@ -153,6 +158,8 @@ object ProducerR { override def partitionsFor(topic: Topic)(implicit trace: Trace): RIO[Any, Seq[PartitionInfo]] = producer.partitionsFor(topic).provideEnvironment(env) + + override def metrics: UIO[Option[Map[MetricName, Metric]]] = producer.metrics } def onShutdown(onShutdown: => UIO[Unit])(implicit trace: Trace): ProducerR[R] = new ProducerR[R] { override def produceAsync( @@ -165,6 +172,8 @@ object ProducerR { override def attributes: Map[String, String] = producer.attributes override def partitionsFor(topic: Topic)(implicit trace: Trace) = producer.partitionsFor(topic) + + override def metrics: UIO[Option[Map[MetricName, Metric]]] = producer.metrics } def tapBoth(onError: (Topic, Cause[ProducerError]) => URIO[R, Unit], onSuccess: RecordMetadata => URIO[R, Unit]) = new ProducerR[R] { @@ -186,6 +195,8 @@ object ProducerR { override def attributes: Map[String, String] = producer.attributes override def partitionsFor(topic: Topic)(implicit trace: Trace) = producer.partitionsFor(topic) + + override def metrics: UIO[Option[Map[MetricName, Metric]]] = producer.metrics } def map(f: ProducerRecord[Chunk[Byte], Chunk[Byte]] => ProducerRecord[Chunk[Byte], Chunk[Byte]]) = new ProducerR[R] { @@ -199,6 +210,8 @@ object ProducerR { override def shutdown(implicit trace: Trace): UIO[Unit] = producer.shutdown override def attributes: Map[String, String] = producer.attributes + + override def metrics: UIO[Option[Map[MetricName, Metric]]] = producer.metrics } } diff --git a/core/src/main/scala/com/wixpress/dst/greyhound/core/producer/ReportingProducer.scala b/core/src/main/scala/com/wixpress/dst/greyhound/core/producer/ReportingProducer.scala index 10eab6a7..69306db1 100644 --- a/core/src/main/scala/com/wixpress/dst/greyhound/core/producer/ReportingProducer.scala +++ b/core/src/main/scala/com/wixpress/dst/greyhound/core/producer/ReportingProducer.scala @@ -5,14 +5,18 @@ import java.util.concurrent.TimeUnit.MILLISECONDS import com.wixpress.dst.greyhound.core.PartitionInfo import com.wixpress.dst.greyhound.core.metrics.{GreyhoundMetric, GreyhoundMetrics} import com.wixpress.dst.greyhound.core.producer.ProducerMetric._ -import zio.{Chunk, IO, RIO, Trace, ULayer, ZIO} +import zio.{Chunk, IO, RIO, Trace, UIO, ULayer, ZIO} import GreyhoundMetrics._ +import org.apache.kafka.common.{Metric, MetricName} import scala.concurrent.duration.FiniteDuration import zio.Clock.currentTime case class ReportingProducer[-R](internal: ProducerR[R], extraAttributes: Map[String, String]) extends ProducerR[GreyhoundMetrics with R] { + + override def metrics: UIO[Option[Map[MetricName, Metric]]] = internal.metrics + override def produceAsync( record: ProducerRecord[Chunk[Byte], Chunk[Byte]] )(implicit trace: Trace): ZIO[GreyhoundMetrics with R, ProducerError, IO[ProducerError, RecordMetadata]] = diff --git a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/BUILD.bazel b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/BUILD.bazel index 90ed4056..3e104e18 100644 --- a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/BUILD.bazel +++ b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/BUILD.bazel @@ -11,17 +11,14 @@ specs2_unit_test( deps = [ "@dev_zio_zio_managed_2_12", "@dev_zio_zio_stacktracer_2_12", - "//core/src/it/scala/com/wixpress/dst/greyhound/testkit", "//core/src/main/scala/com/wixpress/dst/greyhound/core", "//core/src/main/scala/com/wixpress/dst/greyhound/core/consumer", "//core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/domain", "//core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry", "//core/src/main/scala/com/wixpress/dst/greyhound/core/metrics", - "//core/src/main/scala/com/wixpress/dst/greyhound/core/producer", "//core/src/main/scala/com/wixpress/dst/greyhound/core/zioutils", "//core/src/test/resources", "//core/src/test/scala/com/wixpress/dst/greyhound/core/testkit", - "@ch_qos_logback_logback_classic", # "@dev_zio_izumi_reflect_2_12", "@dev_zio_zio_2_12", "@dev_zio_zio_streams_2_12", diff --git a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/EventLoopTest.scala b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/EventLoopTest.scala index 8331d84b..aa335fde 100644 --- a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/EventLoopTest.scala +++ b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/EventLoopTest.scala @@ -9,7 +9,7 @@ import com.wixpress.dst.greyhound.core.consumer.domain.{ConsumerRecord, RecordHa import com.wixpress.dst.greyhound.core.metrics.GreyhoundMetrics import com.wixpress.dst.greyhound.core.testkit.{BaseTest, TestMetrics} import com.wixpress.dst.greyhound.core.zioutils.AwaitShutdown.ShutdownPromise -import com.wixpress.dst.greyhound.core.{Headers, Offset, Topic, TopicPartition} +import com.wixpress.dst.greyhound.core.{Headers, Offset, OffsetAndMetadata, Topic, TopicPartition} import zio._ import java.util.regex.Pattern @@ -139,11 +139,21 @@ trait EmptyConsumer extends Consumer { override def commit(offsets: Map[TopicPartition, Offset])(implicit trace: Trace): Task[Unit] = ZIO.unit + override def commitWithMetadata(offsetsAndMetadata: Map[TopicPartition, OffsetAndMetadata])( + implicit trace: Trace + ): RIO[GreyhoundMetrics, Unit] = + ZIO.unit + override def commitOnRebalance(offsets: Map[TopicPartition, Offset])( implicit trace: Trace ): RIO[GreyhoundMetrics, DelayedRebalanceEffect] = DelayedRebalanceEffect.zioUnit + override def commitWithMetadataOnRebalance(offsets: Map[TopicPartition, OffsetAndMetadata])( + implicit trace: Trace + ): RIO[GreyhoundMetrics, DelayedRebalanceEffect] = + DelayedRebalanceEffect.zioUnit + override def pause(partitions: Set[TopicPartition])(implicit trace: Trace): ZIO[Any, IllegalStateException, Unit] = ZIO.unit @@ -173,4 +183,7 @@ trait EmptyConsumer extends Consumer { override def committedOffsets(partitions: Set[TopicPartition])(implicit trace: Trace): RIO[Any, Map[TopicPartition, Offset]] = ZIO.succeed(Map.empty) + + override def committedOffsetsAndMetadata(partitions: Set[TopicPartition])(implicit trace: Trace): RIO[Any, Map[TopicPartition, OffsetAndMetadata]] = + ZIO.succeed(Map.empty) } diff --git a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsAndGapsTest.scala b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsAndGapsTest.scala new file mode 100644 index 00000000..2a3837ca --- /dev/null +++ b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsAndGapsTest.scala @@ -0,0 +1,84 @@ +package com.wixpress.dst.greyhound.core.consumer + +import com.wixpress.dst.greyhound.core.TopicPartition +import com.wixpress.dst.greyhound.core.consumer.Gap.GAP_SEPARATOR +import com.wixpress.dst.greyhound.core.consumer.OffsetAndGaps.LAST_HANDLED_OFFSET_SEPARATOR +import com.wixpress.dst.greyhound.core.consumer.OffsetGapsTest._ +import com.wixpress.dst.greyhound.core.testkit.BaseTestNoEnv + +class OffsetsAndGapsTestGapsTest extends BaseTestNoEnv { + + "calculate gaps created by handled batch" in { + for { + offsetGaps <- OffsetsAndGaps.make + _ <- offsetGaps.update(topicPartition, Seq(1L, 3L, 7L)) + currentGaps <- offsetGaps.gapsForPartition(topicPartition) + } yield currentGaps must beEqualTo(Seq(Gap(0L, 0L), Gap(2L, 2L), Gap(4L, 6L))) + } + + "update offset and gaps according to handled batch" in { + for { + offsetGaps <- OffsetsAndGaps.make + _ <- offsetGaps.update(topicPartition, Seq(1L, 3L, 7L)) + _ <- offsetGaps.update(topicPartition, Seq(2L, 5L)) + getCommittableAndClear <- offsetGaps.getCommittableAndClear + } yield getCommittableAndClear must havePair(topicPartition -> OffsetAndGaps(7L, Seq(Gap(0L, 0L), Gap(4L, 4L), Gap(6L, 6L)))) + } + + "clear committable offsets" in { + for { + offsetGaps <- OffsetsAndGaps.make + _ <- offsetGaps.update(topicPartition, Seq(1L, 3L, 7L)) + _ <- offsetGaps.getCommittableAndClear + getCommittableAndClear <- offsetGaps.getCommittableAndClear + } yield getCommittableAndClear must beEmpty + } + + "do not clear gaps on retrieving current" in { + for { + offsetGaps <- OffsetsAndGaps.make + _ <- offsetGaps.update(topicPartition, Seq(1L, 3L, 7L)) + _ <- offsetGaps.gapsForPartition(topicPartition) + currentGaps <- offsetGaps.gapsForPartition(topicPartition) + } yield currentGaps must beEqualTo(Seq(Gap(0L, 0L), Gap(2L, 2L), Gap(4L, 6L))) + } + + "update with larger offset" in { + val partition0 = TopicPartition(topic, 0) + val partition1 = TopicPartition(topic, 1) + + for { + offsetGaps <- OffsetsAndGaps.make + _ <- offsetGaps.update(partition0, Seq(1L)) + _ <- offsetGaps.update(partition0, Seq(0L)) + _ <- offsetGaps.update(partition1, Seq(0L)) + current <- offsetGaps.getCommittableAndClear + } yield current must havePairs(partition0 -> OffsetAndGaps(1L, Seq()), partition1 -> OffsetAndGaps(0L, Seq())) + } + + "init with given offsets and calculate subsequent gaps accordingly" in { + val partition0 = TopicPartition(topic, 0) + val partition1 = TopicPartition(topic, 1) + val initialCommittedOffsets = + Map(partition0 -> OffsetAndGaps(100L, committable = false), partition1 -> OffsetAndGaps(200L, committable = false)) + for { + offsetGaps <- OffsetsAndGaps.make + _ <- offsetGaps.init(initialCommittedOffsets) + _ <- offsetGaps.update(partition0, Seq(101L, 102L)) + _ <- offsetGaps.update(partition1, Seq(203L, 204L)) + current <- offsetGaps.getCommittableAndClear + } yield current must havePairs(partition0 -> OffsetAndGaps(102L, Seq()), partition1 -> OffsetAndGaps(204L, Seq(Gap(201L, 202L)))) + } + + "parse gaps from string" in { + val gaps = Seq(s"10${LAST_HANDLED_OFFSET_SEPARATOR}0${GAP_SEPARATOR}1", s"10${LAST_HANDLED_OFFSET_SEPARATOR}", "") + val expected = Seq(Some(OffsetAndGaps(10, Seq(Gap(0, 1)))), Some(OffsetAndGaps(10, Seq())), None) + gaps.map(OffsetsAndGaps.parseGapsString).must(beEqualTo(expected)) + } + +} + +object OffsetGapsTest { + val topic = "some-topic" + val topicPartition = TopicPartition(topic, 0) +} diff --git a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsInitializerTest.scala b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsInitializerTest.scala index d8c0c28e..2b8ffb0f 100644 --- a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsInitializerTest.scala +++ b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/OffsetsInitializerTest.scala @@ -2,7 +2,7 @@ package com.wixpress.dst.greyhound.core.consumer import java.time.{Clock, Duration, ZoneId} import java.util.concurrent.atomic.AtomicReference -import com.wixpress.dst.greyhound.core.TopicPartition +import com.wixpress.dst.greyhound.core.{OffsetAndMetadata, TopicPartition} import com.wixpress.dst.greyhound.core.consumer.ConsumerMetric.{CommittedMissingOffsets, CommittedMissingOffsetsFailed} import com.wixpress.dst.greyhound.core.consumer.SeekTo.{SeekToEnd, SeekToOffset} import com.wixpress.dst.greyhound.core.metrics.GreyhoundMetric @@ -16,7 +16,7 @@ class OffsetsInitializerTest extends SpecificationWithJUnit with Mockito { private val Seq(p1, p2, p3) = Seq("t1" -> 1, "t2" -> 2, "t3" -> 3).map(tp => TopicPartition(tp._1, tp._2)) private val partitions = Set(p1, p2, p3) private val p1Pos, p2Pos, p3Pos = randomInt.toLong - val epochTimeToRewind = 1000L + val epochTimeToRewind = 1000L "do nothing if no missing offsets" in new ctx { @@ -40,8 +40,8 @@ class OffsetsInitializerTest extends SpecificationWithJUnit with Mockito { p3 -> p3Pos ) there was - one(offsetOps).commit( - missingOffsets, + one(offsetOps).commitWithMetadata( + missingOffsets.mapValues(OffsetAndMetadata(_)), timeout ) @@ -57,8 +57,8 @@ class OffsetsInitializerTest extends SpecificationWithJUnit with Mockito { val expected = Map(p1 -> p1Pos, p3 -> p3Pos) there was - one(offsetOps).commit( - expected, + one(offsetOps).commitWithMetadata( + expected.mapValues(OffsetAndMetadata(_)), timeoutIfSeek ) @@ -79,8 +79,8 @@ class OffsetsInitializerTest extends SpecificationWithJUnit with Mockito { val expected = Map(p1 -> p1Pos, p3 -> p3Pos, p2 -> p2Pos) there was - one(offsetOps).commit( - expected, + one(offsetOps).commitWithMetadata( + expected.mapValues(OffsetAndMetadata(_)), timeoutIfSeek ) there was one(offsetOps).seek(Map(p1 -> p1Pos, p2 -> p2Pos)) @@ -93,27 +93,27 @@ class OffsetsInitializerTest extends SpecificationWithJUnit with Mockito { "fail if operation fails and there are relevant seekTo offsets" in new ctx(seekTo = Map(p1 -> SeekToOffset(p1Pos))) { val e = new RuntimeException(randomStr) - offsetOps.committed(any(), any()) throws e + offsetOps.committedWithMetadata(any(), any()) throws e committer.initializeOffsets(partitions) must throwA(e) reported must contain(CommittedMissingOffsetsFailed(clientId, group, partitions, Map.empty, elapsed = Duration.ZERO, e)) } - "report errors in `commit()`, but not fail" in + "report errors in `commitWithMetadata()`, but not fail" in new ctx { val e = new RuntimeException(randomStr) givenCommittedOffsets(partitions)(Map(p1 -> randomInt)) val p2Pos, p3Pos = randomInt.toLong givenPositions(p2 -> p2Pos, p3 -> p3Pos) - offsetOps.commit(any(), any()) throws e + offsetOps.commitWithMetadata(any(), any()) throws e committer.initializeOffsets(partitions) reported must contain(CommittedMissingOffsetsFailed(clientId, group, partitions, Map.empty, elapsed = Duration.ZERO, e)) } - "report errors in `commit()`, but not fail" in + "report errors in `commitWithMetadata()`, but not fail" in new ctx { val e = new RuntimeException(randomStr) givenCommittedOffsets(partitions)(Map(p1 -> randomInt)) @@ -124,71 +124,73 @@ class OffsetsInitializerTest extends SpecificationWithJUnit with Mockito { reported must contain(CommittedMissingOffsetsFailed(clientId, group, partitions, Map.empty, elapsed = Duration.ZERO, e)) } - "rewind uncommitted offsets" in new ctx { - givenCommittedOffsets(partitions)(Map(p2 -> randomInt)) - givenPositions(p2 -> p2Pos, p3 -> p3Pos) - givenOffsetsForTimes(epochTimeToRewind, p1 -> 0L, p2 -> 1L) - - committer.initializeOffsets(partitions) + "rewind uncommitted offsets" in + new ctx { + givenCommittedOffsets(partitions)(Map(p2 -> randomInt)) + givenPositions(p2 -> p2Pos, p3 -> p3Pos) + givenOffsetsForTimes(epochTimeToRewind, p1 -> 0L, p2 -> 1L) - val missingOffsets = Map( - p1 -> p1Pos, - p3 -> p3Pos - ) + committer.initializeOffsets(partitions) - val rewindedOffsets = Map( - p1 -> 0L, - ) + val missingOffsets = Map( + p1 -> p1Pos, + p3 -> p3Pos + ) - there was - one(offsetOps).commit( - missingOffsets ++ rewindedOffsets, - timeout + val rewindedOffsets = Map( + p1 -> 0L ) - } - "rewind to endOffsets for uncommitted partitions when offsetsForTimes return null offsets " in new ctx { - givenCommittedOffsets(partitions)(Map(p2 -> randomInt, p3 -> randomInt)) - givenPositions(p3 -> p3Pos) - givenEndOffsets(partitions, timeout)(Map(p1 -> p1Pos)) - givenOffsetsForTimes(Set(p1))(p1 -> None /*kafka SDK returned null*/) + there was + one(offsetOps).commitWithMetadata( + (missingOffsets ++ rewindedOffsets).mapValues(OffsetAndMetadata(_)), + timeout + ) + } - committer.initializeOffsets(partitions) + "rewind to endOffsets for uncommitted partitions when offsetsForTimes return null offsets " in + new ctx { + givenCommittedOffsets(partitions)(Map(p2 -> randomInt, p3 -> randomInt)) + givenPositions(p3 -> p3Pos) + givenEndOffsets(partitions, timeout)(Map(p1 -> p1Pos)) + givenOffsetsForTimes(Set(p1))(p1 -> None /*kafka SDK returned null*/ ) - val committedOffsets = Map( - p1 -> p1Pos, - ) + committer.initializeOffsets(partitions) - there was - one(offsetOps).commit( - committedOffsets, - timeout + val committedOffsets = Map( + p1 -> p1Pos ) - } - "not rewind uncommitted offsets when offset reset is earliest" in new ctx(offsetReset = OffsetReset.Earliest) { - givenCommittedOffsets(partitions)(Map(p2 -> randomInt)) - givenPositions(p2 -> p2Pos, p3 -> p3Pos) - givenOffsetsForTimes(epochTimeToRewind, p1 -> 0L, p2 -> 1L) + there was + one(offsetOps).commitWithMetadata( + committedOffsets.mapValues(OffsetAndMetadata(_)), + timeout + ) + } - committer.initializeOffsets(partitions) + "not rewind uncommitted offsets when offset reset is earliest" in + new ctx(offsetReset = OffsetReset.Earliest) { + givenCommittedOffsets(partitions)(Map(p2 -> randomInt)) + givenPositions(p2 -> p2Pos, p3 -> p3Pos) + givenOffsetsForTimes(epochTimeToRewind, p1 -> 0L, p2 -> 1L) - val missingOffsets = Map( - p1 -> p1Pos, - p3 -> p3Pos - ) + committer.initializeOffsets(partitions) - val rewindedOffsets = Map( - p1 -> 0L, - ) + val missingOffsets = Map( + p1 -> p1Pos, + p3 -> p3Pos + ) - there was - one(offsetOps).commit( - missingOffsets ++ rewindedOffsets, - timeout + val rewindedOffsets = Map( + p1 -> 0L ) - } + there was + one(offsetOps).commitWithMetadata( + (missingOffsets ++ rewindedOffsets).mapValues(OffsetAndMetadata(_)), + timeout + ) + } class ctx(val seekTo: Map[TopicPartition, SeekTo] = Map.empty, offsetReset: OffsetReset = OffsetReset.Latest) extends Scope { private val metricsLogRef = new AtomicReference(Seq.empty[GreyhoundMetric]) @@ -212,7 +214,8 @@ class OffsetsInitializerTest extends SpecificationWithJUnit with Mockito { if (seekTo == Map.empty) InitialOffsetsSeek.default else (_, _, _, _) => seekTo, rewindUncommittedOffsetsBy = zio.Duration.fromMillis(15 * 60 * 1000), clock, - offsetResetIsEarliest = offsetReset == OffsetReset.Earliest + offsetResetIsEarliest = offsetReset == OffsetReset.Earliest, + false ) def randomOffsets(partitions: Set[TopicPartition]) = partitions.map(p => p -> randomInt.toLong).toMap @@ -220,7 +223,7 @@ class OffsetsInitializerTest extends SpecificationWithJUnit with Mockito { def givenCommittedOffsets(partitions: Set[TopicPartition], timeout: zio.Duration = timeout)( result: Map[TopicPartition, Long] ) = { - offsetOps.committed(partitions, timeout) returns result + offsetOps.committedWithMetadata(partitions, timeout) returns result.mapValues(OffsetAndMetadata(_)) } def givenEndOffsets(partitions: Set[TopicPartition], timeout: zio.Duration = timeout)(result: Map[TopicPartition, Long]) = { @@ -256,4 +259,4 @@ class OffsetsInitializerTest extends SpecificationWithJUnit with Mockito { private def randomStr = Random.alphanumeric.take(5).mkString private def randomInt = Random.nextInt(200) private def randomPartition = TopicPartition(randomStr, randomInt) -} \ No newline at end of file +} diff --git a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/batched/BatchEventLoopTest.scala b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/batched/BatchEventLoopTest.scala index 19f4b43a..bb1144cc 100644 --- a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/batched/BatchEventLoopTest.scala +++ b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/batched/BatchEventLoopTest.scala @@ -1,6 +1,6 @@ package com.wixpress.dst.greyhound.core.consumer.batched -import com.wixpress.dst.greyhound.core.{Offset, Topic, TopicPartition} +import com.wixpress.dst.greyhound.core.{Offset, OffsetAndMetadata, Topic, TopicPartition} import com.wixpress.dst.greyhound.core.consumer.Consumer.Records import com.wixpress.dst.greyhound.core.consumer.batched.BatchConsumer.RecordBatch import com.wixpress.dst.greyhound.core.consumer.batched.BatchEventLoopMetric.{FullBatchHandled, RecordsHandled} @@ -162,6 +162,11 @@ class BatchEventLoopTest extends JUnitRunnableSpec { ZIO.succeed(println(s"commit($offsets)")) *> committedOffsetsRef.update(_ ++ offsets) } + override def commitWithMetadata(offsetsAndMetadata: Map[TopicPartition, OffsetAndMetadata])( + implicit trace: Trace + ): RIO[GreyhoundMetrics, Unit] = + committedOffsetsRef.update(_ ++ offsetsAndMetadata.map { case (tp, om) => tp -> om.offset }) + override def commitOnRebalance( offsets: Map[TopicPartition, Offset] )(implicit trace: Trace): RIO[GreyhoundMetrics, DelayedRebalanceEffect] = { @@ -173,7 +178,18 @@ class BatchEventLoopTest extends JUnitRunnableSpec { ) } } + + override def commitWithMetadataOnRebalance( + offsets: Map[TopicPartition, OffsetAndMetadata] + )(implicit trace: Trace): RIO[GreyhoundMetrics, DelayedRebalanceEffect] = { + ZIO.runtime[Any].flatMap { rt => + ZIO.succeed(DelayedRebalanceEffect(zio.Unsafe.unsafe { implicit s => + rt.unsafe.run(committedOffsetsRef.update(_ ++ offsets.mapValues(_.offset))).getOrThrowFiberFailure() + })) + } + } } + val handler = new BatchRecordHandler[Any, Throwable, Chunk[Byte], Chunk[Byte]] { override def handle(records: RecordBatch): ZIO[Any, HandleError[Throwable], Any] = { ZIO.succeed(println(s"handle($records)")) *> diff --git a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/dispatcher/DispatcherTest.scala b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/dispatcher/DispatcherTest.scala index d4db6885..5a89a1b7 100644 --- a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/dispatcher/DispatcherTest.scala +++ b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/dispatcher/DispatcherTest.scala @@ -29,7 +29,9 @@ class DispatcherTest extends BaseTest[TestMetrics with TestClock] { run(for { promise <- Promise.make[Nothing, Record] ref <- Ref.make[Map[TopicPartition, ShutdownPromise]](Map.empty) - dispatcher <- Dispatcher.make("group", "clientId", promise.succeed, lowWatermark, highWatermark, workersShutdownRef = ref) + init <- getInit + dispatcher <- + Dispatcher.make("group", "clientId", promise.succeed, lowWatermark, highWatermark, workersShutdownRef = ref, init = init) _ <- submit(dispatcher, record) handled <- promise.await } yield handled must equalTo(record)) @@ -43,18 +45,77 @@ class DispatcherTest extends BaseTest[TestMetrics with TestClock] { latch <- CountDownLatch.make(partitions) slowHandler = { _: ConsumerRecord[Chunk[Byte], Chunk[Byte]] => Clock.sleep(1.second) *> latch.countDown } ref <- Ref.make[Map[TopicPartition, ShutdownPromise]](Map.empty) - dispatcher <- Dispatcher.make("group", "clientId", slowHandler, lowWatermark, highWatermark, workersShutdownRef = ref) + init <- getInit + dispatcher <- + Dispatcher.make("group", "clientId", slowHandler, lowWatermark, highWatermark, workersShutdownRef = ref, init = init) _ <- ZIO.foreachDiscard(0 until partitions) { partition => submit(dispatcher, record.copy(partition = partition)) } _ <- TestClock.adjust(1.second) _ <- latch.await } yield ok) // If execution is not parallel, the latch will not be released } + "parallelize single partition handling based on key when using parallel consumer" in + new ctx(highWatermark = 10) { + val numKeys = 8 + val keys = getKeys(numKeys) + + run(for { + latch <- CountDownLatch.make(numKeys) + slowHandler = { _: ConsumerRecord[Chunk[Byte], Chunk[Byte]] => Clock.sleep(1.second) *> latch.countDown } + ref <- Ref.make[Map[TopicPartition, ShutdownPromise]](Map.empty) + init <- getInit + dispatcher <- Dispatcher.make( + "group", + "clientId", + slowHandler, + lowWatermark, + highWatermark, + workersShutdownRef = ref, + consumeInParallel = true, + maxParallelism = 8, + init = init + ) + // produce with unique keys to the same partition + _ <- submitBatch(dispatcher, keys.map(key => record.copy(partition = 0, key = key))) + _ <- TestClock.adjust(1.second) + _ <- latch.await + } yield ok) // if execution is not parallel, the latch will not be released + } + + "parallelize single partition handling of no-key records when using parallel consumer" in + new ctx(highWatermark = 10) { + val numRecords = 8 + + run(for { + latch <- CountDownLatch.make(numRecords) + slowHandler = { _: ConsumerRecord[Chunk[Byte], Chunk[Byte]] => Clock.sleep(1.second) *> latch.countDown } + ref <- Ref.make[Map[TopicPartition, ShutdownPromise]](Map.empty) + init <- getInit + dispatcher <- Dispatcher.make( + "group", + "clientId", + slowHandler, + lowWatermark, + highWatermark, + workersShutdownRef = ref, + consumeInParallel = true, + maxParallelism = numRecords, + init = init + ) + _ <- submitBatch(dispatcher, (1 to numRecords).map(_ => record.copy(partition = 0, key = None))) + _ <- TestClock.adjust(1.second) + _ <- latch.await + } yield ok) // if execution is not parallel, the latch will not be released + } + "reject records when high watermark is reached" in new ctx() { run(for { ref <- Ref.make[Map[TopicPartition, ShutdownPromise]](Map.empty) - dispatcher <- Dispatcher.make[Any]("group", "clientId", _ => ZIO.never, lowWatermark, highWatermark, workersShutdownRef = ref) + init <- getInit + dispatcher <- + Dispatcher + .make[Any]("group", "clientId", _ => ZIO.never, lowWatermark, highWatermark, workersShutdownRef = ref, init = init) _ <- submit(dispatcher, record.copy(offset = 0L)) // Will be polled _ <- submit(dispatcher, record.copy(offset = 1L)) _ <- submit(dispatcher, record.copy(offset = 2L)) @@ -65,19 +126,34 @@ class DispatcherTest extends BaseTest[TestMetrics with TestClock] { } yield (result1 must equalTo(SubmitResult.Rejected)) or (result2 must equalTo(SubmitResult.Rejected))) } + "reject records and return first rejected when high watermark is reached on batch submission" in + new ctx(highWatermark = 5) { + run(for { + ref <- Ref.make[Map[TopicPartition, ShutdownPromise]](Map.empty) + init <- getInit + dispatcher <- + Dispatcher + .make[Any]("group", "clientId", _ => ZIO.never, lowWatermark, highWatermark, workersShutdownRef = ref, init = init) + records = (0 until 7).map(i => record.copy(offset = i.toLong)) + result <- submitBatch(dispatcher, records) + } yield result must beEqualTo(SubmitResult.RejectedBatch(record.copy(offset = 5L)))) + } + "resume paused partitions" in new ctx(lowWatermark = 3, highWatermark = 7) { run( for { queue <- Queue.bounded[Record](1) ref <- Ref.make[Map[TopicPartition, ShutdownPromise]](Map.empty) + init <- getInit dispatcher <- Dispatcher.make[Any]( "group", "clientId", record => queue.offer(record).flatMap(result => ZIO.succeed(println(s"queue.offer result: ${result}"))), lowWatermark, highWatermark, - workersShutdownRef = ref + workersShutdownRef = ref, + init = init ) _ <- ZIO.foreachDiscard(0 to (highWatermark + 1)) { offset => submit( @@ -102,6 +178,7 @@ class DispatcherTest extends BaseTest[TestMetrics with TestClock] { for { queue <- Queue.bounded[Record](1) ref <- Ref.make[Map[TopicPartition, ShutdownPromise]](Map.empty) + init <- getInit dispatcher <- Dispatcher.make[TestClock]( "group", "clientId", @@ -112,7 +189,8 @@ class DispatcherTest extends BaseTest[TestMetrics with TestClock] { lowWatermark, highWatermark, delayResumeOfPausedPartition = 6500, - workersShutdownRef = ref + workersShutdownRef = ref, + init = init ) _ <- ZIO.foreachDiscard(0 to (highWatermark + 1)) { offset => submit( @@ -166,9 +244,18 @@ class DispatcherTest extends BaseTest[TestMetrics with TestClock] { run(for { ref <- Ref.make(0) workersShutdownRef <- Ref.make[Map[TopicPartition, ShutdownPromise]](Map.empty) + init <- getInit dispatcher <- Dispatcher - .make[Any]("group", "clientId", _ => ref.update(_ + 1), lowWatermark, highWatermark, workersShutdownRef = workersShutdownRef) + .make[Any]( + "group", + "clientId", + _ => ref.update(_ + 1), + lowWatermark, + highWatermark, + workersShutdownRef = workersShutdownRef, + init = init + ) _ <- pause(dispatcher) _ <- submit(dispatcher, record) // Will be queued invocations <- ref.get @@ -182,8 +269,10 @@ class DispatcherTest extends BaseTest[TestMetrics with TestClock] { workersShutdownRef <- Ref.make[Map[TopicPartition, ShutdownPromise]](Map.empty) promise <- Promise.make[Nothing, Unit] handler = { _: Record => Clock.sleep(1.second) *> ref.update(_ + 1) *> promise.succeed(()) } - dispatcher <- Dispatcher - .make[Any]("group", "clientId", handler, lowWatermark, highWatermark, workersShutdownRef = workersShutdownRef) + init <- getInit + dispatcher <- + Dispatcher + .make[Any]("group", "clientId", handler, lowWatermark, highWatermark, workersShutdownRef = workersShutdownRef, init = init) _ <- submit(dispatcher, record) // Will be handled _ <- TestMetrics.reported.flatMap(waitUntilRecordHandled(3.seconds)) _ <- pause(dispatcher) @@ -201,7 +290,10 @@ class DispatcherTest extends BaseTest[TestMetrics with TestClock] { workersShutdownRef <- Ref.make[Map[TopicPartition, ShutdownPromise]](Map.empty) promise <- Promise.make[Nothing, Unit] handler = { _: ConsumerRecord[Chunk[Byte], Chunk[Byte]] => ref.update(_ + 1) *> promise.succeed(()) } - dispatcher <- Dispatcher.make("group", "clientId", handler, lowWatermark, highWatermark, workersShutdownRef = workersShutdownRef) + init <- getInit + dispatcher <- + Dispatcher + .make("group", "clientId", handler, lowWatermark, highWatermark, workersShutdownRef = workersShutdownRef, init = init) _ <- pause(dispatcher) _ <- submit(dispatcher, record) _ <- resume(dispatcher) @@ -217,6 +309,12 @@ class DispatcherTest extends BaseTest[TestMetrics with TestClock] { dispatcher.submit(record).tap(_ => TestClock.adjust(10.millis)) } + private def submitBatch( + dispatcher: Dispatcher[Any], + records: Seq[ConsumerRecord[Chunk[Byte], Chunk[Byte]]] + ): URIO[Env, SubmitResult] = + dispatcher.submitBatch(records).tap(_ => TestClock.adjust(10.millis)) + private def waitUntilRecordHandled(timeout: zio.Duration)(metrics: Seq[GreyhoundMetric]) = ZIO .when(metrics.collect { case r: RecordHandled[_, _] => r }.nonEmpty)(ZIO.fail(TimeoutWaitingForHandledMetric)) @@ -235,6 +333,13 @@ class DispatcherTest extends BaseTest[TestMetrics with TestClock] { val partition = 0 val topicPartition = TopicPartition(topic, partition) val record = ConsumerRecord[Chunk[Byte], Chunk[Byte]](topic, partition, 0L, Headers.Empty, None, Chunk.empty, 0L, 0L, 0L) + + def getKeys(numKeys: Int) = (0 until numKeys).map(i => Some(Chunk.fromArray(s"key$i".getBytes))) + + def getInit() = for { + init <- Promise.make[Nothing, Unit] + _ <- init.succeed(()) + } yield init } } diff --git a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/BUILD.bazel b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/BUILD.bazel index b02990a0..dea7a061 100644 --- a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/BUILD.bazel +++ b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/BUILD.bazel @@ -17,7 +17,6 @@ specs2_unit_test( "//core/src/main/scala/com/wixpress/dst/greyhound/core/metrics", "//core/src/main/scala/com/wixpress/dst/greyhound/core/producer", "//core/src/main/scala/com/wixpress/dst/greyhound/core/zioutils", - "//core/src/test/scala/com/wixpress/dst/greyhound/core/consumer", "//core/src/test/scala/com/wixpress/dst/greyhound/core/testkit", # "@dev_zio_izumi_reflect_2_12", "@dev_zio_zio_2_12", diff --git a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryAttemptTest.scala b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryAttemptTest.scala new file mode 100644 index 00000000..9e27c10b --- /dev/null +++ b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryAttemptTest.scala @@ -0,0 +1,94 @@ +package com.wixpress.dst.greyhound.core.consumer.retry + +import com.wixpress.dst.greyhound.core.consumer.domain.ConsumerSubscription +import com.wixpress.dst.greyhound.core.testkit.BaseTest +import zio.test.TestEnvironment + +import java.time.{Duration, Instant} +import scala.util.Random +import scala.concurrent.duration._ + +class RetryAttemptTest extends BaseTest[TestEnvironment] { + + "RetryAttempt.extract" should { + "deserialize attempt from headers for blocking retries" in { + val attempt = randomRetryAttempt + val headers = RetryAttempt.toHeaders(attempt) + val subscription = ConsumerSubscription.Topics(Set(attempt.originalTopic)) + for (result <- RetryAttempt.extract(headers, attempt.originalTopic, randomStr, subscription, None)) + yield result must beSome(attempt) + } + "deserialize attempt from headers and topic for non-blocking retries" in { + val attempt = randomRetryAttempt + // topic and attempt must be extracted from retryTopic + val headers = RetryAttempt.toHeaders(attempt.copy(originalTopic = "", attempt = -1)) + val subscription = ConsumerSubscription.Topics(Set(attempt.originalTopic)) + val group = randomStr + val retryTopic = NonBlockingRetryHelper.fixedRetryTopic(attempt.originalTopic, group, attempt.attempt) + for (result <- RetryAttempt.extract(headers, retryTopic, group, subscription, None)) + yield result must beSome(attempt) + } + "deserialize attempt for non-blocking retry after blocking retries" in { + val attempt = randomRetryAttempt + val headers = RetryAttempt.toHeaders(attempt) + val subscription = ConsumerSubscription.Topics(Set(attempt.originalTopic)) + val group = randomStr + val retries = RetryConfig.blockingFollowedByNonBlockingRetry( + blockingBackoffs = 1.milli :: 1.second :: Nil, + nonBlockingBackoffs = 5.minutes :: Nil, + ) + val retryTopic = NonBlockingRetryHelper.fixedRetryTopic(attempt.originalTopic, group, attempt.attempt) + for (result <- RetryAttempt.extract(headers, retryTopic, group, subscription, Some(retries))) + yield result must beSome(attempt.copy(attempt = attempt.attempt + 2)) // with 2 blocking retries before + } + "In case incorrect originalTopic header propagated from a different consumer group, ignore it," + + "do NOT consider it as if it's a non-blocking retry" in { + val attempt = propagatedRetryAttempt + val headers = RetryAttempt.toHeaders(attempt) + val currentTopic = "relevant-topic" + val subscription = ConsumerSubscription.Topics(Set(currentTopic)) + for (result <- RetryAttempt.extract(headers, currentTopic, randomStr, subscription, None)) + yield result must beNone + } + } + + "RetryAttempt.maxOverallAttempts" should { + "return 0 if no retries configured" in { + RetryAttempt.maxOverallAttempts(randomStr, None) must beSome(0) + } + "return max attempts for blocking retries" in { + val config = RetryConfig.finiteBlockingRetry(1.milli, 1.second) + RetryAttempt.maxOverallAttempts(randomStr, Some(config)) must beSome(2) + } + "return max attempts for non-blocking retries" in { + val config = RetryConfig.nonBlockingRetry(1.milli, 1.second, 5.minutes) + RetryAttempt.maxOverallAttempts(randomStr, Some(config)) must beSome(3) + } + "return max attempts for blocking retries followed by non-blocking" in { + val config = RetryConfig.blockingFollowedByNonBlockingRetry(1.milli :: 2.seconds :: Nil, 1.minute :: Nil) + RetryAttempt.maxOverallAttempts(randomStr, Some(config)) must beSome(3) + } + "return None for infinite blocking retries" in { + val config = RetryConfig.infiniteBlockingRetry(1.milli) + RetryAttempt.maxOverallAttempts(randomStr, Some(config)) must beNone + } + } + + override def env = testEnvironment + + private def randomStr = Random.alphanumeric.take(10).mkString + + private def randomRetryAttempt = RetryAttempt( + originalTopic = randomStr, + attempt = Random.nextInt(1000), + submittedAt = Instant.ofEpochMilli(math.abs(Random.nextLong())), + backoff = Duration.ofMillis(Random.nextInt(100000)) + ) + + private def propagatedRetryAttempt = RetryAttempt( + originalTopic = "some-other-topic", + attempt = Random.nextInt(1000), + submittedAt = Instant.ofEpochMilli(math.abs(Random.nextLong())), + backoff = Duration.ofMillis(Random.nextInt(100000)) + ) +} diff --git a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryConsumerRecordHandlerTest.scala b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryConsumerRecordHandlerTest.scala index c713fb12..ef9fccb1 100644 --- a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryConsumerRecordHandlerTest.scala +++ b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/RetryConsumerRecordHandlerTest.scala @@ -4,8 +4,9 @@ import java.time.Instant import com.wixpress.dst.greyhound.core.Serdes._ import com.wixpress.dst.greyhound.core._ import com.wixpress.dst.greyhound.core.consumer.domain.ConsumerSubscription.Topics -import com.wixpress.dst.greyhound.core.consumer.domain.{ConsumerRecord, ConsumerSubscription, RecordHandler} +import com.wixpress.dst.greyhound.core.consumer.domain.{ConsumerRecord, RecordHandler} import com.wixpress.dst.greyhound.core.consumer.retry.BlockingState.{Blocked, Blocking => InternalBlocking, IgnoringAll, IgnoringOnce} +import com.wixpress.dst.greyhound.core.consumer.retry.RetryConfigForTopic.nonBlockingRetryConfigForTopic import com.wixpress.dst.greyhound.core.consumer.retry.RetryConsumerRecordHandlerTest.{offset, partition, _} import com.wixpress.dst.greyhound.core.consumer.retry.RetryRecordHandlerMetric.{BlockingIgnoredForAllFor, BlockingIgnoredOnceFor, BlockingRetryHandlerInvocationFailed, NoRetryOnNonRetryableFailure} import com.wixpress.dst.greyhound.core.producer.{ProducerError, ProducerRecord} @@ -21,8 +22,6 @@ import zio.Random.{nextBytes, nextIntBounded} import zio.managed.UManaged import zio.test.TestClock -import scala.concurrent.TimeoutException - class RetryConsumerRecordHandlerTest extends BaseTest[TestClock with TestMetrics] { override def env: UManaged[ZEnvironment[TestClock with TestMetrics]] = @@ -52,9 +51,6 @@ class RetryConsumerRecordHandlerTest extends BaseTest[TestClock with TestMetrics _ <- retryHandler.handle(ConsumerRecord(topic, partition, offset, Headers.Empty, Some(key), value, 0L, 0L, 0L)) record <- producer.records.take now <- currentTime - retryAttempt <- IntSerde.serialize(retryTopic, 0) - submittedAt <- InstantSerde.serialize(retryTopic, now) - backoff <- DurationSerde.serialize(retryTopic, 1.second) } yield { record === ProducerRecord( @@ -62,7 +58,7 @@ class RetryConsumerRecordHandlerTest extends BaseTest[TestClock with TestMetrics value, Some(key), partition = None, - headers = Headers("retry-attempt" -> retryAttempt, "retry-submitted-at" -> submittedAt, "retry-backoff" -> backoff) + headers = RetryAttempt.toHeaders(RetryAttempt(topic, 0, now, 1.second)) ) } } @@ -71,7 +67,8 @@ class RetryConsumerRecordHandlerTest extends BaseTest[TestClock with TestMetrics for { producer <- FakeProducer.make topic <- randomTopicName - retryTopic = s"$topic-retry" + attempt = 0 + retryTopic = NonBlockingRetryHelper.fixedRetryTopic(topic, group, attempt) executionTime <- Promise.make[Nothing, Instant] handler = RecordHandler[Clock, HandlerError, Chunk[Byte], Chunk[Byte]] { _ => currentTime.flatMap(executionTime.succeed) } blockingState <- Ref.make[Map[BlockingTarget, BlockingState]](Map.empty) @@ -86,10 +83,7 @@ class RetryConsumerRecordHandlerTest extends BaseTest[TestClock with TestMetrics ) value <- bytes begin <- currentTime - retryAttempt <- IntSerde.serialize(retryTopic, 0) - submittedAt <- InstantSerde.serialize(retryTopic, begin) - backoff <- DurationSerde.serialize(retryTopic, 1.second) - headers = Headers("retry-attempt" -> retryAttempt, "retry-submitted-at" -> submittedAt, "retry-backoff" -> backoff) + headers = RetryAttempt.toHeaders(RetryAttempt(topic, attempt, begin, 1.second)) _ <- retryHandler.handle(ConsumerRecord(retryTopic, partition, offset, headers, None, value, 0L, 0L, 0L)).fork _ <- TestClock.adjust(1.second).repeat(Schedule.once) end <- executionTime.await.disconnect.timeoutFail(TimeoutWaitingForAssertion)(5.seconds) @@ -318,7 +312,10 @@ class RetryConsumerRecordHandlerTest extends BaseTest[TestClock with TestMetrics group, failingHandlerWith(handleCountRef), ZRetryConfig - .blockingFollowedByNonBlockingRetry(List(10.millis, 500.millis), NonBlockingBackoffPolicy(List(1.second))), + .blockingFollowedByNonBlockingRetry( + FiniteBlockingBackoffPolicy(List(10.millis, 500.millis)), + NonBlockingBackoffPolicy(List(1.second)) + ), producer, Topics(Set(topic)), blockingState, @@ -343,9 +340,7 @@ class RetryConsumerRecordHandlerTest extends BaseTest[TestClock with TestMetrics topic <- randomTopicName otherTopic <- randomTopicName blockingState <- Ref.make[Map[BlockingTarget, BlockingState]](Map.empty) - policy = ZRetryConfig.perTopicRetries { - case `otherTopic` => RetryConfigForTopic(() => Nil, NonBlockingBackoffPolicy(1.second :: Nil)) - } + policy = ZRetryConfig.perTopicRetries { case `otherTopic` => nonBlockingRetryConfigForTopic(1.second :: Nil) } retryHandler = RetryRecordHandler.withRetries( group, failingHandler, @@ -404,7 +399,8 @@ class RetryConsumerRecordHandlerTest extends BaseTest[TestClock with TestMetrics producer <- FakeProducer.make topic <- randomTopicName blockingState <- Ref.make[Map[BlockingTarget, BlockingState]](Map.empty) - retryHelper = alwaysBackOffRetryHelper(3.seconds) + retryHelper = FakeRetryHelper(topic) + now <- Clock.instant handling <- AwaitShutdown.makeManaged.flatMap { awaitShutdown => val retryHandler = RetryRecordHandler.withRetries( group, @@ -416,11 +412,12 @@ class RetryConsumerRecordHandlerTest extends BaseTest[TestClock with TestMetrics retryHelper, awaitShutdown = _ => ZIO.succeed(awaitShutdown) ) + val headers = RetryAttempt.toHeaders(RetryAttempt(topic, 0, now, 3.seconds)) for { key <- bytes value <- bytes handling <- retryHandler - .handle(ConsumerRecord(topic, partition, offset, Headers.Empty, Some(key), value, 0L, 0L, 0L)) + .handle(ConsumerRecord(topic, partition, offset, headers, Some(key), value, 0L, 0L, 0L)) .forkDaemon } yield handling } @@ -534,18 +531,6 @@ object RetryConsumerRecordHandlerTest { def randomTopicName = randomStr.map(suffix => s"some-topic-$suffix") val cause = new RuntimeException("cause") - - def alwaysBackOffRetryHelper(backoff: Duration) = { - new FakeNonBlockingRetryHelper { - override val topic: Topic = "" - - override def retryAttempt(topic: Topic, headers: Headers, subscription: ConsumerSubscription)( - implicit trace: Trace - ): UIO[Option[RetryAttempt]] = ZIO.succeed( - Some(RetryAttempt(topic, 1, Instant.now, backoff)) - ) - } - } } object TimeoutWaitingForAssertion extends RuntimeException diff --git a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/ZRetryConfigTest.scala b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/ZRetryConfigTest.scala index ac6dca03..6535207d 100644 --- a/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/ZRetryConfigTest.scala +++ b/core/src/test/scala/com/wixpress/dst/greyhound/core/consumer/retry/ZRetryConfigTest.scala @@ -29,12 +29,12 @@ class ZRetryConfigTest extends SpecificationWithJUnit { val absMult = abs(params.mult) val safeInit = if (init < 10) 10 else init - for (i <- 0 to max) yield { backoffs()(i) } mustEqual (pow(1 + absMult, i) * safeInit).toLong.millis + for (i <- 0 to max) yield { backoffs(i) } mustEqual (pow(1 + absMult, i) * safeInit).toLong.millis val maxMult = math.max(0, max) val lastDurationToCheck = abs(max + 1) * 2 val firstDurationToCheck = math.max(0, max + 1) for (i <- firstDurationToCheck to lastDurationToCheck) - yield backoffs()(i) mustEqual (pow(1 + absMult, maxMult) * safeInit).toLong.millis + yield backoffs(i) mustEqual (pow(1 + absMult, maxMult) * safeInit).toLong.millis } } @@ -57,11 +57,11 @@ class ZRetryConfigTest extends SpecificationWithJUnit { val absMult = abs(params.mult) val safeInit = if (init < 10) 10 else init - for (i <- 0 to maxMult) yield { backoffs()(i) } mustEqual (pow(1 + absMult, i) * safeInit).toLong.millis + for (i <- 0 to maxMult) yield { backoffs(i) } mustEqual (pow(1 + absMult, i) * safeInit).toLong.millis val lastDurationToCheck = abs(maxMult + 1) * 2 val firstDurationToCheck = math.max(0, maxMult + 1) for (i <- firstDurationToCheck to lastDurationToCheck) - yield backoffs()(i) mustEqual (pow(1 + absMult, maxMult) * safeInit).toLong.millis + yield backoffs(i) mustEqual (pow(1 + absMult, maxMult) * safeInit).toLong.millis } } } diff --git a/core/src/test/scala/com/wixpress/dst/greyhound/core/producer/BUILD.bazel b/core/src/test/scala/com/wixpress/dst/greyhound/core/producer/BUILD.bazel index 27f12381..46310b8b 100644 --- a/core/src/test/scala/com/wixpress/dst/greyhound/core/producer/BUILD.bazel +++ b/core/src/test/scala/com/wixpress/dst/greyhound/core/producer/BUILD.bazel @@ -16,7 +16,6 @@ specs2_unit_test( "//core/src/main/scala/com/wixpress/dst/greyhound/core/producer", "//core/src/test/resources", "//core/src/test/scala/com/wixpress/dst/greyhound/core/testkit", - "@ch_qos_logback_logback_classic", # "@dev_zio_izumi_reflect_2_12", "@dev_zio_zio_2_12", "@dev_zio_zio_test_2_12", diff --git a/core/src/test/scala/com/wixpress/dst/greyhound/core/testkit/BUILD.bazel b/core/src/test/scala/com/wixpress/dst/greyhound/core/testkit/BUILD.bazel index 17c68cef..75f65d20 100644 --- a/core/src/test/scala/com/wixpress/dst/greyhound/core/testkit/BUILD.bazel +++ b/core/src/test/scala/com/wixpress/dst/greyhound/core/testkit/BUILD.bazel @@ -15,7 +15,6 @@ scala_library( "//core/src/main/scala/com/wixpress/dst/greyhound/core/consumer/retry", "//core/src/main/scala/com/wixpress/dst/greyhound/core/metrics", "//core/src/main/scala/com/wixpress/dst/greyhound/core/producer", - "//core/src/main/scala/com/wixpress/dst/greyhound/core/zioutils", # "@dev_zio_izumi_reflect_2_12", # "@dev_zio_izumi_reflect_thirdparty_boopickle_shaded_2_12", "@dev_zio_zio_2_12", diff --git a/core/src/test/scala/com/wixpress/dst/greyhound/core/testkit/FakeRetryHelper.scala b/core/src/test/scala/com/wixpress/dst/greyhound/core/testkit/FakeRetryHelper.scala index cee13948..619ada36 100644 --- a/core/src/test/scala/com/wixpress/dst/greyhound/core/testkit/FakeRetryHelper.scala +++ b/core/src/test/scala/com/wixpress/dst/greyhound/core/testkit/FakeRetryHelper.scala @@ -1,35 +1,21 @@ package com.wixpress.dst.greyhound.core.testkit import java.time.Instant -import java.util.concurrent.TimeUnit.MILLISECONDS - -import com.wixpress.dst.greyhound.core.Serdes._ import com.wixpress.dst.greyhound.core._ import com.wixpress.dst.greyhound.core.consumer.domain.{ConsumerRecord, ConsumerSubscription} import com.wixpress.dst.greyhound.core.consumer.retry.RetryDecision.{NoMoreRetries, RetryWith} -import com.wixpress.dst.greyhound.core.consumer.retry.{BlockingHandlerFailed, NonBlockingRetryHelper, RetryAttempt, RetryDecision} +import com.wixpress.dst.greyhound.core.consumer.retry.{BlockingHandlerFailed, NonBlockingRetryHelper, RetryAttempt, RetryDecision, RetryHeader} import com.wixpress.dst.greyhound.core.producer.ProducerRecord import com.wixpress.dst.greyhound.core.testkit.FakeRetryHelper._ import zio._ import zio.Clock -import zio.Clock - trait FakeNonBlockingRetryHelper extends NonBlockingRetryHelper { val topic: Topic override def retryTopicsFor(originalTopic: Topic): Set[Topic] = Set(s"$originalTopic-retry") - override def retryAttempt(topic: Topic, headers: Headers, subscription: ConsumerSubscription)( - implicit trace: Trace - ): UIO[Option[RetryAttempt]] = - (for { - attempt <- headers.get(Header.Attempt, IntSerde) - submittedAt <- headers.get(Header.SubmittedAt, InstantSerde) - backoff <- headers.get(Header.Backoff, DurationSerde) - } yield retryAttemptInternal(topic, attempt, submittedAt, backoff)).orElse(ZIO.none) - override def retryDecision[E]( retryAttempt: Option[RetryAttempt], record: ConsumerRecord[Chunk[Byte], Chunk[Byte]], @@ -38,35 +24,23 @@ trait FakeNonBlockingRetryHelper extends NonBlockingRetryHelper { )(implicit trace: Trace): URIO[Any, RetryDecision] = error match { case RetriableError | BlockingHandlerFailed => - currentTime.flatMap(now => - recordFrom(now, retryAttempt, record) - .fold(_ => NoMoreRetries, RetryWith) + currentTime.map(now => + RetryWith(recordFrom(now, retryAttempt, record)) ) case NonRetriableError => ZIO.succeed(NoMoreRetries) } - private def retryAttemptInternal(topic: Topic, attempt: Option[Int], submittedAt: Option[Instant], backoff: Option[Duration]) = - for { - a <- attempt - s <- submittedAt - b <- backoff - } yield RetryAttempt(topic, a, s, b) - private def recordFrom(now: Instant, retryAttempt: Option[RetryAttempt], record: ConsumerRecord[Chunk[Byte], Chunk[Byte]])( implicit trace: Trace ) = { val nextRetryAttempt = retryAttempt.fold(0)(_.attempt + 1) - for { - retryAttempt <- IntSerde.serialize(topic, nextRetryAttempt) - submittedAt <- InstantSerde.serialize(topic, now) - backoff <- DurationSerde.serialize(topic, 1.second) - } yield ProducerRecord( + ProducerRecord( topic = s"$topic-retry", value = record.value, key = record.key, partition = None, - headers = Headers(Header.Attempt -> retryAttempt, Header.SubmittedAt -> submittedAt, Header.Backoff -> backoff) + headers = RetryAttempt.toHeaders(RetryAttempt(topic, nextRetryAttempt, now, 1.second)) ) } } @@ -75,13 +49,8 @@ case class FakeRetryHelper(topic: Topic) extends FakeNonBlockingRetryHelper object FakeRetryHelper { implicit private val trace = Trace.empty - object Header { - val Attempt = "retry-attempt" - val SubmittedAt = "retry-submitted-at" - val Backoff = "retry-backoff" - } - val currentTime = Clock.currentTime(MILLISECONDS).map(Instant.ofEpochMilli) + val currentTime: UIO[Instant] = Clock.instant } sealed trait HandlerError diff --git a/core/src/test/scala/com/wixpress/dst/greyhound/core/testkit/RecordMatchers.scala b/core/src/test/scala/com/wixpress/dst/greyhound/core/testkit/RecordMatchers.scala index 466636f6..1ea81160 100644 --- a/core/src/test/scala/com/wixpress/dst/greyhound/core/testkit/RecordMatchers.scala +++ b/core/src/test/scala/com/wixpress/dst/greyhound/core/testkit/RecordMatchers.scala @@ -2,6 +2,7 @@ package com.wixpress.dst.greyhound.core.testkit import com.wixpress.dst.greyhound.core.Offset import com.wixpress.dst.greyhound.core.consumer.domain.ConsumerRecord +import com.wixpress.dst.greyhound.core.producer.ProducerRecord import org.specs2.matcher.Matcher import org.specs2.matcher.Matchers._ @@ -14,4 +15,9 @@ object RecordMatchers { def beRecordWithOffset(offset: Offset): Matcher[ConsumerRecord[_, _]] = equalTo(offset) ^^ ((_: ConsumerRecord[_, _]).offset) + + def beRecordsWithKeysAndValues[K, V](records: IndexedSeq[ProducerRecord[K, V]]): Matcher[Seq[ConsumerRecord[K, V]]] = { + val matchers = records.map { r => beRecordWithKey(r.key.get) and beRecordWithValue(r.value.get) } + allOf(matchers: _*) + } } diff --git a/future-interop/src/it/scala/com/wixpress/dst/greyhound/future/BUILD.bazel b/future-interop/src/it/scala/com/wixpress/dst/greyhound/future/BUILD.bazel index 45e8069e..af2ef67c 100644 --- a/future-interop/src/it/scala/com/wixpress/dst/greyhound/future/BUILD.bazel +++ b/future-interop/src/it/scala/com/wixpress/dst/greyhound/future/BUILD.bazel @@ -21,11 +21,8 @@ specs2_ite2e_test( "//core/src/test/resources", "//core/src/test/scala/com/wixpress/dst/greyhound/core/testkit", "//future-interop/src/main/scala/com/wixpress/dst/greyhound/future", - "//java-interop/src/main/java/com/wixpress/dst/greyhound/java", - "@ch_qos_logback_logback_classic", "@dev_zio_izumi_reflect_2_12", "@dev_zio_zio_2_12", "@dev_zio_zio_stacktracer_2_12", - "@org_apache_kafka_kafka_clients", ], ) diff --git a/java-interop/src/main/java/com/wixpress/dst/greyhound/scala/GreyhoundConsumersBuilder.scala b/java-interop/src/main/java/com/wixpress/dst/greyhound/scala/GreyhoundConsumersBuilder.scala index f3924695..484acc95 100644 --- a/java-interop/src/main/java/com/wixpress/dst/greyhound/scala/GreyhoundConsumersBuilder.scala +++ b/java-interop/src/main/java/com/wixpress/dst/greyhound/scala/GreyhoundConsumersBuilder.scala @@ -5,9 +5,9 @@ import com.wixpress.dst.greyhound.core import com.wixpress.dst.greyhound.core.consumer.batched.{BatchConsumer, BatchConsumerConfig, BatchEventLoopConfig} import com.wixpress.dst.greyhound.core.consumer.domain.ConsumerSubscription.Topics import com.wixpress.dst.greyhound.core.consumer.domain.{RecordHandler => CoreRecordHandler} -import com.wixpress.dst.greyhound.core.consumer.retry.{NonBlockingBackoffPolicy, RetryConfig => CoreRetryConfig, RetryConfigForTopic} +import com.wixpress.dst.greyhound.core.consumer.retry.{EmptyInfiniteBlockingBackoffPolicy, FiniteBlockingBackoffPolicy, InfiniteBlockingBackoffPolicy, NonBlockingBackoffPolicy, RetryConfigForTopic, RetryConfig => CoreRetryConfig} import com.wixpress.dst.greyhound.core.consumer.{RecordConsumer, RecordConsumerConfig} -import com.wixpress.dst.greyhound.core.{consumer, Group, NonEmptySet, Topic} +import com.wixpress.dst.greyhound.core.{Group, NonEmptySet, Topic, consumer} import com.wixpress.dst.greyhound.future.GreyhoundRuntime import com.wixpress.dst.greyhound.future.GreyhoundRuntime.Env import zio._ @@ -138,7 +138,11 @@ class GreyhoundConsumersBuilder(val config: GreyhoundConfig) { retryConfig.map(config => { val forTopic = - RetryConfigForTopic(() => config.blockingBackoffs().asScala, NonBlockingBackoffPolicy(config.nonBlockingBackoffs.asScala)) + RetryConfigForTopic( + FiniteBlockingBackoffPolicy(config.blockingBackoffs().asScala.toList), + EmptyInfiniteBlockingBackoffPolicy, + NonBlockingBackoffPolicy(config.nonBlockingBackoffs.asScala.toList) + ) CoreRetryConfig({ case _ => forTopic }, None) }) diff --git a/java-interop/src/main/java/com/wixpress/dst/greyhound/scala/RetryConfigBuilder.scala b/java-interop/src/main/java/com/wixpress/dst/greyhound/scala/RetryConfigBuilder.scala index bd0f5538..795844b1 100644 --- a/java-interop/src/main/java/com/wixpress/dst/greyhound/scala/RetryConfigBuilder.scala +++ b/java-interop/src/main/java/com/wixpress/dst/greyhound/scala/RetryConfigBuilder.scala @@ -44,7 +44,7 @@ object RetryConfigBuilder { } private def fromCoreRetryConfig(coreRetryConfig: com.wixpress.dst.greyhound.core.consumer.retry.RetryConfig): RetryConfig = { - val blocking: util.List[Duration] = seqAsJavaList(coreRetryConfig.blockingBackoffs("").apply) + val blocking: util.List[Duration] = seqAsJavaList(coreRetryConfig.blockingBackoffs("")) val nonBlocking: util.List[Duration] = seqAsJavaList(coreRetryConfig.nonBlockingBackoffs("").intervals) new RetryConfig(blocking, nonBlocking) }