public async Task<IList<IBatchContainer>> GetQueueMessagesAsync(int maxCount)
{
try
{
Task<IEnumerable<Message>> fetchingTask;
IList<IBatchContainer> batches = new List<IBatchContainer>();
using (TimerTimeToGetMessageFromKafka.NewContext(Id.ToString()))
{
fetchingTask = _consumer.FetchMessages(maxCount, CurrentOffset);
await Task.WhenAny(fetchingTask, Task.Delay(_options.ReceiveWaitTimeInMs));
}
// Checking that the task completed successfully
if (!fetchingTask.IsCompleted)
{
_logger.Warn((int)KafkaErrorCodes.KafkaStreamProviderBase,
"KafkaQueueAdapterReceiver - Fetching operation was not completed, tried to fetch {0} messages from offest {1}",
maxCount, CurrentOffset);
return batches;
}
if (fetchingTask.IsFaulted && fetchingTask.Exception != null)
{
_logger.Warn((int)KafkaErrorCodes.KafkaStreamProviderBase,
"KafkaQueueAdapterReceiver - Fetching messages from kafka failed, tried to fetch {0} messages from offest {1}",
maxCount, CurrentOffset);
throw fetchingTask.Exception;
}
if (fetchingTask.Result == null)
{
return batches;
}
var messages = fetchingTask.Result.ToList();
batches = messages.Select(m => _factory.FromKafkaMessage(m, m.Meta.Offset)).ToList();
// No batches, we are done here..
if (batches.Count <= 0) return batches;
_logger.Verbose("KafkaQueueAdapterReceiver - Pulled {0} messages for queue number {1}", batches.Count, Id.GetNumericId());
CurrentOffset += batches.Count;
// Taking a bit of metrics
MeterConsumedMessagesPerSecond.Mark(Id.ToString(), 1);
HistogramConsumedMessagesPerFetch.Update(batches.Count, Id.ToString());
return batches;
}
catch (BufferUnderRunException)
{
// This case the next message in the queue is too big for us to read, so we skip it
_logger.Error((int)KafkaErrorCodes.KafkaStreamProviderBase, $"KafkaQueueAdapterReceiver - A message in the Kafka queue was too big to pull, skipping over it. offset was {CurrentOffset}");
CurrentOffset++;
return new List<IBatchContainer>();
}
}