<commit_before>async function kafka_service(args) {
/*
description("Kafka Service")
base_component_id("kafka_service")
load_once_from_file(true)
only_run_on_server(true)
*/
    var promise = new Promise(async function(returnfn) {
        var kafkaConnection = new Kafka({
            clientId: 'myapp2',
            brokers: ['localhost:9092','localhost:9092']
        })
        console.log(1)
        const producer = kafkaConnection.producer()
        console.log(2)
        const consumer = kafkaConnection.consumer({ groupId: 'test-group2' })
        console.log(3)
        await consumer.connect()
        console.log(4)
        await consumer.subscribe({ topic: 'test', fromBeginning: true })
        console.log(5)

        var dd=[]
        console.log(6)
        await consumer.run({
          eachMessage: async ({ topic, partition, message }) => {
            console.log({
              partition,
              offset: message.offset,
              value: message.value.toString(),
          })
            dd.push(message.value.toString())
            console.log(8)
          }
        })
        returnfn(dd)
    })<commit_msg>Managed to get first message to client<commit_after>async function kafka_service(args) {
/*
description("Kafka Service")
base_component_id("kafka_service")
load_once_from_file(true)
only_run_on_server(true)
*/
    var promise = new Promise(async function(returnfn) {
        var kafkaConnection = new Kafka({
            clientId: 'myapp23',
            brokers: ['localhost:9092','localhost:9092']
        })
        console.log(1)
        const producer = kafkaConnection.producer()
        console.log(2)
        const consumer = kafkaConnection.consumer({ groupId: 'test-group22' })
        console.log(3)
        await consumer.connect()
        console.log(4)
        await consumer.subscribe({ topic: 'test', fromBeginning: true })
        console.log(5)

        var dd=[]
        console.log(6)
        await consumer.run({
          eachMessage: async ({ topic, partition, message }) => {
              consumer.pause([{ topic }])
            console.log({
              partition,
              offset: message.offset,
              value: message.value.toString(),
          })
            dd.push(message.value.toString())
            console.log(8)
            returnfn(dd)
          }
        })

    })
{'fraction_non_alphanumeric': 0.12066045723962743, 'fraction_numerical': 0.015241320914479255, 'mean_word_length': 2.068831168831169, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '248187', 'n_tokens_mistral': 674, 'n_tokens_neox': 660, 'n_words': 130}