<commit_before>// @flow
import Kafka from 'no-kafka';
import moment from 'moment';
import { defineLazyProperty } from 'lazy-object';
import Config from './config';
import type { Reg, ProducerPayload } from '../types';

const Producer = (config: Config, registry: Reg, logger: Object) => {
  const producer: Object = new Kafka.Producer({
    connectionString: config.kafkaConnection,
    codec: config.kafkaCodec,
  });

  const producerPayload = (msg: Object, topic: string) => {
    const timestamp = moment().unix();
    const payload = JSON.stringify(Object.assign({}, msg, { timestamp }));
    logger.info('Payload Size:', topic, payload.length);
    return {
      timestamp,
      topic,
      message: {
        value: payload,
      },
    };
  };

  const initialize = (): Object =><commit_msg>use buffer to calc size<commit_after>// @flow
import Kafka from 'no-kafka';
import moment from 'moment';
import { defineLazyProperty } from 'lazy-object';
import Config from './config';
import type { Reg, ProducerPayload } from '../types';

const Producer = (config: Config, registry: Reg, logger: Object) => {
  const producer: Object = new Kafka.Producer({
    connectionString: config.kafkaConnection,
    codec: config.kafkaCodec,
  });

  const producerPayload = (msg: Object, topic: string) => {
    const timestamp = moment().unix();
    const payload = JSON.stringify(Object.assign({}, msg, { timestamp }));
    const size = Buffer.from(payload, 'utf-8');
    logger.info('Payload Size:', topic, size.length);
    return {
      timestamp,
      topic,
      message: {
        value: payload,
      },
    };
  };

  const initialize = (): Object =>
{'fraction_non_alphanumeric': 0.14629741119807346, 'fraction_numerical': 0.0006020469596628537, 'mean_word_length': 3.3968253968253967, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4962043', 'n_tokens_mistral': 481, 'n_tokens_neox': 478, 'n_words': 144}