<commit_before>package com.eneco.trading.kafka.connect.twitter.domain

import java.text.{DateFormat, SimpleDateFormat}
import java.time.{LocalDate, ZoneOffset}
import java.time.format.DateTimeFormatter
import java.util.{TimeZone, Date}

import org.apache.kafka.connect.data.{Schema, SchemaBuilder, Struct}
import twitter4j.{Status, User}

/**
  * Created by <EMAIL> on 29/02/16. 
  * kafka-connect-twitter
  */
case class TwitterStatus(id: Long, createdAt: String, favoriteCount: Int, text: String, user: TwitterUser)
case class TwitterUser(id: Long, name: String, screenName: String)

object TwitterUser {
  def apply(u: User) = {
    new TwitterUser(id = u.getId, name = u.getName, screenName = u.getScreenName)
  }

  def struct(u: TwitterUser) =
    new Struct(schema)
      .put("id", u.id)
      .put("name", u.name)
      .put("screenName", u.screenName)

  val schema = SchemaBuilder.struct().name("com.eneco.trading.kafka.connect.twitter.TwitterUser")
    .field("id", Schema.INT64_SCHEMA)
    .field("name", Schema.STRING_SCHEMA)
    .field("screenName", Schema.STRING_SCHEMA)
    .build()
}

object TwitterStatus {
  def asIso8601String(d:Date) = {
    val tz = TimeZone.getTimeZone("UTC")
    val df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mmZ")
    df.setTimeZone(tz)
    df.format(new Date())
  }

  def apply(s: Status) = {
    new TwitterStatus(id = s.getId, createdAt = asIso8601String(s.getCreatedAt), favoriteCount = s.getFavoriteCount, text = s.getText, user = TwitterUser(s.getUser))
  }

  def struct(s: TwitterStatus) =
    new Struct(schema)
      .put("id", s.id)
      .put("createdAt", s.createdAt)
      .put("favoriteCount", s.favoriteCount)<commit_msg>fix ISO8601 pattern - add seconds & milliseconds<commit_after>package com.eneco.trading.kafka.connect.twitter.domain

import java.text.{DateFormat, SimpleDateFormat}
import java.time.{LocalDate, ZoneOffset}
import java.time.format.DateTimeFormatter
import java.util.{TimeZone, Date}

import org.apache.kafka.connect.data.{Schema, SchemaBuilder, Struct}
import twitter4j.{Status, User}

/**
  * Created by <EMAIL> on 29/02/16. 
  * kafka-connect-twitter
  */
case class TwitterStatus(id: Long, createdAt: String, favoriteCount: Int, text: String, user: TwitterUser)
case class TwitterUser(id: Long, name: String, screenName: String)

object TwitterUser {
  def apply(u: User) = {
    new TwitterUser(id = u.getId, name = u.getName, screenName = u.getScreenName)
  }

  def struct(u: TwitterUser) =
    new Struct(schema)
      .put("id", u.id)
      .put("name", u.name)
      .put("screenName", u.screenName)

  val schema = SchemaBuilder.struct().name("com.eneco.trading.kafka.connect.twitter.TwitterUser")
    .field("id", Schema.INT64_SCHEMA)
    .field("name", Schema.STRING_SCHEMA)
    .field("screenName", Schema.STRING_SCHEMA)
    .build()
}

object TwitterStatus {
  def asIso8601String(d:Date) = {
    val tz = TimeZone.getTimeZone("UTC")
    val df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ")
    df.setTimeZone(tz)
    df.format(new Date())
  }

  def apply(s: Status) = {
    new TwitterStatus(id = s.getId, createdAt = asIso8601String(s.getCreatedAt), favoriteCount = s.getFavoriteCount, text = s.getText, user = TwitterUser(s.getUser))
  }

  def struct(s: TwitterStatus) =
    new Struct(schema)
      .put("id", s.id)
      .put("createdAt", s.createdAt)
      .put("favoriteCount", s.favoriteCount)
{'fraction_non_alphanumeric': 0.14210217263652378, 'fraction_numerical': 0.011156782149148562, 'mean_word_length': 4.054896142433234, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2565222', 'n_tokens_mistral': 1212, 'n_tokens_neox': 1174, 'n_words': 240}