<commit_before>package org.nryotaro.edgar.text

import org.nryotaro.edgar.exception.EdgarException
import org.springframework.cglib.core.Local
import org.springframework.stereotype.Service
import java.time.LocalDate
import java.time.format.DateTimeFormatter
import java.util.*

@Service
class IndexParser {

    private val pattern = DateTimeFormatter.ofPattern("MMM dd, uuuu", Locale.US)

    fun parse(index: String) :String {
        val lines =  index.split(Regex("\\r?\\n"))
        val c = extractDate(lines)
       return TODO()
    }

    private fun extractDate(lines: List<String>): LocalDate {
        val dateLine: String = lines.first { it.startsWith("Last Data Received:") }
        val dateStr: String? = Regex("Last Data Received:\\s+(\\w.+)").find(dateLine)?.groupValues?.get(1)
        return if(dateStr != null) LocalDate.parse(dateStr, pattern) else throw EdgarException("failed to find date in \"$dateLine\"")
    }
}
<commit_msg>extract<commit_after>package org.nryotaro.edgar.text

import org.nryotaro.edgar.exception.EdgarException
import org.springframework.cglib.core.Local
import org.springframework.stereotype.Service
import java.time.LocalDate
import java.time.format.DateTimeFormatter
import java.util.*

@Service
class IndexParser {

    private val pattern = DateTimeFormatter.ofPattern("MMM dd, uuuu", Locale.US)

    fun parse(index: String) :String {
        val lines =  index.split(Regex("\\r?\\n"))
        val date = extractDate(lines)
        val  indices = extractIndexLine(lines)

       return TODO()
    }

    private fun extractDate(lines: List<String>): LocalDate {
        val dateLine: String = lines.first { it.startsWith("Last Data Received:") }
        val dateStr: String? = Regex("Last Data Received:\\s+(\\w.+)").find(dateLine)?.groupValues?.get(1)
        return if(dateStr != null) LocalDate.parse(dateStr, pattern) else throw EdgarException("failed to find date in \"$dateLine\"")
    }

    private fun extractIndexLine(lines: List<String>): List<String> {
        var start: Int = -1
        var found: Boolean = false
        for(line in lines) {
            start++
            if(line.startsWith("-----")) {
                found = true
                break
            }
        }
        return if(found) lines.subList(start+1, lines.size)  else throw EdgarException("----")
    }
}

{'fraction_non_alphanumeric': 0.12813963388676033, 'fraction_numerical': 0.0017028522775649213, 'mean_word_length': 3.101221640488656, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3517903', 'n_tokens_mistral': 717, 'n_tokens_neox': 700, 'n_words': 165}