<commit_before>package de.jeha.kame.crawler.service.ds;

import com.codahale.metrics.health.HealthCheck;
import de.jeha.kame.crawler.core.types.CrawlResult;
import de.jeha.kame.crawler.service.ds.util.FileNameUtils;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.*;
import java.time.ZonedDateTime;
import java.util.zip.GZIPOutputStream;

/**
 * @author <EMAIL>
 */
public class FileDocumentStore implements DocumentStore {

    private static final Logger LOG = LoggerFactory.getLogger(FileDocumentStore.class);

    private final String path;

    public FileDocumentStore(String path) {
        this.path = path;

        try {
            File root = new File(path);
            if (!root.exists()) {
                FileUtils.forceMkdir(root);
            }
        } catch (IOException e) {
            LOG.error("Could not create root directory", e);
        }
    }

    @Override
    public void save(String crawlId, ZonedDateTime timestamp, CrawlResult result) throws IOException {
        final String filename = String.format("%s-%d-%s.gz", crawlId, timestamp.toEpochSecond(), FileNameUtils.stripToFilename(result.getUrl()));

        try (FileOutputStream output = new FileOutputStream(path + "/" + filename)) {
            try (Writer writer = new OutputStreamWriter(new GZIPOutputStream(output), "UTF-8")) {
                writer.write(result.getContent());
                LOG.debug("file '{}' written", filename);
            }
        }
    }

    @Override
    public HealthCheck.Result checkHealth() {
        File root = new File(path);<commit_msg>store meta record<commit_after>package de.jeha.kame.crawler.service.ds;

import com.codahale.metrics.health.HealthCheck;
import de.jeha.kame.crawler.core.types.CrawlResult;
import de.jeha.kame.crawler.service.ds.util.FileNameUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringEscapeUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.*;
import java.time.ZonedDateTime;
import java.util.zip.GZIPOutputStream;

/**
 * @author <EMAIL>
 */
public class FileDocumentStore implements DocumentStore {

    private static final Logger LOG = LoggerFactory.getLogger(FileDocumentStore.class);

    private final String path;

    public FileDocumentStore(String path) {
        this.path = path;

        try {
            File root = new File(path);
            if (!root.exists()) {
                FileUtils.forceMkdir(root);
            }
        } catch (IOException e) {
            LOG.error("Could not create root directory", e);
        }
    }

    @Override
    public void save(String crawlId, ZonedDateTime timestamp, CrawlResult result) throws IOException {
        final String filenamePrefix = String.format(
                "%s-%d-%s",
                crawlId,
                timestamp.toEpochSecond(),
                FileNameUtils.stripToFilename(result.getUrl())
        );

        saveDocument(filenamePrefix, result);
        saveMetaRecord(filenamePrefix, result);
    }

    private void saveDocument(String filenamePrefix, CrawlResult result) throws IOException {
        final String documentFilename = path + "/" + filenamePrefix + ".gz";
        try (FileOutputStream output = new FileOutputStream(documentFilename)) {
            try (Writer writer = new OutputStreamWriter(new GZIPOutputStream(output), "UTF-8")) {
                writer.write(result.getContent());
                LOG.debug("file '{}' written", documentFilename);
            }
        }
    }

    private void saveMetaRecord(String filenamePrefix, CrawlResult result) throws IOException {
        final String metaFilename = path + "/" + filenamePrefix + ".meta.json";
        final String metaRecord = String.format("{\n  url: %s\n}\n", StringEscapeUtils.escapeJson(result.getUrl()));

        FileUtils.writeStringToFile(new File(metaFilename), metaRecord);
        LOG.debug("file '{}' written", metaFilename);
    }

    @Override
    public HealthCheck.Result checkHealth() {
        File root = new File(path);
{'fraction_non_alphanumeric': 0.11084043848964677, 'fraction_numerical': 0.001705237515225335, 'mean_word_length': 2.840972871842844, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6652402', 'n_tokens_mistral': 1156, 'n_tokens_neox': 1123, 'n_words': 252}