<commit_before>package com.github.dreamhead.moco.util;

import com.google.common.base.Charsets;
import com.google.common.base.Optional;
import com.google.common.collect.ImmutableMap;
import com.google.common.io.Files;

import java.nio.charset.Charset;

import static com.google.common.base.Optional.absent;
import static com.google.common.base.Optional.fromNullable;
import static com.google.common.base.Optional.of;

public final class FileContentType {
    public static final String DEFAULT_CONTENT_TYPE_WITH_CHARSET = "text/plain; charset=UTF-8";
    private static final String DEFAULT_CONTENT_TYPE = "text/plain";

    private static final ImmutableMap<String, String> contentTypeMap = ImmutableMap.<String, String>builder()
            .put("png", "image/png")
            .put("gif", "image/gif")
            .put("jpg", "image/jpeg")
            .put("jpeg", "image/jpeg")
            .put("tiff", "image/tiff")
            .put("css", "text/css")
            .put("html", "text/html")
            .put("txt", "text/plain")
            .put("js", "application/x-javascript")
            .put("json", "application/json")
            .put("pdf", "application/pdf")
            .put("zip", "application/zip")
            .put("xml", "text/xml")
            .build();

    private final String filename;
    private final Optional<Charset> charset;

    public FileContentType(final String filename, final Optional<Charset> charset) {
        this.filename = filename;
        this.charset = charset;
    }

    public String getContentType() {
        Optional<String> optionalType = toContentType(Files.getFileExtension(filename));
        Optional<Charset> charset = toCharset(optionalType);

        String type = optionalType.or(DEFAULT_CONTENT_TYPE);
        if (charset.isPresent()) {
            return type + "; charset=" + charset.get().displayName();
        }

        return type;
    }

    private Optional<Charset> toCharset(final Optional<String> type) {
        if (charset.isPresent()) {
            return charset;
        }

        if (!type.isPresent()) {
            return of(Charsets.UTF_8);
        }

        return absent();
    }

    private Optional<String> toContentType(final String extension) {
        return fromNullable(contentTypeMap.get(extension.toLowerCase()));
    }
}
<commit_msg>applied media type to file content type<commit_after>package com.github.dreamhead.moco.util;

import com.google.common.base.Charsets;
import com.google.common.base.Optional;
import com.google.common.collect.ImmutableMap;
import com.google.common.io.Files;
import com.google.common.net.MediaType;

import java.nio.charset.Charset;

import static com.google.common.base.Optional.absent;
import static com.google.common.base.Optional.of;

public final class FileContentType {
    public static final String DEFAULT_CONTENT_TYPE_WITH_CHARSET = MediaType.PLAIN_TEXT_UTF_8.toString();
    private static final String DEFAULT_CONTENT_TYPE = MediaType.create("text", "plain").toString();

    private static final ImmutableMap<String, MediaType> contentTypeMap = ImmutableMap.<String, MediaType>builder()
            .put("png", MediaType.PNG)
            .put("gif", MediaType.GIF)
            .put("jpg", MediaType.JPEG)
            .put("jpeg", MediaType.JPEG)
            .put("tiff", MediaType.TIFF)
            .put("css", MediaType.CSS_UTF_8)
            .put("html", MediaType.HTML_UTF_8)
            .put("txt", MediaType.create("text", "plain"))
            .put("js", MediaType.JAVASCRIPT_UTF_8)
            .put("json", MediaType.create("application", "json"))
            .put("pdf", MediaType.PDF)
            .put("zip", MediaType.ZIP)
            .put("xml", MediaType.create("text", "xml"))
            .build();

    private final String filename;
    private final Optional<Charset> charset;

    public FileContentType(final String filename, final Optional<Charset> charset) {
        this.filename = filename;
        this.charset = charset;
    }

    public String getContentType() {
        Optional<String> optionalType = toContentType(Files.getFileExtension(filename));
        Optional<Charset> charset = toCharset(optionalType);

        String type = optionalType.or(DEFAULT_CONTENT_TYPE);
        if (charset.isPresent()) {
            return type + "; charset=" + charset.get().displayName();
        }

        return type;
    }

    private Optional<Charset> toCharset(final Optional<String> type) {
        if (charset.isPresent()) {
            return charset;
        }

        if (!type.isPresent()) {
            return of(Charsets.UTF_8);
        }

        return absent();
    }

    private Optional<String> toContentType(final String extension) {
        MediaType mediaType = contentTypeMap.get(extension.toLowerCase());
        if (mediaType == null) {
            return absent();
        }

        return of(mediaType.toString());
    }
}

{'fraction_non_alphanumeric': 0.12688391038696537, 'fraction_numerical': 0.0014256619144602852, 'mean_word_length': 2.7545871559633026, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5046294', 'n_tokens_mistral': 1425, 'n_tokens_neox': 1375, 'n_words': 284}