<commit_before>package com.annosearch.parse;

import com.annosearch.model.AnnotatedDocument;
import com.annosearch.model.Annotation;
import org.apache.commons.lang3.StringUtils;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.*;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import static com.annosearch.parse.AnnotatedDocumentFields.*;

/**
 * @author <NAME> <<EMAIL>>
 * @since 24-Jan-2017
 */
public class AnnotatedDocumentParser {

    private static final Logger LOG = LoggerFactory.getLogger(AnnotatedDocumentParser.class);

    private static final Pattern DOC_ID_PATTERN = Pattern.compile(".*\\/(\\d+)\\.json");

    private String dataSourcePath;

    private List<AnnotatedDocument> annotatedDocuments = new ArrayList<>();

    public static AnnotatedDocumentParser newInstance(String dataSourcePath) {
        return new AnnotatedDocumentParser(dataSourcePath);
    }


    private AnnotatedDocumentParser(String dataSourcePath) {
        this.dataSourcePath = dataSourcePath;
    }


    public List<AnnotatedDocument> getAnnotatedDocuments() {
        return annotatedDocuments;
    }

    public AnnotatedDocumentParser parse() {
        Path path = Paths.get(dataSourcePath);
        if (Files.exists(path)) {
            listJsonFilesAndParse(path.toFile());
        }

        return this;
    }

    private void listJsonFilesAndParse(File file) {<commit_msg>Fix DOC_ID_PATTERN to be Windows compatible<commit_after>package com.annosearch.parse;

import com.annosearch.model.AnnotatedDocument;
import com.annosearch.model.Annotation;
import org.apache.commons.lang3.StringUtils;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.*;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import static com.annosearch.parse.AnnotatedDocumentFields.*;

/**
 * @author <NAME> <<EMAIL>>
 * @since 24-Jan-2017
 */
public class AnnotatedDocumentParser {

    private static final Logger LOG = LoggerFactory.getLogger(AnnotatedDocumentParser.class);

    private static final Pattern DOC_ID_PATTERN = Pattern.compile(".*[\\/\\\\](\\d+)\\.json");

    private String dataSourcePath;

    private List<AnnotatedDocument> annotatedDocuments = new ArrayList<>();

    public static AnnotatedDocumentParser newInstance(String dataSourcePath) {
        return new AnnotatedDocumentParser(dataSourcePath);
    }


    private AnnotatedDocumentParser(String dataSourcePath) {
        this.dataSourcePath = dataSourcePath;
    }


    public List<AnnotatedDocument> getAnnotatedDocuments() {
        return annotatedDocuments;
    }

    public AnnotatedDocumentParser parse() {
        Path path = Paths.get(dataSourcePath);
        if (Files.exists(path)) {
            listJsonFilesAndParse(path.toFile());
        }

        return this;
    }

    private void listJsonFilesAndParse(File file) {
{'fraction_non_alphanumeric': 0.10510863182618908, 'fraction_numerical': 0.005284791544333529, 'mean_word_length': 4.069940476190476, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6005220', 'n_tokens_mistral': 1043, 'n_tokens_neox': 987, 'n_words': 205}