<commit_before>package org.titans.fyp;

import java.io.*;
import java.util.ArrayList;
import java.util.List;

/**
 * Created by Keetmalin on 6/13/2017
 * Project - doc_vector_representation_LegalIR
 */
public class Calculation {

    public static String output_folder = "./LawIE/DocVector/Output";


    public List<List<Double>> CalDocumentVector(List<String> inputWordList, ArrayList<String> vocabulary, double[][] t_matrix) {

        List<List<Double>> DocumentVector = new ArrayList<List<Double>>();

        File f = new File(output_folder);
        FilenameFilter textFilter = new FilenameFilter() {
            public boolean accept(File dir, String name) {
                return name.toLowerCase().endsWith(".txt");
            }
        };

        File[] files = f.listFiles(textFilter);
        for (int docIndex = 0; docIndex < files.length; docIndex++) {
            try {
                String fileName = (files[docIndex]).getCanonicalPath();
                DocumentVector.add(docVector(fileName, docIndex, inputWordList, vocabulary, t_matrix));

            } catch (IOException e) {
                e.printStackTrace();
            }
        }

        return DocumentVector;
    }

    public List<Double> docVector(String fileName, int docIndex, List<String> inputWordList, ArrayList<String> vocabulary, double[][] t_matrix) {

        BufferedReader br = null;
        List<Double> docVector = new ArrayList<Double>();

        try {
            br = new BufferedReader(new FileReader(fileName));

            StringBuilder sb = new StringBuilder();
            String sCurrentLine;
            while ((sCurrentLine = br.readLine()) != null) {
                sb.append(sCurrentLine);
                sb.append(" ");
            }

            String paragraph = sb.toString();
            for (String word : inputWordList) {
                if (paragraph.contains(word)) {
                    int index = vocabulary.indexOf(word);
                    docVector.add(t_matrix[index][docIndex]);
                } else {
                    docVector.add(0.0);
                }
            }

        } catch (FileNotFoundException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }

        return docVector;
    }

    private List<List<Double>> normalize_vectors(List<List<Double>> document_vector) {

        for (int i = 0; i < document_vector.size(); i++) {
            Double sum = 0.0;
            for (int j = 0; j < document_vector.get(i).size(); j++) {
                sum += document_vector.get(i).get(j);
            }<commit_msg>Converting to lower case<commit_after>package org.titans.fyp;

import java.io.*;
import java.util.ArrayList;
import java.util.List;

/**
 * Created by Keetmalin on 6/13/2017
 * Project - doc_vector_representation_LegalIR
 */
public class Calculation {

    public static String output_folder = "./LawIE/DocVector/Output";


    public List<List<Double>> CalDocumentVector(List<String> inputWordList, ArrayList<String> vocabulary, double[][] t_matrix) {

        List<List<Double>> DocumentVector = new ArrayList<List<Double>>();

        File f = new File(output_folder);
        FilenameFilter textFilter = new FilenameFilter() {
            public boolean accept(File dir, String name) {
                return name.toLowerCase().endsWith(".txt");
            }
        };

        File[] files = f.listFiles(textFilter);
        for (int docIndex = 0; docIndex < files.length; docIndex++) {
            try {
                String fileName = (files[docIndex]).getCanonicalPath();
                DocumentVector.add(docVector(fileName, docIndex, inputWordList, vocabulary, t_matrix));

            } catch (IOException e) {
                e.printStackTrace();
            }
        }

        return DocumentVector;
    }

    public List<Double> docVector(String fileName, int docIndex, List<String> inputWordList, ArrayList<String> vocabulary, double[][] t_matrix) {

        BufferedReader br = null;
        List<Double> docVector = new ArrayList<Double>();

        try {
            br = new BufferedReader(new FileReader(fileName));

            StringBuilder sb = new StringBuilder();
            String sCurrentLine;
            while ((sCurrentLine = br.readLine()) != null) {
                sb.append(sCurrentLine.toLowerCase());
                sb.append(" ");
            }

            String paragraph = sb.toString();
            for (String word : inputWordList) {
                if (paragraph.contains(word)) {
                    int index = vocabulary.indexOf(word);
                    docVector.add(t_matrix[index][docIndex]);
                } else {
                    docVector.add(0.0);
                }
            }

        } catch (FileNotFoundException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }

        return docVector;
    }

    private List<List<Double>> normalize_vectors(List<List<Double>> document_vector) {

        for (int i = 0; i < document_vector.size(); i++) {
            Double sum = 0.0;
            for (int j = 0; j < document_vector.get(i).size(); j++) {
                sum += document_vector.get(i).get(j);
            }
{'fraction_non_alphanumeric': 0.11636707663197729, 'fraction_numerical': 0.005298013245033113, 'mean_word_length': 2.018846373500857, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2393134', 'n_tokens_mistral': 1446, 'n_tokens_neox': 1359, 'n_words': 323}