<commit_before>
package dbpedia;

import com.hp.hpl.jena.query.Query;
import com.hp.hpl.jena.query.QueryExecution;
import com.hp.hpl.jena.query.QueryExecutionFactory;
import com.hp.hpl.jena.query.QueryFactory;
import com.hp.hpl.jena.query.QuerySolution;
import com.hp.hpl.jena.query.ResultSet;
import com.hp.hpl.jena.rdf.model.Model;
import com.hp.hpl.jena.rdf.model.ModelFactory;
import com.hp.hpl.jena.rdf.model.RDFNode;
import com.hp.hpl.jena.rdf.model.Resource;
import com.hp.hpl.jena.rdf.model.Statement;
import com.hp.hpl.jena.rdf.model.StmtIterator;
import com.hp.hpl.jena.vocabulary.RDF;
import com.hp.hpl.jena.vocabulary.RDFS;
import edu.stanford.nlp.ling.CoreLabel;
import edu.stanford.nlp.process.CoreLabelTokenFactory;
import edu.stanford.nlp.process.PTBTokenizer;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.jena.riot.RDFDataMgr;
import org.rdfhdt.hdt.hdt.HDT;
import org.rdfhdt.hdt.hdt.HDTManager;
import org.rdfhdt.hdtjena.HDTGraph;

/**
 *
 * @author <NAME> <<EMAIL>>
 * http://dojchinovski.mk
 */
public class PairCountsProcessor {
    
    public void process(String dataLoc) {
        
        PrintWriter out = null;
        try {
            //        String dir = "/Users/Milan/Downloads/db-abstracts-en/";
            out = new PrintWriter(new BufferedWriter(new FileWriter(dataLoc+"train-data/pairCounts", true)));
            HashMap<String,Occurrence> hm = new HashMap();
            File folder = new File(dataLoc+"dbpedia-abstracts/");
            File[] listOfFiles = folder.listFiles();
            System.out.println(dataLoc);
            for (int i = 0; i < listOfFiles.length; i++) {
                if (listOfFiles[i].isFile()) {
                    if(listOfFiles[i].getName().endsWith(".ttl")) {
                        System.out.println("File " + dataLoc+"dbpedia-abstracts/"+listOfFiles[i].getName());
                        convertOneFile(dataLoc+"dbpedia-abstracts/"+listOfFiles[i].getName(), dataLoc, hm);
                    }
                }
            }
            
            Iterator it = hm.entrySet().iterator();
            while (it.hasNext()) {
                Map.Entry pair = (Map.Entry)it.next();
//                System.out.println(pair.getKey() + " = " + pair.getValue());
                Occurrence occ = (Occurrence)pair.getValue();
                out.write(occ.getLabel()+"\t"+occ.getLink()+"\t"+occ.getCount());
                it.remove(); // avoids a ConcurrentModificationException
            }
        } catch (IOException ex) {
            Logger.getLogger(PairCountsProcessor.class.getName()).log(Level.SEVERE, null, ex);<commit_msg>fixed new lines in the output<commit_after>
package dbpedia;

import com.hp.hpl.jena.query.Query;
import com.hp.hpl.jena.query.QueryExecution;
import com.hp.hpl.jena.query.QueryExecutionFactory;
import com.hp.hpl.jena.query.QueryFactory;
import com.hp.hpl.jena.query.QuerySolution;
import com.hp.hpl.jena.query.ResultSet;
import com.hp.hpl.jena.rdf.model.Model;
import com.hp.hpl.jena.rdf.model.ModelFactory;
import com.hp.hpl.jena.rdf.model.RDFNode;
import com.hp.hpl.jena.rdf.model.Resource;
import com.hp.hpl.jena.rdf.model.Statement;
import com.hp.hpl.jena.rdf.model.StmtIterator;
import com.hp.hpl.jena.vocabulary.RDF;
import com.hp.hpl.jena.vocabulary.RDFS;
import edu.stanford.nlp.ling.CoreLabel;
import edu.stanford.nlp.process.CoreLabelTokenFactory;
import edu.stanford.nlp.process.PTBTokenizer;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.jena.riot.RDFDataMgr;
import org.rdfhdt.hdt.hdt.HDT;
import org.rdfhdt.hdt.hdt.HDTManager;
import org.rdfhdt.hdtjena.HDTGraph;

/**
 *
 * @author <NAME> <<EMAIL>>
 * http://dojchinovski.mk
 */
public class PairCountsProcessor {
    
    public void process(String dataLoc) {
        
        PrintWriter out = null;
        try {
            //        String dir = "/Users/Milan/Downloads/db-abstracts-en/";
            out = new PrintWriter(new BufferedWriter(new FileWriter(dataLoc+"train-data/pairCounts", true)));
            HashMap<String,Occurrence> hm = new HashMap();
            File folder = new File(dataLoc+"dbpedia-abstracts/");
            File[] listOfFiles = folder.listFiles();
            System.out.println(dataLoc);
            for (int i = 0; i < listOfFiles.length; i++) {
                if (listOfFiles[i].isFile()) {
                    if(listOfFiles[i].getName().endsWith(".ttl")) {
                        System.out.println("File " + dataLoc+"dbpedia-abstracts/"+listOfFiles[i].getName());
                        convertOneFile(dataLoc+"dbpedia-abstracts/"+listOfFiles[i].getName(), dataLoc, hm);
                    }
                }
            }
            
            Iterator it = hm.entrySet().iterator();
            while (it.hasNext()) {
                Map.Entry pair = (Map.Entry)it.next();
//                System.out.println(pair.getKey() + " = " + pair.getValue());
                Occurrence occ = (Occurrence)pair.getValue();
                out.write(occ.getLabel()+"\t"+occ.getLink()+"\t"+occ.getCount()+"\n");
                it.remove(); // avoids a ConcurrentModificationException
            }
        } catch (IOException ex) {
            Logger.getLogger(PairCountsProcessor.class.getName()).log(Level.SEVERE, null, ex);
{'fraction_non_alphanumeric': 0.14106215446473447, 'fraction_numerical': 0.0003350644999162339, 'mean_word_length': 2.7570799244808057, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '520682', 'n_tokens_mistral': 2006, 'n_tokens_neox': 1979, 'n_words': 300}