<commit_before>package org.phenoscape.obd.loader;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URL;
import java.sql.SQLException;

import org.obd.model.Graph;
import org.obd.model.Node;
import org.obd.query.impl.OBDSQLShard;

public class ZFINPublicationsLoader {

    /** The db-host system property should contain the name of the database server. */
    public static final String DB_HOST = "db-host";
    /** The db-name system property should contain the name of the database. */
    public static final String DB_NAME = "db-name";
    /** The db-user system property should contain the database username. */
    public static final String DB_USER = "db-user";
    /** The db-password system property should contain the database password. */
    public static final String DB_PASSWORD = "<PASSWORD>";
    /** The publications-url system property should contain the URL of the ZFIN phenotypes file. */
    public static final String PUBLICATIONS_URL = "publications-url";

    public void loadPublicationsData() throws IOException, SQLException, ClassNotFoundException {
        final OBDSQLShard shard = this.initializeShard();
        final Graph graph = new Graph();
        final BufferedReader reader = this.getPublicationsData();
        String line;
        while ((line = reader.readLine()) != null) {
            graph.addNode(this.parsePublicationLine(line));
        }
        shard.putGraph(graph);
        shard.disconnect();
    }

    private OBDSQLShard initializeShard() throws SQLException, ClassNotFoundException {
        OBDSQLShard shard = new OBDSQLShard();
        shard.connect("jdbc:postgresql://" + System.getProperty(DB_HOST) + "/" + System.getProperty(DB_NAME), System.getProperty(DB_USER), System.getProperty(DB_PASSWORD));
        return shard;
    }

    private BufferedReader getPublicationsData() throws IOException {
        final URL phenotypeURL = new URL(System.getProperty(PUBLICATIONS_URL));
        return new BufferedReader(new InputStreamReader(phenotypeURL.openStream()));
    }

    private Node parsePublicationLine(String line) {
        final String[] items = line.split("\\t", -1);
        final String pubID = "ZFIN:" + items[0].trim();
        final String authors = items[2].trim();
        final String title = items[3].trim();
        final String journal = items[4].trim(); 
        final String year = items[5].trim();
        final String volume = items[6].trim(); 
        final String pages = items[7].trim();
        final Node pubNode = OBDUtil.createInstanceNode(pubID, Vocab.PUBLICATION_TYPE_ID);
        pubNode.setId(pubID);
        pubNode.setLabel(this.createFullCitation(authors, year, title, journal, volume, pages));
        return pubNode;
    }


    private String createFullCitation(String authors, String year, String title, String journal, String volume, String pages) {
        return String.format("%s. %s. %s. %s %s:%s.", authors, year, title, journal, volume, pages);<commit_msg>Hardcoded character encoding for ZFIN publication download.<commit_after>package org.phenoscape.obd.loader;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URL;
import java.nio.charset.Charset;
import java.sql.SQLException;

import org.obd.model.Graph;
import org.obd.model.Node;
import org.obd.query.impl.OBDSQLShard;

public class ZFINPublicationsLoader {

    /** The db-host system property should contain the name of the database server. */
    public static final String DB_HOST = "db-host";
    /** The db-name system property should contain the name of the database. */
    public static final String DB_NAME = "db-name";
    /** The db-user system property should contain the database username. */
    public static final String DB_USER = "db-user";
    /** The db-password system property should contain the database password. */
    public static final String DB_PASSWORD = "<PASSWORD>";
    /** The publications-url system property should contain the URL of the ZFIN phenotypes file. */
    public static final String PUBLICATIONS_URL = "publications-url";

    public void loadPublicationsData() throws IOException, SQLException, ClassNotFoundException {
        final OBDSQLShard shard = this.initializeShard();
        final Graph graph = new Graph();
        final BufferedReader reader = this.getPublicationsData();
        String line;
        while ((line = reader.readLine()) != null) {
            graph.addNode(this.parsePublicationLine(line));
        }
        shard.putGraph(graph);
        shard.disconnect();
    }

    private OBDSQLShard initializeShard() throws SQLException, ClassNotFoundException {
        OBDSQLShard shard = new OBDSQLShard();
        shard.connect("jdbc:postgresql://" + System.getProperty(DB_HOST) + "/" + System.getProperty(DB_NAME), System.getProperty(DB_USER), System.getProperty(DB_PASSWORD));
        return shard;
    }

    private BufferedReader getPublicationsData() throws IOException {
        final URL phenotypeURL = new URL(System.getProperty(PUBLICATIONS_URL));
        return new BufferedReader(new InputStreamReader(phenotypeURL.openStream(), Charset.forName("ISO-8859-1")));
    }

    private Node parsePublicationLine(String line) {
        final String[] items = line.split("\\t", -1);
        final String pubID = "ZFIN:" + items[0].trim();
        final String authors = items[2].trim();
        final String title = items[3].trim();
        final String journal = items[4].trim(); 
        final String year = items[5].trim();
        final String volume = items[6].trim(); 
        final String pages = items[7].trim();
        final Node pubNode = OBDUtil.createInstanceNode(pubID, Vocab.PUBLICATION_TYPE_ID);
        pubNode.setId(pubID);
        pubNode.setLabel(this.createFullCitation(authors, year, title, journal, volume, pages));
        return pubNode;
    }


    private String createFullCitation(String authors, String year, String title, String journal, String volume, String pages) {
        return String.format("%s. %s. %s. %s %s:%s.", authors, year, title, journal, volume, pages);
{'fraction_non_alphanumeric': 0.10882257541355822, 'fraction_numerical': 0.003405773597145637, 'mean_word_length': 3.4591467823571946, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6984619', 'n_tokens_mistral': 1713, 'n_tokens_neox': 1671, 'n_words': 494}