<commit_before>package ru.dionisius;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.io.InputStream;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.util.Collection;
import java.util.Properties;

/**
 * Created by Dionisius on 26.07.2017.
 * Works with specified database.
 * It can add specified Job class collection
 * to specified database.
 * It can also return the last date from all
 * entries at specified database.
 */
public class DbJobManager {

    /**
     * File with specified database properties.
     */
    private final String propertiesFile;

    /**
     * Logger for database errors.
     */
    private static final Logger LOG = LoggerFactory.getLogger(JobsCollector.class);

    /**
     * Properties for specified database connection.
     */
    private final Properties prs = new Properties();

    /**
     * Connection for specified database.
     */
    private java.sql.Connection conn = null;

    /**
     * Statement for specified connection.
     */
    private PreparedStatement st = null;
    /**
     * ResultSet instance of specified query.
     */
    private ResultSet rs = null;

    /**
     * Constructor.
     * @param propertiesFile file with database properties.
     */
    public DbJobManager(final String propertiesFile) {
        this.propertiesFile = propertiesFile;
        this.connectToDb();
    }

    /**
     * Adds Job class collection to database.
     * @param jobs Job class collection.
     */
    public void add(final Collection<Job> jobs) {
        for(Job job: jobs) {
            try {
                this.st = conn.prepareStatement("INSERT INTO vacancy(title, description, create_date) " +
                        "SELECT ?, ?, ? WHERE NOT EXISTS (SELECT description FROM vacancy WHERE description = ?)");
                st.setString(1, job.getTitle());
                st.setString(2, job.getDescription());
                st.setTimestamp(3, job.getCreate_date());
                st.setString(4, job.getDescription());
                st.executeUpdate();
            } catch (SQLException e) {
                LOG.error(e.getMessage(), e);
            }
        }
    }

    /**
     * Loads properties.
     */
    private void loadProperties() {
        try (InputStream in = getClass().getClassLoader().getResourceAsStream(propertiesFile)) {
            this.prs.load(in);
        } catch (IOException e) {
            LOG.error(e.getMessage(), e);
        }
    }<commit_msg>JSoup all page parser is added<commit_after>package ru.dionisius;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.io.InputStream;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.util.Collection;
import java.util.Properties;

/**
 * Created by Dionisius on 26.07.2017.
 * Works with specified database.
 * It can add specified Job class collection
 * to specified database.
 * It can also return the last date from all
 * entries at specified database.
 */
public class DbJobManager {

    /**
     * File with specified database properties.
     */
    private final String propertiesFile;

    /**
     * Logger for database errors.
     */
    private static final Logger LOG = LoggerFactory.getLogger(JobsCollector.class);

    /**
     * Properties for specified database connection.
     */
    private final Properties prs = new Properties();

    /**
     * Connection for specified database.
     */
    private java.sql.Connection conn = null;

    /**
     * Statement for specified connection.
     */
    private PreparedStatement st = null;
    /**
     * ResultSet instance of specified query.
     */
    private ResultSet rs = null;

    /**
     * Constructor.
     * @param propertiesFile file with database properties.
     */
    public DbJobManager(final String propertiesFile) {
        this.propertiesFile = propertiesFile;
        this.connectToDb();
    }

    /**
     * Adds unique vacancies collection to database.
     * @param jobs Job class collection.
     */
    public void add(final Collection<Job> jobs) {
        for(Job job: jobs) {
            try {
                this.st = conn.prepareStatement("INSERT INTO vacancy(title, description, create_date) " +
                        "SELECT ?, ?, ? WHERE NOT EXISTS (SELECT description FROM vacancy WHERE description = ?)");
                st.setString(1, job.getTitle());
                st.setString(2, job.getDescription());
                st.setTimestamp(3, job.getCreate_date());
                st.setString(4, job.getDescription());
                st.executeUpdate();
            } catch (SQLException e) {
                LOG.error(e.getMessage(), e);
            }
        }
    }

    /**
     * Loads properties.
     */
    private void loadProperties() {
        try (InputStream in = getClass().getClassLoader().getResourceAsStream(propertiesFile)) {
            this.prs.load(in);
        } catch (IOException e) {
            LOG.error(e.getMessage(), e);
        }
    }
{'fraction_non_alphanumeric': 0.10422213769494104, 'fraction_numerical': 0.005325218714340053, 'mean_word_length': 2.2745952677459527, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6815689', 'n_tokens_mistral': 1428, 'n_tokens_neox': 1376, 'n_words': 391}