<commit_before>package org.flymine.dataconversion;

/*
 * Copyright (C) 2002-2003 FlyMine
 *
 * This code may be freely distributed and modified under the
 * terms of the GNU Lesser General Public Licence.  This should
 * be distributed with the code.  See the LICENSE file for more
 * information or http://www.gnu.org/copyleft/lesser.html.
 *
 */

import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Iterator;

import org.flymine.metadata.Model;
import org.flymine.metadata.ClassDescriptor;
import org.flymine.metadata.FieldDescriptor;
import org.flymine.metadata.ReferenceDescriptor;
import org.flymine.sql.Database;
import org.flymine.util.TypeUtil;
import org.flymine.util.DatabaseUtil;
import org.flymine.util.StringUtil;
import org.flymine.model.fulldata.Attribute;
import org.flymine.model.fulldata.Item;
import org.flymine.model.fulldata.Reference;
import org.flymine.model.fulldata.ReferenceList;

import org.apache.log4j.Logger;

/**
 * Class to read a source database and produce a data representation
 *
 * @author <NAME>
 * @author <NAME>
 */
public class DBConverter extends DataConverter
{
    protected static final Logger LOG = Logger.getLogger(DBConverter.class);
    protected Connection c = null;
    protected Model model;
    protected Database db;

    /**
     * Constructor
     *
     * @param model the Model
     * @param db the Database
     * @param writer the ItemWriter used to handle the resultant Items
     */
    protected DBConverter(Model model, Database db, ItemWriter writer) {
        super(writer);
        this.model = model;
        this.db = db;
    }

    /**
     * Process representations of  all the instances of all the classes in the model present in a
     * database
     *
     * @throws Exception if an error occurs in processing
     */
    public void process() throws Exception {
        try {
            c = db.getConnection();
            for (Iterator cldIter = model.getClassDescriptors().iterator(); cldIter.hasNext();) {
                ClassDescriptor cld = (ClassDescriptor) cldIter.next();
                if (!cld.getName().equals("org.flymine.model.FlyMineBusinessObject")) {
                    processClassDescriptor(cld);
                }
            }
        } finally {
            if (c != null) {
                c.close();
            }
            writer.close();
        }
    }

    /**
     * Process a single class from the model
     * @param cld the metadata for the class
     * @throws Exception if an error occurs when reading or writing data
     */
    protected void processClassDescriptor(ClassDescriptor cld) throws Exception {
        String clsName = TypeUtil.unqualifiedName(cld.getName());
        ResultSet r;

        LOG.error("Processing class: " + clsName)

        boolean idsProvided = idsProvided(cld);
        if (idsProvided) {
            r = executeQuery(c, "SELECT * FROM " + clsName + " ORDER BY " + clsName
                             + "_id LIMIT 1");
        } else {
            r = executeQuery(c, "SELECT * FROM " + clsName);
        }

        int identifier = 0;
        while (r.next()) {
            int clsId = idsProvided ? r.getInt(clsName + "_id") : identifier++;
            writer.store(getItem(cld, clsId, r));
            if (idsProvided) {
                r.close();
                r = executeQuery(c, "SELECT * FROM " + clsName + " WHERE " + clsName + "_id > "
                                 + clsId + " ORDER BY " + clsName + "_id LIMIT 1");
            }
        }
     }
<commit_msg>Fix typo<commit_after>package org.flymine.dataconversion;

/*
 * Copyright (C) 2002-2003 FlyMine
 *
 * This code may be freely distributed and modified under the
 * terms of the GNU Lesser General Public Licence.  This should
 * be distributed with the code.  See the LICENSE file for more
 * information or http://www.gnu.org/copyleft/lesser.html.
 *
 */

import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Iterator;

import org.flymine.metadata.Model;
import org.flymine.metadata.ClassDescriptor;
import org.flymine.metadata.FieldDescriptor;
import org.flymine.metadata.ReferenceDescriptor;
import org.flymine.sql.Database;
import org.flymine.util.TypeUtil;
import org.flymine.util.DatabaseUtil;
import org.flymine.util.StringUtil;
import org.flymine.model.fulldata.Attribute;
import org.flymine.model.fulldata.Item;
import org.flymine.model.fulldata.Reference;
import org.flymine.model.fulldata.ReferenceList;

import org.apache.log4j.Logger;

/**
 * Class to read a source database and produce a data representation
 *
 * @author <NAME>
 * @author <NAME>
 */
public class DBConverter extends DataConverter
{
    protected static final Logger LOG = Logger.getLogger(DBConverter.class);
    protected Connection c = null;
    protected Model model;
    protected Database db;

    /**
     * Constructor
     *
     * @param model the Model
     * @param db the Database
     * @param writer the ItemWriter used to handle the resultant Items
     */
    protected DBConverter(Model model, Database db, ItemWriter writer) {
        super(writer);
        this.model = model;
        this.db = db;
    }

    /**
     * Process representations of  all the instances of all the classes in the model present in a
     * database
     *
     * @throws Exception if an error occurs in processing
     */
    public void process() throws Exception {
        try {
            c = db.getConnection();
            for (Iterator cldIter = model.getClassDescriptors().iterator(); cldIter.hasNext();) {
                ClassDescriptor cld = (ClassDescriptor) cldIter.next();
                if (!cld.getName().equals("org.flymine.model.FlyMineBusinessObject")) {
                    processClassDescriptor(cld);
                }
            }
        } finally {
            if (c != null) {
                c.close();
            }
            writer.close();
        }
    }

    /**
     * Process a single class from the model
     * @param cld the metadata for the class
     * @throws Exception if an error occurs when reading or writing data
     */
    protected void processClassDescriptor(ClassDescriptor cld) throws Exception {
        String clsName = TypeUtil.unqualifiedName(cld.getName());
        ResultSet r;

        LOG.error("Processing class: " + clsName);

        boolean idsProvided = idsProvided(cld);
        if (idsProvided) {
            r = executeQuery(c, "SELECT * FROM " + clsName + " ORDER BY " + clsName
                             + "_id LIMIT 1");
        } else {
            r = executeQuery(c, "SELECT * FROM " + clsName);
        }

        int identifier = 0;
        while (r.next()) {
            int clsId = idsProvided ? r.getInt(clsName + "_id") : identifier++;
            writer.store(getItem(cld, clsId, r));
            if (idsProvided) {
                r.close();
                r = executeQuery(c, "SELECT * FROM " + clsName + " WHERE " + clsName + "_id > "
                                 + clsId + " ORDER BY " + clsName + "_id LIMIT 1");
            }
        }
     }

{'fraction_non_alphanumeric': 0.10057787561915245, 'fraction_numerical': 0.003302146395156852, 'mean_word_length': 2.301089918256131, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4739896', 'n_tokens_mistral': 2037, 'n_tokens_neox': 2010, 'n_words': 579}