<commit_before>""" Pandas extension for chemical analysis """
from __future__ import absolute_import
from collections import deque
import pandas as pd

import oddt

pd.set_option("display.max_colwidth", 999999)
image_backend = 'png'  # png or svg


def _mol_reader(fmt='sdf',
                filepath_or_buffer=None,
                usecols=None,
                molecule_column='mol',
                molecule_name_column='mol_name',
                smiles_column=None,
                skip_bad_mols=False,
                chunksize=None,
                **kwargs):

    # capture options for reader
    reader_kwargs = {}
    if 'opt' in kwargs:
        reader_kwargs['opt'] = kwargs.pop('opt')
    if 'sanitize' in kwargs:
        reader_kwargs['sanitize'] = kwargs.pop('sanitize')

    chunk = []
    for n, mol in enumerate(oddt.toolkit.readfile(fmt, filepath_or_buffer, **reader_kwargs)):
        if skip_bad_mols and mol is None:
            continue  # add warning with number of skipped molecules
        if usecols:
            mol_data = dict((k, mol.data[k]) for k in usecols)
        else:
            mol_data = mol.data.to_dict()
        if molecule_column:
            mol_data[molecule_column] = mol
        if molecule_name_column:
            mol_data[molecule_name_column] = mol.title
        if smiles_column:
            mol_data[smiles_column] = mol.write('smi').split()[0]
        chunk.append(mol_data)
        if chunksize and (n + 1) % chunksize == 0:
            yield ChemDataFrame(chunk, **kwargs)
            chunk = []
    if chunk or chunksize is None:
        yield ChemDataFrame(chunk, **kwargs)


def _mol_writer(dataframe,
                fmt='sdf',
                filepath_or_buffer=None,<commit_msg>Speedup reading when no molecules are necesary<commit_after>""" Pandas extension for chemical analysis """
from __future__ import absolute_import
from collections import deque
import pandas as pd

import oddt

pd.set_option("display.max_colwidth", 999999)
image_backend = 'png'  # png or svg


def _mol_reader(fmt='sdf',
                filepath_or_buffer=None,
                usecols=None,
                molecule_column='mol',
                molecule_name_column='mol_name',
                smiles_column=None,
                skip_bad_mols=False,
                chunksize=None,
                **kwargs):

    # capture options for reader
    reader_kwargs = {}
    if 'opt' in kwargs:
        reader_kwargs['opt'] = kwargs.pop('opt')
    if 'sanitize' in kwargs:
        reader_kwargs['sanitize'] = kwargs.pop('sanitize')

    if molecule_column is None:
        if oddt.toolkit.backend == 'ob' and fmt == 'sdf':
            if 'opt' in reader_kwargs:
                reader_kwargs['opt']['P'] = None
            else:
                reader_kwargs['opt'] = {'P': None}
        if oddt.toolkit.backend == 'rdk':
            reader_kwargs['sanitize'] = False

    chunk = []
    for n, mol in enumerate(oddt.toolkit.readfile(fmt, filepath_or_buffer, **reader_kwargs)):
        if skip_bad_mols and mol is None:
            continue  # add warning with number of skipped molecules
        if usecols:
            mol_data = dict((k, mol.data[k]) for k in usecols)
        else:
            mol_data = mol.data.to_dict()
        if molecule_column:
            mol_data[molecule_column] = mol
        if molecule_name_column:
            mol_data[molecule_name_column] = mol.title
        if smiles_column:
            mol_data[smiles_column] = mol.write('smi').split()[0]
        chunk.append(mol_data)
        if chunksize and (n + 1) % chunksize == 0:
            yield ChemDataFrame(chunk, **kwargs)
            chunk = []
    if chunk or chunksize is None:
        yield ChemDataFrame(chunk, **kwargs)


def _mol_writer(dataframe,
                fmt='sdf',
                filepath_or_buffer=None,
{'fraction_non_alphanumeric': 0.09799844034312451, 'fraction_numerical': 0.004678970626462178, 'mean_word_length': 2.177539223781998, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2523825', 'n_tokens_mistral': 1185, 'n_tokens_neox': 1136, 'n_words': 280}