<commit_before>#!/usr/bin/env python2
#-*- coding: utf-8 -*-
# sdf_tools.py
# version: 2014-10-24
# author:  <NAME> (APL)
# contact: firstnamelastname at gmx dot de
# license: BSD, see license.txt in this folder

#==============================================================================
# TODO: implement class-based version
#==============================================================================
# TODO: implement progressbar
# progressbar code:
# import progressbar as pb
# # downloaded from https://github.com/fnoble/python-progressbar/tree/ipython_support
# # location: /home/apl/dev/python/libs/
# # encapsulate in try: ... except:
# widgets = ['Processed: ', pb.Counter("%5d"), ' molecules [', pb.Timer(), ']']
# pbar = pb.ProgressBar(widgets=widgets).start()
# for i in range(80):
#   pbar.update(i+1)
#   time.sleep(0.2)
#==============================================================================

from __future__ import absolute_import, division, print_function # , unicode_literals

from rdkit.Chem import AllChem as Chem
from rdkit.Chem import Draw
import rdkit.Chem.Descriptors as Desc

# imports for similarity search
from rdkit.Chem.Fingerprints import FingerprintMols

# imports for clustering
from rdkit import DataStructs
from rdkit.ML.Cluster import Butina

from PIL import Image, ImageChops

import os
import sys
import random
import math
import csv
import os.path as op

from time import strftime
from collections import Counter

if hasattr(sys, "ps1"): # <True> when called from an interactive session
    print("  > interactive session, trying to import display_png from IPython...", end="")
    try:
        from IPython.core.display import display_png
        IPYTHON = True
        print(" success!")

    except:
        IPYTHON = False
        print()
        print("  * no inline display of molecule structures supported.")

try:
    import pylab
    PYLAB = True

except:
    PYLAB = False
    print("  * failed to import pylab,")
    print("    plotting will not work.")


def set_sdf_report_folder():

    if "SDF_VIEWER_REPORTS" in os.environ:
        folder = os.environ["SDF_VIEWER_REPORTS"]
        print("  > found environment var", folder)
    else:
        if "HOME" in os.environ:
            folder = op.join(os.environ["HOME"], "sdf_reports")
        elif "HOMEPATH" in os.environ: # Windows
            folder = op.join(os.environ["HOMEPATH"], "sdf_reports")
        else:
            folder = None
        print("  > setting default folder", folder)

    if not op.exists(folder):
        print("  * folder does not exist, creating...")
        os.mkdir(folder)

    for directory in ["sdf", "reports", "html", "session"]:
        subfolder = op.join(folder, directory)
        if not op.exists(subfolder):
            print("    - creating subfolder", subfolder)
            os.mkdir(subfolder)

    return folder


MISSING_VAL = -999
POINTSIZE = 40
REPORT_FOLDER = set_sdf_report_folder()


def create_dir_if_not_exist(dir_name):
    if not op.exists(dir_name):
        print("  * target folder does not exist, creating {}...".format(dir_name))
        os.makedirs(dir_name)


def autocrop(im, bgcolor):
    if im.mode != "RGB":
        im = im.convert("RGB")
    bg = Image.new("RGB", im.teiize, bgcolor)
    diff = ImageChops.difference(im, bg)
    bbox = diff.getbbox()
    if bbox:
        return im.crop(bbox)
    return None # no contents


def load_sdf(file_name_or_obj="testset.sdf", large_sdf=False):
    """load small sdf completely in memory as list; return large sdf as file object
    function accepts a string filename or a file object"""

    if type(file_name_or_obj) == str:
        file_obj = open(file_name_or_obj)
    else:
        file_obj = file_name_or_obj

    reader = Chem.ForwardSDMolSupplier(file_obj)

    if large_sdf:
        if type(file_name_or_obj) == str:
            print("  > large sdf {} loaded as file object.".format(file_name_or_obj.split(".")[0]))
        else:
            print("  > large sdf loaded as file object.")

        return reader

    sdf_list = []

    for mol in reader:<commit_msg>fixed typo in autocrop<commit_after>#!/usr/bin/env python2
#-*- coding: utf-8 -*-
# sdf_tools.py
# version: 2014-10-24
# author:  <NAME> (APL)
# contact: firstnamelastname at gmx dot de
# license: BSD, see license.txt in this folder

#==============================================================================
# TODO: implement class-based version
#==============================================================================
# TODO: implement progressbar
# progressbar code:
# import progressbar as pb
# # downloaded from https://github.com/fnoble/python-progressbar/tree/ipython_support
# # location: /home/apl/dev/python/libs/
# # encapsulate in try: ... except:
# widgets = ['Processed: ', pb.Counter("%5d"), ' molecules [', pb.Timer(), ']']
# pbar = pb.ProgressBar(widgets=widgets).start()
# for i in range(80):
#   pbar.update(i+1)
#   time.sleep(0.2)
#==============================================================================

from __future__ import absolute_import, division, print_function # , unicode_literals

from rdkit.Chem import AllChem as Chem
from rdkit.Chem import Draw
import rdkit.Chem.Descriptors as Desc

# imports for similarity search
from rdkit.Chem.Fingerprints import FingerprintMols

# imports for clustering
from rdkit import DataStructs
from rdkit.ML.Cluster import Butina

from PIL import Image, ImageChops

import os
import sys
import random
import math
import csv
import os.path as op

from time import strftime
from collections import Counter

if hasattr(sys, "ps1"): # <True> when called from an interactive session
    print("  > interactive session, trying to import display_png from IPython...", end="")
    try:
        from IPython.core.display import display_png
        IPYTHON = True
        print(" success!")

    except:
        IPYTHON = False
        print()
        print("  * no inline display of molecule structures supported.")

try:
    import pylab
    PYLAB = True

except:
    PYLAB = False
    print("  * failed to import pylab,")
    print("    plotting will not work.")


def set_sdf_report_folder():

    if "SDF_VIEWER_REPORTS" in os.environ:
        folder = os.environ["SDF_VIEWER_REPORTS"]
        print("  > found environment var", folder)
    else:
        if "HOME" in os.environ:
            folder = op.join(os.environ["HOME"], "sdf_reports")
        elif "HOMEPATH" in os.environ: # Windows
            folder = op.join(os.environ["HOMEPATH"], "sdf_reports")
        else:
            folder = None
        print("  > setting default folder", folder)

    if not op.exists(folder):
        print("  * folder does not exist, creating...")
        os.mkdir(folder)

    for directory in ["sdf", "reports", "html", "session"]:
        subfolder = op.join(folder, directory)
        if not op.exists(subfolder):
            print("    - creating subfolder", subfolder)
            os.mkdir(subfolder)

    return folder


MISSING_VAL = -999
POINTSIZE = 40
REPORT_FOLDER = set_sdf_report_folder()


def create_dir_if_not_exist(dir_name):
    if not op.exists(dir_name):
        print("  * target folder does not exist, creating {}...".format(dir_name))
        os.makedirs(dir_name)


def autocrop(im, bgcolor):
    if im.mode != "RGB":
        im = im.convert("RGB")
    bg = Image.new("RGB", im.size, bgcolor)
    diff = ImageChops.difference(im, bg)
    bbox = diff.getbbox()
    if bbox:
        return im.crop(bbox)
    return None # no contents


def load_sdf(file_name_or_obj="testset.sdf", large_sdf=False):
    """load small sdf completely in memory as list; return large sdf as file object
    function accepts a string filename or a file object"""

    if type(file_name_or_obj) == str:
        file_obj = open(file_name_or_obj)
    else:
        file_obj = file_name_or_obj

    reader = Chem.ForwardSDMolSupplier(file_obj)

    if large_sdf:
        if type(file_name_or_obj) == str:
            print("  > large sdf {} loaded as file object.".format(file_name_or_obj.split(".")[0]))
        else:
            print("  > large sdf loaded as file object.")

        return reader

    sdf_list = []

    for mol in reader:
{'fraction_non_alphanumeric': 0.16364300499208573, 'fraction_numerical': 0.005600876658955315, 'mean_word_length': 3.2209660842754366, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 17, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3457545', 'n_tokens_mistral': 2617, 'n_tokens_neox': 2487, 'n_words': 720}