<commit_before>#!/usr/bin/env python
"""
Script for running molecule search
"""
import argparse
import sys

from sm.engine import DB
from sm.engine import ESExporter
from sm.engine import SMDaemonDatasetManager
from sm.engine.util import SMConfig, logger, sm_log_config, init_logger, create_ds_from_files
from sm.engine.search_job import SearchJob


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='SM process dataset at a remote spark location.')
    parser.add_argument('--ds-id', dest='ds_id', type=str, help='Dataset id')
    parser.add_argument('--ds-name', dest='ds_name', type=str, help='Dataset name')
    parser.add_argument('--input-path', type=str, help='Path to a dataset location')
    parser.add_argument('--no-clean', dest='no_clean', action='store_true',
                        help="Don't clean dataset txt files after job is finished")
    parser.add_argument('--config', dest='sm_config_path', default='conf/config.json',
                        type=str, help='SM config path')
    args = parser.parse_args()

    init_logger()
    SMConfig.set_path(args.sm_config_path)
    sm_config = SMConfig.get_conf()
    db = DB(sm_config['db'])
    ds_man = SMDaemonDatasetManager(db, ESExporter(db), mode='local')

    try:
        ds = create_ds_from_files(args.ds_id, args.ds_name, args.input_path)
        ds_man.add(ds, SearchJob, del_first=True)
    except Exception as e:
        logger.error(e)
        sys.exit(1)

    sys.exit()<commit_msg>run molecule search script bug fix<commit_after>#!/usr/bin/env python
"""
Script for running molecule search
"""
import argparse
import sys

from sm.engine import DB
from sm.engine import ESExporter
from sm.engine import SMDaemonDatasetManager
from sm.engine.png_generator import ImageStoreServiceWrapper
from sm.engine.util import SMConfig, logger, sm_log_config, init_logger, create_ds_from_files
from sm.engine.search_job import SearchJob


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='SM process dataset at a remote spark location.')
    parser.add_argument('--ds-id', dest='ds_id', type=str, help='Dataset id')
    parser.add_argument('--ds-name', dest='ds_name', type=str, help='Dataset name')
    parser.add_argument('--input-path', type=str, help='Path to a dataset location')
    parser.add_argument('--no-clean', dest='no_clean', action='store_true',
                        help="Don't clean dataset txt files after job is finished")
    parser.add_argument('--config', dest='sm_config_path', default='conf/config.json',
                        type=str, help='SM config path')
    args = parser.parse_args()

    init_logger()
    SMConfig.set_path(args.sm_config_path)
    sm_config = SMConfig.get_conf()
    db = DB(sm_config['db'])
    img_store = ImageStoreServiceWrapper(sm_config['services']['img_service_url'])
    ds_man = SMDaemonDatasetManager(db, ESExporter(db), img_store, mode='local')

    try:
        ds = create_ds_from_files(args.ds_id, args.ds_name, args.input_path)
        ds_man.add(ds, SearchJob, del_first=True)
    except Exception as e:
        logger.error(e)
        sys.exit(1)

    sys.exit()
{'fraction_non_alphanumeric': 0.12357414448669202, 'fraction_numerical': 0.0006337135614702154, 'mean_word_length': 3.9638364779874213, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4015584', 'n_tokens_mistral': 990, 'n_tokens_neox': 980, 'n_words': 231}