<commit_before>#!/usr/bin/env python3.5


########################################################################################################################
# IMPORTS
########################################################################################################################

import argparse
import glob
import dicom
import datetime
import csv
import logging
import os
import time

from sqlalchemy import create_engine
from sqlalchemy.ext.automap import automap_base
from sqlalchemy.orm import Session
from sqlalchemy import exc

from pymysql import err
from dicom.errors import InvalidDicomError


########################################################################################################################
# CONSTANTS
########################################################################################################################

DEFAULT_HANDEDNESS = 'unknown'
DEFAULT_ROLE = 'U'
DEFAULT_COMMENT = ''


########################################################################################################################
# FUNCTIONS - MAIN
########################################################################################################################

def main():
    logging.basicConfig(level='INFO')
    args = parse_args()

    logging.debug("[START]")
    start_time = time.time()

    logging.info("Connecting to DB")
    db = init_db(args.db)
    db_session = db["session"]
    participant_class = db["participant_class"]
    scan_class = db["scan_class"]
    dicom_class = db["dicom_class"]
    session_class = db["session_class"]
    sequence_type_class = db["sequence_type_class"]
    sequence_class = db["sequence_class"]
    repetition_class = db["repetition_class"]

    checked_folders = dict()
    count_files = 0
    for filename in glob.iglob(args.root + '/**/MR.*', recursive=True):
        try:
            logging.debug("Processing '%s'" % filename)
            count_files += 1
            folder = os.path.split(filename)[0]

            if folder not in checked_folders:
                logging.info("Extracting DICOM headers from '%s'" % filename)
                ds = dicom.read_file(filename)

                participant_id = extract_participant(participant_class, ds, db_session, DEFAULT_HANDEDNESS, args.id)
                scan_id = extract_scan(scan_class, ds, db_session, participant_id, DEFAULT_ROLE, DEFAULT_COMMENT)
                session_id = extract_session(session_class, ds, db_session, scan_id)
                sequence_type_id = extract_sequence_type(sequence_type_class, ds, db_session)
                sequence_id = extract_sequence(sequence_class, db_session, session_id, sequence_type_id)
                repetition_id = extract_repetition(repetition_class, ds, db_session, sequence_id)

                checked_folders[folder] = repetition_id
            extract_dicom(dicom_class, db_session, filename, checked_folders[folder])

        except (InvalidDicomError, IsADirectoryError):
            logging.warning("%s is not a DICOM file !" % filename)

        except (err.IntegrityError, exc.IntegrityError):
            print_db_except()
            db_session.rollback()

    stop_time = time.time()
    logging.debug("[DONE]")

    logging.info("Closing DB connection")
    db_session.close()

    duration = stop_time - start_time
    logging.debug("Processed %s files in %s" % (str(count_files), str(duration)))


########################################################################################################################
# FUNCTIONS - UTILS
########################################################################################################################

def parse_args():

    parser = argparse.ArgumentParser()
    parser.add_argument('root')
    parser.add_argument('db')
    parser.add_argument(
        "-i",
        "--id",
        help="path to CSV file containing unique participants IDs"
    )
    return parser.parse_args()


def format_date(date):
    return datetime.datetime(int(date[:4]), int(date[4:6]), int(date[6:8]))

<commit_msg>use clock instead of time to monitor performance<commit_after>#!/usr/bin/env python3.5


########################################################################################################################
# IMPORTS
########################################################################################################################

import argparse
import glob
import dicom
import datetime
import csv
import logging
import os
import time

from sqlalchemy import create_engine
from sqlalchemy.ext.automap import automap_base
from sqlalchemy.orm import Session
from sqlalchemy import exc

from pymysql import err
from dicom.errors import InvalidDicomError


########################################################################################################################
# CONSTANTS
########################################################################################################################

DEFAULT_HANDEDNESS = 'unknown'
DEFAULT_ROLE = 'U'
DEFAULT_COMMENT = ''


########################################################################################################################
# FUNCTIONS - MAIN
########################################################################################################################

def main():
    logging.basicConfig(level='INFO')
    args = parse_args()

    logging.debug("[START]")
    start_time = time.clock()

    logging.info("Connecting to DB")
    db = init_db(args.db)
    db_session = db["session"]
    participant_class = db["participant_class"]
    scan_class = db["scan_class"]
    dicom_class = db["dicom_class"]
    session_class = db["session_class"]
    sequence_type_class = db["sequence_type_class"]
    sequence_class = db["sequence_class"]
    repetition_class = db["repetition_class"]

    checked_folders = dict()
    count_files = 0

    for filename in glob.iglob(args.root + '/**/MR.*', recursive=True):
        try:
            logging.debug("Processing '%s'" % filename)
            count_files += 1
            folder = os.path.split(filename)[0]

            if folder not in checked_folders:
                logging.info("Extracting DICOM headers from '%s'" % filename)
                ds = dicom.read_file(filename)

                participant_id = extract_participant(participant_class, ds, db_session, DEFAULT_HANDEDNESS, args.id)
                scan_id = extract_scan(scan_class, ds, db_session, participant_id, DEFAULT_ROLE, DEFAULT_COMMENT)
                session_id = extract_session(session_class, ds, db_session, scan_id)
                sequence_type_id = extract_sequence_type(sequence_type_class, ds, db_session)
                sequence_id = extract_sequence(sequence_class, db_session, session_id, sequence_type_id)
                repetition_id = extract_repetition(repetition_class, ds, db_session, sequence_id)

                checked_folders[folder] = repetition_id
            extract_dicom(dicom_class, db_session, filename, checked_folders[folder])

        except (InvalidDicomError, IsADirectoryError):
            logging.warning("%s is not a DICOM file !" % filename)

        except (err.IntegrityError, exc.IntegrityError):
            print_db_except()
            db_session.rollback()

    stop_time = time.clock()
    logging.debug("[DONE]")

    logging.info("Closing DB connection")
    db_session.close()

    duration = stop_time - start_time
    logging.debug("Processed %s files in %s" % (str(count_files), str(duration)))


########################################################################################################################
# FUNCTIONS - UTILS
########################################################################################################################

def parse_args():

    parser = argparse.ArgumentParser()
    parser.add_argument('root')
    parser.add_argument('db')
    parser.add_argument(
        "-i",
        "--id",
        help="path to CSV file containing unique participants IDs"
    )
    return parser.parse_args()


def format_date(date):
    return datetime.datetime(int(date[:4]), int(date[4:6]), int(date[6:8]))


{'fraction_non_alphanumeric': 0.31217964364168904, 'fraction_numerical': 0.00244081034903588, 'mean_word_length': 4.488948425987943, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5590659', 'n_tokens_mistral': 2117, 'n_tokens_neox': 1972, 'n_words': 423}