<commit_before>import openpyxl
from indra.statements import *


def process_sheet(sheet):
    stmts = []
    header = [cell.value for cell in next(sheet.rows)]
    for row in sheet.rows:
        row_values = [r.value for r in row]
        if not any(row_values):
            break
        row_dict = {h: v for h, v in zip(header, row_values)}
        stmts.append(make_stmt(row_dict))
    return stmts


def make_stmt(row_dict):
    # Make subject
    subj_concept = Concept(
        name=row_dict['Source/Cause (Factor A)'],
        db_refs={'WM': row_dict['Source/Cause node (WM ontology node)']})
    if row_dict['Source/Cause polarity']:
        if row_dict['Source/Cause polarity'].lower() == 'increase':
            subj_pol = 1
        elif row_dict['Source/Cause polarity'].lower() == 'decrease':
            subj_pol = -1
    else:
        subj_pol = None
    subj_time = TimeContext(text=row_dict['Original temporal text for cause'],
                            start=row_dict['Cause startTime'],
                            end=row_dict['Cause endTime'])
    subj_context = WorldContext(time=subj_time, geo_location=RefContext(
        name=row_dict['CauseLocation']))
    subj = Event(subj_concept, delta=QualitativeDelta(polarity=subj_pol),
                 context=subj_context)

    # Make object
    obj_concept = Concept(
        row_dict['Target/Effect (Factor B)'],
        db_refs={'WM': row_dict['Target/Effect (WM ontology node)']})
    obj_time = TimeContext(text=row_dict['Original temporal text for effect'],
                           start=row_dict['Effect startTime'],
                           end=row_dict['Effect endTime'])
    # If both source and destination locations are present, make Migration obj
    if row_dict['Effect SourceLocation'] and \
            row_dict['Effect DestinationLocation']:
        obj_context = MovementContext(locations=[
            {'location': RefContext(name=row_dict['Effect SourceLocation']),
             'role': 'origin'},
            {'location': RefContext(name=row_dict['Effect DestinationLocation']),
             'role': 'destination'}], time=obj_time)
        obj = Migration(
            obj_concept, delta=QuantitativeState(
                entity='person', value=row_dict['Relation strength estimate'],
                unit=row_dict['Relation strength unit']), context=obj_context)
    # If only one or no location is present, make Event object
    else:
        if row_dict['Target/Effect polarity']:
            if row_dict['Target/Effect polarity'].lower() == 'increase':
                obj_pol = 1
            elif row_dict['Target/Effect polarity'].lower() == 'decrease':
                obj_pol = -1
        else:
            obj_pol = None
        if row_dict['Effect SourceLocation']:
            obj_loc = RefContext(name=row_dict['Effect SourceLocation'])
        elif row_dict['Effect DestinationLocation']:
            obj_loc = RefContext(name=row_dict['Effect DestinationLocation'])
        else:
            obj_loc = None
        obj_context = WorldContext(time=obj_time, geo_location=obj_loc)
        obj = Event(obj_concept, delta=QualitativeDelta(polarity=obj_pol),
                    context=obj_context)
    # Make evidence
    anns = {'provenance': row_dict['Provenance (file id)'],
            'data_format': row_dict['Data format'],
            'group_id': row_dict['Group ID']}
    if anns['data_format'] == 'text':
        text = row_dict['Sentence(s)/Figure/Table']
    else:
        anns['table_figure_name'] = row_dict['Sentence(s)/Figure/Table']
        text = None
    ev = Evidence(text=text, annotations=anns)
    stmt = Influence(subj, obj, [ev])
    return stmt


if __name__ == '__main__':
    fname = 'Initial annotation exercise for migration use case.xlsx'

    wb = openpyxl.load_workbook(fname, read_only=True)
    sheets = wb.sheetnames
    cag_sheets = [s for s in sheets if 'CAG' in s]

    for sheet_name in cag_sheets:
        sheet = wb[sheet_name]
        process_sheet(sheet)<commit_msg>Save statements to a file<commit_after>import openpyxl
import pickle
from indra.statements import *


def process_sheet(sheet):
    stmts = []
    header = [cell.value for cell in next(sheet.rows)]
    for row in sheet.rows:
        row_values = [r.value for r in row]
        if not any(row_values):
            break
        row_dict = {h: v for h, v in zip(header, row_values)}
        stmts.append(make_stmt(row_dict))
    return stmts


def make_stmt(row_dict):
    # Make subject
    subj_concept = Concept(
        name=row_dict['Source/Cause (Factor A)'],
        db_refs={'WM': row_dict['Source/Cause node (WM ontology node)']})
    if row_dict['Source/Cause polarity']:
        if row_dict['Source/Cause polarity'].lower() == 'increase':
            subj_pol = 1
        elif row_dict['Source/Cause polarity'].lower() == 'decrease':
            subj_pol = -1
    else:
        subj_pol = None
    subj_time = TimeContext(text=row_dict['Original temporal text for cause'],
                            start=row_dict['Cause startTime'],
                            end=row_dict['Cause endTime'])
    subj_context = WorldContext(time=subj_time, geo_location=RefContext(
        name=row_dict['CauseLocation']))
    subj = Event(subj_concept, delta=QualitativeDelta(polarity=subj_pol),
                 context=subj_context)

    # Make object
    obj_concept = Concept(
        row_dict['Target/Effect (Factor B)'],
        db_refs={'WM': row_dict['Target/Effect (WM ontology node)']})
    obj_time = TimeContext(text=row_dict['Original temporal text for effect'],
                           start=row_dict['Effect startTime'],
                           end=row_dict['Effect endTime'])
    # If both source and destination locations are present, make Migration obj
    if row_dict['Effect SourceLocation'] and \
            row_dict['Effect DestinationLocation']:
        obj_context = MovementContext(locations=[
            {'location': RefContext(name=row_dict['Effect SourceLocation']),
             'role': 'origin'},
            {'location': RefContext(name=row_dict['Effect DestinationLocation']),
             'role': 'destination'}], time=obj_time)
        obj = Migration(
            obj_concept, delta=QuantitativeState(
                entity='person', value=row_dict['Relation strength estimate'],
                unit=row_dict['Relation strength unit']), context=obj_context)
    # If only one or no location is present, make Event object
    else:
        if row_dict['Target/Effect polarity']:
            if row_dict['Target/Effect polarity'].lower() == 'increase':
                obj_pol = 1
            elif row_dict['Target/Effect polarity'].lower() == 'decrease':
                obj_pol = -1
        else:
            obj_pol = None
        if row_dict['Effect SourceLocation']:
            obj_loc = RefContext(name=row_dict['Effect SourceLocation'])
        elif row_dict['Effect DestinationLocation']:
            obj_loc = RefContext(name=row_dict['Effect DestinationLocation'])
        else:
            obj_loc = None
        obj_context = WorldContext(time=obj_time, geo_location=obj_loc)
        obj = Event(obj_concept, delta=QualitativeDelta(polarity=obj_pol),
                    context=obj_context)
    # Make evidence
    anns = {'provenance': row_dict['Provenance (file id)'],
            'data_format': row_dict['Data format'],
            'group_id': row_dict['Group ID']}
    if anns['data_format'] == 'text':
        text = row_dict['Sentence(s)/Figure/Table']
    else:
        anns['table_figure_name'] = row_dict['Sentence(s)/Figure/Table']
        text = None
    ev = Evidence(text=text, annotations=anns)
    stmt = Influence(subj, obj, [ev])
    return stmt


if __name__ == '__main__':
    fname = 'Initial annotation exercise for migration use case.xlsx'

    wb = openpyxl.load_workbook(fname, read_only=True)
    sheets = wb.sheetnames
    cag_sheets = [s for s in sheets if 'CAG' in s]

    statements = []
    for sheet_name in cag_sheets:
        sheet = wb[sheet_name]
        new_stmts = process_sheet(sheet)
        statements += new_stmts

    with open('stmts_from_migration_table.pkl', 'wb') as f:
        pickle.dump(statements, f)
{'fraction_non_alphanumeric': 0.11472019464720194, 'fraction_numerical': 0.0009732360097323601, 'mean_word_length': 2.6456762749445675, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6264230', 'n_tokens_mistral': 2489, 'n_tokens_neox': 2381, 'n_words': 560}