<commit_before>#!/usr/bin/env python3
"""
    This file contains the code that:
    1) retrieves data from the summary service on a term by term basis
    2) maintains records for common terms (cache)
    3) manages the provenance for specific heatmaps that have been saved
    4) maintains collapse maps??? or should this happen independently?
    5) calls into the ontology to traverse the graph

"""

import json
from functools import wraps
from os import environ, path
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
from collections import namedtuple, defaultdict

import requests
import psycopg2 as pg
from psycopg2.extras import register_hstore

if environ.get('HEATMAP_PROD',None):  # set in heatmaps.wsgi if not globally
    embed = lambda args: print("THIS IS PRODUCTION AND PRODUCTION DOESNT LIKE IPYTHON ;_;")
else:
    from IPython import embed

from .visualization import applyCollapse, dict_to_matrix,sCollapseToSrcId, sCollapseToSrcName, make_png, sCollToLength, sCollByTermParent
from pyontutils.scigraph_client import Graph, Vocabulary
from pyontutils.hierarchies import creatTree, in_tree
from .scigraph_client import Graph, Vocabulary

# initilaize scigraph services
graph = Graph()
vocab = Vocabulary()

"""
INSERT INTO view_history (id, source_id_order, term_counts) VALUES (
1,
'{"a", "b", "c"}',
'brain => "[1 2 3 4]"'
);

SELECT * FROM view_sources LEFT OUTER JOIN source_entity ON REPLACE(view_sources.src_nif_id,'_','-')=source_entity.nif_id;

SELECT nif_id FROM relation_entity WHERE is_view=TRUE; --burak has a service for this
"""

# sql to update from NULL termcounts to, with an extra check to make sure we never overwrite accidentally
"""
UPDATE term_history SET term_counts=%s WHERE id=%s and term_counts IS NULL;
"""

# sql to retrieve unfinished term counts
"""<commit_msg>updated services to use pyontutils heatmaps instead of the old local copy<commit_after>#!/usr/bin/env python3
"""
    This file contains the code that:
    1) retrieves data from the summary service on a term by term basis
    2) maintains records for common terms (cache)
    3) manages the provenance for specific heatmaps that have been saved
    4) maintains collapse maps??? or should this happen independently?
    5) calls into the ontology to traverse the graph

"""

import json
from functools import wraps
from os import environ, path
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
from collections import namedtuple, defaultdict

import requests
import psycopg2 as pg
from psycopg2.extras import register_hstore

if environ.get('HEATMAP_PROD',None):  # set in heatmaps.wsgi if not globally
    embed = lambda args: print("THIS IS PRODUCTION AND PRODUCTION DOESNT LIKE IPYTHON ;_;")
else:
    from IPython import embed

from .visualization import applyCollapse, dict_to_matrix,sCollapseToSrcId, sCollapseToSrcName, make_png, sCollToLength, sCollByTermParent
from pyontutils.scigraph_client import Graph, Vocabulary
from pyontutils.hierarchies import creatTree, in_tree
from pyontutils.scigraph_client import Graph, Vocabulary

# initilaize scigraph services
graph = Graph()
vocab = Vocabulary()

"""
INSERT INTO view_history (id, source_id_order, term_counts) VALUES (
1,
'{"a", "b", "c"}',
'brain => "[1 2 3 4]"'
);

SELECT * FROM view_sources LEFT OUTER JOIN source_entity ON REPLACE(view_sources.src_nif_id,'_','-')=source_entity.nif_id;

SELECT nif_id FROM relation_entity WHERE is_view=TRUE; --burak has a service for this
"""

# sql to update from NULL termcounts to, with an extra check to make sure we never overwrite accidentally
"""
UPDATE term_history SET term_counts=%s WHERE id=%s and term_counts IS NULL;
"""

# sql to retrieve unfinished term counts
"""
{'fraction_non_alphanumeric': 0.07445099089448312, 'fraction_numerical': 0.006963042313872523, 'mean_word_length': 5.266778523489933, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7445904', 'n_tokens_mistral': 1156, 'n_tokens_neox': 1099, 'n_words': 439}