<commit_before>import os, subprocess, json, codecs, argparse, glob
import urllib
import settings
import time
from nltk.tokenize import sent_tokenize

def _call(cmd):
    done = False
    tries = 0
    while not done and tries < 10:
        try:
            p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            stdout, stderr = p.communicate()
            #print p.returncode
            if stdout != "":
                done = True
            else:
                print "Error"
                tries += 1
        except:
            print "Error"
            tries += 1
    return p.returncode, stdout


def _dbpedia_annotate(sentence):
    cmd = ['curl', '-H', 'Accept: application/json', settings.DBPEDIA_SPOTLIGHT_ENDPOINT, '--data-urlencode', 'text=%s' % sentence,
               '--data', 'confidence=0.2', '--data', 'support=20']
    return _call(cmd)

def _tagme_annotate(text):
    server = "http://tagme.di.unipi.it/tag"
    data = "key="+settings.TAGME_KEY+"&include_categories=true&text="+urllib.quote(text.encode("utf8"))
    cmd = ['curl', '--data', data, server]
    return _call(cmd)
    
def _babelfy_annotate(text):
    server = "https://babelfy.io/v1/disambiguate"
    data = "key="+settings.BABELFY_KEY+"&lang=EN&annType=ALL&text="+urllib.quote(text.encode("utf8"))
    cmd = ['curl', '--data', data, server]
    return _call(cmd)

def create_directories(technique, source):
    if not os.path.exists(settings.PATH+"/entities"):
        os.mkdir(settings.PATH+"/entities")
    if not os.path.exists(settings.PATH+"/entities/"+technique):
        os.mkdir(settings.PATH+"/entities/"+technique)
    if not os.path.exists(settings.PATH+"/entities/"+technique+"/"+source):
        os.mkdir(settings.PATH+"/entities/"+technique+"/"+source)

def spotlight(sentences):
    text = "\n".join(sentences)
    ner_sentences = []
    if text != "":
        code, data = _dbpedia_annotate(text)
        data = eval(data)
        accumulated = 0
        index = 0
        for sentence in sentences:
            entities = []
            if 'Resources' in data:
                for resource in data['Resources']:
                    offset = int(resource['@offset'])
                    if offset >= accumulated and offset < accumulated + len(sentence):
                        entity = dict()
                        entity['uri'] = resource['@URI']
                        entity['types'] = resource['@types']
                        entity['startChar'] = offset - accumulated
                        entity['endChar'] = offset - accumulated + len(resource['@surfaceForm'])
                        entity['confidence'] = resource['@similarityScore']
                        entity['label'] = resource['@surfaceForm']
                        entities.append(entity)
                    if offset > accumulated + len(sentence):
                        break
            accumulated += len(sentence) + 1
            ner_sentences.append({"text":sentence.strip(),"entities":entities,"index":index})
            index += 1
    return ner_sentences

def tagme(sentences):<commit_msg>corrected errors in run_el<commit_after>import os, subprocess, json, codecs, argparse, glob
import urllib
import settings
import time
from nltk.tokenize import sent_tokenize

def _call(cmd):
    done = False
    tries = 0
    while not done and tries < 10:
        try:
            p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            stdout, stderr = p.communicate()
            #print p.returncode
            if stdout != "":
                done = True
            else:
                print "Error"
                tries += 1
            return p.returncode, stdout
        except:
            print "Error"
            tries += 1
            return 0,""

def _dbpedia_annotate(text):
    cmd = ['curl', '-H', 'Accept: application/json', settings.DBPEDIA_SPOTLIGHT_ENDPOINT, '--data-urlencode', 'text=%s' % urllib.quote(text.encode("utf8")),
               '--data', 'confidence=0.2', '--data', 'support=20']
    return _call(cmd)

def _tagme_annotate(text):
    server = "http://tagme.di.unipi.it/tag"
    data = "key="+settings.TAGME_KEY+"&include_categories=true&text="+urllib.quote(text.encode("utf8"))
    cmd = ['curl', '--data', data, server]
    return _call(cmd)
    
def _babelfy_annotate(text):
    server = "https://babelfy.io/v1/disambiguate"
    data = "key="+settings.BABELFY_KEY+"&lang=EN&annType=ALL&text="+urllib.quote(text.encode("utf8"))
    cmd = ['curl', '--data', data, server]
    return _call(cmd)

def create_directories(technique, source):
    if not os.path.exists(settings.PATH+"/entities"):
        os.mkdir(settings.PATH+"/entities")
    if not os.path.exists(settings.PATH+"/entities/"+technique):
        os.mkdir(settings.PATH+"/entities/"+technique)
    if not os.path.exists(settings.PATH+"/entities/"+technique+"/"+source):
        os.mkdir(settings.PATH+"/entities/"+technique+"/"+source)

def spotlight(sentences):
    text = "\n".join(sentences)
    ner_sentences = []
    if text.strip() != "":
        code, data = _dbpedia_annotate(text)
        try:
            data = eval(data)
        except:
            data = []
        accumulated = 0
        index = 0
        for sentence in sentences:
            entities = []
            if 'Resources' in data:
                for resource in data['Resources']:
                    offset = int(resource['@offset'])
                    if offset >= accumulated and offset < accumulated + len(sentence):
                        entity = dict()
                        entity['uri'] = resource['@URI']
                        entity['types'] = resource['@types']
                        entity['startChar'] = offset - accumulated
                        entity['endChar'] = offset - accumulated + len(resource['@surfaceForm'])
                        entity['confidence'] = resource['@similarityScore']
                        entity['label'] = resource['@surfaceForm']
                        entities.append(entity)
                    if offset > accumulated + len(sentence):
                        break
            accumulated += len(sentence) + 1
            ner_sentences.append({"text":sentence.strip(),"entities":entities,"index":index})
            index += 1
    return ner_sentences

def tagme(sentences):
{'fraction_non_alphanumeric': 0.13910719899402704, 'fraction_numerical': 0.00534423137378183, 'mean_word_length': 2.1422222222222222, 'pattern_counts': {'":': 10, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4408509', 'n_tokens_mistral': 1907, 'n_tokens_neox': 1839, 'n_words': 377}