<commit_before>#!/usr/bin/env python
'''Link abbreviations to their full names

Based on

A Simple Algorithm for Identifying Abbreviations Definitions in Biomedical Text
<NAME> and <NAME>
Biocomputing, 2003, pp 451-462.


# License: GNU General Public License, see http://www.clips.ua.ac.be/~vincent/scripts/LICENSE.txt
'''
__author__ = '<NAME>, <NAME>'

import re

def getcandidates(sentence):
  rv = []
  if '-LRB-' in sentence:
    # XXX HACK (?) Johannes
    # The original version checks some sentence properties here first:
    # balanced parentheses and that the first paren is a left one
    # We don't care that much since we only admit one-word abbrevs
    closeindex = -1
    while 1:
      # Look for parenOpen parenthesis
      try:
        openindex = closeindex + 1 + sentence[closeindex + 1:].index('-LRB-')
      except ValueError:
        break
      closeindex = openindex + 2
      
      # XXX HACK (?) Johannes
      # The original version picks up acronyms that include parentheses and multiple words.
      # Since there are no such gene abbreviations, such words won't be confused for being 
      # genes anyways, so I just stop after the first word in the parenthesis
      start = openindex + 1
      stop = start + 1
      if start >= len(sentence):
        break
      abbrev = sentence[start]

      if conditions(abbrev):
        rv.append((start, stop, abbrev))
  return rv

def conditions(string):
  if re.match('([A-Za-z]\. ?){2,}', string.lstrip()):
    return True
  if len(string) < 2 or len(string) > 10:
    return False
  if len(string.split()) > 2:
    return False
  if not re.search('[A-Za-z]', string):
    return False
  if not string[0].isalnum():
    return False

  return True

def getdefinition((startAbbrev, stopAbbrev, abbrev), sentence, stopLastAbbrev):
  ':type candidate: (int, int, list[str])'
  ':type sentence: list[str]'
  startTokens = stopLastAbbrev + 1;
  # Take the tokens in front of the candidate
  tokens = [word.lower() for word in sentence[startTokens:startAbbrev - 1]]

  # the char that we are looking for
  key = abbrev[0].lower()

  # Count the number of tokens that start with the same character as the<commit_msg>One more fix for abbreviations<commit_after>#!/usr/bin/env python
'''Link abbreviations to their full names

Based on

A Simple Algorithm for Identifying Abbreviations Definitions in Biomedical Text
<NAME> and <NAME>
Biocomputing, 2003, pp 451-462.


# License: GNU General Public License, see http://www.clips.ua.ac.be/~vincent/scripts/LICENSE.txt
'''
__author__ = '<NAME>, <NAME>'

import re

def getcandidates(sentence):
  rv = []
  if '-LRB-' in sentence:
    # XXX HACK (?) Johannes
    # The original version checks some sentence properties here first:
    # balanced parentheses and that the first paren is a left one
    # We don't care that much since we only admit one-word abbrevs
    closeindex = -1
    while 1:
      # Look for parenOpen parenthesis
      try:
        openindex = closeindex + 1 + sentence[closeindex + 1:].index('-LRB-')
      except ValueError:
        break
      closeindex = openindex + 2
      
      # XXX HACK (?) Johannes
      # The original version picks up acronyms that include parentheses and multiple words.
      # Since there are no such gene abbreviations, such words won't be confused for being 
      # genes anyways, so I just stop after the first word in the parenthesis
      start = openindex + 1
      stop = start + 1
      if start >= len(sentence):
        break
      abbrev = sentence[start]

      if conditions(abbrev):
        rv.append((start, stop, abbrev))
  return rv

def conditions(string):
  if re.match('([A-Za-z]\. ?){2,}', string.lstrip()):
    return True
  if len(string) < 2 or len(string) > 10:
    return False
  if len(string.split()) > 2:
    return False
  if not re.search('[A-Za-z]', string):
    return False
  if not string[0].isalnum():
    return False

  return True

def getdefinition((startAbbrev, stopAbbrev, abbrev), sentence, stopLastAbbrev):
  ':type candidate: (int, int, list[str])'
  ':type sentence: list[str]'
  startTokens = stopLastAbbrev + 1;
  # Take the tokens in front of the candidate
  tokens = [word.lower() for word in sentence[startTokens:startAbbrev - 1]]

  # the char that we are looking for
  key = abbrev[0].lower()

  if len(tokens) == 0:
    raise ValueError('[SUP] Not enough keys')

  # Count the number of tokens that start with the same character as the
{'fraction_non_alphanumeric': 0.10284561953842707, 'fraction_numerical': 0.011875420120994847, 'mean_word_length': 3.0991735537190084, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '472849', 'n_tokens_mistral': 1440, 'n_tokens_neox': 1313, 'n_words': 509}