<commit_before>##########################################
# your code below

from textblob import TextBlob
# import time

from nltk.tag import StanfordNERTagger
st = StanfordNERTagger('english.all.3class.distsim.crf.ser.gz')
res = st.tag('<NAME> is studying at Stony Brook University in NY'.split())

# res = [('Rami', 'PERSON'), ('Eid', 'PERSON'), ('Stony', 'ORGANIZATION'), ('Brook', 'ORGANIZATION'), ('University', 'ORGANIZATION')]

# join the ner result list into list of entities
def join_by_tags(ners):
  ners.append(('$', 'O'))
  joined = []
  words_to_join = []
  prevE = False
  for w, e in ners:
    if e == 'O':
      if e != prevE:
        if len(words_to_join) > 0:
          joined.append((' '.join(words_to_join), prevE))
        words_to_join = []
    else:
      if e == prevE:
        words_to_join.append(w)
      else:
        if len(words_to_join) > 0:
          joined.append((' '.join(words_to_join), prevE))
          words_to_join = []
        words_to_join.append(w)
    prevE = e
  # termination cond
  ners.pop()
  return joined

print(join_by_tags(res))


# def translate(s):
#   b = TextBlob(s)
#   return b.translate().raw

# print(translate('hola amigos'))

# # Get the TextBlob basic properties in JSON, sent to msg.from via socket
# def getTB(msg):
#   start = time.time()
#   b = TextBlob(msg.get('input'))
#   reply = {
#     'to': msg.get('from'),
#     'from': id,
#     'hash': msg.get('hash'),
#     'correct': b.correct().raw,
#     'lemmatize': list(b.words.lemmatize()),
#     'noun_phrases': b.noun_phrases,
#     'parse': b.parse(),
#     'raw': b.raw,
#     'subjectivity': b.subjectivity,
#     'tags': [list(pair) for pair in b.tags],
#     'time': time.time() - start
#   }
#   return reply

# # !defaults to english for now
# # !hacks: in socketIO-client/transports.py change 'six.b' to 'six.u'. Waiting for author to merge
# def translate(msg):
#   start = time.time()
#   b = TextBlob(msg.get('input'))
#   output = b.translate().raw
#   reply = {
#     'to': msg.get('from'),
#     'from': id,
#     'hash': msg.get('hash'),
#     'output': _translate(msg.get('input')),
#     'time': time.time() - start
#   }
#   return reply<commit_msg>add tb basic with translate, NERTag, TextBlobParse<commit_after># Basic TextBlob wrapped module
# Textblob docs:
# https://textblob.readthedocs.org/en/dev/quickstart.html
# Note that Textblob has some initialization overhead, but subsequent runs will complete in about 0.05s

from textblob import TextBlob
import time
from nltk.tag import StanfordNERTagger
st = StanfordNERTagger('english.all.3class.distsim.crf.ser.gz')


# Helper methods
##########################################

# join the ner result list into list of entities
def join_by_tags(ners):
  ners.append(('$', 'O'))
  joined = []
  words = []
  prevE = False
  for w, e in ners:
    if e == 'O':
      if e != prevE:
        if len(words) > 0:
          joined.append((' '.join(words), prevE))
        words = []
    else:
      if e == prevE:
        words.append(w)
      else:
        if len(words) > 0:
          joined.append((' '.join(words), prevE))
          words = []
        words.append(w)
    prevE = e
  # termination cond
  ners.pop()
  return joined


# Exported methods
##########################################

# Use Google translate, with auto langauge-detection. 
# May optionally specify language code translating to (default='en')
# Language code refer here https://cloud.google.com/translate/v2/using_rest
def translate(s, to='en'):
  b = TextBlob(s)
  if to == 'en':
    return b.translate().raw 
  else:
    from_lang = b.detect_language()
    return b.translate(from_lang=from_lang, to=to).raw


# Uses StanfordNERTagger to extract tag with NER (Named Entity Recognition)
def NERTag(s):
  res = st.tag(s.split())
  return join_by_tags(res)


# Get the TextBlob basic parsed results in JSON
# Note this method has an initialization overhead,
# after the first call it will only take time = 0.05s
def TextBlobParse(sentence):
  start = time.time()
  b = TextBlob(sentence)
  parsed = {
    'tags': b.tags,
    'noun_phrases': b.noun_phrases,
    'correct': b.correct().raw,
    'lemmatize': list(b.words.lemmatize()),
    'parse': b.parse(),
    'raw': b.raw,
    'subjectivity': b.subjectivity,
    'tags': [list(pair) for pair in b.tags],
    'time': time.time() - start
  }
  return parsed



# print(translate('hola amigos'))
# print(translate('hola amigos', 'zh-CN'))

# print(NERTag('<NAME> is studying at Stony Brook University in NY'))

# print(TextBlobParse('<NAME> is studying at Stony Brook University in NY'))
{'fraction_non_alphanumeric': 0.17672600955275727, 'fraction_numerical': 0.002822405557967868, 'mean_word_length': 3.321763602251407, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 10, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1305697', 'n_tokens_mistral': 1594, 'n_tokens_neox': 1563, 'n_words': 420}