<commit_before>from gender_detector import GenderDetector
import nltk

class BylineGender():
  def __init__(self):
    self.detector = GenderDetector('us')

  def gender(self,byline):
    gender_result = {"female":0, "male":0,"unknown":0}
    for name in self.get_first_names(byline):
      gender_result[self.detector.guess(name)] += 1
    return gender_result

  def get_first_names(self, byline):
    print byline
    byline = byline.strip()
    spaces = byline.count(' ')
    commas = byline.count(',')
    conjunctions = byline.count('and')
    bylines_result = []
    if(commas == 0 and conjunctions == 0):
      bylines_result.append(self.get_first_name_from_fullname(byline))
    if(conjunctions >0):
      for name in byline.split('and'):
        bylines_result.append(self.get_first_name_from_fullname(name.strip()))
    if(spaces < 3 and commas == 1):
      bylines_result.append(self.get_first_name_from_reversename(byline))
    return bylines_result

  # assumes there's a single comma
  def get_first_name_from_reversename(self, byline):
    split_byline = [x.strip() for x in byline.split(',')]
    return self.get_first_name_from_fullname(split_byline[1])

  def get_first_name_from_fullname(self, byline):
    tokens = nltk.word_tokenize(byline)
    first_name = ""
    for token in tokens:
      if(token.count(".") > 0 or len(token) == 1):
        continue
      return token
    return byline.split(' ')

  def test_first_names(self):
    test_strings = [
      ["<NAME>", ["Nathan"]],
      ["<NAME>",["Nathan"]],
      ["<NAME>", ["Pamela"]],
      ["<NAME>", ["Pamela"]],
      ["<NAME> and <NAME>", ["Nathan", "Pamela"]],
      ["<NAME> and <NAME>", ["Nathan", "Pamela"]]
    ]
    for test_string in test_strings:
      names = self.get_first_names(test_string[0])
      print 'Got: [%s]' % ', '.join(map(str, names))
      print 'Expected: [%s]' % ', '.join(map(str, test_string[1]))
      print '--------'<commit_msg>corrected an issue in byline_gender that incorrectly assigned gender to surnames in initial-only names<commit_after>from gender_detector import GenderDetector
import nltk

class BylineGender():
  def __init__(self):
    self.detector = GenderDetector('us')

  def byline_gender(self,byline):
    gender_result = {"female":0, "male":0,"unknown":0}
    for name in self.get_first_names(byline):
      print name
      if(name is None):
        gender_result["unknown"] += 1
      else:
        gender_result[self.detector.guess(name)] += 1
    return gender_result

  def get_first_names(self, byline):
    byline = byline.strip()
    spaces = byline.count(' ')
    commas = byline.count(',')
    conjunctions = byline.count('and')
    bylines_result = []
    if(commas == 0 and conjunctions == 0):
      bylines_result.append(self.get_first_name_from_fullname(byline))
    if(conjunctions >0):
      for name in byline.split('and'):
        bylines_result.append(self.get_first_name_from_fullname(name.strip()))
    if(spaces < 3 and commas == 1):
      bylines_result.append(self.get_first_name_from_reversename(byline))
    return bylines_result

  # assumes there's a single comma
  def get_first_name_from_reversename(self, byline):
    split_byline = [x.strip() for x in byline.split(',')]
    # set offset to 0 since the surname has already been stripped
    return self.get_first_name_from_fullname(split_byline[1], 0)

  def get_first_name_from_fullname(self, byline, offset=None):
    if(offset == None):
      offset = -1
    tokens = nltk.word_tokenize(byline)
    first_name = ""
    for i in range(0, (len(tokens)+offset)):
      if(tokens[i].count(".") > 0 or len(tokens[i]) == 1):
        continue
      return tokens[i]
    return None

  def test_first_names(self):
    test_strings = [
      ["<NAME>", ["Nathan"]],
      ["<NAME>", [None]],
      ["<NAME>", [None]],
      ["<NAME>", [None]],
      ["<NAME>", [None]],
      ["<NAME>", ["JN"]],
      ["<NAME>",["Nathan"]],
      ["<NAME>", ["Pamela"]],
      ["<NAME>", ["Pamela"]],
      ["<NAME> and <NAME>", ["Nathan", "Pamela"]],
      ["J. <NAME> and <NAME>", ["Nathan", "Pamela"]]
    ]
    for test_string in test_strings:
      names = self.get_first_names(test_string[0])
      print 'Got: [%s]' % ', '.join(map(str, names))
      print 'Expected: [%s]' % ', '.join(map(str, test_string[1]))
      print '--------'
{'fraction_non_alphanumeric': 0.16282287822878228, 'fraction_numerical': 0.00761070110701107, 'mean_word_length': 3.4989626556016598, 'pattern_counts': {'":': 6, '<': 26, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4990501', 'n_tokens_mistral': 1539, 'n_tokens_neox': 1492, 'n_words': 314}