# Path: prep/parsing.py
# def parse_votes(raw_votes, author):
#     votes = {}
#     str_votes = raw_votes.strip().split(':::')
# 
#     for vote in str_votes:
#       if vote.startswith('</endperson>'):
#         break
#       user, help_vote = vote.split(':')
#       help_vote = int(help_vote)
#       if help_vote < 0 or help_vote > 5:
#         raise Exception()
#       if user not in votes and user != author:
#         # avoid duplication: seems that when there is a also a comment, the vote is duplicated
#         votes[user] = help_vote
#     return votes


from re import match
from networkx import DiGraph
from numpy import mean, nan
from nltk.tokenize import RegexpTokenizer
from nltk.corpus import wordnet
from prep.parsing import parse_votes



_FILE = 'data/reviews.txt'
_NEWFILE = 'data/reviews_filtered.txt'


f = open(_FILE, 'r')
user_votes = {}
for line in f:
  l = line.strip().split('::::')
  votes = parse_votes(l[7], l[0].strip())
  product = l[1].strip()
  for voter in votes:
    if (product, voter) not in user_votes:
      user_votes[(product, voter)] = 0
    user_votes[(product, voter)] += 1

sel_keys = set()
for key in user_votes:
  if user_votes[key] >= 10:
    sel_keys.add(key)

f.seek(0, 0)
output = open(_NEWFILE, 'w')
for line in f:
  l = line.strip().split('::::')
  str_votes = l[7].split(':::') 

# Path: prep/parsing.py
# def parse_votes(raw_votes, author):
#     votes = {}
#     str_votes = raw_votes.strip().split(':::')
# 
#     for vote in str_votes:
#       if vote.startswith('</endperson>'):
#         break
#       user, help_vote = vote.split(':')
#       help_vote = int(help_vote)
#       if help_vote < 0 or help_vote > 5:
#         raise Exception()
#       if user not in votes and user != author:
#         # avoid duplication: seems that when there is a also a comment, the vote is duplicated
#         votes[user] = help_vote
#     return votes


# Path: script/filter_sparse.py
from re import match
from networkx import DiGraph
from numpy import mean, nan
from nltk.tokenize import RegexpTokenizer
from nltk.corpus import wordnet
from prep.parsing import parse_votes




_FILE = 'data/reviews.txt'
_NEWFILE = 'data/reviews_filtered.txt'


f = open(_FILE, 'r')
user_votes = {}
for line in f:
  l = line.strip().split('::::')
  votes = parse_votes(l[7], l[0].strip())
  product = l[1].strip()
  for voter in votes:
    if (product, voter) not in user_votes:
      user_votes[(product, voter)] = 0
    user_votes[(product, voter)] += 1

sel_keys = set()
for key in user_votes:
  if user_votes[key] >= 10:
    sel_keys.add(key)

f.seek(0, 0)
output = open(_NEWFILE, 'w')
for line in f:
  l = line.strip().split('::::')
  str_votes = l[7].split(':::') 

  new_str_votes = []