<commit_before>#encoding=utf-8

"""
get q2q similariy from service
curl -X POST -d '{"query":"你知罪吗", "question":"你知道错了吗"}' http://10.191.15.89:40919/cgi-bin/ranker/q2qsimilarity
warp this comand for whole file
"""

import os
import sys
if sys.version_info[0] == 2:
  reload(sys)
  sys.setdefaultencoding("utf-8")
import time
import json
import codecs
import argparse
import traceback
import multiprocessing as MP
import subprocess


def get_q2q_sim(q0, q1):
  cmd = ''' curl -X POST -d '{{"query":"{}", "question":"{}" }}' http://10.191.15.89:40919/cgi-bin/ranker/q2qsimilarity '''.format(q0, q1)
  # print(cmd)
  pro = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
  outputs, errs = pro.communicate()
  return outputs

def jsonWrite(d, file_path, indent=2):
  with codecs.open(file_path, "w", "utf-8") as f:
    json.dump(d,f,ensure_ascii=False, indent=indent)

def split_join(s):
  return "".join(s.strip().split())

def get_q2q_file_sinle_pro(file_path, save_path, parallels=1, time_dealy=2, join_space=False,
                           delimiter="\t",in_one_line=True):
  # file_path is tokenized file
  f = codecs.open(file_path, "r","utf-8")
  results = []
  pros = []

  begin = time.time()
  while True:
    # handle two conditions:
    # {s0}\t{s1}  ||  {s0}\n{s1}\n
    line0 = f.readline()
    if not line0:
      break
    if in_one_line is True:
      try:
        s, t = line0.split(delimiter)
      except Exception:
        print("errors happen in {}".format(line0))
        continue
    else:
      s = line0
      t = f.readline()
      f.readline()
    s, t = s.strip(), t.strip()
    if join_space:
      s = split_join(s)
      t = split_join(t)
    s = s.replace("SEQUENCE_END", "")
    t = t.replace("SEQUENCE_END", "")
    outputs = get_q2q_sim(s,t)
    tmp_dict = {"source":s, "pred":t}
    try:
      rj = json.loads(outputs.strip())
      if "data" not in rj:
        print("errors, {}".format(rj))
        tmp_dict["score"] = -1
      else:
        if str(rj["data"]["error"]) == "0":
          tmp_dict["score"] = rj["data"]["score"]
        if "score" not in tmp_dict:
          tmp_dict["score"] = -1
    except Exception:
      tmp_dict["score"] = -1
      print(outputs)
      traceback.print_exc()
    results.append(tmp_dict)
    if len(results) % 100000 == 0:
      print("finished {} sents".format(len(results)))
      now_time = time.time()
      print("cost {}".format(now_time - begin))
      time.sleep(time_dealy)

  jsonWrite(results, save_path , indent=2)

def get_q2q_file(file_path, save_path, parallels=MP.cpu_count() - 2, time_dealy=2,
                 in_one_line=False, delimiter="\t", join_space=False):

  # file_path is tokenized
  f = codecs.open(file_path, "r","utf-8")
  results = []
  pool = MP.Pool(parallels)
  pros = []

  while True:
    # handle two conditions:
    # {s0}\t{s1}  ||  {s0}\n{s1}\n
    line0 = f.readline()
    if not line0:
      break
    if in_one_line is True:
      try:
        s, t = line0.split(delimiter)
      except Exception:
        print("errors happen in {}".format(line0))
        continue
    else:
      s = line0
      t = f.readline()<commit_msg>add filter sim<commit_after>#encoding=utf-8

"""
get q2q similariy from service
curl -X POST -d '{"query":"你知罪吗", "question":"你知道错了吗"}' http://10.191.15.89:40919/cgi-bin/ranker/q2qsimilarity
warp this comand for whole file
"""

import os
import sys
if sys.version_info[0] == 2:
  reload(sys)
  sys.setdefaultencoding("utf-8")
import time
import json
import codecs
import argparse
import traceback
import multiprocessing as MP
import subprocess


def get_q2q_sim(q0, q1):
  cmd = ''' curl -X POST -d '{{"query":"{}", "question":"{}" }}' http://10.191.15.89:40919/cgi-bin/ranker/q2qsimilarity '''.format(q0, q1)
  # print(cmd)
  pro = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
  outputs, errs = pro.communicate()
  return outputs

def jsonWrite(d, file_path, indent=2):
  with codecs.open(file_path, "w", "utf-8") as f:
    json.dump(d,f,ensure_ascii=False, indent=indent)

def split_join(s):
  return "".join(s.strip().split())

def get_q2q_file_sinle_pro(file_path, save_path, parallels=1, time_dealy=2, join_space=False,
                           delimiter="\t",in_one_line=True):
  # file_path is tokenized file
  f = codecs.open(file_path, "r","utf-8")
  results = []
  pros = []

  begin = time.time()
  while True:
    # handle two conditions:
    # {s0}\t{s1}  ||  {s0}\n{s1}\n
    line0 = f.readline()
    if not line0:
      break
    if in_one_line is True:
      try:
        s, t = line0.split(delimiter)
      except Exception:
        print("errors happen in {}".format(line0))
        continue
    else:
      s = line0
      t = f.readline()
      f.readline()
    s, t = s.strip(), t.strip()
    if join_space:
      s = split_join(s)
      t = split_join(t)
    s = s.replace("SEQUENCE_END", "")
    t = t.replace("SEQUENCE_END", "")
    outputs = get_q2q_sim(s,t)
    tmp_dict = {"source":s, "pred":t}
    try:
      rj = json.loads(outputs.strip())
      if "data" not in rj:
        print("errors, {}".format(rj))
        tmp_dict["score"] = -1
      else:
        if str(rj["data"]["error"]) == "0":
          tmp_dict["score"] = rj["data"]["score"]
        if "score" not in tmp_dict:
          tmp_dict["score"] = -1
    except Exception:
      tmp_dict["score"] = -1
      print(tmp_dict.values())
      print(outputs)
      traceback.print_exc()
    results.append(tmp_dict)
    if len(results) % 100000 == 0:
      print("finished {} sents".format(len(results)))
      now_time = time.time()
      print("cost {}".format(now_time - begin))
      time.sleep(time_dealy)

  jsonWrite(results, save_path , indent=2)

def get_q2q_file(file_path, save_path, parallels=MP.cpu_count() - 2, time_dealy=2,
                 in_one_line=True, delimiter="\t", join_space=False):

  # file_path is tokenized
  f = codecs.open(file_path, "r","utf-8")
  results = []
  pool = MP.Pool(parallels)
  pros = []

  while True:
    # handle two conditions:
    # {s0}\t{s1}  ||  {s0}\n{s1}\n
    line0 = f.readline()
    if not line0:
      break
    if in_one_line is True:
      try:
        s, t = line0.split(delimiter)
      except Exception:
        print("errors happen in {}".format(line0))
        continue
    else:
      s = line0
      t = f.readline()
{'fraction_non_alphanumeric': 0.1579437800900761, 'fraction_numerical': 0.025159186209038672, 'mean_word_length': 2.872519542994588, 'pattern_counts': {'":': 14, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4409791', 'n_tokens_mistral': 2481, 'n_tokens_neox': 2408, 'n_words': 512}