xu-song commited on
Commit
6f9d07b
1 Parent(s): 0177868
Files changed (2) hide show
  1. app.py +1 -1
  2. util.py +12 -2
app.py CHANGED
@@ -217,7 +217,7 @@ with gr.Blocks(css="style.css") as demo:
217
 
218
  user_input.change(tokenize_pair,
219
  [user_input, tokenizer_type_1, tokenizer_type_2],
220
- [output_text_1, output_table_1, output_text_2, output_table_2])
221
 
222
  tokenizer_type_2.change(tokenize, [user_input, tokenizer_type_2],
223
  [output_text_2, output_table_2])
 
217
 
218
  user_input.change(tokenize_pair,
219
  [user_input, tokenizer_type_1, tokenizer_type_2],
220
+ [output_text_1, output_table_1, output_text_2, output_table_2]) # , pass_request=1
221
 
222
  tokenizer_type_2.change(tokenize, [user_input, tokenizer_type_2],
223
  [output_text_2, output_table_2])
util.py CHANGED
@@ -1,12 +1,13 @@
1
  import gradio as gr
2
  import json
 
3
  import pandas as pd
4
  from vocab import load_tokener
5
  from utils.zh_util import iter_vocab
6
  from utils.log_util import logger
7
 
8
 
9
- def tokenize(text, tokenizer_type, color_num=5, update=True):
10
  """
11
  TODO: cache tokenizer
12
  """
@@ -62,7 +63,16 @@ def tokenize(text, tokenizer_type, color_num=5, update=True):
62
  return pos_tokens, table_df, len(encoding)
63
 
64
 
65
- def tokenize_pair(text, tokenizer_type_1, tokenizer_type_2):
 
 
 
 
 
 
 
 
 
66
  pos_tokens_1, table_df_1 = tokenize(text, tokenizer_type_1)
67
  pos_tokens_2, table_df_2 = tokenize(text, tokenizer_type_2)
68
  return pos_tokens_1, table_df_1, pos_tokens_2, table_df_2
 
1
  import gradio as gr
2
  import json
3
+ import socket
4
  import pandas as pd
5
  from vocab import load_tokener
6
  from utils.zh_util import iter_vocab
7
  from utils.log_util import logger
8
 
9
 
10
+ def tokenize(text, tokenizer_type, color_num=5, update=True):
11
  """
12
  TODO: cache tokenizer
13
  """
 
63
  return pos_tokens, table_df, len(encoding)
64
 
65
 
66
+ def tokenize_pair(text, tokenizer_type_1, tokenizer_type_2, request: gr.Request):
67
+ if request:
68
+ client_ip = request.client.host
69
+ # local_ip = socket.gethostbyname(socket.gethostbyname(""))
70
+ headers = request.kwargs['headers']
71
+ if headers and 'x-forwarded-for' in headers:
72
+ x_forwarded_for = headers['x-forwarded-for']
73
+ client_ip = x_forwarded_for.split(' ')[0] if x_forwarded_for else ""
74
+ logger.info("[client ip]", client_ip, tokenizer_type_1, tokenizer_type_2)
75
+
76
  pos_tokens_1, table_df_1 = tokenize(text, tokenizer_type_1)
77
  pos_tokens_2, table_df_2 = tokenize(text, tokenizer_type_2)
78
  return pos_tokens_1, table_df_1, pos_tokens_2, table_df_2