<commit_before>import random
from datetime import datetime
import tensorflow as tf
from tensorflow import flags
from collections import defaultdict

FLAGS = flags.FLAGS

if __name__=="__main__":
  flags.DEFINE_string("video_id_file", "", "The file in which every line is a video_id.")
  flags.DEFINE_string("input_freq_file", "", "The previous weight of each video.")
  flags.DEFINE_string("input_error_file", "", "The error of each video.")
  flags.DEFINE_string("output_freq_file", "", "Output the corresponding freq of video_ids.")

if __name__=="__main__":
  word_list = []
  freq_dict = defaultdict(lambda: 1.0)<commit_msg>fixed a bug in reweight-sample-freq<commit_after>import random
import math
from datetime import datetime
import tensorflow as tf
from tensorflow import flags
from collections import defaultdict

FLAGS = flags.FLAGS

if __name__=="__main__":
  flags.DEFINE_string("video_id_file", "", "The file in which every line is a video_id.")
  flags.DEFINE_string("input_freq_file", "", "The previous weight of each video.")
  flags.DEFINE_string("input_error_file", "", "The error of each video.")
  flags.DEFINE_string("output_freq_file", "", "Output the corresponding freq of video_ids.")

if __name__=="__main__":
  word_list = []
  freq_dict = defaultdict(lambda: 1.0)
{'fraction_non_alphanumeric': 0.11162790697674418, 'fraction_numerical': 0.0031007751937984496, 'mean_word_length': 5.830687830687831, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2010019', 'n_tokens_mistral': 395, 'n_tokens_neox': 381, 'n_words': 121}