<commit_before># -*- coding: utf-8 -*-
"""
Building bag of words model from raw data.
The model structure is [ { filename: { words: amount }}]
"""

import os
import os.path
import logging
import jieba
import json

from tqdm import tqdm

BASE_DIR = os.path.dirname(os.path.dirname(__file__))
RAW_DATA_DIR = f'{BASE_DIR}/raw_data'
BAG_OF_WORDS = f'{BASE_DIR}/data/bag_of_words.json'

STOP_WORDS = set(line.strip() for line in open(f'{BASE_DIR}/script/stop_words.txt', 'r', encoding='utf-8').readlines())
STOP_WORDS.add("生物谷")


def build_bag_of_words_model():
    """
    build bag of words model
    """
    bag_of_word = []
    count = 0
    err = 0
    for parent, _, file_names in os.walk(RAW_DATA_DIR):
        logging.info(f'got {len(file_names)} files')
        pb_file_names = tqdm(file_names)
        for filename in tqdm(file_names):
            pb_file_names.set_description("Processing %s" % filename)
            full_file_name = os.path.join(parent, filename)
            with open(full_file_name, 'r', encoding='utf-8') as raw_data:
                try:
                    word_list = {}
                    for line in raw_data.readlines():
                        seg_list_stop_words = map(lambda word:word.strip(), jieba.cut(line, cut_all=False))
                        seg_list = filter(lambda word:word and word not in STOP_WORDS, seg_list_stop_words)
                        for word in seg_list:
                            if word in word_list:
                                word_list[word] += 1
                            else:
                                word_list[word] = 1
                    bag_of_word.append({filename: word_list})
                    count += 1
                except Exception as exception:
                    err += 1
                    logging.error(f"Error during process {filename}, error message: {exception}")
    with open(BAG_OF_WORDS, 'w', encoding='utf-8') as result_file:
        json.dump(bag_of_word, result_file, ensure_ascii=False)
    logging.info(f"finish, ${count} success, ${err} failed")

if __name__ == '__main__':
    build_bag_of_words_model()

<commit_msg>fix some bugs<commit_after># -*- coding: utf-8 -*-
"""
Building bag of words model from raw data.
The model structure is [ { filename: { words: amount }}]
"""

import os
import os.path
import jieba
import json
import logging

from tqdm import tqdm

BASE_DIR = os.path.dirname(os.path.dirname(__file__))
print(BASE_DIR)
RAW_DATA_DIR = f'{BASE_DIR}/raw_data'
BAG_OF_WORDS = f'{BASE_DIR}/data/bag_of_words.json'

STOP_WORDS = set(line.strip() for line in open(f'{BASE_DIR}/script/stop_words.txt', 'r', encoding='utf-8').readlines())
STOP_WORDS.add("生物谷")


def build_bag_of_words_model():
    """
    build bag of words model
    """
    bag_of_word = []
    count = 0
    err = 0
    for parent, _, file_names in os.walk(RAW_DATA_DIR):
        print(f'got {len(file_names)} files')
        pb_file_names = tqdm(file_names)
        for filename in pb_file_names:
            pb_file_names.set_description("Processing %s" % filename)
            full_file_name = os.path.join(parent, filename)
            with open(full_file_name, 'r', encoding='utf-8') as raw_data:
                try:
                    word_list = {}
                    for line in raw_data.readlines():
                        seg_list_stop_words = map(lambda word:word.strip(), jieba.cut(line, cut_all=False))
                        seg_list = filter(lambda word:word and word not in STOP_WORDS, seg_list_stop_words)
                        for word in seg_list:
                            if word in word_list:
                                word_list[word] += 1
                            else:
                                word_list[word] = 1
                    bag_of_word.append({filename: word_list})
                    count += 1
                except Exception as exception:
                    err += 1
                    print(f"Error during process {filename}, error message: {exception}")
    with open(BAG_OF_WORDS, 'w', encoding='utf-8') as result_file:
        json.dump(bag_of_word, result_file, ensure_ascii=False)
    print(f"finish, {count} success, {err} failed")

if __name__ == '__main__':
    build_bag_of_words_model()


{'fraction_non_alphanumeric': 0.10949589683470105, 'fraction_numerical': 0.004689331770222743, 'mean_word_length': 2.1344599559147683, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4415777', 'n_tokens_mistral': 1352, 'n_tokens_neox': 1307, 'n_words': 309}