<commit_before># -*- coding: utf-8 -*-

import warnings
import unittest

from cwharaj.parser.base_parser import BaseParser


class BaseParseTest(unittest.TestCase):
    def setUp(self):
        self.comment_header_string = '<div class=" comment_header">  <a href="https://haraj.com.sa/users/iabdulmjeed" class="username">iabdulmjeed</a>   قبل شهر و 2 أسبوع في<a href="https://haraj.com.sa/city/%D8%A7%D9%84%D8%B1%D9%8A%D8%A7%D8%B6" class="city-head">الرياض</a><br>رقم الاعلان : 12880409<br> آخر تحديث قبل ساعه و 4 دقيقه<div class="pull-left"></div></div>'

    def test_regex_get_model_id(self):
        _base_parser = BaseParser()
        published_date = _base_parser.get_published_date(self.comment_header_string)
        expect = '   قبل شهر و 2 أسبوع في'
        self.assertEqual(published_date, expect, "The same published date")<commit_msg>cwharaj.<commit_after># -*- coding: utf-8 -*-

import warnings
import unittest

from cwharaj.parser.base_parser import BaseParser


class BaseParseTest(unittest.TestCase):
    def setUp(self):
        self.comment_header_string = '<div class=" comment_header">  <a href="https://haraj.com.sa/users/iabdulmjeed" class="username">iabdulmjeed</a>   قبل شهر و 2 أسبوع في<a href="https://haraj.com.sa/city/%D8%A7%D9%84%D8%B1%D9%8A%D8%A7%D8%B6" class="city-head">الرياض</a><br>رقم الاعلان : 12880409<br> آخر تحديث قبل ساعه و 4 دقيقه<div class="pull-left"></div></div>'

    def test_regex_get_model_id(self):
        _base_parser = BaseParser()

        published_date = _base_parser.get_published_date(self.comment_header_string)

        expect = '   قبل شهر و 2 أسبوع في'
        self.assertEqual(published_date, expect, "The same published date")
{'fraction_non_alphanumeric': 0.13897102306327616, 'fraction_numerical': 0.02956830277942046, 'mean_word_length': 4.423076923076923, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '355134', 'n_tokens_mistral': 742, 'n_tokens_neox': 671, 'n_words': 117}