<commit_before>import os
import json
import itertools
import datetime

from django.conf import settings
from django.contrib.auth.models import User, Group, Permission
from django.contrib.auth import authenticate
from django.test import Client, TestCase
from django.test.client import BOUNDARY, MULTIPART_CONTENT, encode_multipart
from django.core.urlresolvers import reverse
from django.core.management import call_command
from django.utils import timezone

from rest_framework.authtoken.models import Token

from app.models import Parent, Teacher, Checkcode, Profile, TimeSlot, Order, \
        WeeklyTimeSlot
from app.utils.algorithm import Tree, Node
from app.utils.types import parseInt
from teacher.views import information_complete_percent
from app.models import Region
from app.utils.algorithm import verify_sig
from app.tasks import send_push

app_path = os.path.abspath(os.path.dirname(__file__))


# Create your tests here.
class TestApi(TestCase):
    def setUp(self):
        # 确保单元测试不会发送短信
        self.assertTrue(settings.FAKE_SMS_SERVER)

    def tearDown(self):
        pass

    def test_token_key(self):
        # 测试token是否能正常创建
        user = User.objects.get(username="parent0")
        token = Token.objects.create(user=user)
        self.assertTrue(isinstance(token.key, str))

    def test_concrete_timeslots(self):
        hours = 2
        weekly_time_slots = list(WeeklyTimeSlot.objects.filter(
                weekday=1, start=datetime.time(8, 0)))
        teacher = Teacher.objects.all()[0]
        timeslots = Order.objects.concrete_timeslots(
                hours, weekly_time_slots, teacher)
        self.assertEqual(len(timeslots), 1)
        ts = timeslots[0]
        self.assertEqual(timezone.localtime(ts['start']).hour, 8)

    def test_sms_login(self):
        phone = '0001'
        code = '1111'
        client = Client()
        sms_url = reverse('sms')
        # parent login or register via sms
        parent_group, _new = Group.objects.get_or_create(name="家长")
        self.assertIsNotNone(parent_group)
        # (1) default content_type
        # send
        data = {'action': "send", 'phone': phone}
        response = client.post(sms_url, data=data)
        self.assertEqual(response.status_code, 200)
        json_ret = json.loads(response.content.decode())
        self.assertTrue(json_ret["sent"])
        # verify
        data = {'action': "verify", 'phone': phone, 'code': code}
        response = client.post(sms_url, data=data)
        self.assertEqual(response.status_code, 200)
        json_ret = json.loads(response.content.decode())
        self.assertTrue(json_ret["verified"])
        token = json_ret.get("token")
        self.assertTrue(isinstance(token, str) and token != '')
        # (2) json content_typ
        # send
        content_type = "application/json"
        data = {'action': "send", 'phone': phone}
        data = json.dumps(data)
        response = client.post(sms_url, data=data, content_type=content_type)
        self.assertEqual(response.status_code, 200)
        json_ret = json.loads(response.content.decode())
        self.assertTrue(json_ret["sent"])
        # verify
        data = {'action': "verify", 'phone': phone, 'code': code}
        data = json.dumps(data)
        response = client.post(sms_url, data=data, content_type=content_type)
        self.assertEqual(response.status_code, 200)
        json_ret = json.loads(response.content.decode())
        self.assertTrue(json_ret["verified"])
        token = json_ret.get("token")
        self.assertTrue(isinstance(token, str) and token != '')

    def test_teacher_list(self):
        client = Client()
        url = "/api/v1/teachers"
        response = client.get(url)
        self.assertEqual(response.status_code, 200)
        url = "/api/v1/teachers?grade=4&subject=3&tags=2+6"
        response = client.get(url)
        self.assertEqual(response.status_code, 200)

    def test_teacher_detail(self):
        client = Client()
        url = "/api/v1/teachers"
        response = client.get(url)
        pk = json.loads(response.content.decode())['results'][0]['id']
        url = "/api/v1/teachers/%d" % pk
        response = client.get(url)
        self.assertEqual(response.status_code, 200)

    def test_tag_list(self):
        client = Client()
        url = "/api/v1/tags"
        response = client.get(url)
        self.assertEqual(response.status_code, 200)

    def test_grade_list(self):
        client = Client()
        url = "/api/v1/grades"
        response = client.get(url)
        self.assertEqual(response.status_code, 200)

    def test_memberservice_list(self):
        client = Client()
        url = "/api/v1/memberservices"
        response = client.get(url)
        self.assertEqual(response.status_code, 200)

    def test_weeklytimeslot_list(self):
        client = Client()
        url = "/api/v1/weeklytimeslots"
        response = client.get(url)
        self.assertEqual(response.status_code, 200)

    def test_policy(self):
        client = Client()
        url = "/api/v1/policy"
        response = client.get(url)
        self.assertEqual(response.status_code, 200)

    def test_teacher_weekly_time_slot(self):
        client = Client()
        username = "parent0"
        password = "<PASSWORD>"
        client.login(username=username, password=password)

        url = "/api/v1/teachers/1/weeklytimeslots?school_id=1"
        response = client.get(url)
        self.assertEqual(response.status_code, 200)

    def test_get_token_key(self):
        client = Client()
        request_url = "/api/v1/token-auth"
        username = "parent1"
        password = "<PASSWORD>"
        user = authenticate(username=username, password=password)
        self.assertNotEqual(user, None)
        parent_user = User.objects.get(username=username)
        self.assertEqual(parent_user.is_active, 1)
        response = client.post(request_url, {"username": username,
                                             "password": password})
        self.assertEqual(response.status_code, 200)

        client2 = Client()
        response2 = client2.post(request_url, {"username": username,
                                               "password": password})
        self.assertEqual(response.content, response2.content)

    def test_modify_student_name(self):
        username = "parent1"
        password = "<PASSWORD>"
        user = User.objects.get(username=username)
        parent = user.parent

        client = Client()
        client.login(username=username, password=password)
        request_url = "/api/v1/parents/%d" % (parent.pk,)
        json_data = json.dumps({"student_name": "StudentNewName"})
        response = client.patch(request_url, content_type="application/json",
                                data=json_data)
        self.assertEqual(200, response.status_code)
        json_ret = json.loads(response.content.decode())
        self.assertEqual(json_ret["done"], "true")
        parent_after = Parent.objects.get(user=user)
        self.assertEqual(parent_after.student_name, "StudentNewName")

        parent_after.student_name = ""
        parent_after.save()
        response = client.patch(request_url, content_type="application/json",
                                data=json_data)
        self.assertEqual(200, response.status_code)
        self.assertEqual(response.content.decode(), '{"done":"true"}')
        json_ret = json.loads(response.content.decode())
        self.assertEqual(json_ret["done"], "true")

        request_url = "/api/v1/parents/%d" % (parent.pk,)
        school_name = '洛阳一中'
        json_data = json.dumps({"student_school_name": school_name})
        response = client.patch(request_url, content_type="application/json",
                                data=json_data)
        self.assertEqual(200, response.status_code)
        json_ret = json.loads(response.content.decode())
        self.assertEqual(json_ret["done"], "true")
        parent_after = Parent.objects.get(user=user)
        self.assertEqual(parent_after.student_school_name, school_name)

    def test_modify_user_avatar(self):
        username = "parent1"
        password = "<PASSWORD>"
        user = User.objects.get(username=username)

        change_profile_perm = Permission.objects.get(name='Can change profile')
        user.user_permissions.add(change_profile_perm)
        user.save()

        client = Client()
        client.login(username=username, password=password)
        request_url = "/api/v1/profiles/%d" % (user.profile.pk,)
        img_name = 'img0'  # NOTE: seq is 0 not 1, seq of the user 'parent1'
        img_path = os.path.join(
                app_path, 'migrations', 'avatars', img_name + '.jpg')
        # print(img_path)
        img_fd = open(img_path, 'rb')
        data = {'avatar': img_fd}
        encoded_data = encode_multipart(BOUNDARY, data)
        response = client.patch(request_url, content_type=MULTIPART_CONTENT,
                                data=encoded_data)
        self.assertEqual(200, response.status_code)
        json_ret = json.loads(response.content.decode())
        self.assertEqual(json_ret["done"], "true")
        profile_after = Profile.objects.get(user=user)
        # print(profile_after.avatar_url())
        self.assertTrue(profile_after.avatar.url.find(img_name) >= 0)

    def test_concrete_time_slots(self):
        client = Client()
        url = ("/api/v1/concrete/timeslots" +
                "?hours=100&weekly_time_slots=3+8+18&teacher=1")
        response = client.get(url)
        self.assertEqual(response.status_code, 200)

    def test_send_push(self):
        send_push('Hello')

    def test_create_order(self):
        client = Client()
        username = "parent0"
        password = "<PASSWORD>"
        client.login(username=username, password=password)
        request_url = "/api/v1/orders"
        json_data = json.dumps({
            'teacher': 2, 'school': 1, 'grade': 1, 'subject': 1,
            'coupon': 2, 'hours': 14, 'weekly_time_slots': [3, 8],
            })
        response = client.post(request_url, content_type="application/json",
                               data=json_data,)
        self.assertEqual(201, response.status_code)
        pk = json.loads(response.content.decode())['id']

        request_url = "/api/v1/orders/%d" % pk
        response = client.get(request_url, content_type='application/json')
        self.assertEqual(200, response.status_code)

        json_ret = json.loads(response.content.decode())
        self.assertEqual(json_ret['status'], 'u')

        # Test create charge object
        json_data = json.dumps({
            'action': 'pay', 'channel': 'alipay',
            })
        response = client.patch(request_url, content_type="application/json",
                                data=json_data)
        self.assertEqual(200, response.status_code)

        data = json.loads(response.content.decode())

        charge_id = data['id']

        json_data = json.dumps({
            "id": "evt_ugB6x3K43D16wXCcqbplWAJo",
            "created": 1440407501,
            "livemode": False,
            "type": "charge.succeeded",
            "data": {
                "object": {
                    "id":  charge_id,
                    "object": "charge",
                    "created": 1440407501,
                    "livemode": True,
                    "paid": True,
                    "refunded": False,
                    "app": "app_urj1WLzvzfTK0OuL",
                    "channel": "upacp",
                    "order_no": "123456789",
                    "client_ip": "127.0.0.1",
                    "amount": 100,
                    "amount_settle": 0,
                    "currency": "cny",
                    "subject": "Your Subject",
                    "body": "Your Body",
                    "extra": {
                        },
                    "time_paid": 1440407501,
                    "time_expire": 1440407501,
                    "time_settle": None,
                    "transaction_no": "1224524301201505066067849274",
                    "refunds": {
                        "object": "list",
                        "url": "/v1/charges/ch_Xsr7u35O3m1Ged2ODmi4Lw/refunds",
                        "has_more": False,
                        "data": [
                            ]
                        },
                    "amount_refunded": 0,
                    "failure_code": None,
                    "failure_msg": None,
                    "metadata": {
                        },
                    "credential": {
                        },
                    "description": None
                    }
                },
            "object": "event",
            "pending_webhooks": 0,
            "request": "iar_qH4y1KbTy5eLGm1uHSTS00s"
            })

        request_url = '/api/v1/charge_succeeded'

        response = client.post(
                request_url, content_type="application/json", data=json_data)
        self.assertEqual(200, response.status_code)

        order = Order.objects.get(pk=pk)
        self.assertEqual(order.status, 'p')

        request_url = "/api/v1/subject/1/record"
        response = client.get(request_url)
        self.assertEqual(200, response.status_code)
        json_ret = json.loads(response.content.decode())
        self.assertTrue(json_ret['evaluated'])

        # Available to oneself
        request_url = "/api/v1/teachers/2/weeklytimeslots?school_id=1"
        response = client.get(request_url)
        self.assertEqual(response.status_code, 200)
        data = json.loads(response.content.decode())

        for value in data.values():
            for d in value:
                self.assertTrue(d['available'])

        # Available time for other teacher
        client = Client()
        username = "parent1"
        password = "<PASSWORD>"
        client.login(username=username, password=password)
        request_url = "/api/v1/teachers/2/weeklytimeslots?school_id=1"
        response = client.get(request_url)
        self.assertEqual(response.status_code, 200)
        data = json.loads(response.content.decode())

        for value in data.values():<commit_msg>SERVER-109 Add more unit tests.<commit_after>import os
import json
import itertools
import datetime

from django.conf import settings
from django.contrib.auth.models import User, Group, Permission
from django.contrib.auth import authenticate
from django.test import Client, TestCase
from django.test.client import BOUNDARY, MULTIPART_CONTENT, encode_multipart
from django.core.urlresolvers import reverse
from django.core.management import call_command
from django.utils import timezone

from rest_framework.authtoken.models import Token

from app.models import Parent, Teacher, Checkcode, Profile, TimeSlot, Order, \
        WeeklyTimeSlot
from app.utils.algorithm import Tree, Node
from app.utils.types import parseInt
from teacher.views import information_complete_percent
from app.models import Region
from app.utils.algorithm import verify_sig
from app.tasks import send_push

app_path = os.path.abspath(os.path.dirname(__file__))


# Create your tests here.
class TestApi(TestCase):
    def setUp(self):
        # 确保单元测试不会发送短信
        self.assertTrue(settings.FAKE_SMS_SERVER)

    def tearDown(self):
        pass

    def test_token_key(self):
        # 测试token是否能正常创建
        user = User.objects.get(username="parent0")
        token = Token.objects.create(user=user)
        self.assertTrue(isinstance(token.key, str))

    def test_concrete_timeslots(self):
        hours = 2
        weekly_time_slots = list(WeeklyTimeSlot.objects.filter(
                weekday=1, start=datetime.time(8, 0)))
        teacher = Teacher.objects.all()[0]
        timeslots = Order.objects.concrete_timeslots(
                hours, weekly_time_slots, teacher)
        self.assertEqual(len(timeslots), 1)
        ts = timeslots[0]
        self.assertEqual(timezone.localtime(ts['start']).hour, 8)

    def test_sms_login(self):
        phone = '0001'
        code = '1111'
        client = Client()
        sms_url = reverse('sms')
        # parent login or register via sms
        parent_group, _new = Group.objects.get_or_create(name="家长")
        self.assertIsNotNone(parent_group)
        # (1) default content_type
        # send
        data = {'action': "send", 'phone': phone}
        response = client.post(sms_url, data=data)
        self.assertEqual(response.status_code, 200)
        json_ret = json.loads(response.content.decode())
        self.assertTrue(json_ret["sent"])
        # verify
        data = {'action': "verify", 'phone': phone, 'code': code}
        response = client.post(sms_url, data=data)
        self.assertEqual(response.status_code, 200)
        json_ret = json.loads(response.content.decode())
        self.assertTrue(json_ret["verified"])
        token = json_ret.get("token")
        self.assertTrue(isinstance(token, str) and token != '')
        # (2) json content_typ
        # send
        content_type = "application/json"
        data = {'action': "send", 'phone': phone}
        data = json.dumps(data)
        response = client.post(sms_url, data=data, content_type=content_type)
        self.assertEqual(response.status_code, 200)
        json_ret = json.loads(response.content.decode())
        self.assertTrue(json_ret["sent"])
        # verify
        data = {'action': "verify", 'phone': phone, 'code': code}
        data = json.dumps(data)
        response = client.post(sms_url, data=data, content_type=content_type)
        self.assertEqual(response.status_code, 200)
        json_ret = json.loads(response.content.decode())
        self.assertTrue(json_ret["verified"])
        token = json_ret.get("token")
        self.assertTrue(isinstance(token, str) and token != '')

    def test_teacher_list(self):
        client = Client()
        url = "/api/v1/teachers"
        response = client.get(url)
        self.assertEqual(response.status_code, 200)
        url = "/api/v1/teachers?grade=4&subject=3&tags=2+6"
        response = client.get(url)
        self.assertEqual(response.status_code, 200)

    def test_teacher_detail(self):
        client = Client()
        url = "/api/v1/teachers"
        response = client.get(url)
        pk = json.loads(response.content.decode())['results'][0]['id']
        url = "/api/v1/teachers/%d" % pk
        response = client.get(url)
        self.assertEqual(response.status_code, 200)

    def test_tag_list(self):
        client = Client()
        url = "/api/v1/tags"
        response = client.get(url)
        self.assertEqual(response.status_code, 200)

    def test_grade_list(self):
        client = Client()
        url = "/api/v1/grades"
        response = client.get(url)
        self.assertEqual(response.status_code, 200)

    def test_memberservice_list(self):
        client = Client()
        url = "/api/v1/memberservices"
        response = client.get(url)
        self.assertEqual(response.status_code, 200)

    def test_weeklytimeslot_list(self):
        client = Client()
        url = "/api/v1/weeklytimeslots"
        response = client.get(url)
        self.assertEqual(response.status_code, 200)

    def test_policy(self):
        client = Client()
        url = "/api/v1/policy"
        response = client.get(url)
        self.assertEqual(response.status_code, 200)

    def test_teacher_weekly_time_slot(self):
        client = Client()
        username = "parent0"
        password = "<PASSWORD>"
        client.login(username=username, password=password)

        url = "/api/v1/teachers/1/weeklytimeslots?school_id=1"
        response = client.get(url)
        self.assertEqual(response.status_code, 200)

    def test_get_token_key(self):
        client = Client()
        request_url = "/api/v1/token-auth"
        username = "parent1"
        password = "<PASSWORD>"
        user = authenticate(username=username, password=password)
        self.assertNotEqual(user, None)
        parent_user = User.objects.get(username=username)
        self.assertEqual(parent_user.is_active, 1)
        response = client.post(request_url, {"username": username,
                                             "password": password})
        self.assertEqual(response.status_code, 200)

        client2 = Client()
        response2 = client2.post(request_url, {"username": username,
                                               "password": password})
        self.assertEqual(response.content, response2.content)

    def test_modify_student_name(self):
        username = "parent1"
        password = "<PASSWORD>"
        user = User.objects.get(username=username)
        parent = user.parent

        client = Client()
        client.login(username=username, password=password)
        request_url = "/api/v1/parents/%d" % (parent.pk,)
        json_data = json.dumps({"student_name": "StudentNewName"})
        response = client.patch(request_url, content_type="application/json",
                                data=json_data)
        self.assertEqual(200, response.status_code)
        json_ret = json.loads(response.content.decode())
        self.assertEqual(json_ret["done"], "true")
        parent_after = Parent.objects.get(user=user)
        self.assertEqual(parent_after.student_name, "StudentNewName")

        parent_after.student_name = ""
        parent_after.save()
        response = client.patch(request_url, content_type="application/json",
                                data=json_data)
        self.assertEqual(200, response.status_code)
        self.assertEqual(response.content.decode(), '{"done":"true"}')
        json_ret = json.loads(response.content.decode())
        self.assertEqual(json_ret["done"], "true")

        request_url = "/api/v1/parents/%d" % (parent.pk,)
        school_name = '洛阳一中'
        json_data = json.dumps({"student_school_name": school_name})
        response = client.patch(request_url, content_type="application/json",
                                data=json_data)
        self.assertEqual(200, response.status_code)
        json_ret = json.loads(response.content.decode())
        self.assertEqual(json_ret["done"], "true")
        parent_after = Parent.objects.get(user=user)
        self.assertEqual(parent_after.student_school_name, school_name)

    def test_modify_user_avatar(self):
        username = "parent1"
        password = "<PASSWORD>"
        user = User.objects.get(username=username)

        change_profile_perm = Permission.objects.get(name='Can change profile')
        user.user_permissions.add(change_profile_perm)
        user.save()

        client = Client()
        client.login(username=username, password=password)
        request_url = "/api/v1/profiles/%d" % (user.profile.pk,)
        img_name = 'img0'  # NOTE: seq is 0 not 1, seq of the user 'parent1'
        img_path = os.path.join(
                app_path, 'migrations', 'avatars', img_name + '.jpg')
        # print(img_path)
        img_fd = open(img_path, 'rb')
        data = {'avatar': img_fd}
        encoded_data = encode_multipart(BOUNDARY, data)
        response = client.patch(request_url, content_type=MULTIPART_CONTENT,
                                data=encoded_data)
        self.assertEqual(200, response.status_code)
        json_ret = json.loads(response.content.decode())
        self.assertEqual(json_ret["done"], "true")
        profile_after = Profile.objects.get(user=user)
        # print(profile_after.avatar_url())
        self.assertTrue(profile_after.avatar.url.find(img_name) >= 0)

    def test_concrete_time_slots(self):
        client = Client()
        url = ("/api/v1/concrete/timeslots" +
                "?hours=100&weekly_time_slots=3+8+18&teacher=1")
        response = client.get(url)
        self.assertEqual(response.status_code, 200)

    def test_send_push(self):
        send_push('Hello')

    def test_create_order(self):
        client = Client()
        username = "parent0"
        password = "<PASSWORD>"
        client.login(username=username, password=password)
        request_url = "/api/v1/orders"
        json_data = json.dumps({
            'teacher': 2, 'school': 1, 'grade': 1, 'subject': 1,
            'coupon': 2, 'hours': 14, 'weekly_time_slots': [3, 8],
            })
        response = client.post(request_url, content_type="application/json",
                               data=json_data,)
        self.assertEqual(201, response.status_code)
        pk = json.loads(response.content.decode())['id']

        request_url = "/api/v1/orders/%d" % pk
        response = client.get(request_url, content_type='application/json')
        self.assertEqual(200, response.status_code)

        json_ret = json.loads(response.content.decode())
        self.assertEqual(json_ret['status'], 'u')

        # Test create charge object
        json_data = json.dumps({
            'action': 'pay', 'channel': 'alipay',
            })
        response = client.patch(request_url, content_type="application/json",
                                data=json_data)
        self.assertEqual(200, response.status_code)

        data = json.loads(response.content.decode())

        charge_id = data['id']

        json_data = json.dumps({
            "id": "evt_ugB6x3K43D16wXCcqbplWAJo",
            "created": 1440407501,
            "livemode": False,
            "type": "charge.succeeded",
            "data": {
                "object": {
                    "id":  charge_id,
                    "object": "charge",
                    "created": 1440407501,
                    "livemode": True,
                    "paid": True,
                    "refunded": False,
                    "app": "app_urj1WLzvzfTK0OuL",
                    "channel": "upacp",
                    "order_no": "123456789",
                    "client_ip": "127.0.0.1",
                    "amount": 100,
                    "amount_settle": 0,
                    "currency": "cny",
                    "subject": "Your Subject",
                    "body": "Your Body",
                    "extra": {
                        },
                    "time_paid": 1440407501,
                    "time_expire": 1440407501,
                    "time_settle": None,
                    "transaction_no": "1224524301201505066067849274",
                    "refunds": {
                        "object": "list",
                        "url": "/v1/charges/ch_Xsr7u35O3m1Ged2ODmi4Lw/refunds",
                        "has_more": False,
                        "data": [
                            ]
                        },
                    "amount_refunded": 0,
                    "failure_code": None,
                    "failure_msg": None,
                    "metadata": {
                        },
                    "credential": {
                        },
                    "description": None
                    }
                },
            "object": "event",
            "pending_webhooks": 0,
            "request": "iar_qH4y1KbTy5eLGm1uHSTS00s"
            })

        request_url = '/api/v1/charge_succeeded'

        response = client.post(
                request_url, content_type="application/json", data=json_data)
        self.assertEqual(200, response.status_code)

        order = Order.objects.get(pk=pk)
        self.assertEqual(order.status, 'p')

        request_url = "/api/v1/subject/1/record"
        response = client.get(request_url)
        self.assertEqual(200, response.status_code)
        json_ret = json.loads(response.content.decode())
        self.assertTrue(json_ret['evaluated'])

        # Available to oneself
        request_url = "/api/v1/teachers/2/weeklytimeslots?school_id=1"
        response = client.get(request_url)
        self.assertEqual(response.status_code, 200)
        data = json.loads(response.content.decode())

        for value in data.values():
            for d in value:
                self.assertTrue(d['available'])

        # Concrete time slot
        hours = 6
        weekly_time_slots = list(
                WeeklyTimeSlot.objects.filter(pk__in=[3, 8, 20]))
        teacher = Teacher.objects.get(pk=2)
        timeslots = Order.objects.concrete_timeslots(
                hours, weekly_time_slots, teacher)
        self.assertEqual(len(timeslots), 3)
        wts = weekly_time_slots[2]
        for ts in timeslots:
            self.assertEqual(
                    timezone.localtime(ts['start']).isoweekday(), wts.weekday)
            self.assertEqual(
                    timezone.localtime(ts['start']).time(), wts.start)

        # Available time for other teacher
        client = Client()
        username = "parent1"
        password = "<PASSWORD>"
        client.login(username=username, password=password)
        request_url = "/api/v1/teachers/2/weeklytimeslots?school_id=1"
        response = client.get(request_url)
        self.assertEqual(response.status_code, 200)
        data = json.loads(response.content.decode())

        for value in data.values():
{'fraction_non_alphanumeric': 0.11846677906616661, 'fraction_numerical': 0.019234077693286996, 'mean_word_length': 2.175699300699301, 'pattern_counts': {'":': 94, '<': 15, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7206417', 'n_tokens_mistral': 8873, 'n_tokens_neox': 8417, 'n_words': 1504}