<commit_before>import unittest
import logChunk
from chunkingConstants import *

class logChunktest(unittest.TestCase):

    def readHelper(self,filename):
        inf =open(filename,"r")
        text=""
        for line in inf:
            text+=line

        return text


    def debugFunctions(self, funcList):
        print("===========================================")
        for func in funcList:
            print(func.method)
            print(func.start)
            print(func.end)
            print(func.total_add)
            print(func.total_del)
            print(func.keywordDictionary)           
        print("===========================================")


    def setUp(self):
        self.keyword1 = ["wtc/Assertions.h","excluded","single"]
        self.keyword2 = ["ut_ad","included","single"]
        self.keyword3 = ["try","Included","BLOCK"]
        self.keyword4 = ["for","excludeD","block"]
        self.keyword5 = ["ut_ad included single"]
        self.keyword6 = ["printf(format","str)","included","single"]
        self.keyword7 = ["printf(format, str)","included","single"]
        self.keyword8 = ["assert","incuded","single"]
        self.keyword9 = ["assert","included","lock"]


        self.method1 = "static void blarg() {"
        self.method2 = "int more(int stuff) {"
        self.method3 = "ccv_string * getStuff (int[] stuffToGet) {"
        self.method4 = "int add2to3(int (*functionPtr)(int, int)) {"
        self.method5 = "public static void other(int one, int (*functionPtr)(int, int)) {"
        self.method6 = "static void\n multiline(\n int arg1, string arg2\n) {"
        self.method7 = "int lotsOfSpace     (int stuff) {"
        self.method8 = "                .matrix = {"
        self.method9 = "ccv_string* getStuff (int[] stuffToGet) {"
        self.method10 = "ccv_string *getStuff (int[] stuffToGet) {"
        self.method11 = "void NdbBlob::getBlobEvent(NdbEventImpl& be, const NdbEventImpl* e, const NdbColumnImpl* c) {"
        self.method12 = "bool Repair_mrg_table_error_handler::handle_condition(THD *,uint sql_errno,const char*, MYSQL_ERROR::enum_warning_level level,const char*,MYSQL_ERROR ** cond_hdl) {"
        self.method13 = "(_log2) += 1;  static int CeilingLog2(unsigned int i) {"
        #self.method14 = "(STATEMENT_CONSTRUCTOR_BASE_PARAMETERS, ExpressionPtr exp)   : Statement(STATEMENT_CONSTRUCTOR_BASE_PARAMETER_VALUES),     m_exp(exp) {"
        self.method15 = "static JSC::UString& globalExceptionString(){"
        self.method16 = "(jint) AWT_WINDOW_LOST_FOCUS, (jint) AWT_WINDOW_DEACTIVATED,  static gboolean window_focus_in_cb (GtkWidget * widget, GdkEventFocus *event, jobject peer) {"
        self.method17 = "LinuxPtraceDumper dumper(getpid()); }  TEST(LinuxPtraceDumperTest, FindMappings) {"
        self.method18 = "endif  defined(HAVE_ALTIVEC)   include  elif defined(HAVE_SSE2)   include  endif   ONLY64 inline static int idxof(int i) {"
        self.method19 = "#define UNUSED_VARIABLE(x) (void)(x) const char *sfmt_get_idstring(sfmt_t * sfmt) {"
        self.method20 = "static av_always_inline void hl_motion_420(H264Context *h, uint8_t *dest_y, uint8_t *dest_cb, uint8_t *dest_cr,            qpel_mc_func (*qpix_put)[16], h264_chroma_mc_func (*chroma_put),       qpel_mc_func (*qpix_avg)[16], h264_chroma_mc_func (*chroma_avg),            h264_weight_func *weight_op, h264_biweight_func *weight_avg,             int pixel_shift) {"
        self.method21 = "        for (j = 0; j < i; j++)             if (!memcmp(h->pps.scaling_matrix8[j], h->pps.scaling_matrix8[i],                         64 * sizeof(uint8_t))) {"
        self.method22 = " c_type jl_unbox_##j_type(jl_value_t *v)  {"

        self.testChunk = logChunk.logChunk("", "C++")
        self.testChunk_C = logChunk.logChunk("", "C")

        self.javaMethod1 = "public static Intent createIntent(Context context, String username, String password) {"
        self.javaMethod2 = " public <V> V post(final String uri, final Object params, final Type type) \n throws IOException {"
        self.javaMethod3 = "public static Intent createIntent(final Collection<? extends Issue> issues,\n final Repository repository, final int position) {"
        self.javaMethod4 = "@Override \n public List<User> run(Account account) throws Exception {"
        self.javaMethod5 = "private JClass typeBoundsToJClass(GeneratedClassHolder holder, List<? extends TypeMirror> bounds, Map<String, TypeMirror> actualTypes) {"
        self.javaMethod6 = " public JMethod implementMethod(GeneratedClassHolder holder, List<ExecutableElement> methods, String methodName, String returnType, String... parameterTypes) {"


        #Read in the single tests
        self.chunk1 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk1.txt"), "C++") #Check C++
        self.chunk2 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk2.txt"), "C") #Check C
        self.chunk3 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk3.txt"), "C++") #Check C++
        #self.chunk4 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk4.txt")) #Nope
        #self.chunk5 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk5.txt")) #Nope
        self.chunk6 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk6.txt"), "C++") #Check C++
        self.chunk7 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk7.txt"), "C++") #Check C++
        self.chunk8 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk8.txt"), "C++") #Check C
        self.chunk9 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk9.txt"), "C++") #Check C++
        self.chunk10 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk10.txt"), "C++") #Check C++
        self.chunk11 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk11.txt"), "C++") #Check C++
        self.chunk12 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk12.txt"), "C++") #Check C++
        self.chunk13 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk13.txt"), "C++") #Check C++
        self.chunk14 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk14.txt"), "C++") #Check C++
        self.chunk15 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk15.txt"), "C++") #Check C++
        #self.chunk16 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk16.txt")) #Nope
        #self.chunk17 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk17.txt")) #Nope
        #self.chunk18 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk18.txt")) #Nope
        #self.chunk19 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk19.txt")) #Nope
        #self.chunk20 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk20.txt")) #Nope
        self.chunk21 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk21.txt"), "C++") #Check C++ 
        self.chunk22 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk22.txt"), "C++") #Check C++
        self.chunk23 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk23.txt"), "C++") #Check C++
        self.chunk24 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk24.txt"), "C++") #Check C++
        self.chunk25 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk25.txt"), "C") #Check C
        #self.chunk26 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk26.txt")) #Nope
        self.chunk27 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk27.txt"), "C") #Check C
        #self.chunk28 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk28.txt")) #Nope
        self.chunk29 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk29.txt"), "C") #Check C
        #self.chunk30 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk30.txt")) #Maybe?
        self.chunk31 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk31.txt"), "C++") #Check C++
        self.chunk32 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk32.txt"), "C++") #Check C++
        self.chunk33 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk33.txt"), "C++") #Check C++
        #self.chunk34 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk34.txt")) #Nope
        self.chunk35 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk35.txt"), "C") #Check C
        self.chunk36 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk36.txt"), "C++") #Check C++
        self.chunk37 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk37.txt"), "C++") #Check C++
        self.chunk38 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk38.txt"), "C++") #Check C++
        self.chunk39 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk39.txt"), "C") #C
        self.chunk40 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk40.txt"), "C") #Check C
        self.chunk41 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk41.txt"), "C++") #Check C++
        self.chunk42 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk42.txt"), "C++") # C++
        self.chunk43 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk43.txt"), "C") #Check C
        self.chunk44 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk44.txt"), "C++") # C++
        self.chunk45 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk45.txt"), "C++") # C++ 
        self.chunk46 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk46.txt"), "C++") # C++
        self.chunk47 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk47.txt"), "C++") # C++
        self.chunk48 = logChunk.logChunk(self.readHelper("testfiles/Single/testChunk48.txt"), "C++") # C++




    def test_KeywordValidityCheck(self):
        self.assertTrue(self.testChunk.keywordValidityCheck(self.keyword1))
        self.assertTrue(self.testChunk.keywordValidityCheck(self.keyword2))
        self.assertTrue(self.testChunk.keywordValidityCheck(self.keyword3))
        self.assertTrue(self.testChunk.keywordValidityCheck(self.keyword4))
        self.assertFalse(self.testChunk.keywordValidityCheck(self.keyword5))
        self.assertFalse(self.testChunk.keywordValidityCheck(self.keyword6))
        self.assertTrue(self.testChunk.keywordValidityCheck(self.keyword7))
        self.assertFalse(self.testChunk.keywordValidityCheck(self.keyword8))
        self.assertFalse(self.testChunk.keywordValidityCheck(self.keyword9))

<commit_msg>travis setup trial 21<commit_after>import unittest
import logChunk
from chunkingConstants import *

class logChunktest(unittest.TestCase):

    def readHelper(self,filename):
        inf =open(filename,"r")
        text=""
        for line in inf:
            text+=line

        return text


    def debugFunctions(self, funcList):
        print("===========================================")
        for func in funcList:
            print(func.method)
            print(func.start)
            print(func.end)
            print(func.total_add)
            print(func.total_del)
            print(func.keywordDictionary)           
        print("===========================================")


    def setUp(self):
        self.keyword1 = ["wtc/Assertions.h","excluded","single"]
        self.keyword2 = ["ut_ad","included","single"]
        self.keyword3 = ["try","Included","BLOCK"]
        self.keyword4 = ["for","excludeD","block"]
        self.keyword5 = ["ut_ad included single"]
        self.keyword6 = ["printf(format","str)","included","single"]
        self.keyword7 = ["printf(format, str)","included","single"]
        self.keyword8 = ["assert","incuded","single"]
        self.keyword9 = ["assert","included","lock"]


        self.method1 = "static void blarg() {"
        self.method2 = "int more(int stuff) {"
        self.method3 = "ccv_string * getStuff (int[] stuffToGet) {"
        self.method4 = "int add2to3(int (*functionPtr)(int, int)) {"
        self.method5 = "public static void other(int one, int (*functionPtr)(int, int)) {"
        self.method6 = "static void\n multiline(\n int arg1, string arg2\n) {"
        self.method7 = "int lotsOfSpace     (int stuff) {"
        self.method8 = "                .matrix = {"
        self.method9 = "ccv_string* getStuff (int[] stuffToGet) {"
        self.method10 = "ccv_string *getStuff (int[] stuffToGet) {"
        self.method11 = "void NdbBlob::getBlobEvent(NdbEventImpl& be, const NdbEventImpl* e, const NdbColumnImpl* c) {"
        self.method12 = "bool Repair_mrg_table_error_handler::handle_condition(THD *,uint sql_errno,const char*, MYSQL_ERROR::enum_warning_level level,const char*,MYSQL_ERROR ** cond_hdl) {"
        self.method13 = "(_log2) += 1;  static int CeilingLog2(unsigned int i) {"
        #self.method14 = "(STATEMENT_CONSTRUCTOR_BASE_PARAMETERS, ExpressionPtr exp)   : Statement(STATEMENT_CONSTRUCTOR_BASE_PARAMETER_VALUES),     m_exp(exp) {"
        self.method15 = "static JSC::UString& globalExceptionString(){"
        self.method16 = "(jint) AWT_WINDOW_LOST_FOCUS, (jint) AWT_WINDOW_DEACTIVATED,  static gboolean window_focus_in_cb (GtkWidget * widget, GdkEventFocus *event, jobject peer) {"
        self.method17 = "LinuxPtraceDumper dumper(getpid()); }  TEST(LinuxPtraceDumperTest, FindMappings) {"
        self.method18 = "endif  defined(HAVE_ALTIVEC)   include  elif defined(HAVE_SSE2)   include  endif   ONLY64 inline static int idxof(int i) {"
        self.method19 = "#define UNUSED_VARIABLE(x) (void)(x) const char *sfmt_get_idstring(sfmt_t * sfmt) {"
        self.method20 = "static av_always_inline void hl_motion_420(H264Context *h, uint8_t *dest_y, uint8_t *dest_cb, uint8_t *dest_cr,            qpel_mc_func (*qpix_put)[16], h264_chroma_mc_func (*chroma_put),       qpel_mc_func (*qpix_avg)[16], h264_chroma_mc_func (*chroma_avg),            h264_weight_func *weight_op, h264_biweight_func *weight_avg,             int pixel_shift) {"
        self.method21 = "        for (j = 0; j < i; j++)             if (!memcmp(h->pps.scaling_matrix8[j], h->pps.scaling_matrix8[i],                         64 * sizeof(uint8_t))) {"
        self.method22 = " c_type jl_unbox_##j_type(jl_value_t *v)  {"

        self.testChunk = logChunk.logChunk("", "C++")
        self.testChunk_C = logChunk.logChunk("", "C")

        self.javaMethod1 = "public static Intent createIntent(Context context, String username, String password) {"
        self.javaMethod2 = " public <V> V post(final String uri, final Object params, final Type type) \n throws IOException {"
        self.javaMethod3 = "public static Intent createIntent(final Collection<? extends Issue> issues,\n final Repository repository, final int position) {"
        self.javaMethod4 = "@Override \n public List<User> run(Account account) throws Exception {"
        self.javaMethod5 = "private JClass typeBoundsToJClass(GeneratedClassHolder holder, List<? extends TypeMirror> bounds, Map<String, TypeMirror> actualTypes) {"
        self.javaMethod6 = " public JMethod implementMethod(GeneratedClassHolder holder, List<ExecutableElement> methods, String methodName, String returnType, String... parameterTypes) {"


        #Read in the single tests
        self.chunk1 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk1.txt"), "C++") #Check C++
        self.chunk2 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk2.txt"), "C") #Check C
        self.chunk3 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk3.txt"), "C++") #Check C++
        #self.chunk4 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk4.txt")) #Nope
        #self.chunk5 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk5.txt")) #Nope
        self.chunk6 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk6.txt"), "C++") #Check C++
        self.chunk7 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk7.txt"), "C++") #Check C++
        self.chunk8 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk8.txt"), "C++") #Check C
        self.chunk9 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk9.txt"), "C++") #Check C++
        self.chunk10 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk10.txt"), "C++") #Check C++
        self.chunk11 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk11.txt"), "C++") #Check C++
        self.chunk12 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk12.txt"), "C++") #Check C++
        self.chunk13 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk13.txt"), "C++") #Check C++
        self.chunk14 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk14.txt"), "C++") #Check C++
        self.chunk15 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk15.txt"), "C++") #Check C++
        #self.chunk16 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk16.txt")) #Nope
        #self.chunk17 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk17.txt")) #Nope
        #self.chunk18 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk18.txt")) #Nope
        #self.chunk19 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk19.txt")) #Nope
        #self.chunk20 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk20.txt")) #Nope
        self.chunk21 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk21.txt"), "C++") #Check C++ 
        self.chunk22 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk22.txt"), "C++") #Check C++
        self.chunk23 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk23.txt"), "C++") #Check C++
        self.chunk24 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk24.txt"), "C++") #Check C++
        self.chunk25 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk25.txt"), "C") #Check C
        #self.chunk26 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk26.txt")) #Nope
        self.chunk27 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk27.txt"), "C") #Check C
        #self.chunk28 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk28.txt")) #Nope
        self.chunk29 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk29.txt"), "C") #Check C
        #self.chunk30 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk30.txt")) #Maybe?
        self.chunk31 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk31.txt"), "C++") #Check C++
        self.chunk32 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk32.txt"), "C++") #Check C++
        self.chunk33 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk33.txt"), "C++") #Check C++
        #self.chunk34 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk34.txt")) #Nope
        self.chunk35 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk35.txt"), "C") #Check C
        self.chunk36 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk36.txt"), "C++") #Check C++
        self.chunk37 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk37.txt"), "C++") #Check C++
        self.chunk38 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk38.txt"), "C++") #Check C++
        self.chunk39 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk39.txt"), "C") #C
        self.chunk40 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk40.txt"), "C") #Check C
        self.chunk41 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk41.txt"), "C++") #Check C++
        self.chunk42 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk42.txt"), "C++") # C++
        self.chunk43 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk43.txt"), "C") #Check C
        self.chunk44 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk44.txt"), "C++") # C++
        self.chunk45 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk45.txt"), "C++") # C++ 
        self.chunk46 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk46.txt"), "C++") # C++
        self.chunk47 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk47.txt"), "C++") # C++
        self.chunk48 = logChunk.logChunk(self.readHelper("./src/logChunk/testfiles/Single/testChunk48.txt"), "C++") # C++




    def test_KeywordValidityCheck(self):
        self.assertTrue(self.testChunk.keywordValidityCheck(self.keyword1))
        self.assertTrue(self.testChunk.keywordValidityCheck(self.keyword2))
        self.assertTrue(self.testChunk.keywordValidityCheck(self.keyword3))
        self.assertTrue(self.testChunk.keywordValidityCheck(self.keyword4))
        self.assertFalse(self.testChunk.keywordValidityCheck(self.keyword5))
        self.assertFalse(self.testChunk.keywordValidityCheck(self.keyword6))
        self.assertTrue(self.testChunk.keywordValidityCheck(self.keyword7))
        self.assertFalse(self.testChunk.keywordValidityCheck(self.keyword8))
        self.assertFalse(self.testChunk.keywordValidityCheck(self.keyword9))


{'fraction_non_alphanumeric': 0.15890236119974474, 'fraction_numerical': 0.025070653660315436, 'mean_word_length': 3.881842456608812, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2107491', 'n_tokens_mistral': 7259, 'n_tokens_neox': 6960, 'n_words': 1083}