<commit_before>require 'fiddle'
require 'fiddle/struct'
require 'fiddle/import'
require 'fileutils'
include Fiddle::CParser
include Fiddle::Importer
module Nlpir
  module Ictclas
    NLPIR_FALSE = 0
    NLPIR_TRUE = 1
    POS_MAP_NUMBER = 4
    ICT_POS_MAP_FIRST = 1        #计算所一级标注集
    ICT_POS_MAP_SECOND = 0       #计算所二级标注集
    PKU_POS_MAP_SECOND = 2       #北大二级标注集
    PKU_POS_MAP_FIRST = 3        #北大一级标注集
    POS_SIZE = 40

    Result_t = struct ['int start','int length',"char  sPOS[#{POS_SIZE}]",'int iPOS',
                       'int word_ID','int word_type','int weight']

    GBK_CODE = 0                                                    #默认支持GBK编码
    UTF8_CODE = GBK_CODE + 1                          #UTF8编码
    BIG5_CODE = GBK_CODE + 2                          #BIG5编码
    GBK_FANTI_CODE = GBK_CODE + 3             #GBK编码，里面包含繁体字

    @charset = 'utf-8'

    #提取链接库接口
    libm = Fiddle.dlopen(File.expand_path("../../../bin/libNLPIR.so", __FILE__))

    NLPIR_Init_rb = Fiddle::Function.new(
      libm['NLPIR_Init'],
      [Fiddle::TYPE_VOIDP,Fiddle::TYPE_INT],
      Fiddle::TYPE_INT
    )
    NLPIR_Exit_rb = Fiddle::Function.new(
      libm['NLPIR_Exit'],
      [],
      Fiddle::TYPE_INT
    )
    NLPIR_ImportUserDict_rb = Fiddle::Function.new(
      libm['NLPIR_ImportUserDict'],
      [Fiddle::TYPE_VOIDP],
      Fiddle::TYPE_INT
    )
    NLPIR_ParagraphProcess_rb = Fiddle::Function.new(
      libm['NLPIR_ParagraphProcess'],<commit_msg>added<commit_after>require 'fiddle'
require 'fiddle/struct'
require 'fiddle/import'
require 'fileutils'
include Fiddle::CParser
include Fiddle::Importer
module Nlpir
  module Ictclas
    NLPIR_FALSE = 0
    NLPIR_TRUE = 1
    POS_MAP_NUMBER = 4
    ICT_POS_MAP_FIRST = 1        #计算所一级标注集
    ICT_POS_MAP_SECOND = 0       #计算所二级标注集
    PKU_POS_MAP_SECOND = 2       #北大二级标注集
    PKU_POS_MAP_FIRST = 3        #北大一级标注集
    POS_SIZE = 40

    Result_t = struct ['int start','int length',"char  sPOS[#{POS_SIZE}]",'int iPOS',
                       'int word_ID','int word_type','int weight']

    GBK_CODE = 0                                                    #默认支持GBK编码
    UTF8_CODE = GBK_CODE + 1                          #UTF8编码
    BIG5_CODE = GBK_CODE + 2                          #BIG5编码
    GBK_FANTI_CODE = GBK_CODE + 3             #GBK编码，里面包含繁体字

    @charset = 'utf-8'

    #提取链接库接口
    libm = Fiddle.dlopen(File.expand_path("../../../bin/libNLPIR.so", __FILE__))
    p libm
    NLPIR_Init_rb = Fiddle::Function.new(
      libm['NLPIR_Init'],
      [Fiddle::TYPE_VOIDP,Fiddle::TYPE_INT],
      Fiddle::TYPE_INT
    )
    NLPIR_Exit_rb = Fiddle::Function.new(
      libm['NLPIR_Exit'],
      [],
      Fiddle::TYPE_INT
    )
    NLPIR_ImportUserDict_rb = Fiddle::Function.new(
      libm['NLPIR_ImportUserDict'],
      [Fiddle::TYPE_VOIDP],
      Fiddle::TYPE_INT
    )
    NLPIR_ParagraphProcess_rb = Fiddle::Function.new(
      libm['NLPIR_ParagraphProcess'],
{'fraction_non_alphanumeric': 0.10982264665757162, 'fraction_numerical': 0.01227830832196453, 'mean_word_length': 2.223076923076923, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '73246', 'n_tokens_mistral': 1173, 'n_tokens_neox': 1163, 'n_words': 171}