<commit_before>fs        = require 'fs'
path      = require 'path'
_         = require 'underscore'
mysql     = require 'mysql'
model     = require './model'
stats     = require './stats'

utils =

  ##
  # 把 callback 的写法，作用到 promise 上
  # promise.then(->callback(null)).catch(callback)
  # 目的是为了让callback的写法可以快速对接到 promise 上
  ##
  callback: (promise, callback) ->
    promise.then((args...) ->
      args.unshift(null)
      callback.apply(null, args)
    ).catch(callback)

  ###
  # 将字符串转换为数字
  ###
  intval: (value, mode = 10) ->
    parseInt(value, mode) or 0

  # 根据设置的路径，获取对象
  getModules: (_path) ->
    modules = {}
    for file in utils.readdir(_path, ['coffee', 'js'])
      moduleName = utils.file2Module file
      modules[moduleName] = require "#{_path}/#{file}"

    modules

  ###
  # 从 req 中提取所需的参数
  ###
  pickParams: (req, cols, Model) ->
    attr = {}
    _.each(cols, (x) ->
      return unless req.params.hasOwnProperty(x)
      return unless C = Model.rawAttributes[x]

      value = req.params[x]

      # 如果是数字类型的则数字化
      if C.type.key in ['INTEGER', 'FLOAT']
        value = +value

      # 如果字段允许为空，且默认值为 null 则在等于空字符串的时候赋值为 null
      if (value in ['', null]) and C.hasOwnProperty('defaultValue')
        value = C.defaultValue

      attr[x] = value
    )
    attr

  ###
  # 判断给定ip是否是白名单里的ip地址
  ###
  isPrivateIp: (ip, whiteList) ->
    ip in whiteList<commit_msg>根据字段是否允许为空区别对待,允许为空的值为 null, 不允许为空的值为字段定义的 defaultValue<commit_after>fs        = require 'fs'
path      = require 'path'
_         = require 'underscore'
mysql     = require 'mysql'
model     = require './model'
stats     = require './stats'

utils =

  ##
  # 把 callback 的写法，作用到 promise 上
  # promise.then(->callback(null)).catch(callback)
  # 目的是为了让callback的写法可以快速对接到 promise 上
  ##
  callback: (promise, callback) ->
    promise.then((args...) ->
      args.unshift(null)
      callback.apply(null, args)
    ).catch(callback)

  ###
  # 将字符串转换为数字
  ###
  intval: (value, mode = 10) ->
    parseInt(value, mode) or 0

  # 根据设置的路径，获取对象
  getModules: (_path) ->
    modules = {}
    for file in utils.readdir(_path, ['coffee', 'js'])
      moduleName = utils.file2Module file
      modules[moduleName] = require "#{_path}/#{file}"

    modules

  ###
  # 从 req 中提取所需的参数
  ###
  pickParams: (req, cols, Model) ->
    attr = {}
    _.each(cols, (x) ->
      return unless req.params.hasOwnProperty(x)
      return unless C = Model.rawAttributes[x]

      value = req.params[x]

      # 如果是数字类型的则数字化
      if C.type.key in ['INTEGER', 'FLOAT']
        value = +value

      # 如果字段允许为空，且默认值为 null 则在等于空字符串的时候赋值为 null
      if (value in ['', null]) and C.hasOwnProperty('defaultValue')
        if C.allowNull is no
          value = null
        else
          value = C.defaultValue

      attr[x] = value
    )
    attr

  ###
  # 判断给定ip是否是白名单里的ip地址
  ###
  isPrivateIp: (ip, whiteList) ->
    ip in whiteList
{'fraction_non_alphanumeric': 0.14113597246127366, 'fraction_numerical': 0.002753872633390706, 'mean_word_length': 2.4309327036599764, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '181857', 'n_tokens_mistral': 1159, 'n_tokens_neox': 1218, 'n_words': 229}