<commit_before>'use strict'

/**
 * @license
 * node-scrapy <https://github.com/eeshi/node-scrapy>
 * Copyright <NAME>, <NAME>, and other contributors <https://github.com/eeshi/node-scrapy/graphs/contributors>
 * Released under MIT license <https://github.com/eeshi/node-scrapy/blob/master/LICENSE>
 */

/**
 * Module dependencies
 */

let cssSelect = require('css-select')
let htmlparser2 = require('htmlparser2')
let getters = require('./getters')
let utils = {
  isArray: Array.isArray,
  isPlainObject: require('lodash.isplainobject'),
  isString: require('lodash.isstring')
}

module.exports = exports = extract

/**
 * Given an `html` string/buffer, extract data as described in the `model`.
 * @static
 * @param  {string}           html    HTML string to parse
 * @param  {Object|string}    model   String or object describing the data to be extracted from the given HTML
 * @return {Object}
 */

function extract (html, model) {
  let dom = htmlparser2.parseDOM(html)
  return getItem(dom, model)
}

/**
 * Given a `dom`, traverse it to get the desired item
 * @param  {Object}                 dom       DOM node
 * @param  {(string|Array|Object)}  item      Data to extract
 * @return {string}                           a string or an array of strings with the extracted data
 */

function getItem (dom, item) {
  let data
  let matches

  if (utils.isPlainObject(item)) {
    data = {}
    for (let key in item) data[key] = getItem(dom, item[key])
  } else if (utils.isArray(item)) {
    matches = cssSelect.selectAll(item[0], dom)

    if (utils.isArray(item[1]) || utils.isPlainObject(item[1])) {
      data = matches.map(context => getItem(context, item[1]))
    } else {
      data = matches.map(getters.getTextContent)
    }
  } else if (utils.isString(item)) {
    matches = cssSelect.selectOne(item, dom)
    data = getters.getTextContent(matches)<commit_msg>Use query-parser to get selector<commit_after>'use strict'

/**
 * @license
 * node-scrapy <https://github.com/eeshi/node-scrapy>
 * Copyright <NAME>, <NAME>, and other contributors <https://github.com/eeshi/node-scrapy/graphs/contributors>
 * Released under MIT license <https://github.com/eeshi/node-scrapy/blob/master/LICENSE>
 */

/**
 * Module dependencies
 */

let cssSelect = require('css-select')
let htmlparser2 = require('htmlparser2')
let getters = require('./getters')
let parseQuery = require('./query-parser')
let utils = {
  isArray: Array.isArray,
  isPlainObject: require('lodash.isplainobject'),
  isString: require('lodash.isstring')
}

module.exports = exports = extract

/**
 * Given an `html` string/buffer, extract data as described in the `model`.
 * @static
 * @param  {string}           html    HTML string to parse
 * @param  {Object|string}    model   String or object describing the data to be extracted from the given HTML
 * @return {Object}
 */

function extract (html, model) {
  let dom = htmlparser2.parseDOM(html)
  return getItem(dom, model)
}

/**
 * Given a `dom`, traverse it to get the desired item
 * @param  {Object}                 dom       DOM node
 * @param  {(string|Array|Object)}  item      Data to extract
 * @return {string}                           a string or an array of strings with the extracted data
 */

function getItem (dom, item) {
  let data
  let matches

  if (utils.isPlainObject(item)) {
    data = {}
    for (let key in item) data[key] = getItem(dom, item[key])
  } else if (utils.isArray(item)) {
    matches = cssSelect.selectAll(item[0], dom)

    if (utils.isArray(item[1]) || utils.isPlainObject(item[1])) {
      data = matches.map(context => getItem(context, item[1]))
    } else {
      data = matches.map(getters.getTextContent)
    }
  } else if (utils.isString(item)) {
    let query = parseQuery(item)
    matches = cssSelect.selectOne(query.selector, dom)
    data = getters.getTextContent(matches)
{'fraction_non_alphanumeric': 0.13404200155561316, 'fraction_numerical': 0.003629764065335753, 'mean_word_length': 3.315436241610738, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1014254', 'n_tokens_mistral': 1208, 'n_tokens_neox': 1192, 'n_words': 335}