<commit_before>_ = require 'underscore'
htmlparser = require 'htmlparser2'
{makeSimpleStore} = require './helpers'

LINKS_BEGIN = ['#']
LINKS_CONTAIN = ['/contents/', '.cnx.org']

MEDIA_LINK_EXCLUDES = [
  '.nav'
  '.view-reference-guide'
  '[data-type=footnote-number]'
  '[data-type=footnote-ref]'
  '[data-targeted=media]'
]

buildAllowed = (linksBegin, linksContain) ->
  beginSelectors = _.map linksBegin, (linkString) ->
    "a[href^='#{linkString}']"

  containSelectors = _.map linksContain, (linkString) ->
    "a[href*='#{linkString}']"

  _.union(beginSelectors, containSelectors)
<commit_msg>make selector a little bit more picky<commit_after>_ = require 'underscore'
htmlparser = require 'htmlparser2'
{makeSimpleStore} = require './helpers'

LINKS_BEGIN = ['#']
LINKS_CONTAIN = ['cnx.org/contents/']

MEDIA_LINK_EXCLUDES = [
  '.nav'
  '.view-reference-guide'
  '[data-type=footnote-number]'
  '[data-type=footnote-ref]'
  '[data-targeted=media]'
]

buildAllowed = (linksBegin, linksContain) ->
  beginSelectors = _.map linksBegin, (linkString) ->
    "a[href^='#{linkString}']"

  containSelectors = _.map linksContain, (linkString) ->
    "a[href*='#{linkString}']"

  _.union(beginSelectors, containSelectors)

{'fraction_non_alphanumeric': 0.17766911165444174, 'fraction_numerical': 0.0016299918500407497, 'mean_word_length': 5.0492610837438425, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7281684', 'n_tokens_mistral': 452, 'n_tokens_neox': 449, 'n_words': 73}