<commit_before>package com.continuuity.passport.dal.db;

import com.continuuity.passport.Constants;
import com.continuuity.passport.dal.ProfanityFilter;
import com.google.common.base.Charsets;
import com.google.common.base.Preconditions;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.io.Files;
import com.google.inject.Inject;
import com.google.inject.name.Named;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.util.HashSet;
import java.util.Set;
import java.util.regex.Pattern;

/**
 *
 */
public class ProfanityFilterFileAccess implements ProfanityFilter {

  private static final Logger LOG = LoggerFactory.getLogger(ProfanityFilterFileAccess.class);

  private final Set<String> profanityDictionary;

  //Pattern to look for valid vpc name. letters a-z, A-Z, numbers 0-9 and hypen are supported.
  //The name cannot start or end with a hyphen.
  private Pattern p = Pattern.compile("^[a-zA-Z0-9^][a-zA-Z0-9-]+[a-zA-Z0-9]$");

  @Inject
  public ProfanityFilterFileAccess(@Named(Constants.CFG_PROFANE_WORDS_FILE_PATH)String profaneFilePath) {

    Preconditions.checkNotNull(profaneFilePath);

    Set<String> dictionary = new HashSet<String>();
    try {
      Iterables.addAll(dictionary, Files.readLines(new File(profaneFilePath), Charsets.UTF_8));
    } catch (Exception e) {
      throw Throwables.propagate(e);
    }
    this.profanityDictionary = ImmutableSet.copyOf(dictionary);
    LOG.info(String.format("Loaded %d words into profane dictionary", this.profanityDictionary.size()));
  }
  /**<commit_msg>use static final for pattern<commit_after>package com.continuuity.passport.dal.db;

import com.continuuity.passport.Constants;
import com.continuuity.passport.dal.ProfanityFilter;
import com.google.common.base.Charsets;
import com.google.common.base.Preconditions;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.io.Files;
import com.google.inject.Inject;
import com.google.inject.name.Named;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.util.HashSet;
import java.util.Set;
import java.util.regex.Pattern;

/**
 *
 */
public class ProfanityFilterFileAccess implements ProfanityFilter {

  private static final Logger LOG = LoggerFactory.getLogger(ProfanityFilterFileAccess.class);

  private final Set<String> profanityDictionary;

  //Pattern to look for valid vpc name. letters a-z, A-Z, numbers 0-9 and hypen are supported.
  //The name cannot start or end with a hyphen.
  private static final Pattern p = Pattern.compile("^[a-zA-Z0-9][a-zA-Z0-9-]+[a-zA-Z0-9]$");

  @Inject
  public ProfanityFilterFileAccess(@Named(Constants.CFG_PROFANE_WORDS_FILE_PATH)String profaneFilePath) {

    Preconditions.checkNotNull(profaneFilePath);

    Set<String> dictionary = new HashSet<String>();
    try {
      Iterables.addAll(dictionary, Files.readLines(new File(profaneFilePath), Charsets.UTF_8));
    } catch (Exception e) {
      throw Throwables.propagate(e);
    }
    this.profanityDictionary = ImmutableSet.copyOf(dictionary);
    LOG.info(String.format("Loaded %d words into profane dictionary", this.profanityDictionary.size()));
  }
  /**
{'fraction_non_alphanumeric': 0.11658261643429249, 'fraction_numerical': 0.006526253337288638, 'mean_word_length': 4.9054290718038525, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6038585', 'n_tokens_mistral': 1094, 'n_tokens_neox': 1067, 'n_words': 220}