<commit_before>/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package edu.usc.ir;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.logging.Logger;

import org.kohsuke.args4j.Argument;
import org.kohsuke.args4j.CmdLineException;
import org.kohsuke.args4j.CmdLineParser;
import org.kohsuke.args4j.Option;

import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlAnchor;
import com.gargoylesoftware.htmlunit.html.HtmlElement;
import com.gargoylesoftware.htmlunit.html.HtmlPage;

public class PersonaExtractor {

  private Map<String, String> patterns = new HashMap<String, String>();

  @Option(name = "-p", usage = "Web Page", aliases = {
      "--webPage" }, required = true)
  private File page = null;

  @Option(name = "-c", usage = "Config File with Xpath selectors", aliases = {
      "--configFile" }, required = true)
  private File configFile = null;

  @Option(name = "-h", usage = "Hostname patterns to use to parse this page", aliases = {
      "--hostPatterns" }, required = false)
  private String host;

  // receives other command line parameters than options
  @Argument
  private List<String> arguments = new ArrayList<String>();

  private static final String USAGE = "java PersonaExtractor [options...] arguments...";

  private static Logger LOG = Logger
      .getLogger(PersonaExtractor.class.getName());

  /**
   * Default constructor.
   */
  public PersonaExtractor() {
  }

  public Map<String, Persona> obtainPersonasForAllHosts()
      throws FailingHttpStatusCodeException, MalformedURLException,
      IOException {
    if (this.patterns == null || (this.patterns != null && this.patterns.isEmpty()))
      initPatterns();

    Map<String, Persona> personaMap = new HashMap<String, Persona>();
    for (String host : this.patterns.keySet()) {
      personaMap.put(this.page.toURL().toString(), obtainPersonas(host));
    }

    return personaMap;
  }

  public Persona obtainPersonas(String host)
      throws FailingHttpStatusCodeException, MalformedURLException,
      IOException {
    if (this.patterns == null || (this.patterns != null && this.patterns.isEmpty()))
      initPatterns();
    
    WebClient webClient = new WebClient();
    webClient.getOptions().setJavaScriptEnabled(false);
    webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
    webClient.getOptions().setThrowExceptionOnScriptError(false);
    HtmlPage htmlPage = webClient.getPage(page.toURL());
    Persona persona = new Persona();
    persona.setPageId(page.toURI().toString());
    
    String pattern = patterns.get(host);
    boolean isAnchor = false;
    if (pattern.contains("@href")){
      isAnchor = true;
    }

    List<?> elements = htmlPage
        .getByXPath(patterns.get(host));
    for (int i = 0; i < elements.size(); i++) {
      String username = null;
      if (isAnchor){
        String link = ((HtmlAnchor)elements.get(i)).getHrefAttribute();
        if (isUserLink(link)) {
          int index = link.lastIndexOf('/');
          username = link.substring(index + 1);
        }        
      }
      else{
          username = ((HtmlElement)elements.get(i)).asText();
      }
      persona.getUsernames().add(username);

    
    }

    webClient.close();
    return persona;
  }

  public static void main(String[] args) throws FailingHttpStatusCodeException,
      MalformedURLException, IOException {
    PersonaExtractor extractor = new PersonaExtractor();
    Map<String, Persona> personas = null;
    Persona persona = null;
    try {
      extractor.processArgs(args);
      String host = extractor.getHost();
      if (host == null) {
        LOG.info("Extracting all personas");
        personas = extractor.obtainPersonasForAllHosts();

        for (String personaHost : personas.keySet()) {
          LOG.info("Host: [" + personaHost + "]: Personas: "
              + personas.get(host).getUsernames());
        }
      } else {
        persona = extractor.obtainPersonas(extractor.getHost());<commit_msg>- DomNode not HtmlElement<commit_after>/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package edu.usc.ir;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.MalformedURLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.logging.Logger;

import org.kohsuke.args4j.Argument;
import org.kohsuke.args4j.CmdLineException;
import org.kohsuke.args4j.CmdLineParser;
import org.kohsuke.args4j.Option;

import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.DomNode;
import com.gargoylesoftware.htmlunit.html.HtmlAnchor;
import com.gargoylesoftware.htmlunit.html.HtmlPage;

public class PersonaExtractor {

  private Map<String, String> patterns = new HashMap<String, String>();

  @Option(name = "-p", usage = "Web Page", aliases = {
      "--webPage" }, required = true)
  private File page = null;

  @Option(name = "-c", usage = "Config File with Xpath selectors", aliases = {
      "--configFile" }, required = true)
  private File configFile = null;

  @Option(name = "-h", usage = "Hostname patterns to use to parse this page", aliases = {
      "--hostPatterns" }, required = false)
  private String host;

  // receives other command line parameters than options
  @Argument
  private List<String> arguments = new ArrayList<String>();

  private static final String USAGE = "java PersonaExtractor [options...] arguments...";

  private static Logger LOG = Logger
      .getLogger(PersonaExtractor.class.getName());

  /**
   * Default constructor.
   */
  public PersonaExtractor() {
  }

  public Map<String, Persona> obtainPersonasForAllHosts()
      throws FailingHttpStatusCodeException, MalformedURLException,
      IOException {
    if (this.patterns == null || (this.patterns != null && this.patterns.isEmpty()))
      initPatterns();

    Map<String, Persona> personaMap = new HashMap<String, Persona>();
    for (String host : this.patterns.keySet()) {
      personaMap.put(this.page.toURL().toString(), obtainPersonas(host));
    }

    return personaMap;
  }

  public Persona obtainPersonas(String host)
      throws FailingHttpStatusCodeException, MalformedURLException,
      IOException {
    if (this.patterns == null || (this.patterns != null && this.patterns.isEmpty()))
      initPatterns();
    
    WebClient webClient = new WebClient();
    webClient.getOptions().setJavaScriptEnabled(false);
    webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
    webClient.getOptions().setThrowExceptionOnScriptError(false);
    HtmlPage htmlPage = webClient.getPage(page.toURL());
    Persona persona = new Persona();
    persona.setPageId(page.toURI().toString());
    
    String pattern = patterns.get(host);
    boolean isAnchor = false;
    if (pattern.contains("@href")){
      isAnchor = true;
    }

    List<?> elements = htmlPage
        .getByXPath(patterns.get(host));
    for (int i = 0; i < elements.size(); i++) {
      String username = null;
      if (isAnchor){
        String link = ((HtmlAnchor)elements.get(i)).getHrefAttribute();
        if (isUserLink(link)) {
          int index = link.lastIndexOf('/');
          username = link.substring(index + 1);
        }        
      }
      else{
          username = ((DomNode)elements.get(i)).asText();
      }
      persona.getUsernames().add(username);

    
    }

    webClient.close();
    return persona;
  }

  public static void main(String[] args) throws FailingHttpStatusCodeException,
      MalformedURLException, IOException {
    PersonaExtractor extractor = new PersonaExtractor();
    Map<String, Persona> personas = null;
    Persona persona = null;
    try {
      extractor.processArgs(args);
      String host = extractor.getHost();
      if (host == null) {
        LOG.info("Extracting all personas");
        personas = extractor.obtainPersonasForAllHosts();

        for (String personaHost : personas.keySet()) {
          LOG.info("Host: [" + personaHost + "]: Personas: "
              + personas.get(host).getUsernames());
        }
      } else {
        persona = extractor.obtainPersonas(extractor.getHost());
{'fraction_non_alphanumeric': 0.11360737761208672, 'fraction_numerical': 0.0019621308741293044, 'mean_word_length': 3.5367156208277706, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2642790', 'n_tokens_mistral': 2909, 'n_tokens_neox': 2889, 'n_words': 786}