<commit_before>package org.lantern.httpseverywhere;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;

import javax.xml.xpath.XPathExpressionException;

import org.lantern.LanternUtils;
import org.littleshoot.util.xml.XPathUtils;
import org.littleshoot.util.xml.XmlUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.NamedNodeMap;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;

/**
 * Class for converting requests to HTTPS when we can.
 */
public class HttpsEverywhere {
    
    private static final Logger LOG = 
        LoggerFactory.getLogger(HttpsEverywhere.class);
    
    private static final Map<String, HttpsRuleSet> httpsRules =
        new ConcurrentHashMap<String, HttpsRuleSet>();
    
    static {
        final File httpsDir = new File("https");
        final File[] ruleFiles = httpsDir.listFiles();
        for (final File ruleFile : ruleFiles) {
            try {
                addRuleFile(ruleFile);
            } catch (final XPathExpressionException e) {
                LOG.error("Could not load rule file: "+ruleFile, e);
            } catch (final IOException e) {
                LOG.error("Could not load rule file: "+ruleFile, e);
            } catch (final SAXException e) {
                LOG.error("Could not load rule file: "+ruleFile, e);
            }
        }
    }

    private static void addRuleFile(final File ruleFile) throws IOException, 
        SAXException, XPathExpressionException {
        final InputStream is = new FileInputStream(ruleFile);
        final Document doc = XmlUtils.toDoc(is);
        final XPathUtils utils = XPathUtils.newXPath(doc);
        final Collection<String> targets = 
            utils.getStrings("/ruleset/target/@host");
        
        final Collection<String> exclusions = 
            utils.getStrings("/ruleset/exclusion/@pattern");
        
        final NodeList nodes = utils.getNodes("/ruleset/rule");
        final int length = nodes.getLength();
        
        for (final String target : targets) {
            LOG.info("Checking target: {}", target);
            final HttpsRuleSet ruleSet;
            if (httpsRules.containsKey(target)) {
                ruleSet = httpsRules.get(target);
            } else {
                ruleSet = new HttpsRuleSet(new ArrayList<HttpsRule>(length), exclusions);
                httpsRules.put(target, ruleSet);
            }
            for (int i = 0; i < length; i++) {
                final Node node = nodes.item(i);
                final NamedNodeMap attributes = node.getAttributes();
                final String from = 
                    attributes.getNamedItem("from").getTextContent();
                final String to = 
                    attributes.getNamedItem("to").getTextContent();
                final HttpsRule rule = new HttpsRule(from, to);
                ruleSet.rules.add(rule);
            }
        }
    }
    
    public static String toHttps(final String uri) {
        if (!uri.startsWith("http://")) {
            LOG.info("Not modifying non-http request: {}", uri);
            return uri;
        }
        final Collection<String> candidates = 
            LanternUtils.toHttpsCandidates(uri);
        //LOG.info("Candidats: {}", candidates);
        final Collection<HttpsRuleSet> ruleSets = getRules(candidates);
        if (ruleSets == null || ruleSets.isEmpty()) {
            LOG.info("NO RULES");
            return uri;<commit_msg>Spelling fix<commit_after>package org.lantern.httpseverywhere;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;

import javax.xml.xpath.XPathExpressionException;

import org.lantern.LanternUtils;
import org.littleshoot.util.xml.XPathUtils;
import org.littleshoot.util.xml.XmlUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.NamedNodeMap;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;

/**
 * Class for converting requests to HTTPS when we can.
 */
public class HttpsEverywhere {
    
    private static final Logger LOG = 
        LoggerFactory.getLogger(HttpsEverywhere.class);
    
    private static final Map<String, HttpsRuleSet> httpsRules =
        new ConcurrentHashMap<String, HttpsRuleSet>();
    
    static {
        final File httpsDir = new File("https");
        final File[] ruleFiles = httpsDir.listFiles();
        for (final File ruleFile : ruleFiles) {
            try {
                addRuleFile(ruleFile);
            } catch (final XPathExpressionException e) {
                LOG.error("Could not load rule file: "+ruleFile, e);
            } catch (final IOException e) {
                LOG.error("Could not load rule file: "+ruleFile, e);
            } catch (final SAXException e) {
                LOG.error("Could not load rule file: "+ruleFile, e);
            }
        }
    }

    private static void addRuleFile(final File ruleFile) throws IOException, 
        SAXException, XPathExpressionException {
        final InputStream is = new FileInputStream(ruleFile);
        final Document doc = XmlUtils.toDoc(is);
        final XPathUtils utils = XPathUtils.newXPath(doc);
        final Collection<String> targets = 
            utils.getStrings("/ruleset/target/@host");
        
        final Collection<String> exclusions = 
            utils.getStrings("/ruleset/exclusion/@pattern");
        
        final NodeList nodes = utils.getNodes("/ruleset/rule");
        final int length = nodes.getLength();
        
        for (final String target : targets) {
            LOG.info("Checking target: {}", target);
            final HttpsRuleSet ruleSet;
            if (httpsRules.containsKey(target)) {
                ruleSet = httpsRules.get(target);
            } else {
                ruleSet = new HttpsRuleSet(new ArrayList<HttpsRule>(length), exclusions);
                httpsRules.put(target, ruleSet);
            }
            for (int i = 0; i < length; i++) {
                final Node node = nodes.item(i);
                final NamedNodeMap attributes = node.getAttributes();
                final String from = 
                    attributes.getNamedItem("from").getTextContent();
                final String to = 
                    attributes.getNamedItem("to").getTextContent();
                final HttpsRule rule = new HttpsRule(from, to);
                ruleSet.rules.add(rule);
            }
        }
    }
    
    public static String toHttps(final String uri) {
        if (!uri.startsWith("http://")) {
            LOG.info("Not modifying non-http request: {}", uri);
            return uri;
        }
        final Collection<String> candidates = 
            LanternUtils.toHttpsCandidates(uri);
        //LOG.info("Candidates: {}", candidates);
        final Collection<HttpsRuleSet> ruleSets = getRules(candidates);
        if (ruleSets == null || ruleSets.isEmpty()) {
            LOG.info("NO RULES");
            return uri;
{'fraction_non_alphanumeric': 0.10445757250268528, 'fraction_numerical': 0.0018796992481203006, 'mean_word_length': 2.258530183727034, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4891890', 'n_tokens_mistral': 2091, 'n_tokens_neox': 2044, 'n_words': 488}