<commit_before>package usemp.certh.inference.urlclassifier;

import com.restfb.types.Message;
import com.restfb.types.Post;
import com.restfb.types.StatusMessage;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map.Entry;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.management.Query;

import usemp.certh.scoring.Constants;
import usemp.certh.scoring.ScoringUser;
import usemp.certh.userDataAccess.UserDataAccess;

/**
 *
 * @author gpetkos
 * 
 * This class is the implementation of an inference module that examines the 
 * URLs that a user has posted and associates them to specific privacy attributes.
 * For instance, posting a URL in imdb can be associated to the 
 * attribute "movies series" (under the "Hobbies" dimension)
 * 
 */
public class URLClassifier {

    private HashMap<String,HashSet<String>> mapping;

    private Double defaultConfidence=0.9;


    public URLClassifier() {
        BufferedReader in;
        try {
            ClassLoader classLoader = getClass().getClassLoader();
            in = new BufferedReader(
                    new InputStreamReader(classLoader.getResourceAsStream("urls_to_privacy_values.txt"), "UTF8"));

            String line;
            mapping=new HashMap<String,HashSet<String>>();
            while ((line = in.readLine()) != null) {
                String[] parts = line.split("\t");
                HashSet<String> categories=mapping.get(parts[0]);
                if(categories==null)
                    categories=new HashSet<String>();
                categories.add(parts[1]);
                mapping.put(parts[0], categories);
            }
            in.close();
        } catch (Exception ex) {
            ex.printStackTrace();
        }
        System.out.println("Mapping from URLS to privacy values loaded!");

    }

    private ArrayList<String> extractURLs(String text) {
        ArrayList<String> links = new ArrayList<String>();
        if(text==null) return links;

        String regex = "\\(?\\b(http://|www[.])[-A-Za-z0-9+&amp;@#/%?=~_()|!:,.;]*[-A-Za-z0-9+&amp;@#/%=~_()|]";
        Pattern p = Pattern.compile(regex);
        Matcher m = p.matcher(text);
        while(m.find()) {
            String urlStr = m.group();
            if (urlStr.startsWith("(") && urlStr.endsWith(")")){
                urlStr = urlStr.substring(1, urlStr.length() - 1);
            }
            links.add(urlStr);
        }
        return links;
    }    
    
    
    private HashMap<String,String> getUserURLs(UserDataAccess user_data){
        HashMap<String,String> urlsToContent=new HashMap<String,String>();
        
        HashSet<Post> posts = user_data.getAllPosts();
        for(Post post:posts){
            String text=post.getMessage();
            ArrayList<String> urls=extractURLs(text);
            for(String url:urls)
                urlsToContent.put(url, post.getId());
        }

        HashSet<StatusMessage> statuses=user_data.getAllStatuses();
        for(StatusMessage status:statuses){
            String text=status.getMessage();
            ArrayList<String> urls=extractURLs(text);
            for(String url:urls)
                urlsToContent.put(url, status.getId());
        }

        return urlsToContent;
    }

    
    public HashMap<String,String> getUserURLsText(UserDataAccess user_data){
        HashMap<String,String> urlsToContent=new HashMap<String,String>();
        
        HashSet<Post> posts=user_data.getAllPosts();
        for(Post post:posts){
            String text=post.getMessage();
            ArrayList<String> urls=extractURLs(text);
            for(String url:urls)
                urlsToContent.put(post.getMessage(), url);
        }
        
        HashSet<StatusMessage> statuses = user_data.getAllStatuses();
        for(StatusMessage status:statuses){
            String text=status.getMessage();
            ArrayList<String> urls=extractURLs(text);
            for(String url:urls)
                urlsToContent.put(status.getMessage(), url);
        }

        return urlsToContent;
    }
    
    
    public void classify(UserDataAccess user_data) {
        ScoringUser scoringUser = user_data.getScoringUser();

        HashMap<String,String> urls=getUserURLs(user_data);

        HashMap<String,HashMap<String,HashSet<String>>> counts=new HashMap<String,HashMap<String,HashSet<String>>>();
        
        for(String url:urls.keySet()){
            String urlS=url;
            if(!urlS.contains("://")) urlS="http://"+urlS;
            if(!urlS.endsWith("/")) urlS=urlS+"/";
            HashSet<String> privacyValues=mapping.get(urls);
            if(privacyValues==null){
                try{
                    URL urlO=new URL(urlS);
                    if(urlO!=null){
                        urlS=urlO.getProtocol()+"://"+urlO.getHost()+"/";
                        privacyValues=mapping.get(urlS);<commit_msg>Update URLClassifier.java<commit_after>package usemp.certh.inference.urlclassifier;

import com.restfb.types.Message;
import com.restfb.types.Post;
import com.restfb.types.StatusMessage;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map.Entry;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.management.Query;

import usemp.certh.scoring.Constants;
import usemp.certh.scoring.ScoringUser;
import usemp.certh.userDataAccess.UserDataAccess;

/**
 *
 * @author gpetkos
 * 
 * This class is the implementation of an inference module that examines the 
 * URLs that a user has posted and associates them to specific privacy attributes.
 * For instance, posting a URL in imdb can be associated to the 
 * attribute "movies series" (under the "Hobbies" dimension)
 * 
 */
public class URLClassifier {

    private HashMap<String,HashSet<String>> mapping;

    private Double defaultConfidence=0.9;


    public URLClassifier() {
        BufferedReader in;
        try {
            ClassLoader classLoader = getClass().getClassLoader();
            in = new BufferedReader(
                    new InputStreamReader(classLoader.getResourceAsStream("urls_to_privacy_values.txt"), "UTF8"));

            String line;
            mapping=new HashMap<String,HashSet<String>>();
            while ((line = in.readLine()) != null) {
                String[] parts = line.split("\t");
                HashSet<String> categories=mapping.get(parts[0]);
                if(categories==null)
                    categories=new HashSet<String>();
                categories.add(parts[1]);
                mapping.put(parts[0], categories);
            }
            in.close();
        } catch (Exception ex) {
            ex.printStackTrace();
        }
        System.out.println("Mapping from URLS to privacy values loaded!");

    }

    private ArrayList<String> extractURLs(String text) {
        ArrayList<String> links = new ArrayList<String>();
        if(text==null) return links;

        String regex = "\\(?\\b(http://|www[.])[-A-Za-z0-9+&amp;@#/%?=~_()|!:,.;]*[-A-Za-z0-9+&amp;@#/%=~_()|]";
        Pattern p = Pattern.compile(regex);
        Matcher m = p.matcher(text);
        while(m.find()) {
            String urlStr = m.group();
            if (urlStr.startsWith("(") && urlStr.endsWith(")")){
                urlStr = urlStr.substring(1, urlStr.length() - 1);
            }
            links.add(urlStr);
        }
        return links;
    }    
    
    
    private HashMap<String,String> getUserURLs(UserDataAccess user_data){
        HashMap<String,String> urlsToContent=new HashMap<String,String>();
        
        HashSet<Post> posts = user_data.getAllPosts();
        for(Post post:posts){
            String text=post.getMessage();
            ArrayList<String> urls=extractURLs(text);
            for(String url:urls)
                urlsToContent.put(url, post.getId());
        }

        HashSet<StatusMessage> statuses=user_data.getAllStatuses();
        for(StatusMessage status:statuses){
            String text=status.getMessage();
            ArrayList<String> urls=extractURLs(text);
            for(String url:urls)
                urlsToContent.put(url, status.getId());
        }

        return urlsToContent;
    }

    
    public HashMap<String,String> getUserURLsText(UserDataAccess user_data){
        HashMap<String,String> urlsToContent=new HashMap<String,String>();
        
        HashSet<Post> posts=user_data.getAllPosts();
        for(Post post:posts){
            String text=post.getMessage();
            ArrayList<String> urls=extractURLs(text);
            for(String url:urls)
                urlsToContent.put(post.getMessage(), url);
        }
        
        HashSet<StatusMessage> statuses = user_data.getAllStatuses();
        for(StatusMessage status:statuses){
            String text=status.getMessage();
            ArrayList<String> urls=extractURLs(text);
            for(String url:urls)
                urlsToContent.put(status.getMessage(), url);
        }

        return urlsToContent;
    }
    
    
    public void classify(UserDataAccess user_data) {
        ScoringUser scoringUser = user_data.getScoringUser();

        HashMap<String,String> urls=getUserURLs(user_data);

        HashMap<String,HashMap<String,HashSet<String>>> counts=new HashMap<String,HashMap<String,HashSet<String>>>();
        
        for(String url:urls.keySet()){
            String urlS=url;
            if(!urlS.contains("://")) urlS="http://"+urlS;
            if(!urlS.endsWith("/")) urlS=urlS+"/";
            HashSet<String> privacyValues=mapping.get(urlS);
            if(privacyValues==null){
                try{
                    URL urlO=new URL(urlS);
                    if(urlO!=null){
                        urlS=urlO.getProtocol()+"://"+urlO.getHost()+"/";
                        privacyValues=mapping.get(urlS);
{'fraction_non_alphanumeric': 0.12732121691031212, 'fraction_numerical': 0.002370604504148558, 'mean_word_length': 2.3874205419872867, 'pattern_counts': {'":': 4, '<': 65, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2654126', 'n_tokens_mistral': 2949, 'n_tokens_neox': 2899, 'n_words': 536}