<commit_before>package es.sandwatch.trim;


import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.HttpClientBuilder;
import org.json.JSONException;
import org.json.JSONObject;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.util.*;


/**
 * Main project class.
 *
 * @author <NAME>
 * @version 1.0.0
 */
public class Trim{
    /**
     * Triggers the analysis.
     *
     * @param spec the ApiSpecification object containing all API and model information.
     */
    public static void run(ApiSpecification spec){
        Trim trim = new Trim(spec);
        trim.run();
    }


    private ApiSpecification spec;
    private HttpClient client;


    private Trim(ApiSpecification spec){
        this.spec = spec;
    }

    private void run(){
        //Add all generic headers to all endpoints
        for (Endpoint endpoint:spec.getEndpoints()){
            for (String header:spec.getHeaders().keySet()){
                endpoint.addHeader(header, spec.getHeaders().get(header));
            }
        }

        //Create the http client object
        client = HttpClientBuilder.create().build();

        //Execute the requests to endpoints
        for (Endpoint endpoint:spec.getEndpoints()){
            RequestResult result = getEndpointData(endpoint);

            if (result.is2xx()){
                List<String> keys = getJsonFieldList(result.getResponse());
                for (String key:keys){
                    System.out.println(key);
                }
            }
        }
    }

    private RequestResult getEndpointData(Endpoint endpoint){
        HttpGet request = new HttpGet(endpoint.getUrl());
        for (String header:endpoint.getHeaders().keySet()){
            request.addHeader(header, endpoint.getHeaders().get(header));
        }

        try{
            System.out.println("Executing request for " + endpoint.getUrl());
            HttpResponse response = client.execute(request);

            Reader isr = new InputStreamReader(response.getEntity().getContent());
            BufferedReader reader = new BufferedReader(isr);

            StringBuilder result = new StringBuilder();
            String line;
            while ((line = reader.readLine()) != null){
                result.append(line);
            }

            return new RequestResult(response.getStatusLine().getStatusCode(), result.toString());
        }
        catch (IOException iox){
            iox.printStackTrace();
        }
        return new RequestResult();
    }

    private List<String> getJsonFieldList(String src){
        List<String> fieldSet = new ArrayList<>();
        try{
            JSONObject object = new JSONObject(src);

            Iterator<String> keys = object.keys();
            while (keys.hasNext()){
                fieldSet.add(keys.next());
            }
        }
        catch (JSONException jx){
            jx.printStackTrace();
        }
        return fieldSet;
    }


    /**
     * Class containing the relevant information about the result of an HTTP request.
     *
     * @author <NAME>
     * @version 1.0.0
     */
    private class RequestResult{
        private int statusCode;
        private String response;


        /**
         * Constructor. Call if the request failed.
         */
        private RequestResult(){
            this(-1, "Request failed");
        }

        /**
         * Constructor. Call if the request got through to the server.
         *
         * @param statusCode the status code of the request.
         * @param response the response to the request.
         */
        private RequestResult(int statusCode, String response){
            this.statusCode = statusCode;
            this.response = response;
        }<commit_msg>Extract fields from model classes<commit_after>package es.sandwatch.trim;


import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.HttpClientBuilder;
import org.json.JSONException;
import org.json.JSONObject;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.lang.reflect.Field;
import java.util.*;


/**
 * Main project class.
 *
 * @author <NAME>
 * @version 1.0.0
 */
public class Trim{
    /**
     * Triggers the analysis.
     *
     * @param spec the ApiSpecification object containing all API and model information.
     */
    public static void run(ApiSpecification spec){
        Trim trim = new Trim(spec);
        trim.run();
    }


    private ApiSpecification spec;
    private HttpClient client;


    private Trim(ApiSpecification spec){
        this.spec = spec;
    }

    private void run(){
        //Add all generic headers to all endpoints
        for (Endpoint endpoint:spec.getEndpoints()){
            for (String header:spec.getHeaders().keySet()){
                endpoint.addHeader(header, spec.getHeaders().get(header));
            }
        }

        //Create the http client object
        client = HttpClientBuilder.create().build();

        //Execute the requests to endpoints
        for (Endpoint endpoint:spec.getEndpoints()){
            RequestResult result = getEndpointData(endpoint);

            //If successful
            if (result.is2xx()){
                //Parse the response and create the usage map and the field list
                Set<String> keys = getJsonFieldSet(result.getResponse());
                Map<String, Boolean> usageMap = new HashMap<>();
                List<Field> fields = new ArrayList<>();

                //Populate the field list
                getFieldsOf(endpoint.getModel(), fields);
                for (Field field:fields){
                    System.out.println(field.getName());
                }
            }
        }
    }

    private RequestResult getEndpointData(Endpoint endpoint){
        HttpGet request = new HttpGet(endpoint.getUrl());
        for (String header:endpoint.getHeaders().keySet()){
            request.addHeader(header, endpoint.getHeaders().get(header));
        }

        try{
            System.out.println("Executing request for " + endpoint.getUrl());
            HttpResponse response = client.execute(request);

            Reader isr = new InputStreamReader(response.getEntity().getContent());
            BufferedReader reader = new BufferedReader(isr);

            StringBuilder result = new StringBuilder();
            String line;
            while ((line = reader.readLine()) != null){
                result.append(line);
            }

            return new RequestResult(response.getStatusLine().getStatusCode(), result.toString());
        }
        catch (IOException iox){
            iox.printStackTrace();
        }
        return new RequestResult();
    }

    private Set<String> getJsonFieldSet(String src){
        Set<String> fieldSet = new HashSet<>();
        try{
            JSONObject object = new JSONObject(src);

            Iterator<String> keys = object.keys();
            while (keys.hasNext()){
                fieldSet.add(keys.next());
            }
        }
        catch (JSONException jx){
            jx.printStackTrace();
        }
        return fieldSet;
    }

    private void getFieldsOf(Class<?> targetClass, List<Field> targetList){
        if (!targetClass.equals(Object.class)){
            targetList.addAll(Arrays.asList(targetClass.getDeclaredFields()));
            System.out.println("TargetList size: " + targetList.size());
            getFieldsOf(targetClass.getSuperclass(), targetList);
        }
    }


    /**
     * Class containing the relevant information about the result of an HTTP request.
     *
     * @author <NAME>
     * @version 1.0.0
     */
    private class RequestResult{
        private int statusCode;
        private String response;


        /**
         * Constructor. Call if the request failed.
         */
        private RequestResult(){
            this(-1, "Request failed");
        }

        /**
         * Constructor. Call if the request got through to the server.
         *
         * @param statusCode the status code of the request.
         * @param response the response to the request.
         */
        private RequestResult(int statusCode, String response){
            this.statusCode = statusCode;
            this.response = response;
        }
{'fraction_non_alphanumeric': 0.105011655011655, 'fraction_numerical': 0.0018648018648018648, 'mean_word_length': 2.0472301136363638, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '520834', 'n_tokens_mistral': 2204, 'n_tokens_neox': 2132, 'n_words': 569}