<commit_before>package org.mskcc.cbio.portal.documentation;

import org.springframework.stereotype.Controller;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URL;
import java.net.URLConnection;
import java.net.URLDecoder;
import java.util.Collections;
import java.util.Map;

// Retrieve the content of an external page
// Make it auto-scannable
@Controller
public class ExternalPageController {

    // service name: getexternalpage.json
    // available via GET method
    // sourceURL is required
    @Transactional
    @RequestMapping(value = "/getexternalpage.json", method = {RequestMethod.GET})
    public @ResponseBody Map<String, String> getExternalPage(@RequestParam(required = true) String sourceURL) throws IOException {
        String decodedString, pageText = "";

        // decode the sourceURL and open a connection
        sourceURL = URLDecoder.decode(sourceURL, "UTF-8");
        URL url = new URL(sourceURL);
        URLConnection connection = url.openConnection();

        // create a reader
        BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream()));

        // read
        while ((decodedString = in.readLine()) != null) {
            pageText += decodedString + "\n";
        }
        in.close();

        // turn the pageText into a singletonMap for json and return
        return Collections.singletonMap("response", pageText);
    }
}<commit_msg>added new service that can be used as a proxy for data from other sites<commit_after>package org.mskcc.cbio.portal.documentation;

import org.springframework.stereotype.Controller;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URL;
import java.net.URLConnection;
import java.net.URLDecoder;
import java.util.Collections;
import java.util.Map;

// Retrieve the content of an external page
// Make it auto-scannable
@Controller
public class ExternalPageController {

    // service name: getexternalpage.json
    // available via GET method
    // sourceURL is required
    @Transactional
    @RequestMapping(value = "/getexternalpage.json", method = {RequestMethod.GET})
    public @ResponseBody Map<String, String> getExternalPage(@RequestParam(required = true) String sourceURL) throws IOException {
        // return the contents as a singletonMap for json
        return Collections.singletonMap("response", getExternalPageContents(sourceURL));
    }
    
    // service name: getexternalpage.plain
    // available via GET method
    // sourceURL is required
    @Transactional
    @RequestMapping(value = "/getexternalpage.plain", produces = "text/plain", method = {RequestMethod.GET})
    public @ResponseBody String getExternalPagePlain(@RequestParam(required = true) String sourceURL) throws IOException {
        // return the contents as is
        return getExternalPageContents(sourceURL);
    }
    
    private String getExternalPageContents(String sourceURL) throws IOException {
        String decodedString, pageText = "";

        // decode the sourceURL and open a connection
        sourceURL = URLDecoder.decode(sourceURL, "UTF-8");
        URL url = new URL(sourceURL);
        URLConnection connection = url.openConnection();

        // create a reader
        BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream()));

        // read
        while ((decodedString = in.readLine()) != null) {
            pageText += decodedString + "\n";
        }
        in.close();

        // return the pageText 
        return pageText;
    }
}
{'fraction_non_alphanumeric': 0.09481342407885474, 'fraction_numerical': 0.0004693733865289838, 'mean_word_length': 3.6528384279475983, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5137237', 'n_tokens_mistral': 1093, 'n_tokens_neox': 1072, 'n_words': 294}