Q:

HTTP request multithreading: why not go all the requests?

Good day. Please help me with threads in java. The essence of the following code to get answers from 20 sites, with each site in a separate thread.
The problem is that for some reason, not all flows are processed 20. Sniffer shows 10-14 queries, although supposed to be all 20. 
Attention to the question: why not go all the requests?
import java.net.*;
import java.io.*;
import java.util.*;
import java.util.concurrent.*;

public class ThreadsTest implements Callable<String> {

    private String url;

    public ThreadsTest(String url) {
        this.url = url;
    }

    public String call() throws Exception {

        Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress("127.0.0.1", 7777));

        URLConnection connection = new URL("http://" + url).openConnection(proxy);
        HttpURLConnection stopRedirect = (HttpURLConnection) connection;
        stopRedirect.setInstanceFollowRedirects(false);
        connection.setRequestProperty("User-Agent", "Mozilla/5.0 (compatible; MSIE 5.01; Windows NT 5.0)");
        connection.setRequestProperty("Referer", "http://www.google.co.uk");
        connection.setConnectTimeout(30000);
        connection.setReadTimeout(30000);

        BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream()));

        String inputLine;
        String content = "";

        while ((inputLine = in.readLine()) != null) content+= inputLine;
        in.close();

        return content;
    }

    public static void main(String[] args)  throws Exception {

        String[] sites = {"courseassignment.com", "errocks.com", "villsoft.com", "miraclegarciniacambogiarx.net", "www.mcqplay.com", "www.ajnad-sham.com", "bstore.in", "rajestateagency.in", "royalboutique.in", "www.masiada.com",
                        "awebstreet.com", "adbfc.com", "starhairforum.com",  "akflix.com",  "planetskool.com",  "sceteducation.org",  "www.smarts3i.com",  "www.my.aybhost.com",  "www.svmconstructions.in",  "www.nssj.co.in"
        };

        System.out.println(sites.length);

        ExecutorService service = Executors.newCachedThreadPool();
        List<Future<String>> futures = new ArrayList<Future<String>>();

        for (int i = 0; i < sites.length; i++) {
            //запуск нового потока с параметром из массива
            try{

                Future<String> future = service.submit(new ThreadsTest(sites[i]));
                futures.add(future);

            }
            catch (Exception e) {  }
        }   

        for (Future<String> future : futures) {

            try {

                future.get();

                //System.out.println("get result from Future: " + future.get());

            }
             catch (InterruptedException | ExecutionException e) {
                //e.printStackTrace();
            }

        }

        service.shutdown();
        while (!service.isTerminated()) {

        }

    }
}

A:

Problem
If you uncomment e.printStackTrace() you should see that a certain URL is returning a 403 status code which is causing an exception in your call() method.
Exception in thread "main" java.util.concurrent.ExecutionException: java.io.IOException: Server returned HTTP response code: 403 for URL: http://www.my.aybhost.com

When a client receives a 403 status response header, then there is no response body to read, thus it throws an exception when attempting to create a Reader from the connection's input stream. You probably want to handle non-200 response codes differently.
Solution
public String url;

public ThreadsTest(String url) {
    this.url = url;
}

public String call() throws Exception {
    HttpURLConnection connection = (HttpURLConnection) new URL("http://"
            + url).openConnection();
    connection.setInstanceFollowRedirects(false);
    connection.setRequestProperty("User-Agent",
            "Mozilla/5.0 (compatible; MSIE 5.01; Windows NT 5.0)");
    connection.setRequestProperty("Referer", "http://www.google.co.uk");
    connection.setConnectTimeout(30000);
    connection.setReadTimeout(30000);

    String content = "";
    if (connection.getResponseCode() == 200) {
        BufferedReader in = new BufferedReader(new InputStreamReader(
                connection.getInputStream()));
        String inputLine;

        while ((inputLine = in.readLine()) != null)
            content += inputLine;
        in.close();
        System.out.println(url + " returned "
                + connection.getResponseCode() + " response code");
    } else {
        System.err.println(url + " returned "
                + connection.getResponseCode() + " response code");
    }

    return content;
}

public static void main(String[] args) throws Exception {

    String[] sites = { "courseassignment.com", "errocks.com",
            "villsoft.com", "miraclegarciniacambogiarx.net",
            "www.mcqplay.com", "www.ajnad-sham.com", "bstore.in",
            "rajestateagency.in", "royalboutique.in", "www.masiada.com",
            "awebstreet.com", "adbfc.com", "starhairforum.com",
            "akflix.com", "planetskool.com", "sceteducation.org",
            "www.smarts3i.com", "www.my.aybhost.com",
            "www.svmconstructions.in", "www.nssj.co.in" };

    System.out.println(sites.length);

    ExecutorService service = Executors.newCachedThreadPool();
    List<Future<String>> futures = new ArrayList<Future<String>>();

    for (int i = 0; i < sites.length; i++) {

        Future<String> future = service.submit(new ThreadsTest(sites[i]));
        futures.add(future);

    }

    for (Future<String> future : futures) {
        try {
            future.get();
        } catch (InterruptedException | ExecutionException e) {
            e.printStackTrace();
        }
    }

    service.shutdown();
    if (!service.awaitTermination(10, TimeUnit.SECONDS)) {
        // Timed Out waiting to finish, so force a shutdown
        service.shutdownNow();
    }
}

Output (std error and out)
awebstreet.com returned 301 response code
starhairforum.com returned 200 response code
www.nssj.co.in returned 200 response code
royalboutique.in returned 200 response code
planetskool.com returned 200 response code
www.svmconstructions.in returned 200 response code
sceteducation.org returned 200 response code
bstore.in returned 200 response code
rajestateagency.in returned 200 response code
villsoft.com returned 200 response code
www.mcqplay.com returned 200 response code
www.my.aybhost.com returned 403 response code
errocks.com returned 200 response code
adbfc.com returned 200 response code
miraclegarciniacambogiarx.net returned 200 response code
akflix.com returned 200 response code
www.ajnad-sham.com returned 200 response code
www.smarts3i.com returned 200 response code
www.masiada.com returned 301 response code
courseassignment.com returned 200 response code

Using wireshark, I've also verified that all 20 http requests are being sent out from my machine. So if you still don't see 20 requests being sent out then it's probably not related to the code; may be an issue with your firewall, proxy, java security, or other network settings.

