<commit_before>package cz.muni.fi.datascrapper;

import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import com.google.gson.stream.JsonReader;
import cz.muni.fi.datascrapper.model.Movie;
import cz.muni.fi.datascrapper.model.Person;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.FileReader;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;

/**
 * Created by Rex on 8.11.2016.
 */
public class Main{

    private static String baseActorUrl = "http://www.imdb.com/name/[id]/";
    private static String baseMovieUrl = "http://www.imdb.com/title/[id]/";

    private static String[] actorLists = new String[]{
            "http://www.imdb.com/list/ls058011111/?start=1&view=compact&sort=listorian:asc",
            "http://www.imdb.com/list/ls058011111/?start=251&view=compact&sort=listorian:asc",
            "http://www.imdb.com/list/ls058011111/?start=501&view=compact&sort=listorian:asc",
            "http://www.imdb.com/list/ls058011111/?start=751&view=compact&sort=listorian:asc"
    };

    private static List<Person> baseActors;
    private static List<Movie> movies = new ArrayList<>();

    public static void main(String[] args) throws Exception {
        baseActors = loadOrParseActors();
        System.out.println(baseActors.size());

        loadMoviesForActors();
//        List<Movie> movies = loadPreProcessedMovies();
        System.out.println(movies.size());
    }

    private static List<Movie> loadPreProcessedMovies() throws IOException {
        return new Gson().fromJson(new JsonReader(new FileReader("movies_preprocessed.json")), new TypeToken<ArrayList<Person>>(){}.getType());
    }

    private static void loadMoviesForActors() {
        int buffer = 20;

        int processed = 0;

        for (Person actor : baseActors) {
            try {
                Document doc = Jsoup.connect(baseActorUrl.replace("[id]", actor.getId())).userAgent("Mozilla").timeout(10_000).get();
                Elements movieElements = doc.select("div.filmo-row");
                for (Element movieElement : movieElements) {
                    Element movieRow = movieElement.select("a").first();
                    String id = movieRow.attr("href").replaceAll("/title/(.*?)/.*", "$1");
                    String name = movieRow.text();

                    Movie movie = new Movie(id, name);

                    int i = movies.indexOf(movie);

                    if(i == -1) {
                        movies.add(movie);
                        movie.getActors().add(actor.getId());
                    } else {
                        movie = movies.get(i);
                        if(!movie.getActors().contains(actor.getId())) {
                            movie.getActors().add(actor.getId());
                        }
                    }
                }

                System.out.println("Processed " + ++processed + " actors");

                if(buffer-- == 0 || processed == baseActors.size()) {
                    String moviesJson = new Gson().toJson(movies);
                    Files.write(Paths.get("movies_preprocessed.json"), moviesJson.getBytes());
                    buffer = 20;
                }

                System.out.println("Added movies from actor " + actor.getName());
            } catch (IOException e) {
                System.err.println("Failed to load movies for actor " + actor.getName());
                e.printStackTrace();
            }
        }
    }

    private static List<Person> loadOrParseActors() throws IOException {
        if(Files.exists(Paths.get("actors.json"))) {
            return new Gson().fromJson(new JsonReader(new FileReader("actors.json")), new TypeToken<ArrayList<Person>>(){}.getType());
        } else {
            return parseActorsFromImdb();
        }
    }

    private static List<Person> parseActorsFromImdb() throws IOException {
        List<Person> parsedActors = new ArrayList<>();

        for (String actorList : actorLists) {
            Document doc = Jsoup.connect(actorList).userAgent("Mozilla").timeout(10_000).get();<commit_msg>parsing list of movies from actor lists<commit_after>package cz.muni.fi.datascrapper;

import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import com.google.gson.stream.JsonReader;
import cz.muni.fi.datascrapper.model.Movie;
import cz.muni.fi.datascrapper.model.Person;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.FileReader;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;

/**
 * Created by Rex on 8.11.2016.
 */
public class Main{

    private static volatile int bufferedActors = 20;
    private static volatile int processedActors = 0;

    private static Object writeLock = new Object();

    private static String baseActorUrl = "http://www.imdb.com/name/[id]/";
    private static String baseMovieUrl = "http://www.imdb.com/title/[id]/";


    private static String[] actorLists = new String[]{
            "http://www.imdb.com/list/ls058011111/?start=1&view=compact&sort=listorian:asc",
            "http://www.imdb.com/list/ls058011111/?start=251&view=compact&sort=listorian:asc",
            "http://www.imdb.com/list/ls058011111/?start=501&view=compact&sort=listorian:asc",
            "http://www.imdb.com/list/ls058011111/?start=751&view=compact&sort=listorian:asc"
    };

    private static List<Person> baseActors;
    private static List<Movie> movies = new ArrayList<>();

    public static void main(String[] args) throws Exception {
        baseActors = loadOrParseActors();
        System.out.println(baseActors.size());

        loadMoviesForActors();
        movies = loadPreProcessedMovies();
        System.out.println(movies.size());

        Collections.sort(movies, (o1, o2) -> o1.getActors().size() - o2.getActors().size());

        for (Movie movie : movies) {
            if(movie.getActors().size() > 1)
                System.out.println(movie.getName() + " " + movie.getActors().size());
        }
    }

    private static List<Movie> loadPreProcessedMovies() throws IOException {
        return new Gson().fromJson(new JsonReader(new FileReader("movies_preprocessed.json")), new TypeToken<ArrayList<Movie>>(){}.getType());
    }

    private static void loadMoviesForActors() {
        baseActors.stream().parallel().forEach(actor -> {
            try {
                Document doc = Jsoup.connect(baseActorUrl.replace("[id]", actor.getId())).userAgent("Mozilla").timeout(10_000).get();
                Elements movieElements = doc.select("div.filmo-row");
                for (Element movieElement : movieElements) {
                    Element movieRow = movieElement.select("a").first();
                    String id = movieRow.attr("href").replaceAll("/title/(.*?)/.*", "$1");
                    String name = movieRow.text();

                    Movie movie = new Movie(id, name);

                    int i = movies.indexOf(movie);

                    if (i == -1) {
                        movies.add(movie);
                        movie.getActors().add(actor.getId());
                    } else {
                        movie = movies.get(i);
                        if (!movie.getActors().contains(actor.getId())) {
                            movie.getActors().add(actor.getId());
                        }
                    }
                }

                System.out.println("Processed " + ++processedActors + " actors");

                if (bufferedActors-- == 0 || processedActors == baseActors.size()) {
                    synchronized (writeLock) {
                        String moviesJson = new Gson().toJson(movies);
                        Files.write(Paths.get("movies_preprocessed.json"), moviesJson.getBytes());
                        bufferedActors = 20;
                    }
                }

                System.out.println("Added movies from actor " + actor.getName());
            } catch (IOException e) {
                System.err.println("Failed to load movies for actor " + actor.getName());
                e.printStackTrace();
            }
        });
    }

    private static List<Person> loadOrParseActors() throws IOException {
        if(Files.exists(Paths.get("actors.json"))) {
            return new Gson().fromJson(new JsonReader(new FileReader("actors.json")), new TypeToken<ArrayList<Person>>(){}.getType());
        } else {
            return parseActorsFromImdb();
        }
    }

    private static List<Person> parseActorsFromImdb() throws IOException {
        List<Person> parsedActors = new ArrayList<>();

        for (String actorList : actorLists) {
            Document doc = Jsoup.connect(actorList).userAgent("Mozilla").timeout(10_000).get();
{'fraction_non_alphanumeric': 0.137515211859719, 'fraction_numerical': 0.016262860935944243, 'mean_word_length': 2.2623601587874416, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 12, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6712327', 'n_tokens_mistral': 2729, 'n_tokens_neox': 2482, 'n_words': 465}