<commit_before>package uk.ac.ebi.biosamples.utils.bioschemasrestclient;

import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.function.Consumer;

import org.bson.Document;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.ApplicationArguments;
import org.springframework.boot.ApplicationRunner;
import org.springframework.stereotype.Component;

import com.mongodb.MongoClient;
import com.mongodb.MongoClientURI;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoDatabase;

import uk.ac.ebi.biosamples.utils.AdaptiveThreadPoolExecutor;

@Component
public class BioSchemasRestClientRunner implements ApplicationRunner {
    private static final String LDJSON = ".ldjson";
    private static final String BIOSAMPLES_BASE_URI = "https://www.ebi.ac.uk/biosamples/samples/";
    private static final String BIOSAMPLES = "biosamples";
    private static final String MONGO_SAMPLE = "mongoSample";
    @Value("${spring.data.mongodb.uri}")
    private String mongoUri;
    private static String filePath;
    
    @Override
    public void run(ApplicationArguments args) throws Exception {
        final MongoClientURI uri = new MongoClientURI(mongoUri);
        final MongoClient mongoClient = new MongoClient(uri);
        final MongoDatabase db = mongoClient.getDatabase(BIOSAMPLES);
        final MongoCollection<Document> coll = db.getCollection(MONGO_SAMPLE);

        if(args.getOptionNames().contains("filePath")) {
            filePath = args.getOptionValues("filePath").stream().findFirst().get();
            BioSchemasRdfGenerator.setFilePath(filePath);
        }

        final List<String> listOfAccessions = getAllDocuments(coll);
        System.out.println("List size is : " + listOfAccessions.size());
        mongoClient.close();

        try (AdaptiveThreadPoolExecutor executorService = AdaptiveThreadPoolExecutor.create(100, 10000, true,
                1, 10)) {
            listOfAccessions.forEach(accession -> {
                try {
                    URL url = fetchUrlFromAccession(accession);
                    executorService.submit(new BioSchemasRdfGenerator(url));
                } catch (Exception e) {
                    e.printStackTrace();
                }
            });
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    private URL fetchUrlFromAccession(String accession) throws MalformedURLException {
        return new URL(BIOSAMPLES_BASE_URI + accession + LDJSON);
    }

    private static List<String> getAllDocuments(final MongoCollection<Document> col) throws Exception {
        final List<String> listOfAccessions = new ArrayList<>();

        col.find().forEach((Consumer<? super Document>) doc -> {
            listOfAccessions.add(doc.getString("_id"));
        });

        return listOfAccessions;
    }
}
<commit_msg>code cleanup<commit_after>package uk.ac.ebi.biosamples.utils.bioschemasrestclient;

import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.function.Consumer;

import org.bson.Document;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.ApplicationArguments;
import org.springframework.boot.ApplicationRunner;
import org.springframework.stereotype.Component;

import com.mongodb.MongoClient;
import com.mongodb.MongoClientURI;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoDatabase;

import uk.ac.ebi.biosamples.utils.AdaptiveThreadPoolExecutor;

@Component
public class BioSchemasRestClientRunner implements ApplicationRunner {
    private static final String LDJSON = ".ldjson";
    private static final String BIOSAMPLES_BASE_URI = "https://www.ebi.ac.uk/biosamples/samples/";
    private static final String BIOSAMPLES = "biosamples";
    private static final String MONGO_SAMPLE = "mongoSample";
    @Value("${spring.data.mongodb.uri}")
    private String mongoUri;

    @Override
    public void run(ApplicationArguments args) {
        final MongoClientURI uri = new MongoClientURI(mongoUri);
        final MongoClient mongoClient = new MongoClient(uri);
        final MongoDatabase db = mongoClient.getDatabase(BIOSAMPLES);
        final MongoCollection<Document> coll = db.getCollection(MONGO_SAMPLE);

        if(args.getOptionNames().contains("filePath")) {
            String filePath = args.getOptionValues("filePath").stream().findFirst().get();
            BioSchemasRdfGenerator.setFilePath(filePath);
        }

        final List<String> listOfAccessions = getAllDocuments(coll);
        System.out.println("List size is : " + listOfAccessions.size());
        mongoClient.close();

        try (AdaptiveThreadPoolExecutor executorService = AdaptiveThreadPoolExecutor.create(100, 10000, true,
                1, 10)) {
            listOfAccessions.forEach(accession -> {
                try {
                    URL url = fetchUrlFromAccession(accession);
                    executorService.submit(new BioSchemasRdfGenerator(url));
                } catch (Exception e) {
                    e.printStackTrace();
                }
            });
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    private URL fetchUrlFromAccession(String accession) throws MalformedURLException {
        return new URL(BIOSAMPLES_BASE_URI + accession + LDJSON);
    }

    private static List<String> getAllDocuments(final MongoCollection<Document> col) {
        final List<String> listOfAccessions = new ArrayList<>();

        col.find().forEach((Consumer<? super Document>) doc -> listOfAccessions.add(doc.getString("_id")));

        return listOfAccessions;
    }
}

{'fraction_non_alphanumeric': 0.0979817146800069, 'fraction_numerical': 0.003795066413662239, 'mean_word_length': 3.2979985174203112, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 21, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5147967', 'n_tokens_mistral': 1638, 'n_tokens_neox': 1553, 'n_words': 318}