<commit_before>package com.jbooktrader.platform.util;

import com.jbooktrader.platform.marketdepth.MarketDepthItem;
import com.jbooktrader.platform.model.JBookTraderException;
import com.jbooktrader.platform.startup.JBookTrader;

import java.io.*;
import java.text.*;
import java.util.*;

/**
 * Converts historical market depth data from CME format to JBT format, and writes the data to a file.
 * The created data file can be used for backtesting and optimization of trading strategies.
 */
public class CMEDataConverter {
    private static final String LINE_SEP = System.getProperty("line.separator");
    private final LinkedList<MarketDepthItem> bids, asks;
    private final PrintWriter writer;
    private final BufferedReader reader;
    private final SimpleDateFormat jbtDateFormat, cmeDateFormat;
    private final DecimalFormat decimalFormat;
    private final String contract;
    private long time;
    private long lineNumber;

    public static void main(String[] args) throws JBookTraderException {

        if (args.length != 4) {
            throw new JBookTraderException("Usage: <cmeFileName> <jbtFileName> <contract> <samplingFrequency>");
        }

        CMEDataConverter dataConverter = new CMEDataConverter(args[0], args[1], args[2]);
        long samplingFrequency = Long.valueOf(args[3]);

        dataConverter.convert(samplingFrequency);
    }


    public CMEDataConverter(String cmeFileName, String jbtFileName, String contract) throws JBookTraderException {

        this.contract = contract;
        decimalFormat = NumberFormatterFactory.getNumberFormatter(5);
        jbtDateFormat = new SimpleDateFormat("MMddyy,HH:mm:ss.SSS");
        cmeDateFormat = new SimpleDateFormat("yyyyMMddHHmmssSSS");
        cmeDateFormat.setLenient(false);
        cmeDateFormat.setTimeZone(TimeZone.getTimeZone("America/New_York"));

        bids = new LinkedList<MarketDepthItem>();
        asks = new LinkedList<MarketDepthItem>();

        for (int level = 0; level < 5; level++) {
            bids.add(null);
            asks.add(null);
        }

        try {
            reader = new BufferedReader(new InputStreamReader(new FileInputStream(cmeFileName)));
        } catch (FileNotFoundException fnfe) {
            throw new JBookTraderException("Could not find file " + cmeFileName);
        }

        try {
            writer = new PrintWriter(new BufferedWriter(new FileWriter(jbtFileName, false)));
        } catch (IOException ioe) {
            throw new JBookTraderException("Could not create file " + jbtFileName);
        }
    }


    public void write() {
        StringBuilder sb = new StringBuilder();
        sb.append(jbtDateFormat.format(new Date(time)));
        sb.append(";"); // separator after date and time

        for (MarketDepthItem item : bids) {
            if (item != null) {
                sb.append(item.getSize()).append(",");
                sb.append(decimalFormat.format(item.getPrice())).append(",");
            }
        }
        sb.deleteCharAt(sb.length() - 1);
        sb.append(";"); // separator between bids and asks

        for (MarketDepthItem item : asks) {
            if (item != null) {
                sb.append(item.getSize()).append(",");
                sb.append(decimalFormat.format(item.getPrice())).append(",");
            }
        }
        sb.deleteCharAt(sb.length() - 1);
        writer.println(sb);
    }


    public void convert(long samplingFrequency) {

        String line = null;
<commit_msg>Convert times from Chicago time to New York times.<commit_after>package com.jbooktrader.platform.util;

import com.jbooktrader.platform.marketdepth.MarketDepthItem;
import com.jbooktrader.platform.model.JBookTraderException;
import com.jbooktrader.platform.startup.JBookTrader;

import java.io.*;
import java.text.*;
import java.util.*;

/**
 * Converts historical market depth data from CME format to JBT format, and writes the data to a file.
 * The created data file can be used for backtesting and optimization of trading strategies.
 */
public class CMEDataConverter {
    private static final String LINE_SEP = System.getProperty("line.separator");
    private final LinkedList<MarketDepthItem> bids, asks;
    private final PrintWriter writer;
    private final BufferedReader reader;
    private final SimpleDateFormat jbtDateFormat, cmeDateFormat;
    private final DecimalFormat decimalFormat;
    private final String contract;
    private long time;
    private long lineNumber;

    public static void main(String[] args) throws JBookTraderException {

        if (args.length != 4) {
            throw new JBookTraderException("Usage: <cmeFileName> <jbtFileName> <contract> <samplingFrequency>");
        }

        CMEDataConverter dataConverter = new CMEDataConverter(args[0], args[1], args[2]);
        long samplingFrequency = Long.valueOf(args[3]);

        dataConverter.convert(samplingFrequency);
    }


    public CMEDataConverter(String cmeFileName, String jbtFileName, String contract) throws JBookTraderException {

        this.contract = contract;
        decimalFormat = NumberFormatterFactory.getNumberFormatter(5);
        jbtDateFormat = new SimpleDateFormat("MMddyy,HH:mm:ss.SSS");
        jbtDateFormat.setTimeZone(TimeZone.getTimeZone("America/New_York"));
        cmeDateFormat = new SimpleDateFormat("yyyyMMddHHmmssSSS");
        cmeDateFormat.setLenient(false);
        cmeDateFormat.setTimeZone(TimeZone.getTimeZone("America/Chicago"));

        bids = new LinkedList<MarketDepthItem>();
        asks = new LinkedList<MarketDepthItem>();

        for (int level = 0; level < 5; level++) {
            bids.add(null);
            asks.add(null);
        }

        try {
            reader = new BufferedReader(new InputStreamReader(new FileInputStream(cmeFileName)));
        } catch (FileNotFoundException fnfe) {
            throw new JBookTraderException("Could not find file " + cmeFileName);
        }

        try {
            writer = new PrintWriter(new BufferedWriter(new FileWriter(jbtFileName, false)));
        } catch (IOException ioe) {
            throw new JBookTraderException("Could not create file " + jbtFileName);
        }
    }


    public void write() {
        StringBuilder sb = new StringBuilder();
        sb.append(jbtDateFormat.format(new Date(time)));
        sb.append(";");// separator after date and time

        for (MarketDepthItem item : bids) {
            if (item != null) {
                sb.append(item.getSize()).append(",");
                sb.append(decimalFormat.format(item.getPrice())).append(",");
            }
        }
        sb.deleteCharAt(sb.length() - 1);
        sb.append(";");// separator between bids and asks

        for (MarketDepthItem item : asks) {
            if (item != null) {
                sb.append(item.getSize()).append(",");
                sb.append(decimalFormat.format(item.getPrice())).append(",");
            }
        }
        sb.deleteCharAt(sb.length() - 1);
        writer.println(sb);
    }


    public void convert(long samplingFrequency) {

        String line = null;

{'fraction_non_alphanumeric': 0.10575975214758485, 'fraction_numerical': 0.002816504717645402, 'mean_word_length': 2.859782608695652, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6343432', 'n_tokens_mistral': 1988, 'n_tokens_neox': 1904, 'n_words': 477}