<commit_before>package edu.jhu.hlt.optimize;

import org.apache.log4j.Logger;

/**
 *  SGD-QN: Careful Quasi-Newton Stochastic Gradient Descent
 *  <NAME>, <NAME>, <NAME>; 10(Jul):1737--1754, 2009. JMLR.
 * 
 * TODO: This paper is very careful about taking advantage of sparsity in the online updates.
 *       The current implementation however uses dense updates. All occurences of double [] need to be replaced
 *       with abstract vector operations that can then use sparse math under the hood. 
 * 
 * @author noandrews
 */
public class SGDQN extends    Optimizer<DifferentiableFunction>
                   implements Maximizer<DifferentiableFunction>, 
                              Minimizer<DifferentiableFunction> {
	
    private static final Logger log = Logger.getLogger(SGDQN.class);
	
	int T;
	double lambda;
	double t0;
	int skip;
	
	// Work storage
	double [] prev_gradient;
	
	public SGDQN(DifferentiableFunction f, double lambda, double t0, int T, int skip) {
		super(f);
		this.lambda = lambda;
		this.t0 = t0;
		this.T = T;
		this.skip = skip;
	}
	
	public SGDQN(DifferentiableFunction f, int T) {
		super(f);
		this.T = T;
		this.lambda = 1e-4;
		this.t0 = 1e4;
		this.skip = 16;
	}

	public void updateFunction(int t, double [] B, boolean maximize) {
		double [] pt = f.getPoint();
		for(int i=0; i<pt.length; i++) {
			if(maximize) {
				pt[i] += 1.0/(t+t0)*B[i];
			} else {
				pt[i] -= 1.0/(t+t0)*B[i];
			}
		}
	}
	
	public boolean optimize(boolean maximize) {
		int t = 0;
		int count = skip;
		boolean updateB = false;
		int r = 2;
		double [] B = new double[f.getNumDimensions()];
		for(int i=0; i<B.length; i++) {
			B[i] = lambda;<commit_msg>merging some changes<commit_after>package edu.jhu.hlt.optimize;

import org.apache.log4j.Logger;

/**
 *  SGD-QN: Careful Quasi-Newton Stochastic Gradient Descent
 *  <NAME>, <NAME>, <NAME>; 10(Jul):1737--1754, 2009. JMLR.
 * 
 * TODO: This paper is very careful about taking advantage of sparsity.
 *       The current implementation, however, uses dense updates. 
 *       This should be fixed; all occurences of double [] need to be replaced
 *       with abstract vector operations that can then use sparse math under the hood. 
 * 
 * @author noandrews
 */
public class SGDQN extends    Optimizer<DifferentiableFunction>
                   implements Maximizer<DifferentiableFunction>, 
                              Minimizer<DifferentiableFunction> {
	
    private static final Logger log = Logger.getLogger(SGDQN.class);
	
	int T;
	double lambda;
	double t0;
	int skip;
	
	// Work storage
	double [] prev_gradient;
	
	public SGDQN(DifferentiableFunction f, double lambda, double t0, int T, int skip) {
		super(f);
		this.lambda = lambda;
		this.t0 = t0;
		this.T = T;
		this.skip = skip;
	}
	
	public SGDQN(DifferentiableFunction f, int T) {
		super(f);
		this.T = T;
		this.lambda = 1e-4;
		this.t0 = 1e4;
		this.skip = 16;
	}

	public void updateFunction(int t, double [] B, boolean maximize) {
		double [] pt = f.getPoint();
		for(int i=0; i<pt.length; i++) {
			if(maximize) {
				pt[i] += 1.0/(t+t0)*B[i];
			} else {
				pt[i] -= 1.0/(t+t0)*B[i];
			}
		}
		f.setPoint(pt);
	}
	
	public boolean optimize(boolean maximize) {
		int t = 0;
		int count = skip;
		boolean updateB = false;
		int r = 2;
		double [] B = new double[f.getNumDimensions()];
		for(int i=0; i<B.length; i++) {
			B[i] = lambda;
{'fraction_non_alphanumeric': 0.12636551520519634, 'fraction_numerical': 0.021257750221434897, 'mean_word_length': 2.9212962962962963, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3216971', 'n_tokens_mistral': 1345, 'n_tokens_neox': 1177, 'n_words': 338}