<commit_before>package com.redfin.hudson;

import static hudson.Util.*;
import hudson.model.BuildableItem;
import hudson.model.Item;
import hudson.scheduler.CronTabList;
import hudson.triggers.Trigger;
import hudson.triggers.TriggerDescriptor;
import hudson.util.FormFieldValidator;

import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;

import javax.servlet.ServletException;

import net.sf.json.JSONObject;

import org.kohsuke.stapler.StaplerRequest;
import org.kohsuke.stapler.StaplerResponse;

import antlr.ANTLRException;

/** Triggers a build when the data at a particular URL has changed. */
public class UrlChangeTrigger extends Trigger<BuildableItem> {

    URL url;
    String oldMd5;
    
    private static final int readBufferSize = 8*1024;
    /** Extracts the MD5 string from the specified input stream; this MD5 should
     * be a perfect match for the GNU "md5sum" tool.
     */
    private static String getMd5(InputStream stream) throws IOException {
        MessageDigest messageDigest;
        try {
            messageDigest = MessageDigest.getInstance("MD5");
        } catch (NoSuchAlgorithmException e) {
            throw new RuntimeException(e);
        }
        DigestInputStream dis = new DigestInputStream(stream,
                messageDigest);
        byte[] buf = new byte[readBufferSize];
        while (dis.read(buf, 0, readBufferSize) != -1) {
        ;
        }
        dis.close();
        stream.close();
        byte[] fileDigest = messageDigest.digest ();
        return createDigestString(fileDigest);
    }
    
    /** byte array to string */
    private static String createDigestString(byte[] fileDigest) {
        StringBuffer checksumSb = new StringBuffer();
        for (int i = 0; i < fileDigest.length; i++) {
            String hexStr = Integer.toHexString(0x00ff & fileDigest[i]);
            if (hexStr.length() < 2) {
                checksumSb.append("0");
            }
            checksumSb.append(hexStr);
        }
        return checksumSb.toString();
    }

    public UrlChangeTrigger(String url) throws MalformedURLException {
        this(new URL(url));
    }
    
    public UrlChangeTrigger(URL url) {
        this.url = url;
        try {
            this.tabs = CronTabList.create("* * * * *");
        } catch (ANTLRException e) {
            throw new RuntimeException("Bug! couldn't schedule poll");
        }
    }

    @Override
    public void run() {
        try {
            String currentMd5 = getMd5(url.openStream());
            if (!currentMd5.equals(oldMd5)) {
                oldMd5 = currentMd5;
                job.scheduleBuild();
            }
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    }
    
    public URL getUrl() {
        return url;
    }
    <commit_msg>Issue 2674: The change detection is updated to not stop when reloading or restarting. This also contains a fix to allow restarting or reloading without triggering all jobs. This is done by preserving the signature of the file at the URL in the jobs directory.<commit_after>package com.redfin.hudson;

import static hudson.Util.*;
import hudson.model.BuildableItem;
import hudson.model.Item;
import hudson.scheduler.CronTabList;
import hudson.triggers.Trigger;
import hudson.triggers.TriggerDescriptor;
import hudson.util.FormFieldValidator;

import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintWriter;
import java.net.MalformedURLException;
import java.net.URL;
import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.logging.Level;
import java.util.logging.Logger;

import javax.servlet.ServletException;

import net.sf.json.JSONObject;

import org.kohsuke.stapler.StaplerRequest;
import org.kohsuke.stapler.StaplerResponse;

import antlr.ANTLRException;

/** Triggers a build when the data at a particular URL has changed. */
public class UrlChangeTrigger extends Trigger<BuildableItem> {

    URL url;
    
    private static final int readBufferSize = 8*1024;
    /** Extracts the MD5 string from the specified input stream; this MD5 should
     * be a perfect match for the GNU "md5sum" tool.
     */
    private static String getMd5(InputStream stream) throws IOException {
        MessageDigest messageDigest;
        try {
            messageDigest = MessageDigest.getInstance("MD5");
        } catch (NoSuchAlgorithmException e) {
            throw new RuntimeException(e);
        }
        DigestInputStream dis = new DigestInputStream(stream,
                messageDigest);
        byte[] buf = new byte[readBufferSize];
        while (dis.read(buf, 0, readBufferSize) != -1) {
        ;
        }
        dis.close();
        stream.close();
        byte[] fileDigest = messageDigest.digest ();
        return createDigestString(fileDigest);
    }
    
    /** byte array to string */
    private static String createDigestString(byte[] fileDigest) {
        StringBuffer checksumSb = new StringBuffer();
        for (int i = 0; i < fileDigest.length; i++) {
            String hexStr = Integer.toHexString(0x00ff & fileDigest[i]);
            if (hexStr.length() < 2) {
                checksumSb.append("0");
            }
            checksumSb.append(hexStr);
        }
        return checksumSb.toString();
    }

    public UrlChangeTrigger(String url) throws MalformedURLException {
        this(new URL(url));
    }
    
    public UrlChangeTrigger(URL url) {
        this.url = url;
    }
    
    @Override
    public void start(BuildableItem project, boolean newInstance) {
    	super.start(project, newInstance);
        try {
            this.tabs = CronTabList.create("* * * * *");
        } catch (ANTLRException e) {
            throw new RuntimeException("Bug! couldn't schedule poll");
        }   	
    }

    private static final Logger LOGGER =
        Logger.getLogger(UrlChangeTrigger.class.getName());

    private File getFingerprintFile() {
	return new File(job.getRootDir(), "url-change-trigger-oldmd5");
    }

    @Override
    public void run() {
        try {
            LOGGER.log(Level.FINER, "Testing the file {0}", url);
            String currentMd5 = getMd5(url.openStream());

	    String oldMd5;
	    File file = getFingerprintFile();
	    if (!file.exists()) {
		oldMd5 = "null";
	    } else {
		BufferedReader br = new BufferedReader(new FileReader(file));
		try {
		    oldMd5 = br.readLine();
		} finally {
		    br.close();
		}
	    }
            if (!currentMd5.equals(oldMd5)) {
            	LOGGER.log(Level.FINE, 
            			"Differences found in the file {0}. >{1}< != >{2}<",
            			new Object[] {
                                    url, oldMd5, currentMd5,
            			});
		PrintWriter w = new PrintWriter(new FileOutputStream(file));
		try {
		    w.println(currentMd5);
		} finally {
		    w.close();
		}
                oldMd5 = currentMd5;
                job.scheduleBuild();
            }
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    }
    
    public URL getUrl() {
        return url;
    }
    
{'fraction_non_alphanumeric': 0.09937140564397486, 'fraction_numerical': 0.008559582720342383, 'mean_word_length': 2.474907063197026, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '251326', 'n_tokens_mistral': 2260, 'n_tokens_neox': 2145, 'n_words': 543}