<commit_before>/*				       	-*- c-file-style: "linux" -*-
 *
 * libhsync -- library for network deltas
 * $Id$
 * 
 * Copyright (C) 1999, 2000 by <NAME> <<EMAIL>>
 * Copyright (C) 1999 by <NAME>
 * 
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation; either version 2.1 of the License, or
 * (at your option) any later version.
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 * 
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */


/* 
 * Generate a checksum set, using the newstyle nonblocking
 * arrangement and mapptrs.
 */

#include "config.h"

#include <stdlib.h>
#include <stdio.h>
#include <assert.h>
#include <stdint.h>

#include "hsync.h"
#include "stream.h"
#include "util.h"
#include "protocol.h"
#include "netint.h"


const int       HS_MKSUM_TAG = 123124;

struct hs_mksum_job {
    hs_stream_t    *stream;
    int             dogtag;
    size_t          block_len;
    hs_stats_t      stats;
    size_t          strong_sum_len;

        int             (*statefn)(hs_mksum_job_t *);
};


/*
 * State of trying to send the signature header.
 */
static int _hs_mksum_s_header(hs_mksum_job_t *job)
{
        _hs_squirt_n32(job->stream, HS_SIG_MAGIC);

        return HS_OK;
}


/* Set up a new encoding job. */
hs_mksum_job_t *
hs_mksum_begin(hs_stream_t *stream,
	       size_t new_block_len, size_t strong_sum_len)
{
    hs_mksum_job_t *job;

    job = _hs_alloc_struct(hs_mksum_job_t);

    _hs_stream_check(stream);
    job->stream = stream;
    job->block_len = new_block_len;
    job->dogtag = HS_MKSUM_TAG;

    assert(strong_sum_len > 0 && strong_sum_len <= HS_MD4_LENGTH);
    job->strong_sum_len = strong_sum_len;

    job->statefn = _hs_mksum_s_header;

    return job;
}



int hs_mksum_finish(hs_mksum_job_t * job)
{
        assert(job->dogtag == HS_MKSUM_TAG);
        _hs_bzero(job, sizeof *job);
        free(job);

        return HS_OK;
}



/* 
 * Nonblocking iteration interface for making up a file sum. 
 */
int hs_mksum_iter(hs_mksum_job_t *job, int ending)
{
        int result;

        assert(job->dogtag == HS_MKSUM_TAG);

        do {
                result = _hs_tube_catchup(job->stream);
                if (result != HS_OK)
                        return result;
                
                result = job->statefn(job);
        } while (result == HS_OK);

        return result;
}
<commit_msg>Nonoptimal stream-based mksum code.<commit_after>/*				       	-*- c-file-style: "linux" -*-
 *
 * libhsync -- library for network deltas
 * $Id$
 * 
 * Copyright (C) 1999, 2000 by <NAME> <<EMAIL>>
 * Copyright (C) 1999 by <NAME> <<EMAIL>>
 * 
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation; either version 2.1 of the License, or
 * (at your option) any later version.
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 * 
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */


/* 
 * mksum: Generate and write out checksums using the stream interface.
 */

#include "config.h"

#include <stdlib.h>
#include <stdio.h>
#include <assert.h>
#include <stdint.h>

#include "hsync.h"
#include "stream.h"
#include "util.h"
#include "protocol.h"
#include "netint.h"
#include "trace.h"
#include "checksum.h"

const int       HS_MKSUM_TAG = 123124;

struct hs_mksum_job {
        int          dogtag;
	hs_stream_t *stream;
        enum hs_result (*statefn)(hs_mksum_job_t *);
        size_t          block_len;
        size_t          strong_sum_len;
        int             near_end;
};


/* Possible state functions for signature generation. */
static enum hs_result _hs_mksum_s_header(hs_mksum_job_t *);
static enum hs_result _hs_mksum_s_complete(hs_mksum_job_t *);
static enum hs_result _hs_mksum_s_generate(hs_mksum_job_t *);


                                           
/*
 * State of trying to send the signature header.
 */
static enum hs_result _hs_mksum_s_header(hs_mksum_job_t *job)
{
        _hs_squirt_n32(job->stream, HS_SIG_MAGIC);
        job->statefn = _hs_mksum_s_generate;

        return HS_RUN_OK;
}


static enum hs_result
_hs_mksum_do_block(hs_mksum_job_t *job, const void *block, size_t len)
{
        uint32_t weak_sum;

        weak_sum = _hs_calc_weak_sum(block, len);
        _hs_trace("got weak sum 0x%08x", weak_sum);

        _hs_squirt_n32(job->stream, weak_sum);

        return HS_RUN_OK;
}


/*
 * State of reading a block and trying to generate its sum.
 */
static enum hs_result _hs_mksum_s_generate(hs_mksum_job_t *job)
{
        enum hs_result result;
        int len;
        void *block;
        
        /* must get a whole block, otherwise try again */
        len = job->block_len;
        result = _hs_scoop_read(job->stream, len, &block);
        
        /* unless we're near eof, in which case we'll accept
         * whatever's in there */
        if (result == HS_BLOCKED && job->near_end) {
                result = _hs_scoop_read_rest(job->stream, &len, &block);
                job->statefn = _hs_mksum_s_complete;
        } else if (result != HS_OK) {
                _hs_trace("generate stopped: %s", hs_strerror(result));
                return result;
        }

        _hs_trace("got %d byte block", len);
        fwrite(block, 1, len, stdout);

        return _hs_mksum_do_block(job, block, len);
}


static enum hs_result _hs_mksum_s_complete(hs_mksum_job_t *UNUSED(job))
{
        _hs_trace("signature generation has already finished");

        return HS_OK;
}


/* Set up a new encoding job. */
hs_mksum_job_t * hs_mksum_begin(hs_stream_t *stream,
                                size_t new_block_len, size_t strong_sum_len)
{
        hs_mksum_job_t *job;

        job = _hs_alloc_struct(hs_mksum_job_t);

        _hs_stream_check(stream);
        job->stream = stream;
        job->block_len = new_block_len;
        job->dogtag = HS_MKSUM_TAG;

        assert(strong_sum_len > 0 && strong_sum_len <= HS_MD4_LENGTH);
        job->strong_sum_len = strong_sum_len;

        job->statefn = _hs_mksum_s_header;

        return job;
}



int hs_mksum_finish(hs_mksum_job_t * job)
{
        assert(job->dogtag == HS_MKSUM_TAG);
        _hs_bzero(job, sizeof *job);
        free(job);

        return HS_OK;
}



/* 
 * Nonblocking iteration interface for making up a file sum.
 *
 * ENDING should be true if there is no more data after what's in the
 * input buffer.  The final block checksum will run across whatever's
 * in there, without trying to accumulate anything else.
 */
int hs_mksum_iter(hs_mksum_job_t *job, int ending)
{
        enum hs_result result;

        assert(job->dogtag == HS_MKSUM_TAG);

        if (ending)
                job->near_end = 1;

        while (1) {
                result = _hs_tube_catchup(job->stream);
                if (result != HS_OK)
                        return result;
                
                result = job->statefn(job);
                if (result != HS_RUN_OK)
                        return result;
        } 
}

{'fraction_non_alphanumeric': 0.09499554310454604, 'fraction_numerical': 0.009423150388386604, 'mean_word_length': 2.4953271028037385, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6654659', 'n_tokens_mistral': 2743, 'n_tokens_neox': 2568, 'n_words': 741}