<commit_before>#!/usr/bin/python

# gsocr.py
# uses gstreamer to send frames to gocr

import subprocess
import optparse

import pygtk
pygtk.require ("2.0")
import gobject
gobject.threads_init ()
import pygst
pygst.require ("0.10")
import gst

import gtk
 
def Dictionary():
    dictionary = [w for w in open('dictionary.txt').read().split() if len(w)>3]
    return dictionary

def one_frame( sink,buffer,pad, it):
    # print len(buffer)
    if len(buffer) == 15:
        it.p = subprocess.Popen(['gocr', '-'], stdin=subprocess.PIPE, 
          stdout=subprocess.PIPE,stderr=subprocess.PIPE)
        it.p.stdin.write(buffer)  
        it.f=open('foo.pnm','wb')
        it.f.write(buffer)  

        it.skip=False ## don't seek yet, need to get the 2nd part below

    else:
        it.f.write(buffer)
        it.f.close()

        ocrtext, stderrdata = it.p.communicate(buffer)

        if it.last_ocr != ocrtext:
            it.last_ocr=ocrtext
            words = [w for w in ocrtext.split() if w.upper() in it.dictionary]
            print ocrtext.__repr__()[:70]
            # print "last", it.last_words
            # print "cur", words
            if it.last_words != words:
                it.last_words = words
                print words[:10]

        it.skip=True ## flag to seek

    return   
    
def skip_forward(it):

    if it.skip:
        # print "in skip_forward..."
        it.skip=False
        pos_int = it.pipeline.query_position(it.time_format, None)[0]
        # print "starting at", pos_int
        seek_ns = pos_int + (100 * 1000000000)
        it.pipeline.seek_simple(it.time_format, gst.SEEK_FLAG_FLUSH, seek_ns)

    return True

class Main:

    def __init__(self, filename):<commit_msg>it quits. needs to be reusable<commit_after>#!/usr/bin/python

# gsocr.py
# uses gstreamer to send frames to gocr

import subprocess
import optparse

import pygtk
pygtk.require ("2.0")
import gobject
gobject.threads_init ()
import pygst
pygst.require ("0.10")
import gst

import gtk
 
def Dictionary():
    dictionary = [w.upper() for w in open('dictionary.txt').read().split() if len(w)>3]
    return dictionary

def one_frame( sink,buffer,pad, it):
    # print len(buffer)
    if len(buffer) == 15:
        it.p = subprocess.Popen(['gocr', '-'], stdin=subprocess.PIPE, 
          stdout=subprocess.PIPE,stderr=subprocess.PIPE)
        it.p.stdin.write(buffer)  
        # it.f=open('foo.pnm','wb')
        # it.f.write(buffer)  

        it.skip=False ## don't seek yet, need to get the 2nd part below

    else:
        # it.f.write(buffer)
        # it.f.close()

        ocrtext, stderrdata = it.p.communicate(buffer)

        if it.last_ocr != ocrtext:
            it.last_ocr=ocrtext
            words = [w for w in ocrtext.split() if w.upper() in it.dictionary]
            print ocrtext.__repr__()[:70]
            # print "last", it.last_words
            # print "cur", words
            if it.last_words != words:
                it.last_words = words
                print words[:10]

        it.skip=True ## flag to seek
        gobject.idle_add( skip_forward, p, priority=gobject.PRIORITY_HIGH )

    return   
    
def skip_forward(it):

    if it.skip:
        # print "in skip_forward..."
        it.skip=False
        pos_int = it.pipeline.query_position(it.time_format, None)[0]
        # print "starting at", pos_int
        seek_ns = pos_int + (10 * 1000000000)
        it.pipeline.seek_simple(it.time_format, gst.SEEK_FLAG_FLUSH, seek_ns)

    return False

class Main:

    def __init__(self, filename):
{'fraction_non_alphanumeric': 0.1227080394922426, 'fraction_numerical': 0.014950634696755994, 'mean_word_length': 2.323336457357076, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2704739', 'n_tokens_mistral': 1256, 'n_tokens_neox': 1183, 'n_words': 280}