<commit_before>package main

import "bufio"
import "flag"
import "fmt"
import "log"
import "os"

import "code.google.com/p/lzma"



type Request struct {
	ip string
	path string
}

func read_file(fn string) chan *Request {
	file, err := os.Open(fn)
	if (err != nil) {
		log.Fatal(err)
	}
	// TODO check file format
	r := lzma.NewReader(file)
	r.Close()

	output := make(chan *Request)
	go func() {
		scanner := bufio.NewScanner(file)
		for scanner.Scan() {
			line := scanner.Text()
			r := new(Request)
			r.ip = "TODO: ip"
			r.path = line
			output <- r
		}
		close(output)
	}()
	
	return output
}

func read_files(filenames []string) chan *Request {
	output := make(chan *Request)

	go func() {
		for _, filename := range filenames {<commit_msg>Correct error handling<commit_after>package main

import "bufio"
import "flag"
import "fmt"
import "log"
import "os"


type Request struct {
	ip string
	path string
}

func read_file(fn string) chan *Request {
	file, err := os.Open(fn)
	if (err != nil) {
		log.Fatal(err)
	}
	// TODO check file format

	output := make(chan *Request)
	go func() {
		scanner := bufio.NewScanner(file)
		for scanner.Scan() {
			line := scanner.Text()
			r := new(Request)
			r.ip = "TODO: ip"
			r.path = line
			output <- r
		}
		if (scanner.Err() != nil) {
			log.Fatal(scanner.Err())
		}
		file.Close()
		close(output)
	}()
	
	return output
}

func read_files(filenames []string) chan *Request {
	output := make(chan *Request)

	go func() {
		for _, filename := range filenames {
{'fraction_non_alphanumeric': 0.14814814814814814, 'fraction_numerical': 0.0, 'mean_word_length': 3.322857142857143, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1169096', 'n_tokens_mistral': 627, 'n_tokens_neox': 564, 'n_words': 156}