<commit_before>// Copyright 2015 <NAME>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// 	You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// 	See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bytes"
	"database/sql"
	"flag"
	"io"
	"log"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"time"

	_ "github.com/mattn/go-sqlite3"
	"github.com/ymotongpoo/gopt3rec/epg"
)

const (
	FilePrefixFormat   = "20060102T1504"
	HourMinFormat      = "1504"
	DateHourMinFormat  = "01021504"
	AtCmdFormat        = "0601021504.05"
	EPGInsertStatement = `replace into epg(id, channel, title, detail, start, end, duration) values (?, ?, ?, ?, ?, ?, ?)`
)

var TVChannelMap = map[string]string{
	"1":       "27",
	"nhk":     "27",
	"2":       "26",
	"etv":     "26",
	"4":       "25",
	"ntv":     "25",
	"5":       "24",
	"ex":      "24",
	"6":       "22",
	"tbs":     "22",
	"7":       "23",
	"tx":      "23",
	"8":       "21",
	"cx":      "21",
	"9":       "16",
	"mx":      "16",
	"12":      "28",
	"univ":    "28",
	"nhkbs1":  "BS15_1",
	"nhkbs2":  "BS03_1",
	"bsntv":   "BS13_0",
	"bsex":    "BS01_0",
	"bstbs":   "BS01_1",
	"bsjapan": "BS01_2",
	"bsfuji":  "BS13_1",
}

var replaceChars = map[string]string{
	"!": "！",
	"?": "？",
	"#": "＃",
	"(": "（",<commit_msg>fix NHKBS1 channel<commit_after>// Copyright 2015 <NAME>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// 	You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// 	See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bytes"
	"database/sql"
	"flag"
	"io"
	"log"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"time"

	_ "github.com/mattn/go-sqlite3"
	"github.com/ymotongpoo/gopt3rec/epg"
)

const (
	FilePrefixFormat   = "20060102T1504"
	HourMinFormat      = "1504"
	DateHourMinFormat  = "01021504"
	AtCmdFormat        = "0601021504.05"
	EPGInsertStatement = `replace into epg(id, channel, title, detail, start, end, duration) values (?, ?, ?, ?, ?, ?, ?)`
)

var TVChannelMap = map[string]string{
	"1":       "27",
	"nhk":     "27",
	"2":       "26",
	"etv":     "26",
	"4":       "25",
	"ntv":     "25",
	"5":       "24",
	"ex":      "24",
	"6":       "22",
	"tbs":     "22",
	"7":       "23",
	"tx":      "23",
	"8":       "21",
	"cx":      "21",
	"9":       "16",
	"mx":      "16",
	"12":      "28",
	"univ":    "28",
	"nhkbs1":  "BS15_0",
	"nhkbs2":  "BS03_1",
	"bsntv":   "BS13_0",
	"bsex":    "BS01_0",
	"bstbs":   "BS01_1",
	"bsjapan": "BS01_2",
	"bsfuji":  "BS13_1",
}

var replaceChars = map[string]string{
	"!": "！",
	"?": "？",
	"#": "＃",
	"(": "（",
{'fraction_non_alphanumeric': 0.19323960514507926, 'fraction_numerical': 0.06909961112772958, 'mean_word_length': 2.8569780853517877, 'pattern_counts': {'":': 58, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5167279', 'n_tokens_mistral': 1479, 'n_tokens_neox': 1367, 'n_words': 348}