<commit_before>#!/usr/bin/env python2

# Ampuni aku... :(
import os
import sqlite3

SETTINGS_FILE = "data/settings.sqlite"

def initDB(settingsFile = SETTINGS_FILE):
    with sqlite3.connect(settingsFile) as con:
        cur = con.cursor()
        cur.execute("""
        CREATE TABLE IF NOT EXISTS settings (
            chat_id INT PRIMARY KEY,
            rating TEXT,
            character TEXT,
            copyright TEXT,
            general TEXT)""")

def loadSettings(chat_id, settingsFile = SETTINGS_FILE):
    default = {
        "rating":    set(),
        "character": set(),
        "copyright": set(),
        "general":   set()
    }

    if not os.path.exists(settingsFile):
        return default

    with sqlite3.connect(settingsFile) as con:
        cur = con.cursor()
        cur.execute("""
        SELECT rating, character, copyright, general
          FROM settings
         WHERE chat_id = ?""", (chat_id,))
        data = cur.fetchone()
        if data is None:
            return default

        return {
            "rating":    set(data[0].split(",")),
            "character": set(data[1].split(",")),
            "copyright": set(data[2].split(",")),
            "general":   set(data[3].split(","))
        }

def saveSettings(chat_id, settings, settingsFile = SETTINGS_FILE):
    if not os.path.exists(settingsFile):
        initDB()

    rating    = ",".join(settings["rating"])
    character = ",".join(settings["character"])
    copyright = ",".join(settings["copyright"])
    general   = ",".join(settings["general"])

    with sqlite3.connect(settingsFile) as con:
        cur = con.cursor()<commit_msg>Fix splitting process<commit_after>#!/usr/bin/env python2

# Ampuni aku... :(
import os
import sqlite3

SETTINGS_FILE = "data/settings.sqlite"

def _splitOrEmpty(s, delim):
    if len(s) == 0:
        return []

    return s.split(delim)

def initDB(settingsFile = SETTINGS_FILE):
    with sqlite3.connect(settingsFile) as con:
        cur = con.cursor()
        cur.execute("""
        CREATE TABLE IF NOT EXISTS settings (
            chat_id INT PRIMARY KEY,
            rating TEXT,
            character TEXT,
            copyright TEXT,
            general TEXT)""")

def loadSettings(chat_id, settingsFile = SETTINGS_FILE):
    default = {
        "rating":    set(),
        "character": set(),
        "copyright": set(),
        "general":   set()
    }

    if not os.path.exists(settingsFile):
        return default

    with sqlite3.connect(settingsFile) as con:
        cur = con.cursor()
        cur.execute("""
        SELECT rating, character, copyright, general
          FROM settings
         WHERE chat_id = ?""", (chat_id,))
        data = cur.fetchone()
        if data is None:
            return default

        return {
            "rating":    set(_splitOrEmpty(data[0], ",")),
            "character": set(_splitOrEmpty(data[1], ",")),
            "copyright": set(_splitOrEmpty(data[2], ",")),
            "general":   set(_splitOrEmpty(data[3], ","))
        }

def saveSettings(chat_id, settings, settingsFile = SETTINGS_FILE):
    if not os.path.exists(settingsFile):
        initDB()

    rating    = ",".join(settings["rating"])
    character = ",".join(settings["character"])
    copyright = ",".join(settings["copyright"])
    general   = ",".join(settings["general"])

    with sqlite3.connect(settingsFile) as con:
        cur = con.cursor()
{'fraction_non_alphanumeric': 0.14139643587496348, 'fraction_numerical': 0.005550686532281624, 'mean_word_length': 2.2702960840496655, 'pattern_counts': {'":': 16, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3674579', 'n_tokens_mistral': 1019, 'n_tokens_neox': 983, 'n_words': 236}