<commit_before>import logging
import arrow

from collections import namedtuple
from . import cgi
from .info import URL, DEFAULT_DIR
from .info import WifiMode, WifiModeOnBoot, ModeValue, Operation


logger = logging.getLogger(__name__)


##################
# command.cgi API

def list_files(*filters, remote_dir=DEFAULT_DIR, url=URL):
    response = _get(Operation.list_files, url, DIR=remote_dir)
    files = _split_file_list(response.text)
    return (f for f in files if all(filt(f) for filt in filters))


def count_files(remote_dir=DEFAULT_DIR, url=URL):
    response = _get(Operation.count_files, url, DIR=remote_dir)
    return int(response.text)


def memory_changed(url=URL):
    """Returns True if memory has been written to, False otherwise"""
    response = _get(Operation.memory_changed, url)
    return int(response.text) == 1


def get_ssid(url=URL):
    return _get(Operation.get_ssid, url).text


def get_password(url=URL):
    return _get(Operation.get_password, url).text


def get_mac(url=URL):
    return _get(Operation.get_mac, url).text


def get_browser_lang(url=URL):
    return _get(Operation.get_browser_lang, url).text


def get_fw_version(url=URL):
    return _get(Operation.get_fw_version, url).text


def get_ctrl_image(url=URL):
    return _get(Operation.get_ctrl_image, url).text


def get_wifi_mode(url=URL) -> WifiMode:
    mode_value = int(_get(Operation.get_wifi_mode, url).text)
    all_modes = list(WifiMode) + list(WifiModeOnBoot)
    for mode in all_modes:
        if mode.value == mode_value:
            return mode
    raise ValueError("Uknown mode: {:d}".format(mode_value))
        

#####################
# API implementation

FileInfo = namedtuple(
    "FileInfo", "directory filename size attribute datetime")


def _split_file_list(text):
    lines = text.split("\r\n")
    for line in lines:
        groups = line.split(",")
        if len(groups) == 6:
            directory, filename, *remaining = groups
            remaining = map(int, remaining)
            size, attr_val, date_val, time_val = remaining
            timeinfo = _decode_time(date_val, time_val)
            attribute = _decode_attribute(attr_val)
            yield FileInfo(directory, filename,
                           size, attribute, timeinfo)


def _decode_time(date_val: int, time_val: int):
    year = (date_val >> 9) + 1980  # 0-val is the year 1980
    month = (date_val & (0b1111 << 5)) >> 5
    day = date_val & 0b11111
    hour = time_val >> 11
    minute = ((time_val >> 5) & 0b111111)
    second = (time_val & 0b11111) * 2
    try:
        decoded = arrow.get(year, month, day, hour, minute, second)
    except ValueError:
        logger.warning("Invalid date: {:04d}-{:02d}-{:02d}".format(
                       year, month, day))
        year = max(1980, year)  # FAT32 doesn't go higher<commit_msg>added list_files_raw CGI command for the case where we need a list of files but we don't care about unpacking, attribute, time, and date information<commit_after>import logging
import arrow

from collections import namedtuple
from . import cgi
from .info import URL, DEFAULT_DIR
from .info import WifiMode, WifiModeOnBoot, ModeValue, Operation


logger = logging.getLogger(__name__)


##################
# command.cgi API

def list_files(*filters, remote_dir=DEFAULT_DIR, url=URL):
    response = _get(Operation.list_files, url, DIR=remote_dir)
    files = _split_file_list(response.text)
    return (f for f in files if all(filt(f) for filt in filters))


def list_files_raw(*filters, remote_dir=DEFAULT_DIR, url=URL):
    response = _get(Operation.list_files, url, DIR=remote_dir)
    files = _split_file_list_raw(response.text)
    return (f for f in files if all(filt(f) for filt in filters))


def count_files(remote_dir=DEFAULT_DIR, url=URL):
    response = _get(Operation.count_files, url, DIR=remote_dir)
    return int(response.text)


def memory_changed(url=URL):
    """Returns True if memory has been written to, False otherwise"""
    response = _get(Operation.memory_changed, url)
    return int(response.text) == 1


def get_ssid(url=URL):
    return _get(Operation.get_ssid, url).text


def get_password(url=URL):
    return _get(Operation.get_password, url).text


def get_mac(url=URL):
    return _get(Operation.get_mac, url).text


def get_browser_lang(url=URL):
    return _get(Operation.get_browser_lang, url).text


def get_fw_version(url=URL):
    return _get(Operation.get_fw_version, url).text


def get_ctrl_image(url=URL):
    return _get(Operation.get_ctrl_image, url).text


def get_wifi_mode(url=URL) -> WifiMode:
    mode_value = int(_get(Operation.get_wifi_mode, url).text)
    all_modes = list(WifiMode) + list(WifiModeOnBoot)
    for mode in all_modes:
        if mode.value == mode_value:
            return mode
    raise ValueError("Uknown mode: {:d}".format(mode_value))
        

#####################
# API implementation

FileInfo = namedtuple(
    "FileInfo", "directory filename size attribute datetime")
RawFileInfo = namedtuple(
    "RawFileInfo", "directory filename size attribute date time")


def _split_file_list(text):
    lines = text.split("\r\n")
    for line in lines:
        groups = line.split(",")
        if len(groups) == 6:
            directory, filename, *remaining = groups
            remaining = map(int, remaining)
            size, attr_val, date_val, time_val = remaining
            timeinfo = _decode_time(date_val, time_val)
            attribute = _decode_attribute(attr_val)
            yield FileInfo(directory, filename,
                           size, attribute, timeinfo)


def _split_file_list_raw(text):
    lines = text.split("\r\n")
    for line in lines:
        groups = line.split(",")
        if len(groups) == 6:
            directory, filename, *remaining = groups
            yield RawFileInfo(directory, filename, *map(int, remaining))


def _decode_time(date_val: int, time_val: int):
    year = (date_val >> 9) + 1980  # 0-val is the year 1980
    month = (date_val & (0b1111 << 5)) >> 5
    day = date_val & 0b11111
    hour = time_val >> 11
    minute = ((time_val >> 5) & 0b111111)
    second = (time_val & 0b11111) * 2
    try:
        decoded = arrow.get(year, month, day, hour, minute, second)
    except ValueError:
        logger.warning("Invalid date: {:04d}-{:02d}-{:02d}".format(
                       year, month, day))
        year = max(1980, year)  # FAT32 doesn't go higher
{'fraction_non_alphanumeric': 0.11910322279308734, 'fraction_numerical': 0.016970263116923557, 'mean_word_length': 3.3909774436090228, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '613268', 'n_tokens_mistral': 2157, 'n_tokens_neox': 2036, 'n_words': 536}