<commit_before># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import csv
import json
import shutil
import warnings
from pathlib import Path
from typing import Dict, Iterable, List, TextIO

from .io import line_reader, open_file_like, read_table, temporary_directory


# Any CSV file under 50 MB can use the fast JSON converter
JSON_FAST_CONVERTER_SIZE_BYTES = 50 * 1000 * 1000

# Any CSV file above 150 MB should not be converted to JSON
JSON_MAX_SIZE_BYTES = 150 * 1000 * 1000


def skip_head_reader(file_handle: TextIO, skip_count: int = 1, **read_opts) -> Iterable[str]:
    reader = line_reader(file_handle, **read_opts)
    for _ in range(skip_count):
        next(reader)
    yield from reader

<commit_msg>Increase max JSON file size<commit_after># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import csv
import json
import shutil
import warnings
from pathlib import Path
from typing import Dict, Iterable, List, TextIO

from .io import line_reader, open_file_like, read_table, temporary_directory


# Any CSV file under 50 MB can use the fast in-memory JSON converter
JSON_FAST_CONVERTER_SIZE_BYTES = 50 * 1000 * 1000

# Any CSV file above 500 MB should not be converted to JSON
JSON_MAX_SIZE_BYTES = 500 * 1000 * 1000


def skip_head_reader(file_handle: TextIO, skip_count: int = 1, **read_opts) -> Iterable[str]:
    reader = line_reader(file_handle, **read_opts)
    for _ in range(skip_count):
        next(reader)
    yield from reader


{'fraction_non_alphanumeric': 0.0641945773524721, 'fraction_numerical': 0.027910685805422646, 'mean_word_length': 4.372591006423983, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3683673', 'n_tokens_mistral': 770, 'n_tokens_neox': 691, 'n_words': 323}