<commit_before>#     Copyright 2013, <NAME>, mailto:<EMAIL>
#
#     Part of "Nuitka", an optimizing Python compiler that is compatible and
#     integrates with CPython, but also works on its own.
#
#     Licensed under the Apache License, Version 2.0 (the "License");
#     you may not use this file except in compliance with the License.
#     You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#     Unless required by applicable law or agreed to in writing, software
#     distributed under the License is distributed on an "AS IS" BASIS,
#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#     See the License for the specific language governing permissions and
#     limitations under the License.
#
""" Read source code from files.

This is tremendously more complex than one might think, due to encoding issues
and version differences of Python.
"""

from nuitka import Utils, SyntaxErrors, SourceCodeReferences

import re

def _readSourceCodeFromFilename3( source_filename ):
    with open( source_filename, "rb" ) as source_file:
        source_code = source_file.read()

    if source_code.startswith( b'\xef\xbb\xbf' ):
        return source_code[3:]

    new_line = source_code.find( b"\n" )

    if new_line is not -1:
        line = source_code[ : new_line ]

        line_match = re.search( b"coding[:=]\s*([-\w.]+)", line )

        if line_match:
            encoding = line_match.group(1).decode( "ascii" )

            # Detect encoding problem, as decode won't raise the compatible
            # thing.
            try:
                import codecs
                codecs.lookup( encoding )
            except LookupError:
                SyntaxErrors.raiseSyntaxError(
                    reason       = "unknown encoding: %s" % encoding,
                    source_ref   = SourceCodeReferences.fromFilename(
                        source_filename,
                        None
                    ),
                    display_line = False
                )

            return source_code[ new_line + 1 : ].decode( encoding )

        new_line = source_code.find( b"\n", new_line+1 )

        if new_line is not -1:
            line = source_code[ : new_line ]

            line_match = re.search( b"coding[:=]\s*([-\w.]+)", line )

            if line_match:
                encoding = line_match.group(1).decode( "ascii" )

                return source_code[ new_line + 1 : ].decode( encoding )


    return source_code.decode( "utf-8" )

def _detectEncoding2( source_filename ):
    # Detect the encoding.
    encoding = "ascii"

    with open( source_filename, "rb" ) as source_file:
        line1 = source_file.readline()

        if line1.startswith( b'\xef\xbb\xbf' ):
            encoding = "utf-8"
        else:
            line1_match = re.search( b"coding[:=]\s*([-\w.]+)", line1 )

            if line1_match:
                encoding = line1_match.group(1)
            else:
                line2 = source_file.readline()

                line2_match = re.search( b"coding[:=]\s*([-\w.]+)", line2 )<commit_msg>Python3: Fix, was corrupting line numbers when handling encoding headers.<commit_after>#     Copyright 2013, <NAME>, mailto:<EMAIL>
#
#     Part of "Nuitka", an optimizing Python compiler that is compatible and
#     integrates with CPython, but also works on its own.
#
#     Licensed under the Apache License, Version 2.0 (the "License");
#     you may not use this file except in compliance with the License.
#     You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#     Unless required by applicable law or agreed to in writing, software
#     distributed under the License is distributed on an "AS IS" BASIS,
#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#     See the License for the specific language governing permissions and
#     limitations under the License.
#
""" Read source code from files.

This is tremendously more complex than one might think, due to encoding issues
and version differences of Python.
"""

from nuitka import Utils, SyntaxErrors, SourceCodeReferences

import re

def _readSourceCodeFromFilename3( source_filename ):
    with open( source_filename, "rb" ) as source_file:
        source_code = source_file.read()

    if source_code.startswith( b'\xef\xbb\xbf' ):
        return source_code[3:]

    new_line = source_code.find( b"\n" )

    if new_line is not -1:
        line = source_code[ : new_line ]

        line_match = re.search( b"coding[:=]\s*([-\w.]+)", line )

        if line_match:
            encoding = line_match.group(1).decode( "ascii" )

            # Detect encoding problem, as decode won't raise the compatible
            # thing.
            try:
                import codecs
                codecs.lookup( encoding )
            except LookupError:
                SyntaxErrors.raiseSyntaxError(
                    reason       = "unknown encoding: %s" % encoding,
                    source_ref   = SourceCodeReferences.fromFilename(
                        source_filename,
                        None
                    ),
                    display_line = False
                )

            return source_code[ new_line : ].decode( encoding )

        new_line = source_code.find( b"\n", new_line+1 )

        if new_line is not -1:
            line = source_code[ : new_line ]

            line_match = re.search( b"coding[:=]\s*([-\w.]+)", line )

            if line_match:
                encoding = line_match.group(1).decode( "ascii" )

                return "\n" + source_code[ new_line : ].decode( encoding )


    return source_code.decode( "utf-8" )

def _detectEncoding2( source_filename ):
    # Detect the encoding.
    encoding = "ascii"

    with open( source_filename, "rb" ) as source_file:
        line1 = source_file.readline()

        if line1.startswith( b'\xef\xbb\xbf' ):
            encoding = "utf-8"
        else:
            line1_match = re.search( b"coding[:=]\s*([-\w.]+)", line1 )

            if line1_match:
                encoding = line1_match.group(1)
            else:
                line2 = source_file.readline()

                line2_match = re.search( b"coding[:=]\s*([-\w.]+)", line2 )
{'fraction_non_alphanumeric': 0.10001585037248376, 'fraction_numerical': 0.009351719765414487, 'mean_word_length': 2.051257253384913, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1656525', 'n_tokens_mistral': 1845, 'n_tokens_neox': 1745, 'n_words': 552}