<commit_before>import time
from fontTools.ttLib import TTFont, TTLibError
from fontTools.ttLib.tables._h_e_a_d import mac_epoch_diff
from fontTools.misc.textTools import num2binary
from extractor.exceptions import ExtractorError
from extractor.tools import RelaxedInfo, copyAttr, defaultLeftKerningGroupPrefix, defaultRightKerningGroupPrefix

# ----------------
# Public Functions
# ----------------

def isOpenType(pathOrFile):
    try:
        font = TTFont(pathOrFile)
        del font
    except TTLibError:
        return False
    return True

def extractFontFromOpenType(pathOrFile, destination, doGlyphs=True, doInfo=True, doKerning=True, customFunctions=[]):
    source = TTFont(pathOrFile)
    if doInfo:
        extractOpenTypeInfo(source, destination)
    if doGlyphs:
        extractOpenTypeGlyphs(source, destination)
    if doKerning:
        kerning, groups = extractOpenTypeKerning(source, destination)
        destination.groups.update(groups)
        destination.kerning.clear()
        destination.kerning.update(kerning)
    for function in customFunctions:
        function(source, destination)
    source.close()

# ----
# Info
# ----

def extractOpenTypeInfo(source, destination):
    info = RelaxedInfo(destination.info)
    _extractInfoHead(source, info)
    _extractInfoName(source, info)
    _extracInfoOS2(source, info)
    _extractInfoHhea(source, info)
    _extractInfoVhea(source, info)
    _extractInfoPost(source, info)
    _extractInfoCFF(source, info)

def _extractInfoHead(source, info):
    head = source["head"]
    # version
    version = str(round(head.fontRevision, 3))
    versionMajor, versionMinor = version.split(".")
    info.versionMajor = int(versionMajor)
    info.versionMinor = int(versionMinor)
    # upm
    info.unitsPerEm = head.unitsPerEm
    # created
    format = "%Y/%m/%d %H:%M:%S"
    created = head.created
    created = time.gmtime(max(0, created + mac_epoch_diff))
    info.openTypeHeadCreated = time.strftime(format, created)
    # lowestRecPPEM
    info.openTypeHeadLowestRecPPEM = head.lowestRecPPEM
    # flags
    info.openTypeHeadFlags = binaryToIntList(head.flags)
    # styleMapStyleName
    macStyle = binaryToIntList(head.macStyle)
    styleMapStyleName = "regular"
    if 0 in macStyle and 1 in macStyle:
        styleMapStyleName = "bold italic"
    elif 0 in macStyle:
        styleMapStyleName = "bold"
    elif 1 in macStyle:
        styleMapStyleName = "italic"
    info.styleMapStyleName = styleMapStyleName

def _extractInfoName(source, info):
    nameIDs = {}
    for record in source["name"].names:
        nameIDs[record.nameID, record.platformID, record.platEncID, record.langID] = record.string
    attributes = dict(
        familyName=_priorityOrder(16) + _priorityOrder(1),
        styleName=_priorityOrder(17) + _priorityOrder(2),
        styleMapFamilyName=_priorityOrder(1),
        # styleMapStyleName will be handled in head extraction
        copyright=_priorityOrder(0),
        trademark=_priorityOrder(7),
        openTypeNameDesigner=_priorityOrder(9),
        openTypeNameDesignerURL=_priorityOrder(12),
        openTypeNameManufacturer=_priorityOrder(8),
        openTypeNameManufacturerURL=_priorityOrder(11),
        openTypeNameLicense=_priorityOrder(13),
        openTypeNameLicenseURL=_priorityOrder(14),
        openTypeNameVersion=_priorityOrder(5),
        openTypeNameUniqueID=_priorityOrder(3),
        openTypeNameDescription=_priorityOrder(10),
        openTypeNamePreferredFamilyName=_priorityOrder(16),
        openTypeNamePreferredSubfamilyName=_priorityOrder(17),
        openTypeNameCompatibleFullName=_priorityOrder(18),
        openTypeNameSampleText=_priorityOrder(20),
        openTypeNameWWSFamilyName=_priorityOrder(21),
        openTypeNameWWSSubfamilyName=_priorityOrder(22)
    )
    for attr, priority in attributes.items():
        value = _skimNameIDs(nameIDs, priority)
        if value is not None:
            setattr(info, attr, value)

def _priorityOrder(nameID):
    priority = [
        (nameID, 1, 0, 0),
        (nameID, 1, None, None),
        (nameID, None, None, None)
    ]
    return priority

def _skimNameIDs(nameIDs, priority):
    for (nameID, platformID, platEncID, langID) in priority:
        for (nID, pID, pEID, lID), text in nameIDs.items():
            if nID != nameID:
                continue
            if pID != platformID and platformID is not None:
                continue
            if pEID != platEncID and platEncID is not None:
                continue
            if lID != langID and langID is not None:
                continue
            if pID == 0 or (pID == 3 and pEID in (0, 1)):
                text = text.decode("utf_16_be")
            else:
                text = text.decode("latin1")
            return text

def _extracInfoOS2(source, info):
    os2 = source["OS/2"]
    # openTypeOS2WidthClass
    copyAttr(os2, "usWidthClass", info, "openTypeOS2WidthClass")
    # openTypeOS2WeightClass
    copyAttr(os2, "usWeightClass", info, "openTypeOS2WeightClass")
    # openTypeOS2Selection
    if hasattr(os2, "fsSelection"):
        fsSelection = binaryToIntList(os2.fsSelection)<commit_msg>I've been having trouble decoding from latin1 but macroman seems to work.<commit_after>import time
from fontTools.ttLib import TTFont, TTLibError
from fontTools.ttLib.tables._h_e_a_d import mac_epoch_diff
from fontTools.misc.textTools import num2binary
from extractor.exceptions import ExtractorError
from extractor.tools import RelaxedInfo, copyAttr, defaultLeftKerningGroupPrefix, defaultRightKerningGroupPrefix

# ----------------
# Public Functions
# ----------------

def isOpenType(pathOrFile):
    try:
        font = TTFont(pathOrFile)
        del font
    except TTLibError:
        return False
    return True

def extractFontFromOpenType(pathOrFile, destination, doGlyphs=True, doInfo=True, doKerning=True, customFunctions=[]):
    source = TTFont(pathOrFile)
    if doInfo:
        extractOpenTypeInfo(source, destination)
    if doGlyphs:
        extractOpenTypeGlyphs(source, destination)
    if doKerning:
        kerning, groups = extractOpenTypeKerning(source, destination)
        destination.groups.update(groups)
        destination.kerning.clear()
        destination.kerning.update(kerning)
    for function in customFunctions:
        function(source, destination)
    source.close()

# ----
# Info
# ----

def extractOpenTypeInfo(source, destination):
    info = RelaxedInfo(destination.info)
    _extractInfoHead(source, info)
    _extractInfoName(source, info)
    _extracInfoOS2(source, info)
    _extractInfoHhea(source, info)
    _extractInfoVhea(source, info)
    _extractInfoPost(source, info)
    _extractInfoCFF(source, info)

def _extractInfoHead(source, info):
    head = source["head"]
    # version
    version = str(round(head.fontRevision, 3))
    versionMajor, versionMinor = version.split(".")
    info.versionMajor = int(versionMajor)
    info.versionMinor = int(versionMinor)
    # upm
    info.unitsPerEm = head.unitsPerEm
    # created
    format = "%Y/%m/%d %H:%M:%S"
    created = head.created
    created = time.gmtime(max(0, created + mac_epoch_diff))
    info.openTypeHeadCreated = time.strftime(format, created)
    # lowestRecPPEM
    info.openTypeHeadLowestRecPPEM = head.lowestRecPPEM
    # flags
    info.openTypeHeadFlags = binaryToIntList(head.flags)
    # styleMapStyleName
    macStyle = binaryToIntList(head.macStyle)
    styleMapStyleName = "regular"
    if 0 in macStyle and 1 in macStyle:
        styleMapStyleName = "bold italic"
    elif 0 in macStyle:
        styleMapStyleName = "bold"
    elif 1 in macStyle:
        styleMapStyleName = "italic"
    info.styleMapStyleName = styleMapStyleName

def _extractInfoName(source, info):
    nameIDs = {}
    for record in source["name"].names:
        nameIDs[record.nameID, record.platformID, record.platEncID, record.langID] = record.string
    attributes = dict(
        familyName=_priorityOrder(16) + _priorityOrder(1),
        styleName=_priorityOrder(17) + _priorityOrder(2),
        styleMapFamilyName=_priorityOrder(1),
        # styleMapStyleName will be handled in head extraction
        copyright=_priorityOrder(0),
        trademark=_priorityOrder(7),
        openTypeNameDesigner=_priorityOrder(9),
        openTypeNameDesignerURL=_priorityOrder(12),
        openTypeNameManufacturer=_priorityOrder(8),
        openTypeNameManufacturerURL=_priorityOrder(11),
        openTypeNameLicense=_priorityOrder(13),
        openTypeNameLicenseURL=_priorityOrder(14),
        openTypeNameVersion=_priorityOrder(5),
        openTypeNameUniqueID=_priorityOrder(3),
        openTypeNameDescription=_priorityOrder(10),
        openTypeNamePreferredFamilyName=_priorityOrder(16),
        openTypeNamePreferredSubfamilyName=_priorityOrder(17),
        openTypeNameCompatibleFullName=_priorityOrder(18),
        openTypeNameSampleText=_priorityOrder(20),
        openTypeNameWWSFamilyName=_priorityOrder(21),
        openTypeNameWWSSubfamilyName=_priorityOrder(22)
    )
    for attr, priority in attributes.items():
        value = _skimNameIDs(nameIDs, priority)
        if value is not None:
            setattr(info, attr, value)

def _priorityOrder(nameID):
    priority = [
        (nameID, 1, 0, 0),
        (nameID, 1, None, None),
        (nameID, None, None, None)
    ]
    return priority

def _skimNameIDs(nameIDs, priority):
    for (nameID, platformID, platEncID, langID) in priority:
        for (nID, pID, pEID, lID), text in nameIDs.items():
            if nID != nameID:
                continue
            if pID != platformID and platformID is not None:
                continue
            if pEID != platEncID and platEncID is not None:
                continue
            if lID != langID and langID is not None:
                continue
            if pID == 0 or (pID == 3 and pEID in (0, 1)):
                text = text.decode("utf_16_be")
            else:
                text = text.decode("macroman")
            return text

def _extracInfoOS2(source, info):
    os2 = source["OS/2"]
    # openTypeOS2WidthClass
    copyAttr(os2, "usWidthClass", info, "openTypeOS2WidthClass")
    # openTypeOS2WeightClass
    copyAttr(os2, "usWeightClass", info, "openTypeOS2WeightClass")
    # openTypeOS2Selection
    if hasattr(os2, "fsSelection"):
        fsSelection = binaryToIntList(os2.fsSelection)
{'fraction_non_alphanumeric': 0.09492697924673328, 'fraction_numerical': 0.012682551883166795, 'mean_word_length': 3.165266106442577, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2469713', 'n_tokens_mistral': 3177, 'n_tokens_neox': 3120, 'n_words': 664}