<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/* $Id$ */

package org.apache.fop.fonts.truetype;

import java.io.IOException;
import java.util.BitSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import org.apache.xmlgraphics.fonts.Glyphs;

import org.apache.fop.complexscripts.fonts.AdvancedTypographicTableFormatException;
import org.apache.fop.complexscripts.fonts.GlyphDefinitionTable;
import org.apache.fop.complexscripts.fonts.GlyphPositioningTable;
import org.apache.fop.complexscripts.fonts.GlyphSubstitutionTable;
import org.apache.fop.complexscripts.fonts.OTFAdvancedTypographicTableReader;
import org.apache.fop.fonts.FontUtil;

/**
 * Reads a TrueType file or a TrueType Collection.
 * The TrueType spec can be found at the Microsoft.
 * Typography site: http://www.microsoft.com/truetype/
 */
public class TTFFile {

    static final byte NTABS = 24;
    static final int NMACGLYPHS = 258;
    static final int MAX_CHAR_CODE = 255;
    static final int ENC_BUF_SIZE = 1024;

    private final String encoding = "WinAnsiEncoding";    // Default encoding

    private final short firstChar = 0;

    private boolean useKerning = false;

    private boolean isEmbeddable = true;
    private boolean hasSerifs = true;
    /**
     * Table directory
     */
    protected Map dirTabs;
    private Map<Integer, Map<Integer, Integer>> kerningTab;     // for CIDs
    private Map<Integer, Map<Integer, Integer>> ansiKerningTab; // For winAnsiEncoding
    private List cmaps;
    private Set unicodeMappings;

    private int upem;                                // unitsPerEm from "head" table
    private int nhmtx;                               // Number of horizontal metrics
    private int postFormat;
    private int locaFormat;
    /**
     * Offset to last loca
     */
    protected long lastLoca = 0;
    private int numberOfGlyphs; // Number of glyphs in font (read from "maxp" table)
    private int nmGlyphs;                            // Used in fixWidths - remove?

    /**
     * Contains glyph data
     */
    protected TTFMtxEntry[] mtxTab;                  // Contains glyph data
    private final int[] mtxEncoded = null;

    private String postScriptName = "";
    private String fullName = "";
    private String notice = "";
    private final Set familyNames = new java.util.HashSet(); //Set<String>
    private String subFamilyName = "";

    private long italicAngle = 0;
    private long isFixedPitch = 0;
    private int fontBBox1 = 0;
    private int fontBBox2 = 0;
    private int fontBBox3 = 0;
    private int fontBBox4 = 0;
    private int capHeight = 0;
    private int os2CapHeight = 0;
    private int underlinePosition = 0;
    private int underlineThickness = 0;
    private int xHeight = 0;
    private int os2xHeight = 0;
    //Effective ascender/descender
    private int ascender = 0;
    private int descender = 0;
    //Ascender/descender from hhea table
    private int hheaAscender = 0;
    private int hheaDescender = 0;
    //Ascender/descender from OS/2 table
    private int os2Ascender = 0;
    private int os2Descender = 0;
    private int usWeightClass = 0;

    private short lastChar = 0;

    private int[] ansiWidth;
    private Map ansiIndex;

    // internal mapping of glyph indexes to unicode indexes
    // used for quick mappings in this class
    private final Map glyphToUnicodeMap = new java.util.HashMap();
    private final Map unicodeToGlyphMap = new java.util.HashMap();

    private TTFDirTabEntry currentDirTab;

    private boolean isCFF;

    // advanced typographic table support
    private boolean useAdvanced = false;
    private OTFAdvancedTypographicTableReader advancedTableReader;

    /**
     * logging instance
     */
    protected Log log = LogFactory.getLog(TTFFile.class);

    /**
     * Constructor
     * @param useKerning true if kerning data should be loaded
     * @param useAdvanced true if advanced typographic tables should be loaded
     */
    public TTFFile ( boolean useKerning, boolean useAdvanced ) {
        this.useKerning = useKerning;
        this.useAdvanced = useAdvanced;
    }

    /**
     * Key-value helper class
     */
    class UnicodeMapping implements Comparable {

        private final int unicodeIndex;
        private final int glyphIndex;

        UnicodeMapping(int glyphIndex, int unicodeIndex) {
            this.unicodeIndex = unicodeIndex;
            this.glyphIndex = glyphIndex;
            glyphToUnicodeMap.put(new Integer(glyphIndex), new Integer(unicodeIndex));
            unicodeToGlyphMap.put(new Integer(unicodeIndex), new Integer(glyphIndex));
        }

        /**
         * Returns the glyphIndex.
         * @return the glyph index
         */
        public int getGlyphIndex() {
            return glyphIndex;
        }

        /**
         * Returns the unicodeIndex.
         * @return the Unicode index
         */
        public int getUnicodeIndex() {
            return unicodeIndex;
        }


        /** {@inheritDoc} */
        public int hashCode() {
            int hc = unicodeIndex;
            hc = 19 * hc + ( hc ^ glyphIndex );
            return hc;
        }

        /** {@inheritDoc} */
        public boolean equals ( Object o ) {
            if ( o instanceof UnicodeMapping ) {
                UnicodeMapping m = (UnicodeMapping) o;
                if ( unicodeIndex != m.unicodeIndex ) {
                    return false;
                } else {
                    return ( glyphIndex == m.glyphIndex );
                }
            } else {
                return false;
            }
        }

        /** {@inheritDoc} */
        public int compareTo ( Object o ) {
            if ( o instanceof UnicodeMapping ) {
                UnicodeMapping m = (UnicodeMapping) o;
                if ( unicodeIndex > m.unicodeIndex ) {
                    return 1;
                } else if ( unicodeIndex < m.unicodeIndex ) {
                    return -1;
                } else {
                    return 0;
                }
            } else {
                return -1;
            }
        }
    }

    /**
     * Obtain directory table entry.
     * @param name (tag) of entry
     * @return a directory table entry or null if none found
     */
    public TTFDirTabEntry getDirectoryEntry ( String name ) {
        return (TTFDirTabEntry) dirTabs.get ( name );
    }

    /**
     * Position inputstream to position indicated
     * in the dirtab offset + offset
     * @param in font file reader
     * @param name (tag) of table
     * @param offset from start of table
     * @return true if seek succeeded
     * @throws IOException if I/O exception occurs during seek
     */
    public boolean seekTab(FontFileReader in, String name,
                  long offset) throws IOException {
        TTFDirTabEntry dt = getDirectoryEntry ( name );
        if (dt == null) {
            log.error("Dirtab " + name + " not found.");
            return false;
        } else {
            in.seekSet(dt.getOffset() + offset);
            this.currentDirTab = dt;
        }
        return true;
    }

    /**
     * Convert from truetype unit to pdf unit based on the
     * unitsPerEm field in the "head" table
     * @param n truetype unit
     * @return pdf unit
     */
    public int convertTTFUnit2PDFUnit(int n) {
        int ret;
        if (n < 0) {
            long rest1 = n % upem;
            long storrest = 1000 * rest1;
            long ledd2 = (storrest != 0 ? rest1 / storrest : 0);
            ret = -((-1000 * n) / upem - (int)ledd2);
        } else {
            ret = (n / upem) * 1000 + ((n % upem) * 1000) / upem;
        }

        return ret;
    }

    /**
     * Read the cmap table,
     * return false if the table is not present or only unsupported
     * tables are present. Currently only unicode cmaps are supported.
     * Set the unicodeIndex in the TTFMtxEntries and fills in the
     * cmaps vector.
     */
    private boolean readCMAP(FontFileReader in) throws IOException {

        unicodeMappings = new java.util.TreeSet();

        seekTab(in, "cmap", 2);
        int numCMap = in.readTTFUShort();    // Number of cmap subtables
        long cmapUniOffset = 0;
        long symbolMapOffset = 0;

        if (log.isDebugEnabled()) {
            log.debug(numCMap + " cmap tables");
        }

        //Read offset for all tables. We are only interested in the unicode table
        for (int i = 0; i < numCMap; i++) {
            int cmapPID = in.readTTFUShort();
            int cmapEID = in.readTTFUShort();
            long cmapOffset = in.readTTFULong();

            if (log.isDebugEnabled()) {
                log.debug("Platform ID: " + cmapPID + " Encoding: " + cmapEID);
            }

            if (cmapPID == 3 && cmapEID == 1) {
                cmapUniOffset = cmapOffset;
            }
            if (cmapPID == 3 && cmapEID == 0) {
                symbolMapOffset = cmapOffset;
            }
        }

        if (cmapUniOffset > 0) {
            return readUnicodeCmap(in, cmapUniOffset, 1);
        } else if (symbolMapOffset > 0) {
            return readUnicodeCmap(in, symbolMapOffset, 0);
        } else {
            log.fatal("Unsupported TrueType font: No Unicode or Symbol cmap table"
                    + " not present. Aborting");
            return false;
        }
    }

    private boolean readUnicodeCmap                             // CSOK: MethodLength
        (FontFileReader in, long cmapUniOffset, int encodingID)
            throws IOException {
        //Read CMAP table and correct mtxTab.index
        int mtxPtr = 0;

        // Read unicode cmap
        seekTab(in, "cmap", cmapUniOffset);
        int cmapFormat = in.readTTFUShort();
        /*int cmap_length =*/ in.readTTFUShort(); //skip cmap length

        if (log.isDebugEnabled()) {
            log.debug("CMAP format: " + cmapFormat);
        }

        if (cmapFormat == 4) {
            in.skip(2);    // Skip version number
            int cmapSegCountX2 = in.readTTFUShort();
            int cmapSearchRange = in.readTTFUShort();
            int cmapEntrySelector = in.readTTFUShort();
            int cmapRangeShift = in.readTTFUShort();

            if (log.isDebugEnabled()) {
                log.debug("segCountX2   : " + cmapSegCountX2);
                log.debug("searchRange  : " + cmapSearchRange);
                log.debug("entrySelector: " + cmapEntrySelector);
                log.debug("rangeShift   : " + cmapRangeShift);
            }


            int[] cmapEndCounts = new int[cmapSegCountX2 / 2];
            int[] cmapStartCounts = new int[cmapSegCountX2 / 2];
            int[] cmapDeltas = new int[cmapSegCountX2 / 2];
            int[] cmapRangeOffsets = new int[cmapSegCountX2 / 2];

            for (int i = 0; i < (cmapSegCountX2 / 2); i++) {
                cmapEndCounts[i] = in.readTTFUShort();
            }

            in.skip(2);    // Skip reservedPad

            for (int i = 0; i < (cmapSegCountX2 / 2); i++) {
                cmapStartCounts[i] = in.readTTFUShort();
            }

            for (int i = 0; i < (cmapSegCountX2 / 2); i++) {
                cmapDeltas[i] = in.readTTFShort();
            }

            //int startRangeOffset = in.getCurrentPos();

            for (int i = 0; i < (cmapSegCountX2 / 2); i++) {
                cmapRangeOffsets[i] = in.readTTFUShort();
            }

            int glyphIdArrayOffset = in.getCurrentPos();

            BitSet eightBitGlyphs = new BitSet(256);

            // Insert the unicode id for the glyphs in mtxTab
            // and fill in the cmaps ArrayList

            for (int i = 0; i < cmapStartCounts.length; i++) {

                if (log.isTraceEnabled()) {
                    log.trace(i + ": " + cmapStartCounts[i]
                                                         + " - " + cmapEndCounts[i]);
                }
                if (log.isDebugEnabled()) {
                    if (isInPrivateUseArea(cmapStartCounts[i], cmapEndCounts[i])) {
                        log.debug("Font contains glyphs in the Unicode private use area: "
                                + Integer.toHexString(cmapStartCounts[i]) + " - "
                                + Integer.toHexString(cmapEndCounts[i]));
                    }
                }

                for (int j = cmapStartCounts[i]; j <= cmapEndCounts[i]; j++) {

                    // Update lastChar
                    if (j < 256 && j > lastChar) {
                        lastChar = (short)j;
                    }

                    if (j < 256) {
                        eightBitGlyphs.set(j);
                    }

                    if (mtxPtr < mtxTab.length) {
                        int glyphIdx;
                        // the last character 65535 = .notdef
                        // may have a range offset
                        if (cmapRangeOffsets[i] != 0 && j != 65535) {
                            int glyphOffset = glyphIdArrayOffset
                                + ((cmapRangeOffsets[i] / 2)
                                    + (j - cmapStartCounts[i])
                                    + (i)
                                    - cmapSegCountX2 / 2) * 2;
                            in.seekSet(glyphOffset);
                            glyphIdx = (in.readTTFUShort() + cmapDeltas[i])
                                       & 0xffff;

                            unicodeMappings.add(new UnicodeMapping(glyphIdx, j));
                            mtxTab[glyphIdx].getUnicodeIndex().add(new Integer(j));

                            /*
                             * Removing this feature, since it violates XSL-FO and Unicode
                             * semantics. All characters used in an XSL-FO file (once decoded from
                             * the document encoded) are interpreted as Unicode code points.
                             * This precludes using Unicode code points in the Basic Latin range
                             * for any purpose other than use as Basic Latin characters. In
                             * order to use characters in the Wingding or Symbols font, then
                             * the user must use the specific Unicode code point that is mapped
                             * to the desired glyph via the font's cmap tables. What this means
                             * is that the user must use an appropriate private use character
                             * as defined by the font manufacturer and supported by some cmap
                             * provided in the font. [GA] 2012-04-20
                             * 
                            if (encodingID == 0 && j >= 0xF020 && j <= 0xF0FF) {
                                //Experimental: Mapping 0xF020-0xF0FF to 0x0020-0x00FF
                                //Tested with Wingdings and Symbol TTF fonts which map their
                                //glyphs in the region 0xF020-0xF0FF.
                                int mapped = j - 0xF000;
                                if (!eightBitGlyphs.get(mapped)) {
                                    //Only map if Unicode code point hasn't been mapped before
                                    unicodeMappings.add(new UnicodeMapping(glyphIdx, mapped));
                                    mtxTab[glyphIdx].getUnicodeIndex().add(new Integer(mapped));
                                }
                            }<commit_msg>Tweak comment.<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/* $Id$ */

package org.apache.fop.fonts.truetype;

import java.io.IOException;
import java.util.BitSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import org.apache.xmlgraphics.fonts.Glyphs;

import org.apache.fop.complexscripts.fonts.AdvancedTypographicTableFormatException;
import org.apache.fop.complexscripts.fonts.GlyphDefinitionTable;
import org.apache.fop.complexscripts.fonts.GlyphPositioningTable;
import org.apache.fop.complexscripts.fonts.GlyphSubstitutionTable;
import org.apache.fop.complexscripts.fonts.OTFAdvancedTypographicTableReader;
import org.apache.fop.fonts.FontUtil;

/**
 * Reads a TrueType file or a TrueType Collection.
 * The TrueType spec can be found at the Microsoft.
 * Typography site: http://www.microsoft.com/truetype/
 */
public class TTFFile {

    static final byte NTABS = 24;
    static final int NMACGLYPHS = 258;
    static final int MAX_CHAR_CODE = 255;
    static final int ENC_BUF_SIZE = 1024;

    private final String encoding = "WinAnsiEncoding";    // Default encoding

    private final short firstChar = 0;

    private boolean useKerning = false;

    private boolean isEmbeddable = true;
    private boolean hasSerifs = true;
    /**
     * Table directory
     */
    protected Map dirTabs;
    private Map<Integer, Map<Integer, Integer>> kerningTab;     // for CIDs
    private Map<Integer, Map<Integer, Integer>> ansiKerningTab; // For winAnsiEncoding
    private List cmaps;
    private Set unicodeMappings;

    private int upem;                                // unitsPerEm from "head" table
    private int nhmtx;                               // Number of horizontal metrics
    private int postFormat;
    private int locaFormat;
    /**
     * Offset to last loca
     */
    protected long lastLoca = 0;
    private int numberOfGlyphs; // Number of glyphs in font (read from "maxp" table)
    private int nmGlyphs;                            // Used in fixWidths - remove?

    /**
     * Contains glyph data
     */
    protected TTFMtxEntry[] mtxTab;                  // Contains glyph data
    private final int[] mtxEncoded = null;

    private String postScriptName = "";
    private String fullName = "";
    private String notice = "";
    private final Set familyNames = new java.util.HashSet(); //Set<String>
    private String subFamilyName = "";

    private long italicAngle = 0;
    private long isFixedPitch = 0;
    private int fontBBox1 = 0;
    private int fontBBox2 = 0;
    private int fontBBox3 = 0;
    private int fontBBox4 = 0;
    private int capHeight = 0;
    private int os2CapHeight = 0;
    private int underlinePosition = 0;
    private int underlineThickness = 0;
    private int xHeight = 0;
    private int os2xHeight = 0;
    //Effective ascender/descender
    private int ascender = 0;
    private int descender = 0;
    //Ascender/descender from hhea table
    private int hheaAscender = 0;
    private int hheaDescender = 0;
    //Ascender/descender from OS/2 table
    private int os2Ascender = 0;
    private int os2Descender = 0;
    private int usWeightClass = 0;

    private short lastChar = 0;

    private int[] ansiWidth;
    private Map ansiIndex;

    // internal mapping of glyph indexes to unicode indexes
    // used for quick mappings in this class
    private final Map glyphToUnicodeMap = new java.util.HashMap();
    private final Map unicodeToGlyphMap = new java.util.HashMap();

    private TTFDirTabEntry currentDirTab;

    private boolean isCFF;

    // advanced typographic table support
    private boolean useAdvanced = false;
    private OTFAdvancedTypographicTableReader advancedTableReader;

    /**
     * logging instance
     */
    protected Log log = LogFactory.getLog(TTFFile.class);

    /**
     * Constructor
     * @param useKerning true if kerning data should be loaded
     * @param useAdvanced true if advanced typographic tables should be loaded
     */
    public TTFFile ( boolean useKerning, boolean useAdvanced ) {
        this.useKerning = useKerning;
        this.useAdvanced = useAdvanced;
    }

    /**
     * Key-value helper class
     */
    class UnicodeMapping implements Comparable {

        private final int unicodeIndex;
        private final int glyphIndex;

        UnicodeMapping(int glyphIndex, int unicodeIndex) {
            this.unicodeIndex = unicodeIndex;
            this.glyphIndex = glyphIndex;
            glyphToUnicodeMap.put(new Integer(glyphIndex), new Integer(unicodeIndex));
            unicodeToGlyphMap.put(new Integer(unicodeIndex), new Integer(glyphIndex));
        }

        /**
         * Returns the glyphIndex.
         * @return the glyph index
         */
        public int getGlyphIndex() {
            return glyphIndex;
        }

        /**
         * Returns the unicodeIndex.
         * @return the Unicode index
         */
        public int getUnicodeIndex() {
            return unicodeIndex;
        }


        /** {@inheritDoc} */
        public int hashCode() {
            int hc = unicodeIndex;
            hc = 19 * hc + ( hc ^ glyphIndex );
            return hc;
        }

        /** {@inheritDoc} */
        public boolean equals ( Object o ) {
            if ( o instanceof UnicodeMapping ) {
                UnicodeMapping m = (UnicodeMapping) o;
                if ( unicodeIndex != m.unicodeIndex ) {
                    return false;
                } else {
                    return ( glyphIndex == m.glyphIndex );
                }
            } else {
                return false;
            }
        }

        /** {@inheritDoc} */
        public int compareTo ( Object o ) {
            if ( o instanceof UnicodeMapping ) {
                UnicodeMapping m = (UnicodeMapping) o;
                if ( unicodeIndex > m.unicodeIndex ) {
                    return 1;
                } else if ( unicodeIndex < m.unicodeIndex ) {
                    return -1;
                } else {
                    return 0;
                }
            } else {
                return -1;
            }
        }
    }

    /**
     * Obtain directory table entry.
     * @param name (tag) of entry
     * @return a directory table entry or null if none found
     */
    public TTFDirTabEntry getDirectoryEntry ( String name ) {
        return (TTFDirTabEntry) dirTabs.get ( name );
    }

    /**
     * Position inputstream to position indicated
     * in the dirtab offset + offset
     * @param in font file reader
     * @param name (tag) of table
     * @param offset from start of table
     * @return true if seek succeeded
     * @throws IOException if I/O exception occurs during seek
     */
    public boolean seekTab(FontFileReader in, String name,
                  long offset) throws IOException {
        TTFDirTabEntry dt = getDirectoryEntry ( name );
        if (dt == null) {
            log.error("Dirtab " + name + " not found.");
            return false;
        } else {
            in.seekSet(dt.getOffset() + offset);
            this.currentDirTab = dt;
        }
        return true;
    }

    /**
     * Convert from truetype unit to pdf unit based on the
     * unitsPerEm field in the "head" table
     * @param n truetype unit
     * @return pdf unit
     */
    public int convertTTFUnit2PDFUnit(int n) {
        int ret;
        if (n < 0) {
            long rest1 = n % upem;
            long storrest = 1000 * rest1;
            long ledd2 = (storrest != 0 ? rest1 / storrest : 0);
            ret = -((-1000 * n) / upem - (int)ledd2);
        } else {
            ret = (n / upem) * 1000 + ((n % upem) * 1000) / upem;
        }

        return ret;
    }

    /**
     * Read the cmap table,
     * return false if the table is not present or only unsupported
     * tables are present. Currently only unicode cmaps are supported.
     * Set the unicodeIndex in the TTFMtxEntries and fills in the
     * cmaps vector.
     */
    private boolean readCMAP(FontFileReader in) throws IOException {

        unicodeMappings = new java.util.TreeSet();

        seekTab(in, "cmap", 2);
        int numCMap = in.readTTFUShort();    // Number of cmap subtables
        long cmapUniOffset = 0;
        long symbolMapOffset = 0;

        if (log.isDebugEnabled()) {
            log.debug(numCMap + " cmap tables");
        }

        //Read offset for all tables. We are only interested in the unicode table
        for (int i = 0; i < numCMap; i++) {
            int cmapPID = in.readTTFUShort();
            int cmapEID = in.readTTFUShort();
            long cmapOffset = in.readTTFULong();

            if (log.isDebugEnabled()) {
                log.debug("Platform ID: " + cmapPID + " Encoding: " + cmapEID);
            }

            if (cmapPID == 3 && cmapEID == 1) {
                cmapUniOffset = cmapOffset;
            }
            if (cmapPID == 3 && cmapEID == 0) {
                symbolMapOffset = cmapOffset;
            }
        }

        if (cmapUniOffset > 0) {
            return readUnicodeCmap(in, cmapUniOffset, 1);
        } else if (symbolMapOffset > 0) {
            return readUnicodeCmap(in, symbolMapOffset, 0);
        } else {
            log.fatal("Unsupported TrueType font: No Unicode or Symbol cmap table"
                    + " not present. Aborting");
            return false;
        }
    }

    private boolean readUnicodeCmap                             // CSOK: MethodLength
        (FontFileReader in, long cmapUniOffset, int encodingID)
            throws IOException {
        //Read CMAP table and correct mtxTab.index
        int mtxPtr = 0;

        // Read unicode cmap
        seekTab(in, "cmap", cmapUniOffset);
        int cmapFormat = in.readTTFUShort();
        /*int cmap_length =*/ in.readTTFUShort(); //skip cmap length

        if (log.isDebugEnabled()) {
            log.debug("CMAP format: " + cmapFormat);
        }

        if (cmapFormat == 4) {
            in.skip(2);    // Skip version number
            int cmapSegCountX2 = in.readTTFUShort();
            int cmapSearchRange = in.readTTFUShort();
            int cmapEntrySelector = in.readTTFUShort();
            int cmapRangeShift = in.readTTFUShort();

            if (log.isDebugEnabled()) {
                log.debug("segCountX2   : " + cmapSegCountX2);
                log.debug("searchRange  : " + cmapSearchRange);
                log.debug("entrySelector: " + cmapEntrySelector);
                log.debug("rangeShift   : " + cmapRangeShift);
            }


            int[] cmapEndCounts = new int[cmapSegCountX2 / 2];
            int[] cmapStartCounts = new int[cmapSegCountX2 / 2];
            int[] cmapDeltas = new int[cmapSegCountX2 / 2];
            int[] cmapRangeOffsets = new int[cmapSegCountX2 / 2];

            for (int i = 0; i < (cmapSegCountX2 / 2); i++) {
                cmapEndCounts[i] = in.readTTFUShort();
            }

            in.skip(2);    // Skip reservedPad

            for (int i = 0; i < (cmapSegCountX2 / 2); i++) {
                cmapStartCounts[i] = in.readTTFUShort();
            }

            for (int i = 0; i < (cmapSegCountX2 / 2); i++) {
                cmapDeltas[i] = in.readTTFShort();
            }

            //int startRangeOffset = in.getCurrentPos();

            for (int i = 0; i < (cmapSegCountX2 / 2); i++) {
                cmapRangeOffsets[i] = in.readTTFUShort();
            }

            int glyphIdArrayOffset = in.getCurrentPos();

            BitSet eightBitGlyphs = new BitSet(256);

            // Insert the unicode id for the glyphs in mtxTab
            // and fill in the cmaps ArrayList

            for (int i = 0; i < cmapStartCounts.length; i++) {

                if (log.isTraceEnabled()) {
                    log.trace(i + ": " + cmapStartCounts[i]
                                                         + " - " + cmapEndCounts[i]);
                }
                if (log.isDebugEnabled()) {
                    if (isInPrivateUseArea(cmapStartCounts[i], cmapEndCounts[i])) {
                        log.debug("Font contains glyphs in the Unicode private use area: "
                                + Integer.toHexString(cmapStartCounts[i]) + " - "
                                + Integer.toHexString(cmapEndCounts[i]));
                    }
                }

                for (int j = cmapStartCounts[i]; j <= cmapEndCounts[i]; j++) {

                    // Update lastChar
                    if (j < 256 && j > lastChar) {
                        lastChar = (short)j;
                    }

                    if (j < 256) {
                        eightBitGlyphs.set(j);
                    }

                    if (mtxPtr < mtxTab.length) {
                        int glyphIdx;
                        // the last character 65535 = .notdef
                        // may have a range offset
                        if (cmapRangeOffsets[i] != 0 && j != 65535) {
                            int glyphOffset = glyphIdArrayOffset
                                + ((cmapRangeOffsets[i] / 2)
                                    + (j - cmapStartCounts[i])
                                    + (i)
                                    - cmapSegCountX2 / 2) * 2;
                            in.seekSet(glyphOffset);
                            glyphIdx = (in.readTTFUShort() + cmapDeltas[i])
                                       & 0xffff;

                            unicodeMappings.add(new UnicodeMapping(glyphIdx, j));
                            mtxTab[glyphIdx].getUnicodeIndex().add(new Integer(j));

                            /*
                             * Removing this feature, since it violates XSL-FO and Unicode
                             * semantics. All characters used in an XSL-FO file (once decoded from
                             * the document encoding) are interpreted as Unicode code points.
                             * This precludes using Unicode code points in the Basic Latin range
                             * for any purpose other than use as Basic Latin characters. In
                             * order to use characters in the Wingding or Symbols font, then
                             * the user must use the specific Unicode code point that is mapped
                             * to the desired glyph via the font's cmap tables. What this means
                             * is that the user must use an appropriate private use character
                             * as defined by the font manufacturer and supported by some cmap
                             * provided in the font. [GA] 2012-04-20
                             * 
                             * if (encodingID == 0 && j >= 0xF020 && j <= 0xF0FF) {
                             *    //Experimental: Mapping 0xF020-0xF0FF to 0x0020-0x00FF
                             *    //Tested with Wingdings and Symbol TTF fonts which map their
                             *    //glyphs in the region 0xF020-0xF0FF.
                             *    int mapped = j - 0xF000;
                             *    if (!eightBitGlyphs.get(mapped)) {
                             *        //Only map if Unicode code point hasn't been mapped before
                             *        unicodeMappings.add(new UnicodeMapping(glyphIdx, mapped));
                             *        mtxTab[glyphIdx].getUnicodeIndex().add(new Integer(mapped));
                             *    }
                             * }
{'fraction_non_alphanumeric': 0.08587494651916142, 'fraction_numerical': 0.011124014424546178, 'mean_word_length': 1.6069949012109623, 'pattern_counts': {'":': 2, '<': 39, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 45, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5039019', 'n_tokens_mistral': 9352, 'n_tokens_neox': 8569, 'n_words': 2673}