<commit_before>#include "private.h"

#include <Elementary.h>
#include "termpty.h"
#include "termptydbl.h"

Eina_Bool
_termpty_is_dblwidth_get(Termpty *ty, int g)
{
#if defined(SUPPORT_DBLWIDTH)
   // check for east asian full-width (F), half-width (H), wide (W),
   // narrow (Na) or ambiguous (A) codepoints
   // ftp://ftp.unicode.org/Public/UNIDATA/EastAsianWidth.txt
   
   // optimize for latin1 non-ambiguous
   if (g <= 0xa0)
     return EINA_FALSE;
   // (F)
   if ((g == 0x3000) ||
       ((g >= 0xff01) && (g <= 0xffe6)))
     return EINA_TRUE;
   // (W)
   // 
   if (
       // 1XXX
       (((g >> 12) == 0x1) &&
           (((g >= 0x1100) && (g <= 0x11ff)))) ||
       // 2XXX
       (((g >> 12) == 0x2) &&
           (((g >= 0x2329) && (g <= 0x232A)) ||
            ((g >= 0x2E80) && (g <= 0x4dbf)))) ||
       // 4XXX
       (((g >> 12) == 0x4) &&
           (((g >= 0x4e00) && (g <= 0x9fff)))) ||
       // aXXX
       (((g >> 12) == 0xa) &&
           (((g >= 0xa000) && (g <= 0xa4c6)) ||
            ((g >= 0xa960) && (g <= 0xa97c)) ||
            ((g >= 0xac00) && (g <= 0xd7a3)))) ||
       // dXXX
       (((g >> 12) == 0xd) &&
           (((g >= 0xd7b0) && (g <= 0xd7fb)))) ||
       // fXXX
       (((g >> 12) == 0xf) &&
           (((g >= 0xf900) && (g <= 0xfaff)) ||
            ((g >= 0xfe10) && (g <= 0xfe6b)))) ||
       // 1XXXX
       (((g >> 16) == 0x1) &&
           (((g >= 0x1b000) && (g <= 0x1b001)) ||
            ((g >= 0x1f200) && (g <= 0x1f202)) ||
            ((g >= 0x1f210) && (g <= 0x1f251)))) ||
       // 2XXXX
       (((g >> 16) == 0x2) &&
           (((g >= 0x20000) && (g <= 0x2fffd)))) ||
       // 3XXXX
       (((g >> 12) == 0x3) &&
           (((g >= 0x30000) && (g <= 0x3FFFD)))))
     return EINA_TRUE;
   // FIXME: can optimize by breaking into tree and ranges
   // (A)
   if (ty->state.cjk_ambiguous_wide)
     {
        // grep ';A #' EastAsianWidth.txt | wc -l
        // :(
        if (
            // aX
            (((g >> 4) == 0xa) &&
                ((g == 0x00a1) ||
                 (g == 0x00a4) ||<commit_msg>oops fix an oopsie id id that broke korean double with detection!<commit_after>#include "private.h"

#include <Elementary.h>
#include "termpty.h"
#include "termptydbl.h"

Eina_Bool
_termpty_is_dblwidth_get(Termpty *ty, int g)
{
#if defined(SUPPORT_DBLWIDTH)
   // check for east asian full-width (F), half-width (H), wide (W),
   // narrow (Na) or ambiguous (A) codepoints
   // ftp://ftp.unicode.org/Public/UNIDATA/EastAsianWidth.txt
   
   // optimize for latin1 non-ambiguous
   if (g <= 0xa0)
     return EINA_FALSE;
   // (F)
   if ((g == 0x3000) ||
       ((g >= 0xff01) && (g <= 0xffe6)))
     return EINA_TRUE;
   // (W)
   // 
   if (
       // 1XXX
       (((g >> 12) == 0x1) &&
           (((g >= 0x1100) && (g <= 0x11ff)))) ||
       // 2XXX
       (((g >> 12) == 0x2) &&
           (((g >= 0x2329) && (g <= 0x232A)) ||
            ((g >= 0x2E80) && (g <= 0x4dbf)))) ||
       // 4XXX
       (((g >> 12) == 0x4) &&
           (((g >= 0x4e00) && (g <= 0x9fff)))) ||
       // aXXX
       ((((g >> 12) >= 0xa) && ((g >> 12) <= 0xd)) &&
           (((g >= 0xa000) && (g <= 0xa4c6)) ||
            ((g >= 0xa960) && (g <= 0xa97c)) ||
            ((g >= 0xac00) && (g <= 0xd7a3)) ||
            ((g >= 0xd7b0) && (g <= 0xd7fb)))) ||
       // fXXX
       (((g >> 12) == 0xf) &&
           (((g >= 0xf900) && (g <= 0xfaff)) ||
            ((g >= 0xfe10) && (g <= 0xfe6b)))) ||
       // 1XXXX
       (((g >> 16) == 0x1) &&
           (((g >= 0x1b000) && (g <= 0x1b001)) ||
            ((g >= 0x1f200) && (g <= 0x1f202)) ||
            ((g >= 0x1f210) && (g <= 0x1f251)))) ||
       // 2XXXX
       (((g >> 16) == 0x2) &&
           (((g >= 0x20000) && (g <= 0x2fffd)))) ||
       // 3XXXX
       (((g >> 12) == 0x3) &&
           (((g >= 0x30000) && (g <= 0x3FFFD)))))
     return EINA_TRUE;
   // FIXME: can optimize by breaking into tree and ranges
   // (A)
   if (ty->state.cjk_ambiguous_wide)
     {
        // grep ';A #' EastAsianWidth.txt | wc -l
        // :(
        if (
            // aX
            (((g >> 4) == 0xa) &&
                ((g == 0x00a1) ||
                 (g == 0x00a4) ||
{'fraction_non_alphanumeric': 0.24014336917562723, 'fraction_numerical': 0.08219832735961768, 'mean_word_length': 1.576, 'pattern_counts': {'":': 0, '<': 40, '<?xml version=': 0, '>': 80, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6303692', 'n_tokens_mistral': 1911, 'n_tokens_neox': 1656, 'n_words': 364}