<commit_before>%!PS

% Barcode Writer in Pure PostScript
% https://bwipp.terryburton.co.uk
%
% Copyright (c) 2004-2019 <NAME>
%
% $Id$
%
% Permission is hereby granted, free of charge, to any
% person obtaining a copy of this software and associated
% documentation files (the "Software"), to deal in the
% Software without restriction, including without
% limitation the rights to use, copy, modify, merge,
% publish, distribute, sublicense, and/or sell copies of
% the Software, and to permit persons to whom the Software
% is furnished to do so, subject to the following
% conditions:
%
% The above copyright notice and this permission notice
% shall be included in all copies or substantial portions
% of the Software.
%
% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
% KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
% THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
% PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
% THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
% DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
% CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
% CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
% IN THE SOFTWARE.

% --BEGIN ENCODER pdf417--
% --REQUIRES preamble raiseerror renmatrix--
% --DESC: PDF417
% --EXAM: This is PDF417
% --EXOP: columns=2
% --RNDR: renmatrix
/setpacking where {pop currentpacking true setpacking} if
1 dict
dup /raiseerror dup /uk.co.terryburton.bwipp findresource put
dup /renmatrix dup /uk.co.terryburton.bwipp findresource put
begin
/pdf417 {

    20 dict begin

    /options exch def
    /barcode exch def

    /dontdraw false def
    /compact false def
    /eclevel -1 def
    /columns 0 def
    /rows 0 def
    /rowmult 3 def
    /encoding (auto) def
    /ccc false def
    /raw false def
    /parse false def

    % Parse the input options
    options type /stringtype eq {
        1 dict begin
        options {
            token false eq {exit} if dup length string cvs (=) search
            true eq {cvlit exch pop exch def} {cvlit true def} ifelse
        } loop
        currentdict end /options exch def
    } if
    options {def} forall

    /eclevel eclevel cvi def
    /columns columns cvi def
    /rows rows cvi def
    /rowmult rowmult cvr def

    % Parse ordinals of the form ^NNN to ASCII
    parse {
        /msg barcode length string def
        /j 0 def
        barcode
        { % loop
            (^) search {
                dup msg exch j exch putinterval
                length j add 1 add /j exch def
                pop
                dup 0 3 getinterval cvi msg exch j 1 sub exch put
                dup length 3 sub 3 exch getinterval
            } {
                dup msg exch j exch putinterval
                length j add /j exch def
                /barcode msg 0 j getinterval def
                exit
            } ifelse
        } loop
    } if

    raw {/encoding (raw) def} if

    % Convert input into array of codewords
    encoding (raw) eq {
        /datcws barcode length array def
        /i 0 def /j 0 def
        { % loop
            i barcode length eq {exit} if
            /cw barcode i 1 add 3 getinterval cvi def
            datcws j cw put
            /i i 4 add def
            /j j 1 add def
        } loop
        /datcws datcws 0 j getinterval def
    } if

    /barcode [ barcode {} forall ] def
    /barlen barcode length def

    ccc {/encoding (ccc) def} if

    /encb {
        /in exch def
        /inlen in length def
        /out inlen 6 idiv 5 mul inlen 6 mod add array def
        0 1 inlen 6 idiv 1 sub {
            /k exch def
            /msbs [ in k 6 mul 3 getinterval aload pop ] def
            /mscs [
                msbs aload pop exch 256 mul add exch 65536 mul add
                3 {dup 900 mod exch 900 idiv} repeat
            ] def
            /lsbs [ in k 6 mul 3 add 3 getinterval aload pop ] def
            /lscs [
                lsbs aload pop exch 256 mul add exch 65536 mul add
                3 {dup 900 mod exch 900 idiv} repeat
            ] def
            lscs 0 get mscs 0 get 316 mul add
            out k 5 mul 4 add 2 index 900 mod put
            900 idiv lscs 1 get add mscs 0 get 641 mul add mscs 1 get 316 mul add
            out k 5 mul 3 add 2 index 900 mod put
            900 idiv lscs 2 get add mscs 0 get 20 mul add mscs 1 get 641 mul add mscs 2 get 316 mul add
            out k 5 mul 2 add 2 index 900 mod put
            900 idiv lscs 3 get add mscs 1 get 20 mul add mscs 2 get 641 mul add
            out k 5 mul 1 add 2 index 900 mod put
            900 idiv mscs 2 get 20 mul add
            out k 5 mul 3 -1 roll 900 mod put
        } for
        /rem inlen 6 mod def
        rem 0 ne {
            out out length rem sub
            [ in inlen rem sub rem getinterval aload pop ]
            putinterval
        } if
        out
    } bind def

    encoding (byte) eq encoding (ccc) eq or {
        /datcws barlen 6 idiv 5 mul barlen 6 mod add 1 add array def
        datcws 0 barlen 6 mod 0 eq {924} {901} ifelse put
        datcws 1 barcode encb putinterval
        encoding (ccc) eq {/datcws [920 datcws aload pop] def} if
    } if

    encoding (auto) eq {

        % Modes and text submodes
        /T 0 def  /N 1 def  /B 2 def
        /A 0 def  /L 1 def  /M 2 def  /P 3 def

        % Special function characters for mode switching
        /tl -1 def  /nl -2 def  /bl -3 def  /bl6 -4 def  /bs -5 def

        % Special function characters for text mode
        /al -6 def  /ll -7 def  /ml -8 def  /pl -9 def  /as -10 def  /ps -11 def

        % Character maps for each state
        /charmaps [
          %  A    L    M    P
          [ (A)  (a)  (0)  (;) ]  % 0
          [ (B)  (b)  (1)  (<) ]  % 1
          [ (C)  (c)  (2)  (>) ]  % 2
          [ (D)  (d)  (3)  (@) ]  % 3
          [ (E)  (e)  (4)  ([) ]  % 4
          [ (F)  (f)  (5)   92 ]  % 5
          [ (G)  (g)  (6)  (]) ]  % 6
          [ (H)  (h)  (7)  (_) ]  % 7
          [ (I)  (i)  (8)  (`) ]  % 8
          [ (J)  (j)  (9)  (~) ]  % 9
          [ (K)  (k)  (&)  (!) ]  % 10
          [ (L)  (l)   13   13 ]  % 11
          [ (M)  (m)    9    9 ]  % 12
          [ (N)  (n)  (,)  (,) ]  % 13
          [ (O)  (o)  (:)  (:) ]  % 14
          [ (P)  (p)  (#)   10 ]  % 15
          [ (Q)  (q)  (-)  (-) ]  % 16
          [ (R)  (r)  (.)  (.) ]  % 17
          [ (S)  (s)  ($)  ($) ]  % 18
          [ (T)  (t)  (/)  (/) ]  % 19
          [ (U)  (u)  (+)  (") ]  % 20
          [ (V)  (v)  (%)  (|) ]  % 21
          [ (W)  (w)  (*)  (*) ]  % 22
          [ (X)  (x)  (=)   40 ]  % 23
          [ (Y)  (y)  (^)   41 ]  % 24
          [ (Z)  (z)   pl  (?) ]  % 25
          [ ( )  ( )  ( )  ({) ]  % 26
          [  ll   as   ll  (}) ]  % 27
          [  ml   ml   al  (') ]  % 28
          [  ps   ps   ps   al ]  % 29
        ] def

        % Invert charmaps to give character to value maps for each state
        /charvals [ 30 dict 30 dict 30 dict 30 dict ] def
        /alltext 104 dict def
        0 1 charmaps length 1 sub {
          /i exch def
          /encs charmaps i get def
          0 1 3 {
            /j exch def
            encs j get dup type /stringtype eq {0 get} if  % convert string to ASCII if required
            dup charvals j get exch i put
            alltext exch -1 put
          } for
        } for

        /e 10000 def  % "Empty"

        /latlen [  % Bit length of latch between submodes
        % To:  A  L  M  P      From
            [  0  1  1  2 ]  % A
            [  2  0  1  2 ]  % L
            [  1  1  0  1 ]  % M
            [  1  2  2  0 ]  % P
        ] def

        /latseq [  % Latch sequences between submodes
        % To: A       L       M       P               From
            [ []      [ll]    [ml]    [ml pl]    ]  % A
            [ [ml al] []      [ml]    [ml pl]    ]  % L
            [ [al]    [ll]    []      [pl]       ]  % M
            [ [al]    [al ll] [al ml] []         ]  % P
        ] def

        /shftlen [  % Bit length of shift to submode
        % To:  A  L  M  P      From
            [  e  e  e  1 ]  % A
            [  1  e  e  1 ]  % L
            [  e  e  e  1 ]  % M
            [  e  e  e  e ]  % P
        ] def

        % Determine runlengths of digits
        /numdigits [ barlen {0} repeat 0 ] def
        /numtext   [ barlen {0} repeat 0 ] def
        /numbytes  [ barlen {0} repeat 0 ] def
        barlen 1 sub -1 0 {
            /i exch def
            barcode i get dup 48 ge exch 57 le and {
                numdigits i numdigits i 1 add get 1 add put
            } if
            alltext barcode i get known numdigits i get 13 lt and {
                numtext i numtext i 1 add get 1 add put
            } if
            numtext i get 5 lt numdigits i get 13 lt and {
                numbytes i numbytes i 1 add get 1 add put
            } if
        } for
        /numdigits numdigits 0 barlen getinterval def
        /numtext   numtext   0 barlen getinterval def
        /numbytes  numbytes  0 barlen getinterval def

        /seq [] def /seqlen 0 def /state T def /p 0 def {  % loop
            p barlen eq {exit} if
            /n numdigits p get def
            n 13 ge {
                /seq [
                    seq aload pop
                    nl
                    [ barcode p n getinterval aload pop ]
                ] def
                /state N def
                /p p n add def
                /seqlen seqlen 1 add n add def
            } {  % next
            /t numtext p get def
            t 5 ge {
                /seq [
                    seq aload pop
                    state T ne {tl} if
                    [ barcode p t getinterval aload pop ]
                ] def
                /state T def
                /p p t add def
                /seqlen seqlen 1 add t add def  % ish
            } {  % next
            /b numbytes p get def
            b 1 eq state T eq and {
                /seq [
                    seq aload pop
                    bs
                    [ barcode p get ]
                ] def
                /p p b add def
                /seqlen seqlen 2 add def
            } {  % next
                /seq [
                    seq aload pop
                    b 6 mod 0 ne {bl} {bl6} ifelse
                    [ barcode p b getinterval aload pop ]
                ] def
                /state B def
                /p p b add def
                /seqlen seqlen 1 add b add def
            } ifelse } ifelse } ifelse
        } loop

        /latchcws <<
            tl 900  bl 901  bl6 924  nl 902  bs 913
        >> def

        % Submode encoding functions
        /enca {charvals A get exch get} bind def
        /encl {charvals L get exch get} bind def
        /encm {charvals M get exch get} bind def
        /encp {charvals P get exch get} bind def
        /textencfuncs [ /enca /encl /encm /encp ] def

        /addtotext {
            text exch l exch put
            /l l 1 add def
        } bind def<commit_msg>pdf417: Add ECI parsing (no encoding yet)<commit_after>%!PS

% Barcode Writer in Pure PostScript
% https://bwipp.terryburton.co.uk
%
% Copyright (c) 2004-2019 <NAME>
%
% $Id$
%
% Permission is hereby granted, free of charge, to any
% person obtaining a copy of this software and associated
% documentation files (the "Software"), to deal in the
% Software without restriction, including without
% limitation the rights to use, copy, modify, merge,
% publish, distribute, sublicense, and/or sell copies of
% the Software, and to permit persons to whom the Software
% is furnished to do so, subject to the following
% conditions:
%
% The above copyright notice and this permission notice
% shall be included in all copies or substantial portions
% of the Software.
%
% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
% KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
% THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
% PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
% THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
% DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
% CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
% CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
% IN THE SOFTWARE.

% --BEGIN ENCODER pdf417--
% --REQUIRES preamble raiseerror renmatrix--
% --DESC: PDF417
% --EXAM: This is PDF417
% --EXOP: columns=2
% --RNDR: renmatrix
/setpacking where {pop currentpacking true setpacking} if
1 dict
dup /raiseerror dup /uk.co.terryburton.bwipp findresource put
dup /renmatrix dup /uk.co.terryburton.bwipp findresource put
begin
/pdf417 {

    20 dict begin

    /options exch def
    /barcode exch def

    /dontdraw false def
    /compact false def
    /eclevel -1 def
    /columns 0 def
    /rows 0 def
    /rowmult 3 def
    /encoding (auto) def
    /ccc false def
    /raw false def
    /parse false def
    /parsefnc false def

    % Parse the input options
    options type /stringtype eq {
        1 dict begin
        options {
            token false eq {exit} if dup length string cvs (=) search
            true eq {cvlit exch pop exch def} {cvlit true def} ifelse
        } loop
        currentdict end /options exch def
    } if
    options {def} forall

    /eclevel eclevel cvi def
    /columns columns cvi def
    /rows rows cvi def
    /rowmult rowmult cvr def

    % Parse ordinals of the form ^NNN to ASCII
    parse {
        /msg barcode length string def
        /j 0 def
        barcode
        { % loop
            (^) search {
                dup msg exch j exch putinterval
                length j add 1 add /j exch def
                pop
                dup 0 3 getinterval cvi msg exch j 1 sub exch put
                dup length 3 sub 3 exch getinterval
            } {
                dup msg exch j exch putinterval
                length j add /j exch def
                /barcode msg 0 j getinterval def
                exit
            } ifelse
        } loop
    } if

    /barlen barcode length def

    raw {/encoding (raw) def} if

    % Convert input into array of codewords
    encoding (raw) eq {
        /datcws barlen array def
        /i 0 def /j 0 def
        { % loop
            i barlen eq {exit} if
            /cw barcode i 1 add 3 getinterval cvi def
            datcws j cw put
            /i i 4 add def
            /j j 1 add def
        } loop
        /datcws datcws 0 j getinterval def
    } if

    ccc {/encoding (ccc) def} if

    /encb {
        /in exch def
        /inlen in length def
        /out inlen 6 idiv 5 mul inlen 6 mod add array def
        0 1 inlen 6 idiv 1 sub {
            /k exch def
            /msbs [ in k 6 mul 3 getinterval aload pop ] def
            /mscs [
                msbs aload pop exch 256 mul add exch 65536 mul add
                3 {dup 900 mod exch 900 idiv} repeat
            ] def
            /lsbs [ in k 6 mul 3 add 3 getinterval aload pop ] def
            /lscs [
                lsbs aload pop exch 256 mul add exch 65536 mul add
                3 {dup 900 mod exch 900 idiv} repeat
            ] def
            lscs 0 get mscs 0 get 316 mul add
            out k 5 mul 4 add 2 index 900 mod put
            900 idiv lscs 1 get add mscs 0 get 641 mul add mscs 1 get 316 mul add
            out k 5 mul 3 add 2 index 900 mod put
            900 idiv lscs 2 get add mscs 0 get 20 mul add mscs 1 get 641 mul add mscs 2 get 316 mul add
            out k 5 mul 2 add 2 index 900 mod put
            900 idiv lscs 3 get add mscs 1 get 20 mul add mscs 2 get 641 mul add
            out k 5 mul 1 add 2 index 900 mod put
            900 idiv mscs 2 get 20 mul add
            out k 5 mul 3 -1 roll 900 mod put
        } for
        /rem inlen 6 mod def
        rem 0 ne {
            out out length rem sub
            [ in inlen rem sub rem getinterval aload pop ]
            putinterval
        } if
        out
    } bind def

    encoding (byte) eq encoding (ccc) eq or {
        /datcws barlen 6 idiv 5 mul barlen 6 mod add 1 add array def
        datcws 0 barlen 6 mod 0 eq {924} {901} ifelse put
        datcws 1 barcode encb putinterval
        encoding (ccc) eq {/datcws [920 datcws aload pop] def} if
    } if

    encoding (auto) eq {

        % Convert input into bytes accounting for FNC characters
        /fncvals <<
            % Predates FNC non-data codewords, but supports ECI
        >> def
        /msg barlen array def
        /i 0 def /j 0 def {
            i barlen eq {exit} if
            /char barcode i get def
            /i i 1 add def
            parsefnc char 94 eq and i barlen lt and {
                barcode i get 94 ne {  % "^FNCx" -> fncvals{FNCx}
                    i barlen 3 sub ge {
                        /bwipp.truncatedFNC (Function character truncated) //raiseerror exec
                    } if
                    barcode i 4 getinterval
                    dup 0 3 getinterval (ECI) ne {
                        dup fncvals exch known not {
                            dup length 28 add string dup 28 4 -1 roll putinterval
                            dup 0 (Unknown function character: ) putinterval
                            /bwipp.unknownFNC exch //raiseerror exec
                        } if
                        fncvals exch get
                        /i i 4 add def
                    } {  % "^ECInnnnnn" -> -nnnnnn
                        pop
                        i barlen 8 sub ge {
                            /bwipp.truncatedECI (ECI truncated) //raiseerror exec
                        } if
                        barcode i 3 add 6 getinterval
                        dup {
                            dup 48 lt exch 57 gt or {
                                /bwipp.invalidECI (ECI must be 000000 to 999999) //raiseerror exec
                            } if
                        } forall
                        0 exch {48 sub sub 10 mul} forall 10 idiv 1000000 sub
                        /i i 9 add def
                    } ifelse
                    /char exch def
                } {  % "^^" -> "^"
                    /i i 1 add def
                } ifelse
            } if
            msg j char put
            /j j 1 add def
        } loop
        /msg msg 0 j getinterval def
        /msglen msg length def

        % Modes and text submodes
        /T 0 def  /N 1 def  /B 2 def
        /A 0 def  /L 1 def  /M 2 def  /P 3 def

        % Special function characters for mode switching
        /tl -1 def  /nl -2 def  /bl -3 def  /bl6 -4 def  /bs -5 def

        % Special function characters for text mode
        /al -6 def  /ll -7 def  /ml -8 def  /pl -9 def  /as -10 def  /ps -11 def

        % Character maps for each state
        /charmaps [
          %  A    L    M    P
          [ (A)  (a)  (0)  (;) ]  % 0
          [ (B)  (b)  (1)  (<) ]  % 1
          [ (C)  (c)  (2)  (>) ]  % 2
          [ (D)  (d)  (3)  (@) ]  % 3
          [ (E)  (e)  (4)  ([) ]  % 4
          [ (F)  (f)  (5)   92 ]  % 5
          [ (G)  (g)  (6)  (]) ]  % 6
          [ (H)  (h)  (7)  (_) ]  % 7
          [ (I)  (i)  (8)  (`) ]  % 8
          [ (J)  (j)  (9)  (~) ]  % 9
          [ (K)  (k)  (&)  (!) ]  % 10
          [ (L)  (l)   13   13 ]  % 11
          [ (M)  (m)    9    9 ]  % 12
          [ (N)  (n)  (,)  (,) ]  % 13
          [ (O)  (o)  (:)  (:) ]  % 14
          [ (P)  (p)  (#)   10 ]  % 15
          [ (Q)  (q)  (-)  (-) ]  % 16
          [ (R)  (r)  (.)  (.) ]  % 17
          [ (S)  (s)  ($)  ($) ]  % 18
          [ (T)  (t)  (/)  (/) ]  % 19
          [ (U)  (u)  (+)  (") ]  % 20
          [ (V)  (v)  (%)  (|) ]  % 21
          [ (W)  (w)  (*)  (*) ]  % 22
          [ (X)  (x)  (=)   40 ]  % 23
          [ (Y)  (y)  (^)   41 ]  % 24
          [ (Z)  (z)   pl  (?) ]  % 25
          [ ( )  ( )  ( )  ({) ]  % 26
          [  ll   as   ll  (}) ]  % 27
          [  ml   ml   al  (') ]  % 28
          [  ps   ps   ps   al ]  % 29
        ] def

        % Invert charmaps to give character to value maps for each state
        /charvals [ 30 dict 30 dict 30 dict 30 dict ] def
        /alltext 104 dict def
        0 1 charmaps length 1 sub {
          /i exch def
          /encs charmaps i get def
          0 1 3 {
            /j exch def
            encs j get dup type /stringtype eq {0 get} if  % convert string to ASCII if required
            dup charvals j get exch i put
            alltext exch -1 put
          } for
        } for

        /e 10000 def  % "Empty"

        /latlen [  % Bit length of latch between submodes
        % To:  A  L  M  P      From
            [  0  1  1  2 ]  % A
            [  2  0  1  2 ]  % L
            [  1  1  0  1 ]  % M
            [  1  2  2  0 ]  % P
        ] def

        /latseq [  % Latch sequences between submodes
        % To: A       L       M       P               From
            [ []      [ll]    [ml]    [ml pl]    ]  % A
            [ [ml al] []      [ml]    [ml pl]    ]  % L
            [ [al]    [ll]    []      [pl]       ]  % M
            [ [al]    [al ll] [al ml] []         ]  % P
        ] def

        /shftlen [  % Bit length of shift to submode
        % To:  A  L  M  P      From
            [  e  e  e  1 ]  % A
            [  1  e  e  1 ]  % L
            [  e  e  e  1 ]  % M
            [  e  e  e  e ]  % P
        ] def

        % Determine runlengths of digits
        /numdigits [ msglen {0} repeat 0 ] def
        /numtext   [ msglen {0} repeat 0 ] def
        /numbytes  [ msglen {0} repeat 0 ] def
        msglen 1 sub -1 0 {
            /i exch def
            msg i get dup 48 ge exch 57 le and {
                numdigits i numdigits i 1 add get 1 add put
            } if
            alltext msg i get known numdigits i get 13 lt and {
                numtext i numtext i 1 add get 1 add put
            } if
            numtext i get 5 lt numdigits i get 13 lt and {
                numbytes i numbytes i 1 add get 1 add put
            } if
        } for
        /numdigits numdigits 0 msglen getinterval def
        /numtext   numtext   0 msglen getinterval def
        /numbytes  numbytes  0 msglen getinterval def

        /seq [] def /seqlen 0 def /state T def /p 0 def {  % loop
            p msglen eq {exit} if
            /n numdigits p get def
            n 13 ge {
                /seq [
                    seq aload pop
                    nl
                    [ msg p n getinterval aload pop ]
                ] def
                /state N def
                /p p n add def
                /seqlen seqlen 1 add n add def
            } {  % next
            /t numtext p get def
            t 5 ge {
                /seq [
                    seq aload pop
                    state T ne {tl} if
                    [ msg p t getinterval aload pop ]
                ] def
                /state T def
                /p p t add def
                /seqlen seqlen 1 add t add def  % ish
            } {  % next
            /b numbytes p get def
            b 1 eq state T eq and {
                /seq [
                    seq aload pop
                    bs
                    [ msg p get ]
                ] def
                /p p b add def
                /seqlen seqlen 2 add def
            } {  % next
                /seq [
                    seq aload pop
                    b 6 mod 0 ne {bl} {bl6} ifelse
                    [ msg p b getinterval aload pop ]
                ] def
                /state B def
                /p p b add def
                /seqlen seqlen 1 add b add def
            } ifelse } ifelse } ifelse
        } loop

        /latchcws <<
            tl 900  bl 901  bl6 924  nl 902  bs 913
        >> def

        % Submode encoding functions
        /enca {charvals A get exch get} bind def
        /encl {charvals L get exch get} bind def
        /encm {charvals M get exch get} bind def
        /encp {charvals P get exch get} bind def
        /textencfuncs [ /enca /encl /encm /encp ] def

        /addtotext {
            text exch l exch put
            /l l 1 add def
        } bind def
{'fraction_non_alphanumeric': 0.0758137790188805, 'fraction_numerical': 0.03413495602884174, 'mean_word_length': 1.134507606084868, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 16, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6459224', 'n_tokens_mistral': 8299, 'n_tokens_neox': 7286, 'n_words': 3168}