<commit_before>module Flows

import 
    Base: (*), +, -, string, show

export TimeExpression, TimeVariable, TimeLinearCombination
export coefficient, substitute, simplify


###################################################################################################

abstract TimeExpression

type TimeVariable <: TimeExpression
   name::AbstractString
   latex::AbstractString
end 

function TimeVariable(name::AbstractString; 
                      latex::AbstractString=name)
    TimeVariable(name, latex)
end   

string(t::TimeVariable) = t.name
show(io::IO, t::TimeVariable) = print(io, string(t))

type TimeLinearCombination <: TimeExpression
   terms :: Array{Tuple{TimeExpression, Real},1}
end

TimeLinearCombination(x...) = simplify(TimeLinearCombination([ (x[i],x[i+1]) for i=1:2:length(x) ]))

+(a::TimeExpression, b::TimeExpression) = TimeLinearCombination(a,1, b, 1)
-(a::TimeExpression, b::TimeExpression) = TimeLinearCombination(a,1, b,-1)
-(a::TimeExpression) = (-1)*a
*(f::Real, ex::TimeVariable) = TimeLinearCombination(ex,f)
*(f::Real, ex::TimeLinearCombination) = TimeLinearCombination( [ (x, f*c) for (x, c) in ex.terms ] )
*(ex::TimeExpression, f::Real) = f*ex

function string(ex::TimeLinearCombination) 
    if length(ex.terms) == 0 
        return "0"  #empty linear combination
    else    
        s = join([join([c>=0?"+":"-", abs(c)==1?"":abs(c),
            typeof(x)!=TimeVariable?"(":"", 
            string(x), 
            typeof(x)!=TimeVariable?")":"", 
        ]) for (x, c) in ex.terms])
        return s[1]=='+' ? s[2:end] : s
    end    
end   

show(io::IO, t::TimeLinearCombination) = print(io, string(t))

function _expand(ex::TimeLinearCombination)
    TimeLinearCombination(
        vcat([ typeof(x)==TimeVariable ? (x, c) : (c*_expand(x)).terms for (x, c) in ex.terms ]...))
end

function _collect(ex::TimeLinearCombination)
    d = Dict(ex.terms)
    for key in keys(d)
        d[key] = 0
    end    
    for (key, val) in ex.terms
        d[key] += val
    end
    for (key, val) in d
        if val == 0
            delete!(d, key)
        end
    end    
    TimeLinearCombination([(key,val) for (key, val) in d])
end

simplify(v::TimeVariable) = v
simplify(ex::TimeLinearCombination) = _collect(_expand(ex))

coefficient(ex::TimeVariable, v::TimeVariable) = (ex==v ? 1 : 0)

function coefficient(ex::TimeLinearCombination, v::TimeVariable)
    c = 0
    for (ex1, c1) in ex.terms
       c += c1*coefficient(ex1, v)
    end
    c
end 

substitute(ex::TimeVariable, this::TimeVariable, by::TimeExpression) = (ex==this ? by : ex)

function substitute(ex::TimeLinearCombination, this::TimeVariable, by::TimeExpression)
    TimeLinearCombination([(substitute(x, this, by), c) for (x, c) in ex.terms])
end


###################################################################################################

type AutonomousFunction
   name::AbstractString
   latex::AbstractString
end

function AutonomousFunction(name::AbstractString; 
                            latex::AbstractString=name)
    AutonomousFunction(name, latex)
end   

string(t::TimeVariable) = t.name
show(io::IO, t::TimeVariable) = print(io, string(t))

###################################################################################################
# Essentially the same stuff again with Time substituted by Space...

abstract SpaceExpression

type SpaceVariable <: SpaceExpression
   name::AbstractString
   latex::AbstractString
end 

function SpaceVariable(name::AbstractString; 
                      latex::AbstractString=name)
    SpaceVariable(name, latex)
end   

string(t::SpaceVariable) = t.name
show(io::IO, t::SpaceVariable) = print(io, string(t))


type SpaceLinearCombination <: SpaceExpression
   terms :: Array{Tuple{SpaceExpression, Real},1}
end

SpaceLinearCombination(x...) = simplify(SpaceLinearCombination([ (x[i],x[i+1]) for i=1:2:length(x) ]))

+(a::SpaceExpression, b::SpaceExpression) = SpaceLinearCombination(a,1, b, 1)
-(a::SpaceExpression, b::SpaceExpression) = SpaceLinearCombination(a,1, b,-1)
-(a::SpaceExpression) = (-1)*a
*(f::Real, ex::SpaceVariable) = SpaceLinearCombination(ex,f)
*(f::Real, ex::SpaceLinearCombination) = SpaceLinearCombination( [ (x, f*c) for (x, c) in ex.terms ] )
*(ex::SpaceExpression, f::Real) = f*ex

function string(ex::SpaceLinearCombination) 
    if length(ex.terms) == 0 
        return "0"  #empty linear combination
    else    
        s = join([join([c>=0?"+":"-", abs(c)==1?"":abs(c),
            typeof(x)!=SpaceVariable?"(":"", #TODO: which other types don't need parantheses?
            string(x), 
            typeof(x)!=SpaceVariable?")":"", 
        ]) for (x, c) in ex.terms])
        return s[1]=='+' ? s[2:end] : s
    end    
end   

show(io::IO, t::SpaceLinearCombination) = print(io, string(t))


substitute(ex::SpaceVariable, this::SpaceVariable, by::SpaceExpression) = (ex==this ? by : ex)
substitute(ex::SpaceVariable, this::TimeVariable, by::TimeExpression) = ex 

function substitute(ex::SpaceLinearCombination, this::SpaceVariable, by::SpaceExpression)
    TimeLinearCombination([(substitute(x, this, by), c) for (x, c) in ex.terms])
end

function substitute(ex::SpaceLinearCombination, this::TimeVariable, by::TimeExpression)
    TimeLinearCombination([(substitute(x, this, by), c) for (x, c) in ex.terms])
end



###################################################################################################




end #module Flows<commit_msg>register handling<commit_after>module Flows

import 
    Base: (*), +, -, string, show

export TimeExpression, TimeVariable, TimeLinearCombination
export SpaceExpression, SpaceVariable, SpaceLinearCombination
export coefficient, substitute, simplify
export print_time_expression_register, print_space_expression_register

export _str, _expand, _collect
export _register, _str_flat_arg_name, _get_register_key, _register
export _time_expression_index, _time_expression_register
export _space_expression_index, _space_expression_register



###################################################################################################

abstract TimeExpression

type TimeVariable <: TimeExpression
   name::AbstractString
   latex::AbstractString
end 

function TimeVariable(name::AbstractString; 
                      latex::AbstractString=name)
    TimeVariable(name, latex)
end   

string(t::TimeVariable) = t.name
show(io::IO, t::TimeVariable) = print(io, string(t))

type TimeLinearCombination <: TimeExpression
   terms :: Array{Tuple{TimeExpression, Real},1}
end

TimeLinearCombination(x...) = simplify(TimeLinearCombination([ (x[i],x[i+1]) for i=1:2:length(x) ]))

+(a::TimeExpression, b::TimeExpression) = TimeLinearCombination(a,1, b, 1)
-(a::TimeExpression, b::TimeExpression) = TimeLinearCombination(a,1, b,-1)
-(a::TimeExpression) = (-1)*a
*(f::Real, ex::TimeVariable) = TimeLinearCombination(ex,f)
*(f::Real, ex::TimeLinearCombination) = TimeLinearCombination( [ (x, f*c) for (x, c) in ex.terms ] )
*(ex::TimeExpression, f::Real) = f*ex

function _str(ex::TimeLinearCombination; flat::Bool=true) 
    if length(ex.terms) == 0 
        return "0"  #empty linear combination
    else    
        s = join([join([c>=0?"+":"-", abs(c)==1?"":abs(c),
            typeof(x)!=TimeVariable?"(":"", 
            flat?_str_flat_arg_name(x):string(x), 
            typeof(x)!=TimeVariable?")":"", 
        ]) for (x, c) in ex.terms])
        return s[1]=='+' ? s[2:end] : s
    end    
end   

string(ex::TimeLinearCombination) = _str(ex, flat=false)
show(io::IO, t::TimeLinearCombination) = print(io, string(t))

function _expand(ex::TimeLinearCombination)
    TimeLinearCombination(
        vcat([ typeof(x)==TimeVariable ? (x, c) : (c*_expand(x)).terms for (x, c) in ex.terms ]...))
end

function _collect(ex::TimeLinearCombination)
    d = Dict(ex.terms)
    for key in keys(d)
        d[key] = 0
    end    
    for (key, val) in ex.terms
        d[key] += val
    end
    for (key, val) in d
        if val == 0
            delete!(d, key)
        end
    end    
    if length(d) == 1 
        #Each linear combination consisting of only one term with coefficient 1 
        #is replaced by this term.
        #Note that this term must have been already registered.
        for (key, val) in d
            if val==1
                return key
            end    
        end
    end    
    return _register(TimeLinearCombination([(key,val) for (key, val) in d]))
end

simplify(v::TimeVariable) = v
simplify(ex::TimeLinearCombination) = _collect(_expand(ex))

coefficient(ex::TimeVariable, v::TimeVariable) = (ex==v ? 1 : 0)

function coefficient(ex::TimeLinearCombination, v::TimeVariable)
    c = 0
    for (ex1, c1) in ex.terms
       c += c1*coefficient(ex1, v)
    end
    c
end 

substitute(ex::TimeVariable, this::TimeVariable, by::TimeExpression) = (ex==this ? by : ex)

function substitute(ex::TimeLinearCombination, this::TimeVariable, by::TimeExpression)
    TimeLinearCombination([(substitute(x, this, by), c) for (x, c) in ex.terms])
end


global _time_expression_index = Dict{TimeExpression,Int}()
global _time_expression_register = Dict{ASCIIString,Tuple{TimeExpression,Int}}()

function _get_register_key(ex::TimeLinearCombination)
    join(['L', join([join([hex(Int(pointer_from_objref(x))), ':', c]) 
        for (x,c) in sort(ex.terms, 
            lt = (a,b) -> pointer_from_objref(a[1])<pointer_from_objref(b[1])) ],'|')])
end    

function _register(ex::TimeExpression)
    key = _get_register_key(ex)
    (ex,i) = get!(_time_expression_register, key) do
        i = length(_time_expression_index)
        _time_expression_index[ex] = i 
        (ex, i)
    end
    ex
end

_str_flat_arg_name(ex::TimeExpression) = join(["#", _time_expression_index[ex]])
_str_flat_arg_name(ex::TimeVariable) = ex.name

function print_time_expression_register()
    for (ex, i) in sort(collect(values(_time_expression_register)), lt = (a,b)-> a[2]<b[2])
        println("#", i, "\t" , _str(ex, flat=true))
     end
end

###################################################################################################

type AutonomousFunction
   name::AbstractString
   latex::AbstractString
end

function AutonomousFunction(name::AbstractString; 
                            latex::AbstractString=name)
    AutonomousFunction(name, latex)
end   

string(t::TimeVariable) = t.name
show(io::IO, t::TimeVariable) = print(io, string(t))

###################################################################################################
# Essentially the same stuff again with Time substituted by Space...

abstract SpaceExpression

type SpaceVariable <: SpaceExpression
   name::AbstractString
   latex::AbstractString
end 

function SpaceVariable(name::AbstractString; 
                      latex::AbstractString=name)
    SpaceVariable(name, latex)
end   

string(t::SpaceVariable) = t.name
show(io::IO, t::SpaceVariable) = print(io, string(t))


type SpaceLinearCombination <: SpaceExpression
   terms :: Array{Tuple{SpaceExpression, Real},1}
end

SpaceLinearCombination(x...) = simplify(SpaceLinearCombination([ (x[i],x[i+1]) for i=1:2:length(x) ]))

+(a::SpaceExpression, b::SpaceExpression) = SpaceLinearCombination(a,1, b, 1)
-(a::SpaceExpression, b::SpaceExpression) = SpaceLinearCombination(a,1, b,-1)
-(a::SpaceExpression) = (-1)*a
*(f::Real, ex::SpaceVariable) = SpaceLinearCombination(ex,f)
*(f::Real, ex::SpaceLinearCombination) = SpaceLinearCombination( [ (x, f*c) for (x, c) in ex.terms ] )
*(ex::SpaceExpression, f::Real) = f*ex

function _str(ex::SpaceLinearCombination; flat::Bool=true) 
    if length(ex.terms) == 0 
        return "0"  #empty linear combination
    else    
        s = join([join([c>=0?"+":"-", abs(c)==1?"":abs(c),
            typeof(x)!=SpaceVariable?"(":"", #TODO: which other types don't need parantheses?
            flat?_str_flat_arg_name(x):string(x), 
            typeof(x)!=SpaceVariable?")":"", 
        ]) for (x, c) in ex.terms])
        return s[1]=='+' ? s[2:end] : s
    end    
end   

string(ex::SpaceLinearCombination) = _str(ex, flat=false)
show(io::IO, t::SpaceLinearCombination) = print(io, string(t))


substitute(ex::SpaceVariable, this::SpaceVariable, by::SpaceExpression) = (ex==this ? by : ex)
substitute(ex::SpaceVariable, this::TimeVariable, by::TimeExpression) = ex 

function substitute(ex::SpaceLinearCombination, this::SpaceVariable, by::SpaceExpression)
    SpaceLinearCombination([(substitute(x, this, by), c) for (x, c) in ex.terms])
end

function substitute(ex::SpaceLinearCombination, this::TimeVariable, by::TimeExpression)
    SpaceLinearCombination([(substitute(x, this, by), c) for (x, c) in ex.terms])
end


global _space_expression_index = Dict{SpaceExpression,Int}()
global _space_expression_register = Dict{ASCIIString,Tuple{SpaceExpression,Int}}()

function _get_register_key(ex::SpaceLinearCombination)
    join(['L', join([join([hex(Int(pointer_from_objref(x))), ':', c]) 
        for (x,c) in sort(ex.terms, 
            lt = (a,b) -> pointer_from_objref(a[1])<pointer_from_objref(b[1])) ],'|')])
end    

function _register(ex::SpaceExpression)
    key = _get_register_key(ex)
    (ex,i) = get!(_space_expression_register, key) do
        i = length(_space_expression_index)
        _space_expression_index[ex] = i 
        (ex, i)
    end
    ex
end

_str_flat_arg_name(ex::SpaceExpression) = join(["#", _space_expression_index[ex]])
_str_flat_arg_name(ex::SpaceVariable) = ex.name

function print_space_expression_register()
    for (ex, i) in sort(collect(values(_space_expression_register)), lt = (a,b)-> a[2]<b[2])
        println("#", i, "\t" , _str(ex, flat=true))
     end
end

###################################################################################################




end #module Flows
{'fraction_non_alphanumeric': 0.20059510849843964, 'fraction_numerical': 0.006459104434284055, 'mean_word_length': 4.554212011285772, 'pattern_counts': {'":': 16, '<': 15, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7338478', 'n_tokens_mistral': 4488, 'n_tokens_neox': 4290, 'n_words': 923}