<commit_before>using LightXML
using CSV
using DataFrames

"""
    dict_to_array(dict::Dict)

Given a dictionary, returns a tuple of arrays with the keys and values.
"""
function dict_to_array(dict::Dict)
    keys = []
    vals = []

    for (key, val) in dict
        push!(keys, key)
        push!(vals, val)
    end

    return keys, vals
end

"""
    strip_newline(val::String)

Replaces new line characters with spaces.
"""
function strip_newline(val::String)
    res = replace(val, "\n" => " ")
    res = replace(res, "\r" => " ")
    return res
end
strip_newline(::Missing) = missing

"""
    parse_MedlineDate(ml_dt::String)

Parses the contents of the MedlineDate element and returns a tuple of the year and month.
"""
function parse_MedlineDate(ml_dt::String)
    year = ""
    month = ""
    matches = split(ml_dt, " ", limit = 2)
    try
        year = matches[1]
        month = matches[2]
    catch
        @debug "Couldn't fully parse date: $ml_dt"
    end

    return year, month
end

"""
    parse_year(yr::String)

Parses the string year and returns an integer with the first year in range.
"""
function parse_year(yr::AbstractString)
    try
        Base.parse(Int64, yr[1:4])
    catch
        missing
    end
end

"""
    parse_month(mon::String)

Parses the string month (month or season) and returns an integer with the first month in range.
"""
function parse_month(mon::AbstractString)
    try
        trim_mon = lowercase(mon[1:3])

        if trim_mon == "jan" || trim_mon == "win"
            1
        elseif trim_mon == "feb"
            2
        elseif trim_mon == "mar"
            3
        elseif trim_mon == "apr" || trim_mon == "spr"
            4
        elseif trim_mon == "may"
            5
        elseif trim_mon == "jun"
            6
        elseif trim_mon == "jul" || trim_mon == "sum"
            7
        elseif trim_mon == "aug"
            8
        elseif trim_mon == "sep"
            9
        elseif trim_mon == "oct" || trim_mon == "fal"
            10
        elseif trim_mon == "nov"
            11
        elseif trim_mon == "dec"
            12
        else
            missing
        end

    catch
        missing
    end

end

"""
    parse_orcid(raw_orc::String)

Takes a string containing an ORC ID (url, 16 digit string) and returns a formatted ID (0000-1111-2222-3333).
"""
function parse_orcid(raw_orc::String)
    if occursin(r"^[0-9]{16}$", raw_orc)
        return "$(raw_orc[1:4])-$(raw_orc[5:8])-$(raw_orc[9:12])-$(raw_orc[13:16])"
    else
        reg = match(r"^.*([0-9]{4}-[0-9]{4}-[0-9]{4}-[0-9a-zA-Z]{4}).*$", raw_orc)

        return reg == nothing ? "PARSE_ERROR" : reg.captures[1]
    end
end

"""
    parse_author

Takes xml for author, and returns parsed elements
"""
function parse_author(xml::LightXML.XMLElement)

    first_name = missing :: Union{Missing,String}
    initials = missing :: Union{Missing,String}
    last_name = missing :: Union{Missing,String}
    suffix = missing :: Union{Missing,String}
    orcid = missing :: Union{Missing,String}
    collective = missing :: Union{Missing,String}
    affs = ""

    for names in child_elements(xml)
        names_name = name(names)
        if names_name == "LastName"
            last_name = content(names)
        elseif names_name == "ForeName"
            first_name = content(names)
        elseif names_name == "Initials"
            initials = content(names)
        elseif names_name == "Suffix"
            suffix = content(names)
        elseif names_name == "Identifier" && attribute(names, "Source") == "ORCID"
            orcid = parse_orcid(content(names))
        elseif names_name == "CollectiveName"
            collective = content(names)
        elseif names_name == "AffiliationInfo"
            for affil_info in child_elements(names)
                if name(affil_info) == "Affiliation"
                    affs *= content(affil_info) * "; "
                end
            end
        end
    end

    affil_str = (affs == "" ? missing : affs[1:end-2])

    return last_name, first_name, initials, suffix, orcid, collective, affil_str
end

# Note: If needed it could be further refactored to to that author, journal is a type
"""
    parse_articles(xml)

Parses a PubMedArticleSet that matches the NCBI-XML format
"""
function parse_articles(xml::LightXML.XMLElement)

    articles = get_elements_by_tagname(xml,"PubmedArticle")
    n_articles = length(articles)

    #basic
    pmid = Vector{Int64}(undef, n_articles)
    url = Vector{Union{Missing, String}}(undef, n_articles)
    title = Vector{String}(undef, n_articles)
    auth_cite = Vector{Union{Missing, String}}(undef, n_articles)
    pub_year = Vector{Union{Missing, Int64}}(undef, n_articles)
    pub_month = Vector{Union{Missing, Int64}}(undef, n_articles)
    pub_dt_desc = Vector{String}(undef, n_articles)
    journal_title = Vector{Union{Missing, String}}(undef, n_articles)
    journal_iso_abbrv = Vector{Union{Missing, String}}(undef, n_articles)
    journal_issn = Vector{Union{Missing, String}}(undef, n_articles)<commit_msg>author parsing string index fix<commit_after>using LightXML
using CSV
using DataFrames

"""
    dict_to_array(dict::Dict)

Given a dictionary, returns a tuple of arrays with the keys and values.
"""
function dict_to_array(dict::Dict)
    keys = []
    vals = []

    for (key, val) in dict
        push!(keys, key)
        push!(vals, val)
    end

    return keys, vals
end

"""
    strip_newline(val::String)

Replaces new line characters with spaces.
"""
function strip_newline(val::String)
    res = replace(val, "\n" => " ")
    res = replace(res, "\r" => " ")
    return res
end
strip_newline(::Missing) = missing

"""
    parse_MedlineDate(ml_dt::String)

Parses the contents of the MedlineDate element and returns a tuple of the year and month.
"""
function parse_MedlineDate(ml_dt::String)
    year = ""
    month = ""
    matches = split(ml_dt, " ", limit = 2)
    try
        year = matches[1]
        month = matches[2]
    catch
        @debug "Couldn't fully parse date: $ml_dt"
    end

    return year, month
end

"""
    parse_year(yr::String)

Parses the string year and returns an integer with the first year in range.
"""
function parse_year(yr::AbstractString)
    try
        Base.parse(Int64, yr[1:4])
    catch
        missing
    end
end

"""
    parse_month(mon::String)

Parses the string month (month or season) and returns an integer with the first month in range.
"""
function parse_month(mon::AbstractString)
    try
        trim_mon = lowercase(mon[1:3])

        if trim_mon == "jan" || trim_mon == "win"
            1
        elseif trim_mon == "feb"
            2
        elseif trim_mon == "mar"
            3
        elseif trim_mon == "apr" || trim_mon == "spr"
            4
        elseif trim_mon == "may"
            5
        elseif trim_mon == "jun"
            6
        elseif trim_mon == "jul" || trim_mon == "sum"
            7
        elseif trim_mon == "aug"
            8
        elseif trim_mon == "sep"
            9
        elseif trim_mon == "oct" || trim_mon == "fal"
            10
        elseif trim_mon == "nov"
            11
        elseif trim_mon == "dec"
            12
        else
            missing
        end

    catch
        missing
    end

end

"""
    parse_orcid(raw_orc::String)

Takes a string containing an ORC ID (url, 16 digit string) and returns a formatted ID (0000-1111-2222-3333).
"""
function parse_orcid(raw_orc::String)
    if occursin(r"^[0-9]{16}$", raw_orc)
        return "$(raw_orc[1:4])-$(raw_orc[5:8])-$(raw_orc[9:12])-$(raw_orc[13:16])"
    else
        reg = match(r"^.*([0-9]{4}-[0-9]{4}-[0-9]{4}-[0-9a-zA-Z]{4}).*$", raw_orc)

        return reg == nothing ? "PARSE_ERROR" : reg.captures[1]
    end
end

"""
    parse_author

Takes xml for author, and returns parsed elements
"""
function parse_author(xml::LightXML.XMLElement)

    first_name = missing :: Union{Missing,String}
    initials = missing :: Union{Missing,String}
    last_name = missing :: Union{Missing,String}
    suffix = missing :: Union{Missing,String}
    orcid = missing :: Union{Missing,String}
    collective = missing :: Union{Missing,String}
    affs = ""

    for names in child_elements(xml)
        names_name = name(names)
        if names_name == "LastName"
            last_name = content(names)
        elseif names_name == "ForeName"
            first_name = content(names)
        elseif names_name == "Initials"
            initials = content(names)
        elseif names_name == "Suffix"
            suffix = content(names)
        elseif names_name == "Identifier" && attribute(names, "Source") == "ORCID"
            orcid = parse_orcid(content(names))
        elseif names_name == "CollectiveName"
            collective = content(names)
        elseif names_name == "AffiliationInfo"
            for affil_info in child_elements(names)
                if name(affil_info) == "Affiliation"
                    affs *= content(affil_info) * "; "
                end
            end
        end
    end

    affil_str = (affs == "" ? missing : affs[1:prevind(affs,lastindex(affs),2)])

    return last_name, first_name, initials, suffix, orcid, collective, affil_str
end

# Note: If needed it could be further refactored to to that author, journal is a type
"""
    parse_articles(xml)

Parses a PubMedArticleSet that matches the NCBI-XML format
"""
function parse_articles(xml::LightXML.XMLElement)

    articles = get_elements_by_tagname(xml,"PubmedArticle")
    n_articles = length(articles)

    #basic
    pmid = Vector{Int64}(undef, n_articles)
    url = Vector{Union{Missing, String}}(undef, n_articles)
    title = Vector{String}(undef, n_articles)
    auth_cite = Vector{Union{Missing, String}}(undef, n_articles)
    pub_year = Vector{Union{Missing, Int64}}(undef, n_articles)
    pub_month = Vector{Union{Missing, Int64}}(undef, n_articles)
    pub_dt_desc = Vector{String}(undef, n_articles)
    journal_title = Vector{Union{Missing, String}}(undef, n_articles)
    journal_iso_abbrv = Vector{Union{Missing, String}}(undef, n_articles)
    journal_issn = Vector{Union{Missing, String}}(undef, n_articles)
{'fraction_non_alphanumeric': 0.12015351308797481, 'fraction_numerical': 0.015351308797480812, 'mean_word_length': 2.629642857142857, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 12}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '636429', 'n_tokens_mistral': 3325, 'n_tokens_neox': 3200, 'n_words': 829}