<commit_before>import module namespace companies = "http://xbrl.io/modules/bizql/profiles/sec/companies";
import module namespace filings = "http://xbrl.io/modules/bizql/profiles/sec/filings";
import module namespace archives = "http://xbrl.io/modules/bizql/archives";
import module namespace fiscal = "http://xbrl.io/modules/bizql/profiles/sec/fiscal/core";

import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace session = "http://apps.28.io/session";

let $format      := lower-case(request:param-values("format")[1])
let $ciks        := distinct-values(companies:eid(request:param-values("cik")))
let $tags        := distinct-values(request:param-values("tag") ! upper-case($$))
let $tickers     := distinct-values(request:param-values("ticker"))
let $sics        := distinct-values(request:param-values("sic"))
let $fiscalPeriods := distinct-values(let $fp := request:param-values("fiscalPeriod", "FY")
                      return
                        if (($fp ! lower-case($$)) = "all")
                        then $fiscal:ALL_FISCAL_PERIODS
                        else if (($fp ! lower-case($$)) = "fy")
                        then distinct-values(("FY", "Q4", $fp))
                        else $fp)
let $aids     := request:param-values("aid")
let $companies := companies:companies($ciks, $tags, $tickers, $sics)
let $fiscalYears := distinct-values(
                    for $y in request:param-values("fiscalYear", "LATEST")
                    return
                        if ($y eq "ALL")
                        then $fiscal:ALL_FISCAL_YEARS
                        else if ($y eq "LATEST")
                          then for $cik in $companies 
                               for $fp in $fiscalPeriods
                               return
                                   if ($fiscal:ALL_FISCAL_PERIODS eq $fp)
                                   then
                                       (fiscal:latest-reported-fiscal-period($cik).year) ! ($$ cast as integer) 
                                   else 
                                       (fiscal:latest-reported-fiscal-period($cik, $fp).year) ! ($$ cast as integer) 
                        else if ($y castable as integer)
                        then $y cast as integer
                        else ()
                )
let $archives := (archives:archives($aids),
                    for $fp in $fiscalPeriods, $fy in $fiscalYears
                    return
                      fiscal:filings-for-entities-and-fiscal-periods-and-years($companies, $fp, $fy)) 
return
    if (session:only-dow30($companies) or session:valid())
    then {
        let $summaries := filings:summaries($archives) 
        return
            switch ($format)
            case "xml"  return {
                response:serialization-parameters({"omit-xml-declaration" : false, indent : true });
                ( session:comment("xml", {
                    NumArchives: count($summaries),
                    TotalNumArchives: session:num-archives(),<commit_msg>use filings:filings instead of archive module make sure only dow30 entities are allowed<commit_after>import module namespace companies = "http://xbrl.io/modules/bizql/profiles/sec/companies";
import module namespace filings = "http://xbrl.io/modules/bizql/profiles/sec/filings";
import module namespace fiscal = "http://xbrl.io/modules/bizql/profiles/sec/fiscal/core";

import module namespace request = "http://www.28msec.com/modules/http-request";
import module namespace response = "http://www.28msec.com/modules/http-response";
import module namespace session = "http://apps.28.io/session";

let $format      := lower-case(request:param-values("format")[1])
let $ciks        := distinct-values(companies:eid(request:param-values("cik")))
let $tags        := distinct-values(request:param-values("tag") ! upper-case($$))
let $tickers     := distinct-values(request:param-values("ticker"))
let $sics        := distinct-values(request:param-values("sic"))
let $aids        := distinct-values(request:param-values("aid"))
let $companies   := companies:companies($ciks, $tags, $tickers, $sics)

let $fiscalPeriods := distinct-values(let $fp := request:param-values("fiscalPeriod", "FY")
                      return
                        if (($fp ! lower-case($$)) = "all")
                        then $fiscal:ALL_FISCAL_PERIODS
                        else if (($fp ! lower-case($$)) = "fy")
                        then distinct-values(("FY", "Q4", $fp))
                        else $fp)
let $fiscalYears := distinct-values(
                    for $y in request:param-values("fiscalYear", "LATEST")
                    return
                        if ($y eq "ALL")
                        then $fiscal:ALL_FISCAL_YEARS
                        else if ($y eq "LATEST")
                          then for $cik in $companies 
                               for $fp in $fiscalPeriods
                               return
                                   if ($fiscal:ALL_FISCAL_PERIODS eq $fp)
                                   then
                                       (fiscal:latest-reported-fiscal-period($cik).year) ! ($$ cast as integer) 
                                   else 
                                       (fiscal:latest-reported-fiscal-period($cik, $fp).year) ! ($$ cast as integer) 
                        else if ($y castable as integer)
                        then $y cast as integer
                        else ()
                )
let $archives := (filings:filings($aids),
                    for $fp in $fiscalPeriods, $fy in $fiscalYears
                    return
                      fiscal:filings-for-entities-and-fiscal-periods-and-years($companies, $fp, $fy)) 
let $companies   := companies:companies($archives.Entity)
return
    if (session:only-dow30($companies) or session:valid())
    then {
        let $summaries := filings:summaries($archives) 
        return
            switch ($format)
            case "xml"  return {
                response:serialization-parameters({"omit-xml-declaration" : false, indent : true });
                ( session:comment("xml", {
                    NumArchives: count($summaries),
                    TotalNumArchives: session:num-archives(),
{'fraction_non_alphanumeric': 0.1263899765074393, 'fraction_numerical': 0.0034455755677368833, 'mean_word_length': 1.777729447585907, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '218042', 'n_tokens_mistral': 1867, 'n_tokens_neox': 1750, 'n_words': 367}