<commit_before><#--
  This is a freemarker template which will generate an HQL script which is run at download time.
  When run in Hive as a parameterized query, this will create a set of tables ...
-->
<#-- Required syntax to escape Hive parameters. Outputs "USE ${hiveDB};" -->
USE ${r"${hiveDB}"};

-- don't run joins locally, else risk running out of memory
SET hive.auto.convert.join=false;

-- setup for our custom, combinable deflated compression
-- See https://github.com/gbif/occurrence/issues/28#issuecomment-432958372
SET hive.exec.compress.output=true;
SET io.seqfile.compression.type=BLOCK;
SET mapred.output.compression.codec=org.gbif.hadoop.compress.d2.D2Codec;
SET io.compression.codecs=org.gbif.hadoop.compress.d2.D2Codec;
SET hive.merge.mapfiles=false;
SET hive.merge.mapredfiles=false;


-- Extension Tables
<#list extensions as extension>

-- ${extension.extension} extension
CREATE TABLE IF NOT EXISTS ${downloadTableName}_ext_${extension.hiveTableName}
ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' TBLPROPERTIES ("serialization.null.format"="");
AS SELECT ext.${extension.interpretedFields?join(", ext.")}
FROM ${tableName}_ext_${extension.hiveTableName} ext
JOIN ${interpretedTable} ON ${interpretedTable}.gbifid = ext.gbifid;
</#list>
<commit_msg>removing ;<commit_after><#--
  This is a freemarker template which will generate an HQL script which is run at download time.
  When run in Hive as a parameterized query, this will create a set of tables ...
-->
<#-- Required syntax to escape Hive parameters. Outputs "USE ${hiveDB};" -->
USE ${r"${hiveDB}"};

-- don't run joins locally, else risk running out of memory
SET hive.auto.convert.join=false;

-- setup for our custom, combinable deflated compression
-- See https://github.com/gbif/occurrence/issues/28#issuecomment-432958372
SET hive.exec.compress.output=true;
SET io.seqfile.compression.type=BLOCK;
SET mapred.output.compression.codec=org.gbif.hadoop.compress.d2.D2Codec;
SET io.compression.codecs=org.gbif.hadoop.compress.d2.D2Codec;
SET hive.merge.mapfiles=false;
SET hive.merge.mapredfiles=false;


-- Extension Tables
<#list extensions as extension>

-- ${extension.extension} extension
CREATE TABLE IF NOT EXISTS ${downloadTableName}_ext_${extension.hiveTableName}
ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' TBLPROPERTIES ("serialization.null.format"="")
AS SELECT ext.${extension.interpretedFields?join(", ext.")}
FROM ${tableName}_ext_${extension.hiveTableName} ext
JOIN ${interpretedTable} ON ${interpretedTable}.gbifid = ext.gbifid;
</#list>

{'fraction_non_alphanumeric': 0.12824547600314712, 'fraction_numerical': 0.011801730920535013, 'mean_word_length': 5.232843137254902, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7491753', 'n_tokens_mistral': 843, 'n_tokens_neox': 814, 'n_words': 227}