<commit_before>"""Retriever script for the Portal Project mammals

Dataset published by Ernest et al. 2009 in Ecological Archives.

"""

#TO DO - confirm column reversal with authors and correct

from retriever.lib.templates import BasicTextTemplate
from retriever.lib.models import Table, Cleanup, correct_invalid_value

VERSION = '0.5'

SCRIPT = BasicTextTemplate(
                           name="Portal Project Mammals (Ecological Archives 2002)",
                           description="<NAME>, <NAME>, and <NAME>. 2009. Long-term monitoring and experimental manipulation of a Chihuahuan Desert ecosystem near Portal, Arizona, USA. Ecology 90:1708.",
                           shortname="PortalMammals",
                           ref="http://esapubs.org/archive/ecol/E090/118/",
                           tables = {
                                     "plots": Table("plots", 
                                                    delimiter=',', 
                                                    contains_pk=True,
                                                    header_rows=0,
                                                    columns=[("PlotID"            , ("pk-auto",) ),
                                                             ("PlotTypeAlphaCode" , ("char", 2)  ),
                                                             ("PlotTypeNumCode"   , ("int",)     ),
                                                             ("PlotTypeDescript"  , ("char", 30) )]
                                                    ),
                                     "main": Table("main", 
                                                   cleanup=Cleanup(correct_invalid_value,
                                                                   nulls=[])
                                                   ),
                                     },
                           urls = {
                                   "main": "http://esapubs.org/archive/ecol/E090/118/Portal_rodent_19772002.csv",
                                   "species": "http://wiki.ecologicaldata.org/sites/default/files/portal_species.txt",
                                   "plots": "http://wiki.ecologicaldata.org/sites/default/files/portal_plots.txt"<commit_msg>Typo correction in Portal Mammal script. Publication date was 2009, not 2002.<commit_after>"""Retriever script for the Portal Project mammals

Dataset published by Ernest et al. 2009 in Ecological Archives.

"""

#TO DO - confirm column reversal with authors and correct

from retriever.lib.templates import BasicTextTemplate
from retriever.lib.models import Table, Cleanup, correct_invalid_value

VERSION = '0.5'

SCRIPT = BasicTextTemplate(
                           name="Portal Project Mammals (Ecological Archives 2009)",
                           description="<NAME>, <NAME>, and <NAME>. 2009. Long-term monitoring and experimental manipulation of a Chihuahuan Desert ecosystem near Portal, Arizona, USA. Ecology 90:1708.",
                           shortname="PortalMammals",
                           ref="http://esapubs.org/archive/ecol/E090/118/",
                           tables = {
                                     "plots": Table("plots", 
                                                    delimiter=',', 
                                                    contains_pk=True,
                                                    header_rows=0,
                                                    columns=[("PlotID"            , ("pk-auto",) ),
                                                             ("PlotTypeAlphaCode" , ("char", 2)  ),
                                                             ("PlotTypeNumCode"   , ("int",)     ),
                                                             ("PlotTypeDescript"  , ("char", 30) )]
                                                    ),
                                     "main": Table("main", 
                                                   cleanup=Cleanup(correct_invalid_value,
                                                                   nulls=[])
                                                   ),
                                     },
                           urls = {
                                   "main": "http://esapubs.org/archive/ecol/E090/118/Portal_rodent_19772002.csv",
                                   "species": "http://wiki.ecologicaldata.org/sites/default/files/portal_species.txt",
                                   "plots": "http://wiki.ecologicaldata.org/sites/default/files/portal_plots.txt"
{'fraction_non_alphanumeric': 0.08874836458787615, 'fraction_numerical': 0.020933275185346708, 'mean_word_length': 0.9305555555555556, 'pattern_counts': {'":': 10, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 56, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2947394', 'n_tokens_mistral': 1020, 'n_tokens_neox': 892, 'n_words': 202}