<commit_before>#!/usr/bin/env python

'''
Tiger
'''

import json
import os
import subprocess
from collections import OrderedDict
from tasks.util import (LoadPostgresFromURL, classpath, TempTableTask,
                        sql_to_cartodb_table, grouper, shell,
                        underscore_slugify, TableTask, ColumnTarget,
                        ColumnsTask
                       )
from tasks.meta import (OBSColumnTable, OBSColumn, current_session,
                        OBSColumnTag, OBSColumnToColumn, current_session)
from tasks.tags import CategoryTags
from tasks.carto import Import as CartoImport

from luigi import (Task, WrapperTask, Parameter, LocalTarget, BooleanParameter,
                   IntParameter)
from psycopg2 import ProgrammingError


class ClippedGeomColumns(ColumnsTask):

    def version(self):
        return 4

    def requires(self):
        return {
            'geom_columns': GeomColumns(),
            'tags': CategoryTags(),
        }

    def columns(self):
        cols = OrderedDict()
        tags = self.input()['tags']
        for colname, coltarget in self.input()['geom_columns'].iteritems():
            col = coltarget._column
            cols[colname + '_clipped'] = OBSColumn(
                type='Geometry',
                name='Shoreline clipped ' + col.name,
                weight=col.weight,
                description='A cartography-ready version of {name}'.format(
                    name=col.name),
                targets={col: 'cartography'},
                tags=[tags['boundary']],
                weight=col.weight
            )

        return cols


class GeomColumns(ColumnsTask):

    def version(self):
        return 6

    def requires(self):
        return {
            'tags': CategoryTags(),
        }<commit_msg>fix bug with double-entry in tiger<commit_after>#!/usr/bin/env python

'''
Tiger
'''

import json
import os
import subprocess
from collections import OrderedDict
from tasks.util import (LoadPostgresFromURL, classpath, TempTableTask,
                        sql_to_cartodb_table, grouper, shell,
                        underscore_slugify, TableTask, ColumnTarget,
                        ColumnsTask
                       )
from tasks.meta import (OBSColumnTable, OBSColumn, current_session,
                        OBSColumnTag, OBSColumnToColumn, current_session)
from tasks.tags import CategoryTags
from tasks.carto import Import as CartoImport

from luigi import (Task, WrapperTask, Parameter, LocalTarget, BooleanParameter,
                   IntParameter)
from psycopg2 import ProgrammingError


class ClippedGeomColumns(ColumnsTask):

    def version(self):
        return 4

    def requires(self):
        return {
            'geom_columns': GeomColumns(),
            'tags': CategoryTags(),
        }

    def columns(self):
        cols = OrderedDict()
        tags = self.input()['tags']
        for colname, coltarget in self.input()['geom_columns'].iteritems():
            col = coltarget._column
            cols[colname + '_clipped'] = OBSColumn(
                type='Geometry',
                name='Shoreline clipped ' + col.name,
                weight=col.weight,
                description='A cartography-ready version of {name}'.format(
                    name=col.name),
                targets={col: 'cartography'},
                tags=[tags['boundary']]
            )

        return cols


class GeomColumns(ColumnsTask):

    def version(self):
        return 6

    def requires(self):
        return {
            'tags': CategoryTags(),
        }
{'fraction_non_alphanumeric': 0.0893355667225014, 'fraction_numerical': 0.0016750418760469012, 'mean_word_length': 2.0083963056255247, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '629526', 'n_tokens_mistral': 967, 'n_tokens_neox': 905, 'n_words': 234}