<commit_before>#!/usr/bin/env python

"""
csv_filter: filter a csv file by one or more other csv files

@author: mitzip
@contact: http://github.com/mitzip/csv_filter
@license: Public Domain
@version: 1.0.0
@todo: add argument to toggle case-sensitivity
"""

import sys
import csv
import argparse

# Define accepted command-line arguments
parser = argparse.ArgumentParser()
parser.add_argument('--filter', type=file, required=True,
                    help='CSV file to be filtered')
parser.add_argument('--by', type=file, required=True, action='append',
                    help='CSV file to filter by')
parser.add_argument('--output', nargs='?',
                    type=argparse.FileType('w'), default=sys.stdout,
                    help='Filtered output CSV filename, omit for stdout')
parser.add_argument('--filter-col', default=3, type=int,
                    help='Column in CSV to filter to match, default 3')
parser.add_argument('--by-col', default=0, type=int,
                    help='Column in CSV to filter by to match, default 0')
args = parser.parse_args()

# Open CSV file to be filtered read-only
filter_csv = csv.reader(args.filter)

# Open output CSV file as writable, or stdout
filtered_csv = csv.writer(args.output)

# Open each CSV file to filter by read-only and add to a unique
# set of field values for rows you do NOT want in filtered output.
to_remove = set()
for by_file in args.by:
    by_csv = csv.reader(by_file)
    next(by_csv)
    to_remove.update({row[args.by_col].lower() for row in by_csv if row})

# Check each row in CSV you want filtered for field values in to_remove set
for row in filter_csv:
    if row and row[args.filter_col].lower() not in to_remove:
        filtered_csv.writerow(row)<commit_msg>fixed file open mode to universal-newline mode to not blow up on newlines in unquoted field<commit_after>#!/usr/bin/env python

"""
csv_filter: filter a csv file by one or more other csv files

@author: mitzip
@contact: http://github.com/mitzip/csv_filter
@license: Public Domain
@version: 1.0.0
@todo: add argument to toggle case-sensitivity
"""

import sys
import csv
import argparse

# Define accepted command-line arguments
parser = argparse.ArgumentParser()
parser.add_argument('--filter', type=argparse.FileType('rU'), required=True,
                    help='CSV file to be filtered')
parser.add_argument('--by', type=argparse.FileType('rU'), required=True,
                    action='append', help='CSV file to filter by')
parser.add_argument('--output', nargs='?',
                    type=argparse.FileType('w'), default=sys.stdout,
                    help='Filtered output CSV filename, omit for stdout')
parser.add_argument('--filter-col', default=3, type=int,
                    help='Column in CSV to filter to match, default 3')
parser.add_argument('--by-col', default=0, type=int,
                    help='Column in CSV to filter by to match, default 0')
args = parser.parse_args()

# Open CSV file to be filtered read-only
filter_csv = csv.reader(args.filter)

# Open output CSV file as writable, or stdout
filtered_csv = csv.writer(args.output)

# Open each CSV file to filter by read-only and add to a unique
# set of field values for rows you do NOT want in filtered output.
to_remove = set()
for by_file in args.by:
    by_csv = csv.reader(by_file)
    next(by_csv)
    to_remove.update({row[args.by_col].lower() for row in by_csv if row})

# Check each row in CSV you want filtered for field values in to_remove set
totalRemoved = 0
for row in filter_csv:
    if row and row[args.filter_col].lower() not in to_remove:
        filtered_csv.writerow(row)
    else:
        # TODO: Add option to output removed rows
        totalRemoved += 1

print "Total Removed:", totalRemoved
{'fraction_non_alphanumeric': 0.10755270883373365, 'fraction_numerical': 0.004270082732852949, 'mean_word_length': 3.4042303172737953, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '709049', 'n_tokens_mistral': 1167, 'n_tokens_neox': 1105, 'n_words': 396}