<commit_before>#!/usr/bin/env python
#=========================================================================
#
#  Copyright Insight Software Consortium
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0.txt
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#
#=========================================================================

import re, getopt, sys, csv
import SimpleITK
import itk


class bcolors:
    """A class to print colored text."""
    HEADER='\033[95m';  OKBLUE='\033[94m'; OKGREEN='\033[92m';
    WARNING='\033[93m'; FAIL='\033[91m';   ENDC='\033[0m';


class FilterSet:
    """The main data structure for holding the filter data"""
    filters = list()    # a list of all the filters
    itk     = set()     # a set of all the ITK filters
    sitk    = set()     # a set of all the SimpleITK filters
    todo    = set()     # a set of the ITK filters not yet in SimpleITK but to be done
    remarks = dict()    # a dictionary of remarks, indexed by filter name


# Global variables
fs              = FilterSet()   # the filter data structure
remarkFile      = ""
onlyRemarksFlag = False
sortByType      = False
quietMode       = False
writelessMode   = False

fieldnames      = ( 'Filter', 'ITK', 'SITK', 'Remark', 'ToDo' )    # fields in the CSV file


#
#
def writeCSV(name):
    """Write the results out to a common-spaced-value file (readable by any spreadsheet program)"""
    try:
        with open(name, "w") as fp:
            writer = csv.DictWriter(fp, fieldnames=fieldnames)
            headers = dict( (n,n) for n in fieldnames)
            writer.writerow(headers)
            for filt in fs.filters:
                if filt in fs.remarks:
                    rem = fs.remarks[filt]
                else:
                    rem = ""
                if len(rem) or not onlyRemarksFlag:
                    writer.writerow( { fieldnames[0]:filt,
                                       fieldnames[1]:filt in fs.itk,
                                       fieldnames[2]:filt in fs.sitk,
                                       fieldnames[3]:rem,
                                       fieldnames[4]:filt in fs.todo,
                                       } )
    except:
        print bcolors.FAIL, "Error:", bcolors.ENDC, "Couldn't write output file", name, "\n"
        sys.exit(-1)
<commit_msg>Updated script to output Unix newlines<commit_after>#!/usr/bin/env python
#=========================================================================
#
#  Copyright Insight Software Consortium
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0.txt
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#
#=========================================================================

import re, getopt, sys, csv
import SimpleITK
import itk


class bcolors:
    """A class to print colored text."""
    HEADER='\033[95m';  OKBLUE='\033[94m'; OKGREEN='\033[92m';
    WARNING='\033[93m'; FAIL='\033[91m';   ENDC='\033[0m';


class FilterSet:
    """The main data structure for holding the filter data"""
    filters = list()    # a list of all the filters
    itk     = set()     # a set of all the ITK filters
    sitk    = set()     # a set of all the SimpleITK filters
    todo    = set()     # a set of the ITK filters not yet in SimpleITK but to be done
    remarks = dict()    # a dictionary of remarks, indexed by filter name


# Global variables
fs              = FilterSet()   # the filter data structure
remarkFile      = ""
onlyRemarksFlag = False
sortByType      = False
quietMode       = False
writelessMode   = False

fieldnames      = ( 'Filter', 'ITK', 'SITK', 'Remark', 'ToDo' )    # fields in the CSV file


#
#
class unix_dialect(csv.Dialect):
    """Describe the usual properties of unix-generated CSV files."""
    delimiter = ','
    quotechar = '"'
    doublequote = True
    skipinitialspace = False
    lineterminator = '\n'
    quoting = csv.QUOTE_MINIMAL

def writeCSV(name):
    """Write the results out to a common-spaced-value file (readable by any spreadsheet program)"""
    csv.register_dialect("unix_dialect", unix_dialect)
    try:
        with open(name, "w") as fp:
            writer = csv.DictWriter(fp, fieldnames=fieldnames, dialect="unix_dialect")
            headers = dict( (n,n) for n in fieldnames)
            writer.writerow(headers)
            for filt in fs.filters:
                if filt in fs.remarks:
                    rem = fs.remarks[filt]
                else:
                    rem = ""
                if len(rem) or not onlyRemarksFlag:
                    writer.writerow( { fieldnames[0]:filt,
                                       fieldnames[1]:filt in fs.itk,
                                       fieldnames[2]:filt in fs.sitk,
                                       fieldnames[3]:rem,
                                       fieldnames[4]:filt in fs.todo,
                                       } )
    except:
        print bcolors.FAIL, "Error:", bcolors.ENDC, "Couldn't write output file", name, "\n"
        sys.exit(-1)

{'fraction_non_alphanumeric': 0.14201383032551865, 'fraction_numerical': 0.013155675493337831, 'mean_word_length': 2.222826086956522, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1993381', 'n_tokens_mistral': 1726, 'n_tokens_neox': 1616, 'n_words': 570}