<commit_before><commit_msg>new python sample code pfilter<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
""" filter point file
"""

import sys
import re

def GetNext(fp, separator=';'):
    """ get next line from file and parse """
    buf = fp.readline().strip()
    if buf:     # None returned on end of file
        fields = [f.strip() for f in buf.split(separator) if len(f)]
        if len(fields) > 2:
            return fields[0], [float(f) for f in fields[1:]]
    return None

def GetCoo(fn, sep=';'):
    """ load coordinates int oa dictionary
    """
    fp = open(fn, 'r')
    plist = {}
    l = GetNext(fp, sep)
    while l:
        plist[l[0]] = l[1]
        l = GetNext(fp, sep)
    return plist

def Filter(pl, pattern):
    """ filter points 
    """
    pat = re.compile(pattern)
    res = {}
    for key in pl:
        if pat.search(key):
            #res[key] = pl[key]   # !!!! danger !!!!
            res[key] = pl[key][:]
    return res

def Area(pl, ps):
    """ calculate area of points given in ps by name
    """
    n = len(ps)
    are = 0
    for i in range(n):
        j = (i + 1) % n
        if ps[i] in pl and ps[j] in pl:
           are += (pl[ps[i]][0] + pl[ps[j]][0]) * (pl[ps[i]][1] - pl[ps[j]][1])
    return are / 2

pp = GetCoo('labor.csv', ';')
print(Filter(pp, '^1'))
print(Area(pp, ['501', '507', '509', '515']))
{'fraction_non_alphanumeric': 0.18667642752562225, 'fraction_numerical': 0.01903367496339678, 'mean_word_length': 2.278177458033573, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7581613', 'n_tokens_mistral': 515, 'n_tokens_neox': 495, 'n_words': 149}