<commit_before>#!/usr/bin/env python
#
# $Id$
#

"""Routines common to all posix systems."""

import os
import errno
import subprocess
import psutil
import socket
import re
import sys
import warnings

try:
    from collections import namedtuple
except ImportError:
    from psutil.compat import namedtuple  # python < 2.6

from psutil.error import AccessDenied, NoSuchProcess


def pid_exists(pid):
    """Check whether pid exists in the current process table."""
    if pid < 0:
        return False
    try:
        os.kill(pid, 0)
    except OSError, e:
        return e.errno == errno.EPERM
    else:
        return True


class LsofParser:
    """A wrapper for lsof command line utility.
    Executes lsof in subprocess and parses its output.
    """
    socket_table = {'TCP' : socket.SOCK_STREAM,
                    'UDP' : socket.SOCK_DGRAM,
                    'IPv4' : socket.AF_INET,
                    'IPv6' : socket.AF_INET6}
    _connection_ntuple = namedtuple('connection', 'family type local_address '
                                                  'remote_address status fd')
                                                  
    def __init__(self, pid, name):
        self.pid = pid
        self.process_name = name

    def get_process_open_files(self):
        """Return files opened by process by parsing lsof output."""
        # Options:
        # -i == network files only
        # -a == ANDing of all options
        # -p == process with given PID only
        # -n == do not resolve IP addresses
        # -P == do not resolve port numbers
        # -w == suppresses warnings
        # -F0nPt == (0) separate lines with "\x00"
        #           (n) file name
        #           (t) file type
        #           (f) file descriptr
        cmd = "lsof -a -p %s -n -P -F0ftn" % self.pid
        stdout = self.runcmd(cmd)
        if not stdout:
            return []
        files = []
        lines = stdout.split("\n")
        del lines[0]  # first line contains the PID
        for line in lines:
            if not line:
                continue
            line = line.strip("\x00")
            fields = {}
            for field in line.split("\x00"):
                key, value = field[0], field[1:]
                fields[key] = value
            _type = fields['t']
            fd = fields['f']<commit_msg>fix open_files() on BSD where it seems sometimes lsoft return file objects without type specified<commit_after>#!/usr/bin/env python
#
# $Id$
#

"""Routines common to all posix systems."""

import os
import errno
import subprocess
import psutil
import socket
import re
import sys
import warnings

try:
    from collections import namedtuple
except ImportError:
    from psutil.compat import namedtuple  # python < 2.6

from psutil.error import AccessDenied, NoSuchProcess


def pid_exists(pid):
    """Check whether pid exists in the current process table."""
    if pid < 0:
        return False
    try:
        os.kill(pid, 0)
    except OSError, e:
        return e.errno == errno.EPERM
    else:
        return True


class LsofParser:
    """A wrapper for lsof command line utility.
    Executes lsof in subprocess and parses its output.
    """
    socket_table = {'TCP' : socket.SOCK_STREAM,
                    'UDP' : socket.SOCK_DGRAM,
                    'IPv4' : socket.AF_INET,
                    'IPv6' : socket.AF_INET6}
    _connection_ntuple = namedtuple('connection', 'family type local_address '
                                                  'remote_address status fd')
                                                  
    def __init__(self, pid, name):
        self.pid = pid
        self.process_name = name

    def get_process_open_files(self):
        """Return files opened by process by parsing lsof output."""
        # Options:
        # -i == network files only
        # -a == ANDing of all options
        # -p == process with given PID only
        # -n == do not resolve IP addresses
        # -P == do not resolve port numbers
        # -w == suppresses warnings
        # -F0nPt == (0) separate lines with "\x00"
        #           (n) file name
        #           (t) file type
        #           (f) file descriptr
        cmd = "lsof -a -p %s -n -P -F0ftn" % self.pid
        stdout = self.runcmd(cmd)
        if not stdout:
            return []
        files = []
        lines = stdout.split("\n")
        del lines[0]  # first line contains the PID
        for line in lines:
            if not line:
                continue
            line = line.strip("\x00")
            fields = {}
            for field in line.split("\x00"):
                key, value = field[0], field[1:]
                fields[key] = value
            if not 't' in fields:
                continue
            _type = fields['t']
            fd = fields['f']
{'fraction_non_alphanumeric': 0.09489352904692991, 'fraction_numerical': 0.00785610915856936, 'mean_word_length': 1.6165494862087615, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4289091', 'n_tokens_mistral': 1395, 'n_tokens_neox': 1335, 'n_words': 459}