<commit_before>from __future__ import print_function
import sys
import os
import pprint
from getopt import getopt
import json
import pickle
import traceback
import marshal
import time

import yara
from yara import scan


"""
Scanner command line interface.

[<EMAIL>]
"""


__help__ = """
NAME yara-ctypes Scan files or processes against YARA rules

SYNOPSIS
    yara-ctypes [OPTIONS]... [FILE(s)|PID(s)]...

DESCRIPTION

Scanner control:
    --mode=default mode is stdin or file
        mode options [file|chunk|proc|stdin]
        file  -queues file paths found in FILE(s)
        chunk -queues chunks of data read from files found in FILE(s)
        proc  -enqueues PID(s) for process scanning
        stdin -read from the stdin stream. 

    --thread-pool=%s
        size of the thread pool used in the Scanner instance 

    --chunk-size=%s
        size of data in bytes to read and enqueue from data read from a stream

    --chunk-overlap=%s
        percentage from 0 - 99 %% of data to be reprocesses on block boundaries

    --readhead-limit=%s
        maximum number of bytes to read ahead when reading data from a stream


File control:
    --recurse-dirs 
        recurse directories specified in FILE(s) 

    --exclude-filesize-lt=
        exclude files which are less then this value

    --exclude-filesize-gt=
        exclude files which are greater then then value

    --path-end-include=[end1,end2,end3, ...]
        path endings inclusion filter (comma separate list)

    --path-end-exclude=[end1,end2,end3, ...]
        path endings exclusion filter (comma separate list)

    --path-contains-exclude=[str1,str2,str3, ...]
        exclude path's that contain str (comma separate list)

    --path-contains-include=[str1,str2,str3, ...]
        include path's that contain str (comma separate list)

Rules control:
  rule:  
    --fast
        fast matching mode

    -d <identifier>=<value>
        define external variable.

  namespace: 
    --list
        list available YARA namespaces

    -w, --whitelist=
        whitelist of comma separated YARA namespaces to include in scan

    -b, --blacklist=
        blacklist of comma separated YARA namespaces to exclude from scan

    --root=(env[YARA_RULES] or <pkg>/yara/rules/)
        set the YARA_RULES path (path to the root of the rules directory)
  
  yarafile:
    -r, --rule=
        Use the rule file specified by this input argument and ignore the
        YARA namespaces

Web control:
    --max-post-size=%s

Scan output control:
    --fmt=dict
        output format [dict|pprint|json|pickle|marshal]
    -o
        outfile path -> redirect stdout results to outfile

    -t  [tag1,tag2,tag3, ...]
        print matches that contain specific tags and filter out the rest

    -i  [ident1,ident2,ident3, ...]
        print matches that contain specific identifiers and filter out the rest

    --simple 
        print the filepath and the rule which was hit

    --error-log=
        write scan errors out to this error log

Other:
    --version 
        print the version of yara-ctypes

    -h, --help 
        print this help
""" % (scan.DEFAULT_THREAD_POOL,
       scan.DEFAULT_STREAM_CHUNK_SIZE,
       scan.DEFAULT_STREAM_CHUNK_OVERLAP,
       scan.DEFAULT_STREAM_READAHEAD_LIMIT,
       scan.MAX_POST_SIZE)


def match_filter(tags_filter, idents_filter, res):
    if tags_filter is not None:
        new_res = {}
        for ns, matches in res.iteritems():
            for match in matches:
                if tags_filter.intersection(match['tags']):
                    mlist = new_res.get(ns, [])
                    mlist.append(match)
                    new_res[ns] = mlist
        res = new_res
    if idents_filter is not None:
        new_res = {}
        for ns, matches in res.iteritems():
            for match in matches:
                idents = [s['identifier'] for s in match['strings']]
                if idents_filter.intersection(idents):
                    mlist = new_res.get(ns, [])
                    mlist.append(match)
                    new_res[ns] = mlist
        res = new_res
    return res<commit_msg>cli doc<commit_after>from __future__ import print_function
import sys
import os
import pprint
from getopt import getopt
import json
import pickle
import traceback
import marshal
import time

import yara
from yara import scan


"""
Scanner command line interface.

[<EMAIL>]
"""


__help__ = """
NAME yara-ctypes Scan files or processes against YARA rules

SYNOPSIS
    yara-ctypes [OPTIONS]... [FILE(s)|PID(s)]...

DESCRIPTION

Scanner control:
    --mode=default mode is stdin or file
        mode options [file|chunk|proc|stdin]
        file  -queues file paths found in FILE(s)
        chunk -queues chunks of data read from files found in FILE(s)
        proc  -enqueues PID(s) for process scanning
        stdin -read from the stdin stream. 

    --thread-pool=%s
        size of the thread pool used in the Scanner instance 

    --chunk-size=%s
        size of data in bytes to read and enqueue from data read from a stream

    --chunk-overlap=%s
        percentage from 0 - 99 %% of data to be reprocesses on block boundaries

    --readhead-limit=%s
        maximum number of bytes to read ahead when reading data from a stream

File control:
    --recurse-dirs 
        recurse directories specified in FILE(s) 

    --exclude-filesize-lt=
        exclude files which are less then this value

    --exclude-filesize-gt=
        exclude files which are greater then then value

    --path-end-include=[end1,end2,end3, ...]
        path endings inclusion filter (comma separate list)

    --path-end-exclude=[end1,end2,end3, ...]
        path endings exclusion filter (comma separate list)

    --path-contains-exclude=[str1,str2,str3, ...]
        exclude path's that contain str (comma separate list)

    --path-contains-include=[str1,str2,str3, ...]
        include path's that contain str (comma separate list)

Rules control:
  rule:  
    --fast
        fast matching mode

    -d <identifier>=<value>
        define external variable.

  namespace: 
    --list
        list available YARA namespaces

    -w, --whitelist=
        whitelist of comma separated YARA namespaces to include in scan

    -b, --blacklist=
        blacklist of comma separated YARA namespaces to exclude from scan

    --root=(env[YARA_RULES] or <pkg>/yara/rules/)
        set the YARA_RULES path (path to the root of the rules directory)
  
  yarafile:
    -r, --rule=
        Use the rule file specified by this input argument and ignore the
        YARA namespaces

Scan output control:
    --fmt=dict
        output format [dict|pprint|json|pickle|marshal]
    -o
        outfile path -> redirect stdout results to outfile

    -t  [tag1,tag2,tag3, ...]
        print matches that contain specific tags and filter out the rest

    -i  [ident1,ident2,ident3, ...]
        print matches that contain specific identifiers and filter out the rest

    --simple 
        print the filepath and the rule which was hit

    --error-log=
        write scan errors out to error-log

Other:
    --version 
        print the version of yara-ctypes

    -h, --help 
        print this help
""" % (scan.DEFAULT_THREAD_POOL,
       scan.DEFAULT_STREAM_CHUNK_SIZE,
       scan.DEFAULT_STREAM_CHUNK_OVERLAP,
       scan.DEFAULT_STREAM_READAHEAD_LIMIT)


def match_filter(tags_filter, idents_filter, res):
    if tags_filter is not None:
        new_res = {}
        for ns, matches in res.iteritems():
            for match in matches:
                if tags_filter.intersection(match['tags']):
                    mlist = new_res.get(ns, [])
                    mlist.append(match)
                    new_res[ns] = mlist
        res = new_res
    if idents_filter is not None:
        new_res = {}
        for ns, matches in res.iteritems():
            for match in matches:
                idents = [s['identifier'] for s in match['strings']]
                if idents_filter.intersection(idents):
                    mlist = new_res.get(ns, [])
                    mlist.append(match)
                    new_res[ns] = mlist
        res = new_res
    return res
{'fraction_non_alphanumeric': 0.08292563600782779, 'fraction_numerical': 0.005136986301369863, 'mean_word_length': 2.205409643277146, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6288442', 'n_tokens_mistral': 2476, 'n_tokens_neox': 2352, 'n_words': 832}