<commit_before>#!/usr/bin/env python3

import os, re, sys
from prjxray import util


def maketodo(pipfile, dbfile, strict=True):
    '''Print name of all pips in pipfile but not dbfile'''
    todos = set()
    with open(pipfile, "r") as f:
        for line in f:
            line = line.split()
            todos.add(line[0])
    print('%s: %u entries' % (pipfile, len(todos)), file=sys.stderr)
    with open(dbfile, "r") as f:
        for line in f:
            line = line.split()
            pip = line[0]
            try:
                todos.remove(pip)
            except KeyError:
                # DB (incorrectly) had multiple entries
                # Workaround for testing on old DB revision
                if strict:
                    raise
                print(
                    'WARNING: failed to remove pip %s' % pip, file=sys.stderr)
    print('Remove %s: %u entries' % (dbfile, len(todos)), file=sys.stderr)
    drops = 0
    lines = 0
    for line in todos:
        if line.endswith(".VCC_WIRE"):
            drops += 1
            continue
        print(line)
        lines += 1
    print('Print %u entries w/ %u drops' % (lines, drops), file=sys.stderr)


def run(strict=True):
    maketodo(
        "pips_int_l.txt",
        "%s/%s/segbits_int_l.db" %
        (os.getenv("XRAY_DATABASE_DIR"), os.getenv("XRAY_DATABASE")),
        strict=strict)
    maketodo(
        "pips_int_r.txt",<commit_msg>rempips: support generate without existing db<commit_after>#!/usr/bin/env python3

import os, re, sys
from prjxray import util


def maketodo(pipfile, dbfile, strict=True):
    '''Print name of all pips in pipfile but not dbfile'''
    todos = set()
    with open(pipfile, "r") as f:
        for line in f:
            line = line.split()
            todos.add(line[0])
    print('%s: %u entries' % (pipfile, len(todos)), file=sys.stderr)
    # Support generate without existing DB
    if os.path.exists(dbfile) or strict:
        with open(dbfile, "r") as f:
            for line in f:
                line = line.split()
                pip = line[0]
                try:
                    todos.remove(pip)
                except KeyError:
                    # DB (incorrectly) had multiple entries
                    # Workaround for testing on old DB revision
                    if strict:
                        raise
                    print(
                        'WARNING: failed to remove pip %s' % pip,
                        file=sys.stderr)
    print('Remove %s: %u entries' % (dbfile, len(todos)), file=sys.stderr)
    drops = 0
    lines = 0
    for line in todos:
        if line.endswith(".VCC_WIRE"):
            drops += 1
            continue
        print(line)
        lines += 1
    print('Print %u entries w/ %u drops' % (lines, drops), file=sys.stderr)


def run(strict=True):
    maketodo(
        "pips_int_l.txt",
        "%s/%s/segbits_int_l.db" %
        (os.getenv("XRAY_DATABASE_DIR"), os.getenv("XRAY_DATABASE")),
        strict=strict)
    maketodo(
        "pips_int_r.txt",
{'fraction_non_alphanumeric': 0.11720288903479974, 'fraction_numerical': 0.004596191726854892, 'mean_word_length': 1.7108540925266904, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5426279', 'n_tokens_mistral': 993, 'n_tokens_neox': 963, 'n_words': 267}