<commit_before>from bs4 import BeautifulSoup as bs
import os
import argparse
from datetime import datetime
import zip_io
import paths
import artifacts

# Do not import pandas into this module.

D = 2**20


def write_features(data, outfile):
  print 'Feature space dimension: %d' % D
  start = datetime.now()
  outpath = os.path.join(paths.PROCESSED, outfile + '.libsvm')
  with open(outpath, 'w') as f_out:
    for (k, page_tuple) in enumerate(data):
      label = str(page_tuple[1])
      page = page_tuple[2]
      row = {}
      tag_counts(row, page)
      # at 20k, it was better without these
      #tag_bigrams(row, page)
      #tag_trigrams(row, page)
      tag_attrs(row, page)
      tag_attr_vals(row, page)
      #attrs(row, page)
      f_out.write(label + ' ')
      for key in sorted(row.keys()):
        f_out.write('%s:%d ' % (key, row[key]))
      f_out.write('\n')
      if (k + 1) % 1000 == 0:
        print '%d lines read' % (k + 1)
  finish = datetime.now()
  print 'Elapsed time: %d sec.' % (finish - start).seconds


def tag_counts(row, page):
  tags = page.find_all(True)
  for tag in tags:
    key = abs(hash(tag.name)) % D
    ct = row.get(key, 0)
    row[key] = ct + 1

def tag_bigrams(row, page):
  tags = page.find_all(True)
  for tag in tags:
    children = tag.find_all(True, recursive=False)
    for child in children:
      key = abs(hash(tag.name + '-' + child.name)) % D
      ct = row.get(key, 0)
      row[key] = ct + 1

def tag_trigrams(row, page):
  tags = page.find_all(True)
  for tag in tags:
    children = tag.find_all(True, recursive=False)
    for child in children:
      grandkids = child.find_all(True, recursive=False)
      for grandkid in grandkids:
        key = abs(hash(tag.name + '-' + child.name + '_' + grandkid.name)) % D
        ct = row.get(key, 0)
        row[key] = ct + 1


def tag_attrs(row, page):
  tags = page.find_all(True)
  for tag in tags:
    for a in tag.attrs:
      key = abs(hash(tag.name + '-' + a)) % D
      ct = row.get(key, 0)
      row[key] = ct + 1

<commit_msg>added 2 types of features, but they are not called because they did not help any<commit_after>from bs4 import BeautifulSoup as bs
import os
import argparse
from datetime import datetime
import zip_io
import paths
import artifacts

# Do not import pandas into this module.

D = 2**20


def write_features(data, outfile):
  print 'Feature space dimension: %d' % D
  start = datetime.now()
  outpath = os.path.join(paths.PROCESSED, outfile + '.libsvm')
  with open(outpath, 'w') as f_out:
    for (k, page_tuple) in enumerate(data):
      label = str(page_tuple[1])
      page = page_tuple[2]
      row = {}
      tag_counts(row, page)
      #parents(row, page)
      #tag_children(row, page)
      #tag_bigrams(row, page)
      #tag_trigrams(row, page)
      tag_attrs(row, page)
      tag_attr_vals(row, page)
      #attrs(row, page)
      f_out.write(label + ' ')
      for key in sorted(row.keys()):
        f_out.write('%s:%d ' % (key, row[key]))
      f_out.write('\n')
      if (k + 1) % 1000 == 0:
        print '%d lines read' % (k + 1)
  finish = datetime.now()
  print 'Elapsed time: %d sec.' % (finish - start).seconds


def tag_counts(row, page):
  tags = page.find_all(True)
  for tag in tags:
    key = abs(hash(tag.name)) % D
    ct = row.get(key, 0)
    row[key] = ct + 1


def parents(row, page):
  tags = page.find_all(True)
  for tag in tags:
    parents = len(list(tag.parents))
    key = abs(hash(tag.name + 'child' +  str(parents))) % D
    ct = row.get(key, 0)
    row[key] = ct + 1


def tag_children(row, page):
  tags = page.find_all(True)
  for tag in tags:
    children = len(tag.find_all(True, recursive=False))
    key = abs(hash(tag.name + 'child' +  str(children))) % D
    ct = row.get(key, 0)
    row[key] = ct + 1


def tag_bigrams(row, page):
  tags = page.find_all(True)
  for tag in tags:
    children = tag.find_all(True, recursive=False)
    for child in children:
      key = abs(hash(tag.name + '-' + child.name)) % D
      ct = row.get(key, 0)
      row[key] = ct + 1


def tag_trigrams(row, page):
  tags = page.find_all(True)
  for tag in tags:
    children = tag.find_all(True, recursive=False)
    for child in children:
      grandkids = child.find_all(True, recursive=False)
      for grandkid in grandkids:
        key = abs(hash(tag.name + '-' + child.name + '_' + grandkid.name)) % D
        ct = row.get(key, 0)
        row[key] = ct + 1


def tag_attrs(row, page):
  tags = page.find_all(True)
  for tag in tags:
    for a in tag.attrs:
      key = abs(hash(tag.name + '-' + a)) % D
      ct = row.get(key, 0)
      row[key] = ct + 1


{'fraction_non_alphanumeric': 0.14014251781472684, 'fraction_numerical': 0.010580868063053337, 'mean_word_length': 2.714514835605453, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '873408', 'n_tokens_mistral': 1787, 'n_tokens_neox': 1679, 'n_words': 473}