<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import division, print_function

import pdb
import re
import string
import tarfile
import fnmatch
import requests
import feedparser
import numpy as np
import os
import sys
from astropy.table import Table

from contextlib import closing

URL = "http://arxiv.org/rss/astro-ph"
COMMENT_RE = re.compile(r"(?<!\\)%")
AMP_RE = re.compile(r"(?<!\\)&")
DATA_DIR = os.environ.get("ARXIV_DATA_DIR", "data")

class table_stub():
    def __init__(self):
        self.colnames = []
    def __getitem__(self,field):
        return [0.]

def clean_string(tab):
  # First deal with multiple spaces at front of line
  tab = tab.expandtabs()
  tab = re.sub("\\\\hline","",tab)
  tab = re.sub("\$","",tab)
  tab = re.sub("\\%","",tab)
  tab = re.sub(" *\n","\n",tab)
  tab = re.sub("(?<!\\\\\\\\)\n","",tab)
  tab = re.sub("(?<!\n) *\\\\caption","\n\\\\caption",tab)
  tab = re.sub("(?<!\n) *\\\\centering","\n\\\\centering",tab)
  tab = re.sub("(?<!\n) *\\\\label","\n\\\\label",tab)
  tab = re.sub("(?<!\n) *\\\\item","\n\\\\item",tab)
  tab = re.sub("(?<!\n) *\\\\begin","\n\\\\begin",tab)
  tab = re.sub("(?<=[lcr])r}(?!\n)","r}\n",tab)
  tab = re.sub("(?<=[lcr])l}(?!\n)","l}\n",tab)
  tab = re.sub("(?<=[lcr])c}(?!\n)","c}\n",tab)
  tab = re.sub("(?<!\n) *\\\\end","\n\\\\end",tab)
  tab = re.sub("\n  *","\n",tab)
  tab = re.sub("^\n","",tab)
  tab = re.sub("\n\n*","\n",tab)
  return tab


def split_errors(table):
    # Here we find string columns with \pm and split them into new columns
    repl_str = np.str_("\pm")
    for col_name in table.colnames:
        if type(table[col_name][0]) == type(np.str_()) :
            do_split = False
            for entry in table[col_name]:
                if string.find(entry,repl_str) >= 0:
                    do_split = True
                    break
            if do_split :
                new_col_name = col_name + "Data"
                new_col_error_name = new_col_name + "Err"
                table[new_col_name] = np.zeros(len(table[col_name]),dtype=np.float64)
                table[new_col_error_name] = np.zeros(len(table[col_name]),dtype=np.float64)
                new_index = 0
                for entry in table[col_name]:
                    if string.find(entry,repl_str) < 0:
                        try:
                            new_data = np.float64(entry)
                        except:
                            new_data = np.nan
                        new_error = np.nan
                    else:<commit_msg>Fixed string.find to be compatible with py3<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import division, print_function

import pdb
import re
import string
import tarfile
import fnmatch
import requests
import feedparser
import numpy as np
import os
import sys
from astropy.table import Table

from contextlib import closing

URL = "http://arxiv.org/rss/astro-ph"
COMMENT_RE = re.compile(r"(?<!\\)%")
AMP_RE = re.compile(r"(?<!\\)&")
DATA_DIR = os.environ.get("ARXIV_DATA_DIR", "data")

class table_stub():
    def __init__(self):
        self.colnames = []
    def __getitem__(self,field):
        return [0.]

def clean_string(tab):
  # First deal with multiple spaces at front of line
  tab = tab.expandtabs()
  tab = re.sub("\\\\hline","",tab)
  tab = re.sub("\$","",tab)
  tab = re.sub("\\%","",tab)
  tab = re.sub(" *\n","\n",tab)
  tab = re.sub("(?<!\\\\\\\\)\n","",tab)
  tab = re.sub("(?<!\n) *\\\\caption","\n\\\\caption",tab)
  tab = re.sub("(?<!\n) *\\\\centering","\n\\\\centering",tab)
  tab = re.sub("(?<!\n) *\\\\label","\n\\\\label",tab)
  tab = re.sub("(?<!\n) *\\\\item","\n\\\\item",tab)
  tab = re.sub("(?<!\n) *\\\\begin","\n\\\\begin",tab)
  tab = re.sub("(?<=[lcr])r}(?!\n)","r}\n",tab)
  tab = re.sub("(?<=[lcr])l}(?!\n)","l}\n",tab)
  tab = re.sub("(?<=[lcr])c}(?!\n)","c}\n",tab)
  tab = re.sub("(?<!\n) *\\\\end","\n\\\\end",tab)
  tab = re.sub("\n  *","\n",tab)
  tab = re.sub("^\n","",tab)
  tab = re.sub("\n\n*","\n",tab)
  return tab


def split_errors(table):
    # Here we find string columns with \pm and split them into new columns
    repl_str = np.str_("\pm")
    for col_name in table.colnames:
        if type(table[col_name][0]) == type(np.str_()) :
            do_split = False
            for entry in table[col_name]:
                if entry.find(repl_str) >= 0:
                    do_split = True
                    break
            if do_split :
                new_col_name = col_name + "Data"
                new_col_error_name = new_col_name + "Err"
                table[new_col_name] = np.zeros(len(table[col_name]),dtype=np.float64)
                table[new_col_error_name] = np.zeros(len(table[col_name]),dtype=np.float64)
                new_index = 0
                for entry in table[col_name]:
                    if entry.find(repl_str) < 0:
                        try:
                            new_data = np.float64(entry)
                        except:
                            new_data = np.nan
                        new_error = np.nan
                    else:
{'fraction_non_alphanumeric': 0.20831706364701288, 'fraction_numerical': 0.004880905896134323, 'mean_word_length': 2.101089588377724, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2679460', 'n_tokens_mistral': 1892, 'n_tokens_neox': 1797, 'n_words': 365}