<commit_before># -*- coding: utf-8 -*-
"""Imports for the WISeREP spectroscopic repository.
"""
import json
import os
import warnings
from glob import glob
from html import unescape

from astrocats.catalog.utils import is_number, pbar, pbar_strings, uniq_cdl
from astrocats.catalog.source import SOURCE
from astropy.time import Time as astrotime

from ..supernova import SUPERNOVA


def do_wiserep_spectra(catalog):
    task_str = catalog.get_current_task_str()
    secondaryreference = 'WISeREP'
    secondaryrefurl = 'http://wiserep.weizmann.ac.il/'
    secondarybibcode = '2012PASP..124..668Y'
    wiserepcnt = 0

    # These are known to be in error on the WISeREP page, either fix or ignore
    # them.
    wiserepbibcorrectdict = {'2000AJ....120..367G]': '2000AJ....120..367G',
                             'Harutyunyan et al. 2008': '2008A&A...488..383H',
                             '0609268': '2007AJ....133...58K',
                             '2006ApJ...636...400Q': '2006ApJ...636..400Q',
                             '2011ApJ...741...76': '2011ApJ...741...76C',
                             '2016PASP...128...961': '2016PASP..128...961',
                             '2002AJ....1124..417H': '2002AJ....1124.417H',
                             '2013ApJ…774…58D': '2013ApJ...774...58D',
                             '2011Sci.333..856S': '2011Sci...333..856S',
                             '2014MNRAS.438,368': '2014MNRAS.438..368T',
                             '2012MNRAS.420.1135': '2012MNRAS.420.1135S',
                             '2012Sci..337..942D': '2012Sci...337..942D',
                             'stt1839': '2013MNRAS.436.3614S',
                             'arXiv:1605.03136': '2016arXiv160503136T',
                             '10.1093/mnras/stt1839': '2013MNRAS.436.3614S'}

    file_names = list(
        glob(os.path.join(
            catalog.get_current_task_repo(), '*')))
    for folder in pbar_strings(file_names, task_str):
        if '.txt' in folder:
            continue
        name = os.path.basename(folder).strip()
        if name.startswith('sn'):
            name = 'SN' + name[2:]
        if (name.startswith(('CSS', 'SSS', 'MLS')) and
                ':' not in name):
            name = name.replace('-', ':', 1)
        if name.startswith('MASTERJ'):
            name = name.replace('MASTERJ', 'MASTER OT J')
        if name.startswith('PSNJ'):
            name = name.replace('PSNJ', 'PSN J')
        name = catalog.add_entry(name)

        secondarysource = catalog.entries[name].add_source(
            name=secondaryreference,
            url=secondaryrefurl,
            bibcode=secondarybibcode, secondary=True)
        catalog.entries[name].add_quantity(
            SUPERNOVA.ALIAS, name, secondarysource)

        with open(os.path.join(folder, 'README.json'), 'r') as f:
            fileinfo = json.loads(f.read())

        files = glob(folder + '/*')
        for fname in pbar(files, task_str):
            if "README.json" in fname:
                continue
            specfile = os.path.basename(fname)
            claimedtype = fileinfo[specfile]["Type"]
            instrument = fileinfo[specfile]["Instrument"]
            epoch = fileinfo[specfile]["Obs. Date"]
            observer = fileinfo[specfile]["Observer"]
            reducer = ''
            bibcode = fileinfo[specfile]["Bibcode"]
            redshift = fileinfo[specfile]["Redshift"]
            survey = fileinfo[specfile]["Program"]
            reduction = fileinfo[specfile]["Reduction Status"]

            if bibcode:
                newbibcode = bibcode
                if bibcode in wiserepbibcorrectdict:
                    newbibcode = wiserepbibcorrectdict[bibcode]
                if newbibcode and len(newbibcode) == 19:<commit_msg>MAINT: exclude README.json from list of files so pbar shows correct #<commit_after># -*- coding: utf-8 -*-
"""Imports for the WISeREP spectroscopic repository.
"""
import json
import os
import warnings
from glob import glob
from html import unescape

from astrocats.catalog.utils import is_number, pbar, pbar_strings, uniq_cdl
from astrocats.catalog.source import SOURCE
from astropy.time import Time as astrotime

from ..supernova import SUPERNOVA


def do_wiserep_spectra(catalog):
    task_str = catalog.get_current_task_str()
    secondaryreference = 'WISeREP'
    secondaryrefurl = 'http://wiserep.weizmann.ac.il/'
    secondarybibcode = '2012PASP..124..668Y'
    wiserepcnt = 0

    # These are known to be in error on the WISeREP page, either fix or ignore
    # them.
    wiserepbibcorrectdict = {'2000AJ....120..367G]': '2000AJ....120..367G',
                             'Harutyunyan et al. 2008': '2008A&A...488..383H',
                             '0609268': '2007AJ....133...58K',
                             '2006ApJ...636...400Q': '2006ApJ...636..400Q',
                             '2011ApJ...741...76': '2011ApJ...741...76C',
                             '2016PASP...128...961': '2016PASP..128...961',
                             '2002AJ....1124..417H': '2002AJ....1124.417H',
                             '2013ApJ…774…58D': '2013ApJ...774...58D',
                             '2011Sci.333..856S': '2011Sci...333..856S',
                             '2014MNRAS.438,368': '2014MNRAS.438..368T',
                             '2012MNRAS.420.1135': '2012MNRAS.420.1135S',
                             '2012Sci..337..942D': '2012Sci...337..942D',
                             'stt1839': '2013MNRAS.436.3614S',
                             'arXiv:1605.03136': '2016arXiv160503136T',
                             '10.1093/mnras/stt1839': '2013MNRAS.436.3614S'}

    file_names = list(
        glob(os.path.join(
            catalog.get_current_task_repo(), '*')))
    for folder in pbar_strings(file_names, task_str):
        if '.txt' in folder:
            continue
        name = os.path.basename(folder).strip()
        if name.startswith('sn'):
            name = 'SN' + name[2:]
        if (name.startswith(('CSS', 'SSS', 'MLS')) and
                ':' not in name):
            name = name.replace('-', ':', 1)
        if name.startswith('MASTERJ'):
            name = name.replace('MASTERJ', 'MASTER OT J')
        if name.startswith('PSNJ'):
            name = name.replace('PSNJ', 'PSN J')
        name = catalog.add_entry(name)

        secondarysource = catalog.entries[name].add_source(
            name=secondaryreference,
            url=secondaryrefurl,
            bibcode=secondarybibcode, secondary=True)
        catalog.entries[name].add_quantity(
            SUPERNOVA.ALIAS, name, secondarysource)

        with open(os.path.join(folder, 'README.json'), 'r') as f:
            fileinfo = json.loads(f.read())

        files = list(set(glob(folder + '/*')) -
                     set(glob(folder + '/README.json')))
        for fname in pbar(files, task_str):
            specfile = os.path.basename(fname)
            claimedtype = fileinfo[specfile]["Type"]
            instrument = fileinfo[specfile]["Instrument"]
            epoch = fileinfo[specfile]["Obs. Date"]
            observer = fileinfo[specfile]["Observer"]
            reducer = ''
            bibcode = fileinfo[specfile]["Bibcode"]
            redshift = fileinfo[specfile]["Redshift"]
            survey = fileinfo[specfile]["Program"]
            reduction = fileinfo[specfile]["Reduction Status"]

            if bibcode:
                newbibcode = bibcode
                if bibcode in wiserepbibcorrectdict:
                    newbibcode = wiserepbibcorrectdict[bibcode]
                if newbibcode and len(newbibcode) == 19:
{'fraction_non_alphanumeric': 0.1367038739330269, 'fraction_numerical': 0.07984241628365069, 'mean_word_length': 1.792812614594793, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '301428', 'n_tokens_mistral': 2812, 'n_tokens_neox': 2271, 'n_words': 445}