<commit_before>from portality import models
import os, json
from copy import deepcopy

def minimise_article(full_article):
    # we want to keep the id and the bibjson
    id = full_article.id
    bibjson = deepcopy(full_article.bibjson())
    
    # remove the issns from the bibjson
    bibjson.remove_identifiers(idtype=bibjson.P_ISSN)
    bibjson.remove_identifiers(idtype=bibjson.E_ISSN)
    
    # remove all the journal metadata
    bibjson.remove_journal_metadata()
    
    # remove all the subject classifications
    bibjson.remove_subjects()
    
    # remove the year and the month (they are held elsewhere in this case)
    del bibjson.month
    del bibjson.year
    
    # create a minimised version of the article
    minimised = models.Article()
    minimised.set_id(id)
    minimised.set_bibjson(bibjson)
    
    return minimised
    

if __name__ == "__main__":
    skipped = 0
    js = 0
    volumes = 0
    
    journals = models.Journal.iterall(page_size=1000)
    for journal in journals:
        issns = journal.known_issns()
        known_volumes = models.Article.list_volumes(issns)
        print "Generating ToC for journal", journal.id, "with ISSNs", issns, "and volumes", known_volumes
        
        if len(known_volumes) == 0:
            skipped += 1
            continue
        js += 1
        
        for kv in known_volumes:
            print "...Generating Volume ToC for", kv
            volumes += 1
            articles = models.Article.get_by_volume(issns, kv)
            
            table = models.JournalVolumeToC()
            table.set_about(journal.id)
            table.set_issn(issns)
            table.set_volume(kv)
            
            for article in articles:
                bj = article.bibjson()
                
                # get the issue number, or "unknown" if there isn't one
                num = bj.number
                if num is None:
                    num = "unknown"
                
                # there may already be an issue for this number.  If not
                # make a new one and add it
                iss = table.get_issue(num)
                if iss is None:
                    iss = models.JournalIssueToC()
                    iss.number = num
                    if bj.year is not None:
                        iss.year = bj.year
                    if bj.month is not None:
                        iss.month = bj.month
                    print "......Generating Issue ToC for number", num, "(year:", bj.year, "month:", bj.month, ")"
                    table.add_issue(iss)
                
                # iss is now bound to the toc, so we can update it without
                # adding it to the table again
                iss.add_article(minimise_article(article))<commit_msg>more robus logging fo toc migration script<commit_after>from portality import models
import os, json
from copy import deepcopy

def minimise_article(full_article):
    # we want to keep the id and the bibjson
    id = full_article.id
    bibjson = deepcopy(full_article.bibjson())
    
    # remove the issns from the bibjson
    bibjson.remove_identifiers(idtype=bibjson.P_ISSN)
    bibjson.remove_identifiers(idtype=bibjson.E_ISSN)
    
    # remove all the journal metadata
    bibjson.remove_journal_metadata()
    
    # remove all the subject classifications
    bibjson.remove_subjects()
    
    # remove the year and the month (they are held elsewhere in this case)
    del bibjson.month
    del bibjson.year
    
    # create a minimised version of the article
    minimised = models.Article()
    minimised.set_id(id)
    minimised.set_bibjson(bibjson)
    
    return minimised
    

if __name__ == "__main__":
    skipped = 0
    js = 0
    volumes = 0
    
    journals = models.Journal.iterall(page_size=1000)
    for journal in journals:
        issns = journal.known_issns()
        known_volumes = models.Article.list_volumes(issns)
        print "Generating ToC for journal", journal.id, "with ISSNs", issns#, "and volumes", known_volumes
        
        if len(known_volumes) == 0:
            skipped += 1
            continue
        js += 1
        
        for kv in known_volumes:
            print "...Generating Volume ToC for", unicode(kv).encode("utf-8", "replace")
            volumes += 1
            articles = models.Article.get_by_volume(issns, kv)
            
            table = models.JournalVolumeToC()
            table.set_about(journal.id)
            table.set_issn(issns)
            table.set_volume(kv)
            
            for article in articles:
                bj = article.bibjson()
                
                # get the issue number, or "unknown" if there isn't one
                num = bj.number
                if num is None:
                    num = "unknown"
                
                # there may already be an issue for this number.  If not
                # make a new one and add it
                iss = table.get_issue(num)
                if iss is None:
                    iss = models.JournalIssueToC()
                    iss.number = num
                    if bj.year is not None:
                        iss.year = bj.year
                    if bj.month is not None:
                        iss.month = bj.month
                    print "......Generating Issue ToC for number", unicode(num).encode("utf-8", "replace")#, "(year:", bj.year, "month:", bj.month, ")"
                    table.add_issue(iss)
                
                # iss is now bound to the toc, so we can update it without
                # adding it to the table again
                iss.add_article(minimise_article(article))
{'fraction_non_alphanumeric': 0.07533884879422637, 'fraction_numerical': 0.004224608343601478, 'mean_word_length': 1.6124137931034483, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1040453', 'n_tokens_mistral': 1657, 'n_tokens_neox': 1546, 'n_words': 473}