<commit_before>/*
 ** 2013 August 10
 **
 ** The author disclaims copyright to this source code.  In place of
 ** a legal notice, here is a blessing:
 **    May you do good and not evil.
 **    May you find forgiveness for yourself and forgive others.
 **    May you share freely, never taking more than you give.
 */
package info.ata4.unity.struct.db;

import static java.nio.file.StandardOpenOption.*;
import static java.nio.file.StandardCopyOption.*;
import info.ata4.unity.asset.Asset;
import info.ata4.unity.struct.FieldType;
import info.ata4.unity.struct.TypeTree;
import info.ata4.unity.util.ClassID;
import info.ata4.util.collection.Pair;
import info.ata4.util.io.DataInputReader;
import info.ata4.util.io.DataOutputWriter;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;

/**
 *
 * @author <NAME> <barracuda415 at yahoo.de>
 */
public class StructDatabase {
    
    private static final Logger L = Logger.getLogger(StructDatabase.class.getName());
    private static final int VERSION = 1;
    private static StructDatabase instance;

    public static StructDatabase getInstance() {
        if (instance == null) {
            instance = new StructDatabase();
        }
        return instance;
    }
    
    private FieldTypeMapper ftm = new FieldTypeMapper();
    private Path dbFile = Paths.get("resources", "structdb.dat");
    private Path dbFileBackup = Paths.get("resources", "structdb.dat.1");
    private int learnedTotal;
    
    private StructDatabase() {
        load();
    }
    
    public int getLearned() {
        return learnedTotal;
    }
    
    public FieldTypeMapper getTypeMapper() {
        return ftm;
    }
    
    private void load() {
        L.info("Loading struct database");
        
        // read database file if existing
        if (Files.exists(dbFile)) {
            try (InputStream fis = Files.newInputStream(dbFile, READ)) {
                DataInputReader in = new DataInputReader(new DataInputStream(new BufferedInputStream(fis)));
                
                // read header
                int version = in.readInt();

                if (version != VERSION) {
                    throw new RuntimeException("Wrong database version");
                }

                // read field node table
                int fieldNodeSize = in.readInt();
                List<FieldType> fieldNodes = new ArrayList<>(fieldNodeSize);

                for (int i = 0; i < fieldNodeSize; i++) {
                    FieldType fieldNode = new FieldType();
                    fieldNode.read(in);
                    fieldNodes.add(fieldNode);
                }

                // read revision string table
                int revisionSize = in.readInt();
                List<String> revisions = new ArrayList<>(revisionSize);

                for (int i = 0; i < revisionSize; i++) {
                    revisions.add(in.readStringNull());
                }

                // read mapping data
                int fieldNodeKeySize = in.readInt();

                for (int i = 0; i < fieldNodeKeySize; i++) {
                    int index = in.readInt();
                    int classID = in.readInt();
                    int revisionIndex = in.readInt();
                    String revision = revisions.get(revisionIndex);
                    FieldType fieldNode = fieldNodes.get(index);

                    ftm.add(classID, revision, fieldNode);
                }
            } catch (IOException ex) {
                L.log(Level.SEVERE, "Can't read struct database", ex);
            }
        }
    }
    
    private void save() {
        L.info("Saving struct database");
        
        // create database backup
        if (Files.exists(dbFile)) {
            try {
                Files.move(dbFile, dbFileBackup, REPLACE_EXISTING);
            } catch (IOException ex) {
                L.log(Level.WARNING, "Can't create struct database backup", ex);
            }
        }
        
        // write updated database file
        try (OutputStream fos = Files.newOutputStream(dbFile, CREATE, WRITE)) {
            DataOutputWriter out = new DataOutputWriter(new DataOutputStream(new BufferedOutputStream(fos)));
            
            // write header
            out.writeInt(VERSION);

            // write field node table
            Set<FieldType> fieldNodes = new HashSet<>(ftm.values());
            Map<FieldType, Integer> fieldNodeMap = new HashMap<>();

            out.writeInt(fieldNodes.size());

            int index = 0;
            for (FieldType fieldNode : fieldNodes) {
                fieldNodeMap.put(fieldNode, index++);
                fieldNode.write(out);
            }

            Set<Pair<Integer, String>> fieldNodeKeys = ftm.keySet();

            // write revision string table
            Set<String> revisions = new HashSet<>();
            Map<String, Integer> revisionMap = new HashMap<>();

            for (Map.Entry<Pair<Integer, String>, FieldType> entry : ftm.entrySet()) {
                revisions.add(entry.getKey().getRight());
            }

            out.writeInt(revisions.size());

            index = 0;
            for (String revision : revisions) {
                revisionMap.put(revision, index++);
                out.writeStringNull(revision);
            }

            // write mapping data
            out.writeInt(fieldNodeKeys.size());

            for (Map.Entry<Pair<Integer, String>, FieldType> entry : ftm.entrySet()) {
                index = fieldNodeMap.get(entry.getValue());
                Pair<Integer, String> fieldNodeKey = entry.getKey();

                int classID = fieldNodeKey.getLeft();<commit_msg>Fixed incompletely written database because of using the wrong output stream in the try-with-resources statement<commit_after>/*
 ** 2013 August 10
 **
 ** The author disclaims copyright to this source code.  In place of
 ** a legal notice, here is a blessing:
 **    May you do good and not evil.
 **    May you find forgiveness for yourself and forgive others.
 **    May you share freely, never taking more than you give.
 */
package info.ata4.unity.struct.db;

import static java.nio.file.StandardOpenOption.*;
import static java.nio.file.StandardCopyOption.*;
import info.ata4.unity.asset.Asset;
import info.ata4.unity.struct.FieldType;
import info.ata4.unity.struct.TypeTree;
import info.ata4.unity.util.ClassID;
import info.ata4.util.collection.Pair;
import info.ata4.util.io.DataInputReader;
import info.ata4.util.io.DataOutputWriter;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;

/**
 *
 * @author <NAME> <barracuda415 at yahoo.de>
 */
public class StructDatabase {
    
    private static final Logger L = Logger.getLogger(StructDatabase.class.getName());
    private static final int VERSION = 1;
    private static StructDatabase instance;

    public static StructDatabase getInstance() {
        if (instance == null) {
            instance = new StructDatabase();
        }
        return instance;
    }
    
    private FieldTypeMapper ftm = new FieldTypeMapper();
    private Path dbFile = Paths.get("resources", "structdb.dat");
    private Path dbFileBackup = Paths.get("resources", "structdb.dat.1");
    private int learnedTotal;
    
    private StructDatabase() {
        load();
    }
    
    public int getLearned() {
        return learnedTotal;
    }
    
    public FieldTypeMapper getTypeMapper() {
        return ftm;
    }
    
    private void load() {
        L.info("Loading struct database");
        
        // read database file if existing
        if (Files.exists(dbFile)) {
            try (BufferedInputStream bis = new BufferedInputStream(Files.newInputStream(dbFile, READ))) {
                DataInputReader in = new DataInputReader(new DataInputStream(bis));
                
                // read header
                int version = in.readInt();

                if (version != VERSION) {
                    throw new RuntimeException("Wrong database version");
                }

                // read field node table
                int fieldNodeSize = in.readInt();
                List<FieldType> fieldNodes = new ArrayList<>(fieldNodeSize);

                for (int i = 0; i < fieldNodeSize; i++) {
                    FieldType fieldNode = new FieldType();
                    fieldNode.read(in);
                    fieldNodes.add(fieldNode);
                }

                // read revision string table
                int revisionSize = in.readInt();
                List<String> revisions = new ArrayList<>(revisionSize);

                for (int i = 0; i < revisionSize; i++) {
                    revisions.add(in.readStringNull());
                }

                // read mapping data
                int fieldNodeKeySize = in.readInt();

                for (int i = 0; i < fieldNodeKeySize; i++) {
                    int index = in.readInt();
                    int classID = in.readInt();
                    int revisionIndex = in.readInt();
                    String revision = revisions.get(revisionIndex);
                    FieldType fieldNode = fieldNodes.get(index);

                    ftm.add(classID, revision, fieldNode);
                }
            } catch (IOException ex) {
                L.log(Level.SEVERE, "Can't read struct database", ex);
            }
        }
    }
    
    private void save() {
        L.info("Saving struct database");
        
        // create database backup
        if (Files.exists(dbFile)) {
            try {
                Files.move(dbFile, dbFileBackup, REPLACE_EXISTING);
            } catch (IOException ex) {
                L.log(Level.WARNING, "Can't create struct database backup", ex);
            }
        }
        
        // write updated database file
        try (BufferedOutputStream bos = new BufferedOutputStream(Files.newOutputStream(dbFile, CREATE, WRITE))) {
            DataOutputWriter out = new DataOutputWriter(new DataOutputStream(bos));
            
            // write header
            out.writeInt(VERSION);

            // write field node table
            Set<FieldType> fieldNodes = new HashSet<>(ftm.values());
            Map<FieldType, Integer> fieldNodeMap = new HashMap<>();

            out.writeInt(fieldNodes.size());

            int index = 0;
            for (FieldType fieldNode : fieldNodes) {
                fieldNodeMap.put(fieldNode, index++);
                fieldNode.write(out);
            }

            // write revision string table
            Set<String> revisions = new HashSet<>();
            Map<String, Integer> revisionMap = new HashMap<>();

            for (Map.Entry<Pair<Integer, String>, FieldType> entry : ftm.entrySet()) {
                revisions.add(entry.getKey().getRight());
            }

            out.writeInt(revisions.size());

            index = 0;
            for (String revision : revisions) {
                revisionMap.put(revision, index++);
                out.writeStringNull(revision);
            }

            // write mapping data
            out.writeInt(ftm.entrySet().size());

            for (Map.Entry<Pair<Integer, String>, FieldType> entry : ftm.entrySet()) {
                index = fieldNodeMap.get(entry.getValue());
                Pair<Integer, String> fieldNodeKey = entry.getKey();

                int classID = fieldNodeKey.getLeft();
{'fraction_non_alphanumeric': 0.10452253131608914, 'fraction_numerical': 0.003904343582235237, 'mean_word_length': 1.9477343562694798, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6124016', 'n_tokens_mistral': 3310, 'n_tokens_neox': 3144, 'n_words': 837}