<jupyter_start><jupyter_text>The plan We scan the list in two passes.* In the first pass, we collect the known types and occupations for each node.* In the second pass, for each row, we look up the matching type/occupation and record a count of this property on that type. Then we can use this to write out a table of type-property-counts:     type    property    count We can then use this table to rate nodes.  Configuration<jupyter_code>// data source
var edges = "/scratch/trh6u/db_csv/WD_entity_prop_entity.csv";
// property of interest
const int p_InstanceOf = 31;
const int p_SubclassOf = 279;
var propertyOfInterest =  p_InstanceOf;
    // instance of 31;
var test_path = $@"/scratch/jag2j/final_data/test1000_10plus.csv";
var testIds =
    File.ReadLines(test_path)
        .Skip(1) // header
        .Select(x => x.Split(",")[1])
        .Select(long.Parse)
        .ToHashSet();
testIds.Take(5)<jupyter_output><empty_output><jupyter_text>Data Model  Reader functions<jupyter_code>// always empty list
var empty = new List<long>();
class Edge {
    public long S;
    public long P;
    public long O;
    public Edge(long s, long p, long o) {
        S = s; P = p; O = o;
    }
}
// Read the lines of a file, passing them through a parse function
// and returning the results for those successfully parsed.
//
// You can optional set a maximum line count.
IEnumerable<T> Read<T>(TextReader tr, Func<string, T> parse, int maxCount = -1) 
{
   for(var i = 0; maxCount == -1 || i < maxCount; i++)
   {
      var line = tr.ReadLine();
      if (line == null) yield break;
      var parsed = parse(line);
      if(parsed != null) {
         yield return parsed;
      }
   }
}
// Parse an edge from the edge file (src, dst).
// Return null if the line is corrupt or edges are not numeric.
 Edge ParseEdge(string line) {
   var fields = line.Split(',');
   if(fields.Length < 3) return null;
   if(!long.TryParse(fields[0], out var e1)) return null;
   if(!long.TryParse(fields[1], out var e2)) return null;
   if(!long.TryParse(fields[2], out var e3)) return null;
   return new Edge(e1, e2, e3);
}
// Open a reader on a file
TextReader GetReader(string path)
{
   var fr = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
   var tr = new StreamReader(fr); // auto-close fr when tr closes
   return tr;
}<jupyter_output><empty_output><jupyter_text>Build type map<jupyter_code>// entites :: entity ID -> ID of entity representing type (i.e., the superentity)
var testTypes = new Dictionary<long,List<long>>();
var entities = new Dictionary<long, long>(500_000_000);

using(var r = GetReader(edges)) {
    foreach(var res in Read(r, ParseEdge))
    {
        // capture test type
        if(testIds.Contains(res.S) && res.P == propertyOfInterest)
        {
            if(!testTypes.TryGetValue(res.S, out var testEntity)) {
                testTypes[res.S] = new List<long> { res.O };
            }
            else testTypes[res.S].Add(res.O);
        }
        
        // for processing training entities list, drop test data
        if(testIds.Contains(res.S) || testIds.Contains(res.P))
        {
            continue; // skip the below processing to exclude test data from training data
        }
        
        if(!entities.TryGetValue(res.S, out var entity)) {
            entities[res.S] = 0; // default to 0
        }

        if(res.P == propertyOfInterest) {
            entities[res.S] = res.O;            
        }
    }
}
entities.Take(10)
testTypes.Take(5)<jupyter_output><empty_output><jupyter_text>Let's make sure the ID is skipped.<jupyter_code>var sampleTestId = testIds.FirstOrDefault();    
sampleTestId
entities.Where(e => e.Key == sampleTestId || e.Value == sampleTestId)
testTypes.Where(e=>e.Key == sampleTestId || e.Value.Contains(sampleTestId))
using(var f = new FileStream($"/scratch/jag2j/final_data/{propertyOfInterest}-test-types_10plus.csv", FileMode.Create)) 
using(var tw = new StreamWriter(f))
{
    tw.AutoFlush = true;
    tw.WriteLine("entity,type");
    foreach(var kv in testTypes)
    foreach(var v in kv.Value.Distinct())
    {
      var entity = kv.Key;
      var type = v;
      tw.WriteLine($"{entity},{type}");
    }
}
using(var f = new FileStream($"/scratch/jag2j/final_data/{propertyOfInterest}-train-types_10plus.csv", FileMode.Create)) 
using(var tw = new StreamWriter(f))
{
    tw.AutoFlush = true;
    tw.WriteLine("entity,type");
    foreach(var kv in entities)
    {
      var entity = kv.Key;
      var type = kv.Value;
      tw.WriteLine($"{entity},{type}");
    }
}<jupyter_output><empty_output><jupyter_text>Build type/prop count<jupyter_code>// set { (subject, prop) }
var seen = new HashSet<Tuple<long, long>>();

// (type, prop) -> count (= sp_mat)
var typeprops = new Dictionary<Tuple<long, long>, long>();

using(var r = GetReader(edges)) {
    foreach(var res in Read(r, ParseEdge)) {
        if(testIds.Contains(res.S) || testIds.Contains(res.P)) continue; // skip test data
        
        var type = entities[res.S];
        var key = Tuple.Create(type, res.P);
        
        // skip if this subj/prop is not new
        var subjProp = Tuple.Create(res.S, res.P);
        if(!seen.Add(subjProp)) {
            continue; 
        }
        
        if(!typeprops.TryGetValue(key, out var cnt)) {
            typeprops[key] = 1;
        }
        else typeprops[key] += 1;
    }
}

// permit seen memory to be freed
seen = null;
typeprops.Take(10)
using(var f = new FileStream($"/scratch/jag2j/final_data/aux/{propertyOfInterest}_type-property-count_10plus.csv", FileMode.Create)) 
using(var tw = new StreamWriter(f))
{
    tw.AutoFlush = true;
    tw.WriteLine("type,property,count");
    foreach(var kv in typeprops) {
        var type = kv.Key.Item1;
        var prop = kv.Key.Item2;
        var cnt = kv.Value;
        tw.WriteLine($"{type},{prop},{cnt}");
    }
}<jupyter_output><empty_output><jupyter_text>Compute totals and frequency matrix<jupyter_code>// type -> # of entities with the type (= se_sum)
var totals =
    // type x entity
    entities.GroupBy(e => e.Value)
        .ToDictionary(e => e.Key, e => e.Count());

using(var f = new FileStream($"/scratch/jag2j/final_data/aux/{propertyOfInterest}_se_sum_10plus.csv", FileMode.Create)) 
using(var tw = new StreamWriter(f))
{
    tw.AutoFlush = true;
    tw.WriteLine("type,count");
    foreach(var kv in totals) {
        tw.WriteLine($"{kv.Key},{kv.Value}");

    }
}
var freqList = new List<(long type, long prop, float freq)>();

using(var f = new FileStream($"/scratch/jag2j/final_data/{propertyOfInterest}-train_freq_matrix_10plus.csv", FileMode.Create)) 
using(var tw = new StreamWriter(f))
{
    tw.AutoFlush = true;
    tw.WriteLine("type,property,frequency");
    foreach(var kv in typeprops) {
        var type = kv.Key.Item1;
        var prop = kv.Key.Item2;
        float freq = kv.Value;
        if(totals.TryGetValue(type, out var typeSubjectCount)) {
            if(typeSubjectCount > 0)
              freq /= typeSubjectCount;
        }
        freqList.Add((type, prop, freq));
        tw.WriteLine($"{type},{prop},{freq}");
    }
}
freqList
freqList.Count()
typeprops.Count()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13542825361512792, 'fraction_numerical': 0.009593993325917686, 'mean_word_length': 2.9521978021978024, 'pattern_counts': {'":': 0, '<': 40, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '381103', 'license': 'MIT', 'n_tokens_mistral': 2420, 'n_tokens_neox': 2367, 'n_words': 621, 'path': '200 Create frequency and related matrices.ipynb', 'repo_name': 'jag2j/capstone-wikidata-2021', 'size': '7192'}