<commit_before>package main

import (
  "fmt"
  "time"
)

func BuildLocationsList() (locations Locations, err error) {
  miners, err := MinersCollection()
  locations = Locations{}
  locationsAdded := map[string]Location {}

  // Add the default 'all' location
  allLocations := Location{
    Name: "all",
  }
  locations = append(locations, allLocations)

  locationsAdded["all"] = allLocations
  for _, miner := range miners {
    if _, exists := locationsAdded[miner.Location]; !exists {
        locationsAdded[miner.Location] = Location{
          Name: miner.Location,
          GeoCoord: miner.GeoCoord,
        }
        locations = append(locations, locationsAdded[miner.Location])
      }
  }

  return
}


func WordCountRootCollection(location string, source string, fromParam string, toParam string, interval int, limit int) (sortedCounts WordCounts,  collectionErr error) {

  defer func() {
        if r := recover(); r != nil {
            var ok bool
            collectionErr, ok = r.(error)
            if !ok {
                collectionErr = fmt.Errorf("WordCountRootCollection: %v", r)
            }
        }
    }()

  wordCounts := WordCounts {}

  if location == "all" {
    location = ""
  }

  t := time.Now()
  if fromParam == "" {
    from := t.Add(-24 * time.Hour)
    fromParam = from.Format("200601021504")
  }

  if toParam == "" {
    toParam = t.Format("200601021504")
  }

  fromTime, err := time.Parse("200601021504", fromParam)
  if err != nil {
      fmt.Errorf("invalid from date: %v", err)
  }

  toTime, err := time.Parse("200601021504", toParam)
  if err != nil {
      fmt.Errorf("invalid to date: %v", err)
  }

  duration := toTime.Sub(fromTime)
  duration  = duration / time.Duration(interval)

  for i := 0; i < interval; i++ {

    toTime = fromTime.Add(duration)
    toParam = toTime.Format("200601021504")

    rows, err := QueryTerms(source, location, "", fromParam, toParam)
    checkErr(err)

    for rows.Next() {
      var uid int
      var postid int
      var term string
      var wordcount int
      var posted time.Time
      var termLocation string
      var locationHash int
      var termSource string
      err := rows.Scan(&uid, &postid, &term, &wordcount, &posted, &termLocation, &locationHash, &termSource)
      checkErr(err)
      wordCount := WordCount {
        Term: term,
        Occurrences: wordcount,
        Series: []int{wordcount},
        Sequence: i,
      }
      wordCounts = append(wordCounts, wordCount)
    }
<commit_msg>added hardcoded blacklist<commit_after>package main

import (
  "fmt"
  "time"
  "strings"
)

func BuildLocationsList() (locations Locations, err error) {
  miners, err := MinersCollection()
  locations = Locations{}
  locationsAdded := map[string]Location {}

  // Add the default 'all' location
  allLocations := Location{
    Name: "all",
  }
  locations = append(locations, allLocations)

  locationsAdded["all"] = allLocations
  for _, miner := range miners {
    if _, exists := locationsAdded[miner.Location]; !exists {
        locationsAdded[miner.Location] = Location{
          Name: miner.Location,
          GeoCoord: miner.GeoCoord,
        }
        locations = append(locations, locationsAdded[miner.Location])
      }
  }

  return
}


func WordCountRootCollection(location string, source string, fromParam string, toParam string, interval int, limit int) (sortedCounts WordCounts,  collectionErr error) {

  defer func() {
        if r := recover(); r != nil {
            var ok bool
            collectionErr, ok = r.(error)
            if !ok {
                collectionErr = fmt.Errorf("WordCountRootCollection: %v", r)
            }
        }
    }()

  wordCounts := WordCounts {}

  if location == "all" {
    location = ""
  }

  t := time.Now()
  if fromParam == "" {
    from := t.Add(-24 * time.Hour)
    fromParam = from.Format("200601021504")
  }

  if toParam == "" {
    toParam = t.Format("200601021504")
  }

  fromTime, err := time.Parse("200601021504", fromParam)
  if err != nil {
      fmt.Errorf("invalid from date: %v", err)
  }

  toTime, err := time.Parse("200601021504", toParam)
  if err != nil {
      fmt.Errorf("invalid to date: %v", err)
  }

  duration := toTime.Sub(fromTime)
  duration  = duration / time.Duration(interval)

  for i := 0; i < interval; i++ {

    toTime = fromTime.Add(duration)
    toParam = toTime.Format("200601021504")

    rows, err := QueryTerms(source, location, "", fromParam, toParam)
    checkErr(err)

    for rows.Next() {
      var uid int
      var postid int
      var term string
      var wordcount int
      var posted time.Time
      var termLocation string
      var locationHash int
      var termSource string
      err := rows.Scan(&uid, &postid, &term, &wordcount, &posted, &termLocation, &locationHash, &termSource)
      checkErr(err)
      wordCount := WordCount {
        Term: term,
        Occurrences: wordcount,
        Series: []int{wordcount},
        Sequence: i,
      }

      if !strings.ContainsAny(term, "<>[]/:;()=\"") && !strings.Contains(term, "http") {
        wordCounts = append(wordCounts, wordCount)
      }

    }

{'fraction_non_alphanumeric': 0.12648375170266588, 'fraction_numerical': 0.024518388791593695, 'mean_word_length': 2.6146272855133614, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1393613', 'n_tokens_mistral': 1709, 'n_tokens_neox': 1588, 'n_words': 407}