<commit_before>import { readDataByYear } from '../service/dbService'
import { sumCount, filterByEnrolment } from './scriptUtils/aggDataUtil'
import { umiAvg, dispersionIndex } from '../utils/calculate'
import R from 'ramda'

const umi6AvgByClassSize = (data) => {
  const tenAndUnder = filterByEnrolment(0, 10)(data)
  const tenToTwenty = filterByEnrolment(11, 20)(data)
  const twentyToThirty = filterByEnrolment(21, 30)(data)
  const thirtyToFourty = filterByEnrolment(31, 40)(data)
  const fourtyToFifty = filterByEnrolment(41, 50)(data)
  const fiftyToSixty = filterByEnrolment(51, 60)(data)
  const sixtyToSeventy = filterByEnrolment(61, 70)(data)
  const seventyToEighty = filterByEnrolment(71, 80)(data)
  const eightyToNinety = filterByEnrolment(81, 90)(data)
  const ninetyToOneHundred = filterByEnrolment(91, 100)(data)
  const over100 = filterByEnrolment(101, 10000)(data)

  const dataArray = [tenAndUnder, tenToTwenty, twentyToThirty, thirtyToFourty, fourtyToFifty, fiftyToSixty, sixtyToSeventy, seventyToEighty, eightyToNinety, ninetyToOneHundred, over100]

  dataArray.map(arr =>
    console.log(
      'UMI6 average: ' + umiAvg(sumCount(arr.map(section => section.UMI6.count))),
      'Dispersion Index: ' + dispersionIndex(sumCount(arr.map(section => section.UMI6.count))),
      'Number of Courses: ' + arr.length
    )
  )
}

const umi6AvgByDept = (data) => {
  const depts = R.uniq(data.map(section => section.dept))
  console.log(depts)
}

readDataByYear('2016', 'aggregatedData', (data) => {
  // umi6AvgByClassSize(data)
  umi6AvgByDept(data)
})<commit_msg>implemented stats by dept<commit_after>import { readDataByYear } from '../service/dbService'
import { sumCount, filterByEnrolment } from './scriptUtils/aggDataUtil'
import { umiAvg, dispersionIndex } from '../utils/calculate'
import R from 'ramda'

const byClassSize = (data) => {
  const tenAndUnder = filterByEnrolment(0, 10)(data)
  const tenToTwenty = filterByEnrolment(11, 20)(data)
  const twentyToThirty = filterByEnrolment(21, 30)(data)
  const thirtyToFourty = filterByEnrolment(31, 40)(data)
  const fourtyToFifty = filterByEnrolment(41, 50)(data)
  const fiftyToSixty = filterByEnrolment(51, 60)(data)
  const sixtyToSeventy = filterByEnrolment(61, 70)(data)
  const seventyToEighty = filterByEnrolment(71, 80)(data)
  const eightyToNinety = filterByEnrolment(81, 90)(data)
  const ninetyToOneHundred = filterByEnrolment(91, 100)(data)
  const over100 = filterByEnrolment(101, 10000)(data)

  const dataArray = [tenAndUnder, tenToTwenty, twentyToThirty, thirtyToFourty, fourtyToFifty, fiftyToSixty, sixtyToSeventy, seventyToEighty, eightyToNinety, ninetyToOneHundred, over100]

  dataArray.map(arr =>
    console.log(
      'UMI6 average: ' + umiAvg(sumCount(arr.map(section => section.UMI6.count))),
      'Dispersion Index: ' + dispersionIndex(sumCount(arr.map(section => section.UMI6.count))),
      'Number of Courses: ' + arr.length
    )
  )
}

const byDept = (data) => {
  const depts = R.uniq(data.map(section => section.dept))
  const groupedByDepts = depts.map(dept => data.filter(section => section.dept === dept))
  const sumCountByDept = groupedByDepts.map(aGroup => ({
    avg: umiAvg(sumCount(aGroup.map(x => x.UMI6.count))),
    dispersion: dispersionIndex(sumCount(aGroup.map(x => x.UMI6.count))),
    length: aGroup.length,
    dept: aGroup[0].dept
  })).sort((a, b) => a.avg - b.avg)
  console.log(sumCountByDept)
}

readDataByYear('2016', 'aggregatedData', (data) => {
  // byClassSize(data)
  byDept(data)
})
{'fraction_non_alphanumeric': 0.1338068181818182, 'fraction_numerical': 0.03664772727272727, 'mean_word_length': 5.33273381294964, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1849847', 'n_tokens_mistral': 1396, 'n_tokens_neox': 1260, 'n_words': 236}