<commit_before>=begin

Copyright 2012 Shared Learning Collaborative, LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

=end

require 'date'
require 'logger'

require_relative "../OutputGeneration/XML/educationOrganizationGenerator.rb"
require_relative "../OutputGeneration/XML/educationOrgCalendarGenerator.rb"
require_relative "../OutputGeneration/XML/masterScheduleGenerator.rb"
require_relative "../Shared/EntityClasses/enum/GradeLevelType.rb"
require_relative "../Shared/EntityClasses/enum/GradingPeriodType.rb"
require_relative "../Shared/EntityClasses/schoolEducationOrganization.rb"
require_relative "../Shared/data_utility.rb"
require_relative "../Shared/date_interval.rb"
require_relative "../Shared/date_utility.rb"

# World Builder
# -> intent is to create 'scaffolding' that represents a detailed, time-sensitive view of the world
#
# generation strategies:
# (1) create world using number of students + time information (begin year, number of years)
# (2) create world using number of schools  + time information (begin year, number of years) [not supported]
class WorldBuilder
  def initialize
    small_batch_size  = 500
    medium_batch_size = 5000
    large_batch_size  = 25000

    $stdout.sync = true
    @log         = Logger.new($stdout)
    @log.level   = Logger::INFO

    @breakdown                        = Hash.new
    @edOrgs                           = Hash.new
    @edOrgs["seas"]                   = []
    @edOrgs["leas"]                   = []
    @edOrgs["elementary"]             = []
    @edOrgs["middle"]                 = []
    @edOrgs["high"]                   = []
    @edOrgs["students"]               = Hash.new
    @edOrgs["students"]["elementary"] = Hash.new
    @edOrgs["students"]["middle"]     = Hash.new
    @edOrgs["students"]["high"]       = Hash.new

    @education_organization_writer = EducationOrganizationGenerator.new(medium_batch_size)
    @education_org_calendar_writer = EducationOrgCalendarGenerator.new(medium_batch_size)
    @master_schedule_writer        = MasterScheduleGenerator.new(medium_batch_size)
  end

  # Builds the initial snapshot of the world
  # -> generates SEA(s), LEAs, Schools, Course [Education Organization interchange]
  # -> Session, GradingPeriod, CalendarDate [Education Organization Calendar interchange]
  # -> generates Staff, Teacher, StaffEdOrgAssignmentAssociation, TeacherSchoolAssociation [Staff Association interchange]
  # -> generates CourseOffering [Master Schedule interchange]
  # -> out of scope: Program, StaffProgramAssociation, StudentProgramAssociation, Cohort, StaffCohortAssociation, StudentCohortAssociation
  # -> returns education organization structure built
  def build(rand, yaml)
    if !yaml["studentCount"].nil?
      build_world_from_students(rand, yaml)
    #elsif !yaml["schoolCount"].nil?
    #  build_world_from_edOrgs(rand, yaml)
    #  --> not supported yet	
    else
      @log.error "studentCount or schoolCount must be set for a world to be created --> Exiting..."
    end

    close_interchanges
    return @edOrgs
  end

  # Builds world using the specified number of students as the driving criteria
  # -> ignores schoolCount, if specified in yml file
  # -> creates elementary, middle, and high schools from number of students
  # -> walks back up ed-fi graph to create LEAs and SEA(s) from created schools
  def build_world_from_students(rand, yaml)
    num_students = yaml["studentCount"]
    @log.info "Creating world from initial number of students: #{num_students}"

    # create grade breakdown from total number of students
    # populate education organization structure using breakdown (number of students per grade)
    # update structure with time information
    # use time information to create master schedule (course offerings, sections)
    # finally, write interchanges
    compute_grade_breakdown(rand, yaml, num_students)
    @log.info "breakdown by grade: #{@breakdown} --> total number of students: #{@breakdown.values.inject(:+)}"
    create_edOrgs_using_breakdown(rand, yaml)
    add_time_information_to_edOrgs(rand, yaml)
    create_master_schedule(rand, yaml)
    write_interchanges(rand, yaml)
  end

  # iterates through the set of ordered grade levels (Kindergarten through 12th grade) and uses
  # the default percentages (as defined in yaml configuration file) for minimum and maximum per grade
  # to compute the number of students that will initially populate the specified grade level.
  def compute_grade_breakdown(rand, yaml, num_students)
    students_so_far = 0
    GradeLevelType::get_ordered_grades.each do |grade|
      num_students_this_grade = get_num_students_per_grade(rand, yaml, num_students)      
      num_students_this_grade = num_students - students_so_far if grade == :TWELFTH_GRADE
      @breakdown[grade]       = num_students_this_grade
      students_so_far         += num_students_this_grade
    end
  end

  # randomly select the number of students for the current grade by using the minimum percentage,
  # maximum percentage, and total number of students
  def get_num_students_per_grade(rand, yaml, num_students)
    min = yaml["MINIMUM_GRADE_PERCENTAGE"]
    max = yaml["MAXIMUM_GRADE_PERCENTAGE"]

    if min.nil?
      @log.error "MINIMUM_GRADE_PERCENTAGE must be set for a world to be created --> Exiting..."
      abort
    end
    if max.nil?
      @log.error "MAXIMUM_GRADE_PERCENTAGE must be set for a world to be created --> Exiting..."
      abort
    end

    ((random_on_interval(rand, min, max) / 100) * num_students).round
  end

  # Uses the breakdown hash to bucketize number of students according to type of school
  # they would be enrolled at, and then computes the number of schools required to hold those
  # students according to national averages (average students per elementary school, ...).
  def create_edOrgs_using_breakdown(rand, yaml)
    num_elementary_school_students = 0
    num_middle_school_students     = 0
    num_high_school_students       = 0

    GradeLevelType::elementary.each do |level| 
      num_elementary_school_students += @breakdown[level]
    end

    GradeLevelType::middle.each do |level|
      num_middle_school_students += @breakdown[level]
    end

    GradeLevelType::high.each do |level|
      num_high_school_students += @breakdown[level]
    end

    num_schools   = 0
    num_schools   = create_schools(rand, yaml, "elementary", num_schools, num_elementary_school_students)
    num_schools   = create_schools(rand, yaml, "middle", num_schools, num_middle_school_students)
    num_schools   = create_schools(rand, yaml, "high", num_schools, num_high_school_students)
    num_districts = update_schools_with_districts(rand, yaml, num_schools)
    update_districts_with_states(rand, yaml, num_districts)
  end

  # uses the total number of students, as well as range of [min, max] to compute students at a school of type 'tag', and then
  # populates the edOrgs structure according to that tag (which should be "elementary", "middle", or "high").
  def create_schools(rand, yaml, tag, num_schools, total_num_students)
    if tag == "elementary"
      avg_num_students = yaml["AVERAGE_ELEMENTARY_SCHOOL_NUM_STUDENTS"]
    elsif tag == "middle"
      avg_num_students = yaml["AVERAGE_MIDDLE_SCHOOL_NUM_STUDENTS"]
    elsif tag == "high"
      avg_num_students = yaml["AVERAGE_HIGH_SCHOOL_NUM_STUDENTS"]
    end
    avg_num_students_threshold = yaml["AVERAGE_NUM_STUDENTS_THRESHOLD"]
    min                        = avg_num_students - (avg_num_students * avg_num_students_threshold)
    max                        = avg_num_students + (avg_num_students * avg_num_students_threshold)
    school_counter             = num_schools
    student_counter            = 0
    while student_counter < total_num_students
      current_students = random_on_interval(rand, min, max).round
      school_counter  += 1
      if current_students > (total_num_students - student_counter)        
        current_students = (total_num_students - student_counter)
      end
      student_counter += current_students
      
      edOrg              = Hash.new
      edOrg["id"]        = school_counter
      edOrg["parent"]    = nil
      edOrg["programs"]  = []
      edOrg["sessions"]  = []
      edOrg["staff"]     = create_staff_for_school(rand, yaml)
      edOrg["offerings"] = Hash.new
      edOrg["students"]  = Hash.new
      begin_year         = yaml["beginYear"]
      edOrg["students"][begin_year] = current_students

      @edOrgs[tag]       << edOrg
    end
    return school_counter
  end

  # updates the populated edOrgs arrays for elementary, middle, and high schools by determining how many schools are to be
  # contained in a given district (looks at yaml configuration file for average number of schools per district and threshold),
  # and then actually going and updating the schools to reference back to newly created local education agencies
  def update_schools_with_districts(rand, yaml, num_schools)
    avg_num_schools_per_district = yaml["AVERAGE_NUM_SCHOOLS_PER_DISTRICT"]
    avg_num_schools_threshold    = yaml["AVERAGE_SCHOOLS_THRESHOLD"]
    min_num_schools_per_district = avg_num_schools_per_district - (avg_num_schools_per_district * avg_num_schools_threshold)
    max_num_schools_per_district = avg_num_schools_per_district + (avg_num_schools_per_district * avg_num_schools_threshold)
    all_schools      = (1..num_schools).to_a
    district_counter = num_schools
    school_counter   = 0
    while all_schools.size > 0
      district_counter += 1
      num_schools_in_this_district = random_on_interval(rand, min_num_schools_per_district, max_num_schools_per_district).round
      
      if num_schools_in_this_district > (num_schools - school_counter)
        num_schools_in_this_district = (num_schools - school_counter)
      end

      schools_in_this_district = all_schools.pop(num_schools_in_this_district)      
      update_schools_with_district_id(district_counter, schools_in_this_district)
      
      edOrg             = Hash.new
      edOrg["id"]       = district_counter
      edOrg["parent"]   = nil
      edOrg["programs"] = []
      edOrg["sessions"] = []
      edOrg["staff"]    = create_staff_for_local_education_agency(rand, yaml)
      @edOrgs["leas"]   << edOrg
      school_counter    += num_schools_in_this_district
    end
    return district_counter
  end

  # actually does the work to iterate through the edOrgs structure (specifically the arrays
  # contained in the tags: "elementary", "middle", and "high" schools), finding edOrgs whose
  # unique "id" is contained within "schools_in_this_district", and sets the "parent" attribute
  # of those matching edOrgs to the district id.
  def update_schools_with_district_id(district_id, schools_in_this_district)
    # check in elementary schools
    @edOrgs["elementary"].each do |edOrg|
      edOrg["parent"] = district_id if schools_in_this_district.include?(edOrg["id"])        
    end
    # check in middle schools
    @edOrgs["middle"].each do |edOrg|
      edOrg["parent"] = district_id if schools_in_this_district.include?(edOrg["id"])
    end
    # check in high schools
    @edOrgs["high"].each do |edOrg|
      edOrg["parent"] = district_id if schools_in_this_district.include?(edOrg["id"])
    end
  end

  # updates the populated edOrgs arrays for leas (local education agencies) by determining how many districts are to be
  # contained in a given state. current implementation assumes:
  # - single tier for local education agencies
  # - all local education agencies are contained within a single state
  # 
  # future implementation should create more 'layers' within the local education agency 'tier'
  # future implementation should look at yaml for average number of districts in a state and create multiple 
  #  state education agencies, as needed
  def update_districts_with_states(rand, yaml, num_districts)
    state_id          = num_districts + 1
    edOrg             = Hash.new
    edOrg["id"]       = state_id
    edOrg["courses"]  = create_courses(yaml)
    edOrg["staff"]    = create_staff_for_state_education_agency(rand, yaml)
    edOrg["programs"] = []
    @edOrgs["seas"]  << edOrg
    
    @edOrgs["leas"].each do |edOrg| 
      edOrg["parent"] = state_id
    end
  end

  # creates staff members at the state education agency level
  def create_staff_for_state_education_agency(rand, yaml)
    members = []
    members
  end

  # creates staff members at the local education agency level
  def create_staff_for_local_education_agency(rand, yaml)
    members = []
    members
  end  

  # creates staff members at the school level
  def create_staff_for_school(rand, yaml)
    members = []
    members
  end

  def build_world_from_edOrgs(rand, yaml)
  	num_schools = yaml["schoolCount"]
    @log.info "Creating world from initial number of schools: #{num_schools}"
    # NOT CURRENTLY SUPPORTED
    # update structure with time information
    add_time_information_to_edOrgs(rand, yaml)
  end

  # creates sessions for each local education agency
  # -> uses 'beginYear' and 'numYears' properties specified in yaml configuration file (defaults to current year and 1, respectively, if not specified)
  # -> iterates from begin year to (begin year + num years), creating Sessions, Grading Periods, and Calendar Dates
  # -> each Session stores a date interval (contains start date, end date, number of instructional days, and holidays for that interval)
  def add_time_information_to_edOrgs(rand, yaml)
    begin_year   = yaml["beginYear"]
    num_years    = yaml["numYears"]
    
    if begin_year.nil?
      this_year = Date.today.year
      @log.info "Property: beginYear --> not set for scenario. Using default: #{this_year}"
      begin_year = this_year
    end

    if num_years.nil?
      @log.info "Property: numYears --> not set for scenario. Using default: 1"
      num_years = 1
    end

    # loop over years updating infrastructure and population
    for year in begin_year..(begin_year+num_years-1) do
      # create session for LEAs
      # -> create grading period(s) for session
      # -> create calendar date(s) for grading period
      # -> [not implemented] iterate through schools and determine with some probability if school overrides existing lea session
      @log.info "creating session information for school year: #{year}-#{year+1}"
      @edOrgs["leas"].each_index do |index|
        edOrg = @edOrgs["leas"][index]
        state_organization_id = DataUtility.get_local_education_agency_id(edOrg["id"])
        start_date = DateUtility.random_school_day_on_interval(rand, Date.new(year, 8, 25), Date.new(year, 9, 10))
        interval   = DateInterval.create_using_start_and_num_days(rand, start_date, 180)

        session             = Hash.new
        session["term"]     = :YEAR_ROUND
        session["year"]     = year
        session["name"]     = year.to_s + "-" + (year+1).to_s + " " + SchoolTerm.to_string(:YEAR_ROUND) + " session: " + state_organization_id
        session["interval"] = interval
        session["edOrgId"]  = state_organization_id
        @edOrgs["leas"][index]["sessions"] << session
      end
    end
    update_schools_with_sessions
  end

  # iterate through each type of school and set the sessions to be used
  def update_schools_with_sessions
    # update elementary schools
    @edOrgs["elementary"].each do |edOrg|
      edOrg["sessions"] = get_sessions_to_be_used_by_school(edOrg)
    end
    # update middle schools
    @edOrgs["middle"].each do |edOrg|
      edOrg["sessions"] = get_sessions_to_be_used_by_school(edOrg)
    end
    # update high schools
    @edOrgs["high"].each_index do |index|
      school = @edOrgs["high"][index]
      @edOrgs["high"][index]["sessions"] = get_sessions_to_be_used_by_school(school)
    end
  end
  
  # create master schedule interchange
  # -> create course offerings
  # -> create sections (using course offerings and breakdown of students per grade)
  # --> do NOT create teachers for course offerings at elementary school (this should be done for sections)
  # --> in middle and high schools, create teacher for course offering (teachers can teach multiple sections)
  def create_master_schedule(rand, yaml)
    offering_id = 0
    create_course_offerings_for_school("elementary", offering_id)
    create_course_offerings_for_school("middle", offering_id)
    create_course_offerings_for_school("high", offering_id)

    #update_shuffled_students(rand, yaml, "elementary")
    #update_shuffled_students(rand, yaml, "middle")
    #update_shuffled_students(rand, yaml, "high")
    #puts "all students shuffled during course of simulation: #{@edOrgs["students"]}"

    #update_student_counts_at_schools(rand, yaml)
  end

  # updates the number of students shuffled out of each type of school
  # --> used for determining number of incoming students for given year to next school type ('middle' --> 'high')
  # --> allows for consistent number of students to remain in system at any given time
  def update_shuffled_students(rand, yaml, type)
    @edOrgs[type].each_index do |index|
      school       = @edOrgs[type][index]<commit_msg>code cleanup in world builder. updating comments as well<commit_after>=begin

Copyright 2012 Shared Learning Collaborative, LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

=end

require 'date'
require 'logger'

require_relative "../OutputGeneration/XML/educationOrganizationGenerator.rb"
require_relative "../OutputGeneration/XML/educationOrgCalendarGenerator.rb"
require_relative "../OutputGeneration/XML/masterScheduleGenerator.rb"
require_relative "../Shared/EntityClasses/enum/GradeLevelType.rb"
require_relative "../Shared/EntityClasses/enum/GradingPeriodType.rb"
require_relative "../Shared/EntityClasses/schoolEducationOrganization.rb"
require_relative "../Shared/data_utility.rb"
require_relative "../Shared/date_interval.rb"
require_relative "../Shared/date_utility.rb"

# World Builder
# -> intent is to create 'scaffolding' that represents a detailed, time-sensitive view of the world
#
# generation strategies:
# (1) create world using number of students + time information (begin year, number of years)
# (2) create world using number of schools  + time information (begin year, number of years) [not supported]
class WorldBuilder
  def initialize
    small_batch_size  = 500
    medium_batch_size = 5000
    large_batch_size  = 25000

    $stdout.sync = true
    @log         = Logger.new($stdout)
    @log.level   = Logger::INFO

    @breakdown                        = Hash.new
    @edOrgs                           = Hash.new
    @edOrgs["seas"]                   = []
    @edOrgs["leas"]                   = []
    @edOrgs["elementary"]             = []
    @edOrgs["middle"]                 = []
    @edOrgs["high"]                   = []
    @edOrgs["students"]               = Hash.new
    @edOrgs["students"]["elementary"] = Hash.new
    @edOrgs["students"]["middle"]     = Hash.new
    @edOrgs["students"]["high"]       = Hash.new

    @education_organization_writer = EducationOrganizationGenerator.new(medium_batch_size)
    @education_org_calendar_writer = EducationOrgCalendarGenerator.new(medium_batch_size)
    @master_schedule_writer        = MasterScheduleGenerator.new(medium_batch_size)
  end

  # Builds the initial snapshot of the world
  # -> generates SEA(s), LEAs, Schools, Course [Education Organization interchange]
  # -> Session, GradingPeriod, CalendarDate [Education Organization Calendar interchange]
  # -> generates Staff, Teacher, StaffEdOrgAssignmentAssociation, TeacherSchoolAssociation [Staff Association interchange]
  # -> generates CourseOffering [Master Schedule interchange]
  # -> out of scope: Program, StaffProgramAssociation, StudentProgramAssociation, Cohort, StaffCohortAssociation, StudentCohortAssociation
  # -> returns education organization structure built
  def build(rand, yaml)
    if !yaml["studentCount"].nil?
      build_world_from_students(rand, yaml)
    #elsif !yaml["schoolCount"].nil?
    #  build_world_from_edOrgs(rand, yaml)
    #  --> not supported yet	
    else
      @log.error "studentCount or schoolCount must be set for a world to be created --> Exiting..."
    end

    close_interchanges
    return @edOrgs
  end

  # Builds world using the specified number of students as the driving criteria
  # -> ignores schoolCount, if specified in yml file
  # -> creates elementary, middle, and high schools from number of students
  # -> walks back up ed-fi graph to create LEAs and SEA(s) from created schools
  def build_world_from_students(rand, yaml)
    num_students = yaml["studentCount"]
    @log.info "Creating world from initial number of students: #{num_students}"

    # create grade breakdown from total number of students
    # populate education organization structure using breakdown (number of students per grade)
    # update structure with time information
    # use time information to create master schedule (course offerings, sections)
    # finally, write interchanges
    compute_grade_breakdown(rand, yaml, num_students)
    @log.info "breakdown by grade: #{@breakdown} --> total number of students: #{@breakdown.values.inject(:+)}"
    create_edOrgs_using_breakdown(rand, yaml)
    add_time_information_to_edOrgs(rand, yaml)
    create_master_schedule(rand, yaml)
    write_interchanges(rand, yaml)
  end

  # iterates through the set of ordered grade levels (Kindergarten through 12th grade) and uses
  # the default percentages (as defined in yaml configuration file) for minimum and maximum per grade
  # to compute the number of students that will initially populate the specified grade level.
  def compute_grade_breakdown(rand, yaml, num_students)
    students_so_far = 0
    GradeLevelType::get_ordered_grades.each do |grade|
      num_students_this_grade = get_num_students_per_grade(rand, yaml, num_students)      
      num_students_this_grade = num_students - students_so_far if grade == :TWELFTH_GRADE
      @breakdown[grade]       = num_students_this_grade
      students_so_far         += num_students_this_grade
    end
  end

  # randomly select the number of students for the current grade by using the minimum percentage,
  # maximum percentage, and total number of students
  def get_num_students_per_grade(rand, yaml, num_students)
    min = yaml["MINIMUM_GRADE_PERCENTAGE"]
    max = yaml["MAXIMUM_GRADE_PERCENTAGE"]

    if min.nil?
      @log.error "MINIMUM_GRADE_PERCENTAGE must be set for a world to be created --> Exiting..."
      abort
    end
    if max.nil?
      @log.error "MAXIMUM_GRADE_PERCENTAGE must be set for a world to be created --> Exiting..."
      abort
    end

    ((random_on_interval(rand, min, max) / 100) * num_students).round
  end

  # Uses the breakdown hash to bucketize number of students according to type of school
  # they would be enrolled at, and then computes the number of schools required to hold those
  # students according to national averages (average students per elementary school, ...).
  def create_edOrgs_using_breakdown(rand, yaml)
    num_elementary_school_students = 0
    num_middle_school_students     = 0
    num_high_school_students       = 0

    GradeLevelType::elementary.each do |level| 
      num_elementary_school_students += @breakdown[level]
    end

    GradeLevelType::middle.each do |level|
      num_middle_school_students += @breakdown[level]
    end

    GradeLevelType::high.each do |level|
      num_high_school_students += @breakdown[level]
    end

    num_schools   = 0
    num_schools   = create_schools(rand, yaml, "elementary", num_schools, num_elementary_school_students)
    num_schools   = create_schools(rand, yaml, "middle", num_schools, num_middle_school_students)
    num_schools   = create_schools(rand, yaml, "high", num_schools, num_high_school_students)
    num_districts = update_schools_with_districts(rand, yaml, num_schools)
    update_districts_with_states(rand, yaml, num_districts)
  end

  # uses the total number of students, as well as range of [min, max] to compute students at a school of type 'tag', and then
  # populates the edOrgs structure according to that tag (which should be "elementary", "middle", or "high").
  def create_schools(rand, yaml, tag, num_schools, total_num_students)
    if tag == "elementary"
      avg_num_students = yaml["AVERAGE_ELEMENTARY_SCHOOL_NUM_STUDENTS"]
    elsif tag == "middle"
      avg_num_students = yaml["AVERAGE_MIDDLE_SCHOOL_NUM_STUDENTS"]
    elsif tag == "high"
      avg_num_students = yaml["AVERAGE_HIGH_SCHOOL_NUM_STUDENTS"]
    end
    avg_num_students_threshold = yaml["AVERAGE_NUM_STUDENTS_THRESHOLD"]
    min                        = avg_num_students - (avg_num_students * avg_num_students_threshold)
    max                        = avg_num_students + (avg_num_students * avg_num_students_threshold)
    school_counter             = num_schools
    student_counter            = 0
    while student_counter < total_num_students
      current_students = random_on_interval(rand, min, max).round
      school_counter  += 1
      if current_students > (total_num_students - student_counter)        
        current_students = (total_num_students - student_counter)
      end
      student_counter += current_students
      
      edOrg              = Hash.new
      edOrg["id"]        = school_counter
      edOrg["parent"]    = nil
      edOrg["programs"]  = []
      edOrg["sessions"]  = []
      edOrg["staff"]     = create_staff_for_school(rand, yaml)
      edOrg["offerings"] = Hash.new
      edOrg["students"]  = Hash.new
      begin_year         = yaml["beginYear"]
      edOrg["students"][begin_year] = current_students

      @edOrgs[tag]       << edOrg
    end
    return school_counter
  end

  # updates the populated edOrgs arrays for elementary, middle, and high schools by determining how many schools are to be
  # contained in a given district (looks at yaml configuration file for average number of schools per district and threshold),
  # and then actually going and updating the schools to reference back to newly created local education agencies
  def update_schools_with_districts(rand, yaml, num_schools)
    avg_num_schools_per_district = yaml["AVERAGE_NUM_SCHOOLS_PER_DISTRICT"]
    avg_num_schools_threshold    = yaml["AVERAGE_SCHOOLS_THRESHOLD"]
    min_num_schools_per_district = avg_num_schools_per_district - (avg_num_schools_per_district * avg_num_schools_threshold)
    max_num_schools_per_district = avg_num_schools_per_district + (avg_num_schools_per_district * avg_num_schools_threshold)
    all_schools      = (1..num_schools).to_a
    district_counter = num_schools
    school_counter   = 0
    while all_schools.size > 0
      district_counter += 1
      num_schools_in_this_district = random_on_interval(rand, min_num_schools_per_district, max_num_schools_per_district).round
      
      if num_schools_in_this_district > (num_schools - school_counter)
        num_schools_in_this_district = (num_schools - school_counter)
      end

      schools_in_this_district = all_schools.pop(num_schools_in_this_district)      
      update_schools_with_district_id(district_counter, schools_in_this_district)
      
      edOrg             = Hash.new
      edOrg["id"]       = district_counter
      edOrg["parent"]   = nil
      edOrg["programs"] = []
      edOrg["sessions"] = []
      edOrg["staff"]    = create_staff_for_local_education_agency(rand, yaml)
      @edOrgs["leas"]   << edOrg
      school_counter    += num_schools_in_this_district
    end
    return district_counter
  end

  # actually does the work to iterate through the edOrgs structure (specifically the arrays
  # contained in the tags: "elementary", "middle", and "high" schools), finding edOrgs whose
  # unique "id" is contained within "schools_in_this_district", and sets the "parent" attribute
  # of those matching edOrgs to the district id.
  def update_schools_with_district_id(district_id, schools_in_this_district)
    # check in elementary schools
    @edOrgs["elementary"].each do |edOrg|
      edOrg["parent"] = district_id if schools_in_this_district.include?(edOrg["id"])        
    end
    # check in middle schools
    @edOrgs["middle"].each do |edOrg|
      edOrg["parent"] = district_id if schools_in_this_district.include?(edOrg["id"])
    end
    # check in high schools
    @edOrgs["high"].each do |edOrg|
      edOrg["parent"] = district_id if schools_in_this_district.include?(edOrg["id"])
    end
  end

  # updates the populated edOrgs arrays for leas (local education agencies) by determining how many districts are to be
  # contained in a given state. current implementation assumes:
  # - single tier for local education agencies
  # - all local education agencies are contained within a single state
  # 
  # future implementation should create more 'layers' within the local education agency 'tier'
  # future implementation should look at yaml for average number of districts in a state and create multiple 
  #  state education agencies, as needed
  def update_districts_with_states(rand, yaml, num_districts)
    state_id          = num_districts + 1
    edOrg             = Hash.new
    edOrg["id"]       = state_id
    edOrg["courses"]  = create_courses(yaml)
    edOrg["staff"]    = create_staff_for_state_education_agency(rand, yaml)
    edOrg["programs"] = []
    @edOrgs["seas"]  << edOrg
    
    @edOrgs["leas"].each do |edOrg| 
      edOrg["parent"] = state_id
    end
  end

  # creates staff members at the state education agency level
  def create_staff_for_state_education_agency(rand, yaml)
    members = []
    members
  end

  # creates staff members at the local education agency level
  def create_staff_for_local_education_agency(rand, yaml)
    members = []
    members
  end  

  # creates staff members at the school level
  def create_staff_for_school(rand, yaml)
    members = []
    members
  end

  def build_world_from_edOrgs(rand, yaml)
  	num_schools = yaml["schoolCount"]
    @log.info "Creating world from initial number of schools: #{num_schools}"
    # NOT CURRENTLY SUPPORTED
    # update structure with time information
    add_time_information_to_edOrgs(rand, yaml)
  end

  # creates sessions for each local education agency
  # -> uses 'beginYear' and 'numYears' properties specified in yaml configuration file (defaults to current year and 1, respectively, if not specified)
  # -> iterates from begin year to (begin year + num years), creating Sessions, Grading Periods, and Calendar Dates
  # -> each Session stores a date interval (contains start date, end date, number of instructional days, and holidays for that interval)
  def add_time_information_to_edOrgs(rand, yaml)
    begin_year   = yaml["beginYear"]
    num_years    = yaml["numYears"]
    
    if begin_year.nil?
      this_year = Date.today.year
      @log.info "Property: beginYear --> not set for scenario. Using default: #{this_year}"
      begin_year = this_year
    end

    if num_years.nil?
      @log.info "Property: numYears --> not set for scenario. Using default: 1"
      num_years = 1
    end

    # loop over years updating infrastructure and population
    for year in begin_year..(begin_year+num_years-1) do
      # create session for LEAs
      # -> create grading period(s) for session
      # -> create calendar date(s) for grading period
      # -> [not implemented] iterate through schools and determine with some probability if school overrides existing lea session
      @log.info "creating session information for school year: #{year}-#{year+1}"
      @edOrgs["leas"].each_index do |index|
        edOrg = @edOrgs["leas"][index]
        state_organization_id = DataUtility.get_local_education_agency_id(edOrg["id"])
        start_date = DateUtility.random_school_day_on_interval(rand, Date.new(year, 8, 25), Date.new(year, 9, 10))
        interval   = DateInterval.create_using_start_and_num_days(rand, start_date, 180)

        session             = Hash.new
        session["term"]     = :YEAR_ROUND
        session["year"]     = year
        session["name"]     = year.to_s + "-" + (year+1).to_s + " " + SchoolTerm.to_string(:YEAR_ROUND) + " session: " + state_organization_id
        session["interval"] = interval
        session["edOrgId"]  = state_organization_id
        @edOrgs["leas"][index]["sessions"] << session
      end
    end

    # update each of the schools already in existence with the sessions that were just created at the local education agency level
    # -> suggest implementing school 'extending' inherited sessions in these functions rather than above
    update_schools_with_sessions("elementary")
    update_schools_with_sessions("middle")
    update_schools_with_sessions("high")
  end

  # iterate through schools of matching 'type' and set the sessions to be used
  def update_schools_with_sessions(type)
    @edOrgs[type].each_index do |index|
      school = @edOrgs[type][index]
      @edOrgs[type][index]["sessions"] = get_sessions_to_be_used_by_school(school)
    end
  end
  
  # create master schedule interchange
  # -> create course offerings
  # -> create sections (using course offerings and breakdown of students per grade)
  # --> do NOT create teachers for course offerings at elementary school (this should be done for sections)
  # --> in middle and high schools, create teacher for course offering (teachers can teach multiple sections)
  def create_master_schedule(rand, yaml)
    offering_id = 0
    create_course_offerings_for_school("elementary", offering_id)
    create_course_offerings_for_school("middle", offering_id)
    create_course_offerings_for_school("high", offering_id)

    #update_shuffled_students(rand, yaml, "elementary")
    #update_shuffled_students(rand, yaml, "middle")
    #update_shuffled_students(rand, yaml, "high")
    #puts "all students shuffled during course of simulation: #{@edOrgs["students"]}"

    #update_student_counts_at_schools(rand, yaml)
  end

  # updates the number of students shuffled out of each type of school
  # --> used for determining number of incoming students for given year to next school type ('middle' --> 'high')
  # --> allows for consistent number of students to remain in system at any given time
  def update_shuffled_students(rand, yaml, type)
    @edOrgs[type].each_index do |index|
      school       = @edOrgs[type][index]
{'fraction_non_alphanumeric': 0.08183287492556782, 'fraction_numerical': 0.003119063146851164, 'mean_word_length': 3.6626123744050765, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 70, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2642944', 'n_tokens_mistral': 10843, 'n_tokens_neox': 10227, 'n_words': 2993}