<commit_before>################################################################################
## Toolbox: Add GTFS to a Network Dataset / Transit Analysis Tools
## Tool name: Calculate Accessibility Matrix
## Created by: <NAME>, Esri, <EMAIL>
## Last updated: 28 July 2017
################################################################################
'''Count the number of destinations reachable from each origin by transit and 
walking. The tool calculates an Origin-Destination Cost Matrix for each start 
time within a time window because the reachable destinations change depending 
on the time of day because of the transit schedules.  The output gives the 
total number of destinations reachable at least once as well as the number of 
destinations reachable at least 10%, 20%, ...90% of start times during the time 
window.  The number of reachable destinations can be weighted based on a field, 
such as the number of jobs available at each destination.  The tool also 
calculates the percentage of total destinations reachable.'''
################################################################################
'''Copyright 2017 Esri
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at
       http://www.apache.org/licenses/LICENSE-2.0
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.'''
################################################################################

import arcpy
import AnalysisHelpers
arcpy.env.overwriteOutput = True

class CustomError(Exception):
    pass

try:

    #Check out the Network Analyst extension license
    if arcpy.CheckExtension("Network") == "Available":
        arcpy.CheckOutExtension("Network")
    else:
        arcpy.AddError("You must have a Network Analyst license to use this tool.")
        raise CustomError


    # ----- Get and process inputs -----

    # OD layer from the map or .lyr file that with all the desired settings
    # (except time of day - we'll adjust that in this script)
    # Does not need Origins and Destinations loaded. We'll do that in the script.
    input_network_analyst_layer = arcpy.GetParameter(0)
    desc = arcpy.Describe(input_network_analyst_layer)
    if desc.dataType != "NALayer" or desc.solverName != "OD Cost Matrix Solver":
        arcpy.AddError("Input layer must be an OD Cost Matrix layer.")
        raise CustomError
    
    # Origins and Destinations
    origins_feature_class = arcpy.GetParameterAsText(1)
    destinations_feature_class = arcpy.GetParameterAsText(2)
    destinations_weight_field = arcpy.GetParameterAsText(3)

    # Start and end day and time
    start_day_input = arcpy.GetParameterAsText(4)
    end_day_input = arcpy.GetParameterAsText(6)
    start_time_input = arcpy.GetParameterAsText(5)
    end_time_input = arcpy.GetParameterAsText(7)
    increment_input = arcpy.GetParameter(8)

    # Make list of times of day to run the analysis
    timelist = AnalysisHelpers.make_analysis_time_of_day_list(start_day_input, end_day_input, start_time_input, end_time_input, increment_input)

    
    # ----- Add Origins and Destinations to the OD layer -----

    arcpy.AddMessage("Adding Origins and Destinations to OD Cost Matrix Layer...")

    # Get the sublayer names and objects for use later
    sublayer_names = arcpy.na.GetNAClassNames(input_network_analyst_layer) # To ensure compatibility with localized software
    origins_sublayer_name = sublayer_names["Origins"]
    destinations_sublayer_name = sublayer_names["Destinations"]
    lines_sublayer_name = sublayer_names["ODLines"]
    origins_subLayer = arcpy.mapping.ListLayers(input_network_analyst_layer, origins_sublayer_name)[0]
    destinations_subLayer = arcpy.mapping.ListLayers(input_network_analyst_layer, destinations_sublayer_name)[0]
    lines_subLayer = arcpy.mapping.ListLayers(input_network_analyst_layer, lines_sublayer_name)[0]

    # Keep track of the ObjectID field of the input
    desc = arcpy.Describe(origins_feature_class)
    origins_objectID = desc.OIDFieldName
    desc = arcpy.Describe(destinations_sublayer_name)
    destinations_objectID = desc.OIDFieldName
    arcpy.na.AddFieldToAnalysisLayer(input_network_analyst_layer, origins_sublayer_name, "InputOID", "LONG")
    fieldMappings_origins = arcpy.na.NAClassFieldMappings(input_network_analyst_layer, origins_sublayer_name)
    fieldMappings_origins["InputOID"].mappedFieldName = origins_objectID
    arcpy.na.AddFieldToAnalysisLayer(input_network_analyst_layer, destinations_sublayer_name, "InputOID", "LONG")
    fieldMappings_destinations = arcpy.na.NAClassFieldMappings(input_network_analyst_layer, destinations_sublayer_name)
    fieldMappings_destinations["InputOID"].mappedFieldName = destinations_objectID

    # Add origins and destinations
    arcpy.na.AddLocations(input_network_analyst_layer, origins_sublayer_name, origins_feature_class, fieldMappings_origins, "", append="CLEAR")
    arcpy.na.AddLocations(input_network_analyst_layer, destinations_sublayer_name, destinations_feature_class, fieldMappings_destinations, "", append="CLEAR")

    # Create dictionary linking the ObjectID fields of the input feature classes and the NA sublayers
    # We need to do this because, particularly when the NA layer already had data in it, the ObjectID
    # values don't always start with 1.
    origins_oid_dict = {} # {Input feature class Object ID: Origins sublayer OID}
    origin_ids = []
    with arcpy.da.SearchCursor(origins_subLayer, ["OID@", "InputOID"]) as cur:
        for row in cur:
            origin_ids.append(row[0])
            origins_oid_dict[row[1]] = row[0]
    destinations_oid_dict = {} # {Destination sublayer OID, Input feature class Object ID: }
    with arcpy.da.SearchCursor(destinations_subLayer, ["OID@", "InputOID"]) as cur:
         for row in cur:
             destinations_oid_dict[row[0]] = row[1]


    # ----- Solve NA layer in a loop for each time of day -----

    # Initialize a dictionary for counting the number of times each destination is reached by each origin
    OD_count_dict = {} # {Origin OID: {Destination OID: Number of times reached}}
    for oid in origin_ids:
        OD_count_dict[oid] = {}
        for did in destinations_oid_dict:
            OD_count_dict[oid][did] = 0

    # Grab the solver properties object from the NA layer so we can set the time of day
    solverProps = arcpy.na.GetSolverProperties(input_network_analyst_layer)

    # Solve for each time of day and save output
    arcpy.AddMessage("Solving OD Cost matrix at time...")
    for t in timelist:
        arcpy.AddMessage(str(t))
        
        # Switch the time of day
        solverProps.timeOfDay = t
        
        # Solve the OD Cost Matrix
        arcpy.na.Solve(input_network_analyst_layer)

        # Read the OD matrix output and increment the dictionary
        # There is one entry in Lines for each OD pair that was reached within the cutoff time
        with arcpy.da.SearchCursor(lines_subLayer, ["OriginID", "DestinationID"]) as cur:
            for line in cur:
                OD_count_dict[line[0]][line[1]] += 1


    # ----- Calculate statistics and generate output -----

    arcpy.AddMessage("Calculating statistics and writing results...")

    # If the destinations are weighted (eg, number of jobs at each destination), track them here
    destination_weight_dict = {} # {Input Destinations feature class ObjectID: Weight}
    num_dests = 0
    if destinations_weight_field:
        with arcpy.da.SearchCursor(destinations_feature_class, ["OID@", destinations_weight_field]) as cur:
            for row in cur:
                destination_weight_dict[row[0]] = row[1]
                num_dests += row[1]
    else:
        num_dests = len(destinations_oid_dict)

    # Add fields to input origins for output statistics. If the fields already exist, this will do nothing.
    arcpy.management.AddField(origins_feature_class, "TotalDests", "LONG")
    arcpy.management.AddField(origins_feature_class, "PercDests", "DOUBLE")
    stats_fields = ["TotalDests", "PercDests"]
    for i in range(1, 10):
        dest_field = "DsAL%i0Perc" % i
        perc_field = "PsAL%i0Perc" % i
        stats_fields.append(dest_field)
        stats_fields.append(perc_field)<commit_msg>Memory improvement and error handling<commit_after>################################################################################
## Toolbox: Add GTFS to a Network Dataset / Transit Analysis Tools
## Tool name: Calculate Accessibility Matrix
## Created by: <NAME>, Esri, <EMAIL>
## Last updated: 28 July 2017
################################################################################
'''Count the number of destinations reachable from each origin by transit and 
walking. The tool calculates an Origin-Destination Cost Matrix for each start 
time within a time window because the reachable destinations change depending 
on the time of day because of the transit schedules.  The output gives the 
total number of destinations reachable at least once as well as the number of 
destinations reachable at least 10%, 20%, ...90% of start times during the time 
window.  The number of reachable destinations can be weighted based on a field, 
such as the number of jobs available at each destination.  The tool also 
calculates the percentage of total destinations reachable.'''
################################################################################
'''Copyright 2017 Esri
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at
       http://www.apache.org/licenses/LICENSE-2.0
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.'''
################################################################################

import arcpy
import AnalysisHelpers
arcpy.env.overwriteOutput = True

class CustomError(Exception):
    pass

try:

    #Check out the Network Analyst extension license
    if arcpy.CheckExtension("Network") == "Available":
        arcpy.CheckOutExtension("Network")
    else:
        arcpy.AddError("You must have a Network Analyst license to use this tool.")
        raise CustomError


    # ----- Get and process inputs -----

    # OD layer from the map or .lyr file that with all the desired settings
    # (except time of day - we'll adjust that in this script)
    # Does not need Origins and Destinations loaded. We'll do that in the script.
    input_network_analyst_layer = arcpy.GetParameter(0)
    desc = arcpy.Describe(input_network_analyst_layer)
    if desc.dataType != "NALayer" or desc.solverName != "OD Cost Matrix Solver":
        arcpy.AddError("Input layer must be an OD Cost Matrix layer.")
        raise CustomError
    
    # Origins and Destinations
    origins_feature_class = arcpy.GetParameterAsText(1)
    destinations_feature_class = arcpy.GetParameterAsText(2)
    destinations_weight_field = arcpy.GetParameterAsText(3)

    # Start and end day and time
    start_day_input = arcpy.GetParameterAsText(4)
    end_day_input = arcpy.GetParameterAsText(6)
    start_time_input = arcpy.GetParameterAsText(5)
    end_time_input = arcpy.GetParameterAsText(7)
    increment_input = arcpy.GetParameter(8)

    # Make list of times of day to run the analysis
    timelist = AnalysisHelpers.make_analysis_time_of_day_list(start_day_input, end_day_input, start_time_input, end_time_input, increment_input)

    
    # ----- Add Origins and Destinations to the OD layer -----

    arcpy.AddMessage("Adding Origins and Destinations to OD Cost Matrix Layer...")

    # Get the sublayer names and objects for use later
    sublayer_names = arcpy.na.GetNAClassNames(input_network_analyst_layer) # To ensure compatibility with localized software
    origins_sublayer_name = sublayer_names["Origins"]
    destinations_sublayer_name = sublayer_names["Destinations"]
    lines_sublayer_name = sublayer_names["ODLines"]
    origins_subLayer = arcpy.mapping.ListLayers(input_network_analyst_layer, origins_sublayer_name)[0]
    destinations_subLayer = arcpy.mapping.ListLayers(input_network_analyst_layer, destinations_sublayer_name)[0]
    lines_subLayer = arcpy.mapping.ListLayers(input_network_analyst_layer, lines_sublayer_name)[0]

    # Keep track of the ObjectID field of the input
    desc = arcpy.Describe(origins_feature_class)
    origins_objectID = desc.OIDFieldName
    desc = arcpy.Describe(destinations_sublayer_name)
    destinations_objectID = desc.OIDFieldName
    arcpy.na.AddFieldToAnalysisLayer(input_network_analyst_layer, origins_sublayer_name, "InputOID", "LONG")
    fieldMappings_origins = arcpy.na.NAClassFieldMappings(input_network_analyst_layer, origins_sublayer_name)
    fieldMappings_origins["InputOID"].mappedFieldName = origins_objectID
    arcpy.na.AddFieldToAnalysisLayer(input_network_analyst_layer, destinations_sublayer_name, "InputOID", "LONG")
    fieldMappings_destinations = arcpy.na.NAClassFieldMappings(input_network_analyst_layer, destinations_sublayer_name)
    fieldMappings_destinations["InputOID"].mappedFieldName = destinations_objectID

    # Add origins and destinations
    arcpy.na.AddLocations(input_network_analyst_layer, origins_sublayer_name, origins_feature_class, fieldMappings_origins, "", append="CLEAR")
    arcpy.na.AddLocations(input_network_analyst_layer, destinations_sublayer_name, destinations_feature_class, fieldMappings_destinations, "", append="CLEAR")

    # Create dictionary linking the ObjectID fields of the input feature classes and the NA sublayers
    # We need to do this because, particularly when the NA layer already had data in it, the ObjectID
    # values don't always start with 1.
    origins_oid_dict = {} # {Input feature class Object ID: Origins sublayer OID}
    origin_ids = []
    with arcpy.da.SearchCursor(origins_subLayer, ["OID@", "InputOID"]) as cur:
        for row in cur:
            origin_ids.append(row[0])
            origins_oid_dict[row[1]] = row[0]
    destinations_oid_dict = {} # {Destination sublayer OID, Input feature class Object ID: }
    with arcpy.da.SearchCursor(destinations_subLayer, ["OID@", "InputOID"]) as cur:
         for row in cur:
             destinations_oid_dict[row[0]] = row[1]


    # ----- Solve NA layer in a loop for each time of day -----

    # Initialize a dictionary for counting the number of times each destination is reached by each origin
    OD_count_dict = {} # {Origin OID: {Destination OID: Number of times reached}}

    # Grab the solver properties object from the NA layer so we can set the time of day
    solverProps = arcpy.na.GetSolverProperties(input_network_analyst_layer)

    # Solve for each time of day and save output
    arcpy.AddMessage("Solving OD Cost matrix at time...")
    for t in timelist:
        arcpy.AddMessage(str(t))
        
        # Switch the time of day
        solverProps.timeOfDay = t
        
        # Solve the OD Cost Matrix
        try:
            arcpy.na.Solve(input_network_analyst_layer)
        except:
            # Solve failed.  It could be that no destinations were reachable within the time limit,
            # or it could be another error.  Running out of memory is a distinct possibility.
            errs = arcpy.GetMessages(2)
            if "No solution found" not in errs:
                # Only alert them if it's some weird error.
                arcpy.AddMessage("Solve failed.  Errors: %s. Continuing to next time of day." % errs)
            continue

        # Read the OD matrix output and increment the dictionary
        # There is one entry in Lines for each OD pair that was reached within the cutoff time
        with arcpy.da.SearchCursor(lines_subLayer, ["OriginID", "DestinationID"]) as cur:
            for line in cur:
                if line[0] not in OD_count_dict:
                    OD_count_dict[line[0]] = {}
                if line[1] not in OD_count_dict[line[0]]:
                    OD_count_dict[line[0]][line[1]] = 1
                else:
                    OD_count_dict[line[0]][line[1]] += 1


    # ----- Calculate statistics and generate output -----

    arcpy.AddMessage("Calculating statistics and writing results...")

    # If the destinations are weighted (eg, number of jobs at each destination), track them here
    destination_weight_dict = {} # {Input Destinations feature class ObjectID: Weight}
    num_dests = 0
    if destinations_weight_field:
        with arcpy.da.SearchCursor(destinations_feature_class, ["OID@", destinations_weight_field]) as cur:
            for row in cur:
                destination_weight_dict[row[0]] = row[1]
                num_dests += row[1]
    else:
        num_dests = len(destinations_oid_dict)

    # Add fields to input origins for output statistics. If the fields already exist, this will do nothing.
    arcpy.management.AddField(origins_feature_class, "TotalDests", "LONG")
    arcpy.management.AddField(origins_feature_class, "PercDests", "DOUBLE")
    stats_fields = ["TotalDests", "PercDests"]
    for i in range(1, 10):
        dest_field = "DsAL%i0Perc" % i
        perc_field = "PsAL%i0Perc" % i
        stats_fields.append(dest_field)
        stats_fields.append(perc_field)
{'fraction_non_alphanumeric': 0.10824163485290815, 'fraction_numerical': 0.006119470020211093, 'mean_word_length': 3.8013477088948786, 'pattern_counts': {'":': 4, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2403272', 'n_tokens_mistral': 5067, 'n_tokens_neox': 4862, 'n_words': 1597}