<commit_before>//
//  AWSService.swift
//  AWSSDKSwift
//
//  Created by <NAME> on 2017/03/28.
//
//

import Foundation
import SwiftyJSON
import AWSSDKSwiftCore

enum AWSServiceError: Error {
    case eventStreamingCodeGenerationsAreUnsupported
}

struct AWSService {
    let apiJSON: JSON
    let docJSON: JSON
    let endpointJSON: JSON
    var shapes = [Shape]()
    var operations: [Operation] = []
    var errorShapeNames = [String]()
    var shapeDoc: [String: [String: String]] = [:]

    var version: String {
        return apiJSON["metadata"]["apiVersion"].stringValue
    }

    var jsonVersion: ServiceProtocol.Version? {
        if let version = apiJSON["metadata"]["jsonVersion"].string {
            let componets = version.components(separatedBy: ".")
            return ServiceProtocol.Version(major: Int(componets[0])!, minor: Int(componets[1])!)
        }

        return nil
    }

    var serviceProtocol: ServiceProtocol {
        if let version = jsonVersion {
            return ServiceProtocol(name: apiJSON["metadata"]["protocol"].stringValue, version: version)
        }
        return ServiceProtocol(name: apiJSON["metadata"]["protocol"].stringValue)
    }

    var endpointPrefix: String {
        return apiJSON["metadata"]["endpointPrefix"].stringValue
    }

    var serviceName: String {
        return apiJSON["serviceName"].stringValue.toSwiftClassCase()
    }

    var serviceErrorName: String {
        return serviceName+"ErrorType"
    }

    var serviceDescription: String {
        return docJSON["service"].stringValue.tagStriped()
    }

    var endpoint: JSON {
        return endpointJSON["partitions"].arrayValue[0]["services"][endpointPrefix]
    }

    var serviceEndpoints: [String: String] {
        var endpointMap: [String: String] = [:]
        endpoint["endpoints"].dictionaryValue.forEach {
            if let hostname = $0.value["hostname"].string {
                endpointMap[$0.key] = hostname
            }
        }
        return endpointMap
    }

    var partitionEndpoint: String? {
        return endpoint.dictionaryValue["partitionEndpoint"]?.string
    }

    init(fromAPIJSON apiJSON: JSON, docJSON: JSON, endpointJSON: JSON) throws {
        self.apiJSON = patch(apiJSON)
        self.docJSON = docJSON
        self.endpointJSON = endpointJSON
        self.shapes = try parseShapes()
        (self.operations, self.errorShapeNames) = try parseOperation(shapes: shapes)
        self.shapeDoc = parseDoc()
    }

    private func shapeType(from json: JSON, level: Int = 0) throws -> ShapeType {
        // TODO 10 is unspecific number to break
        if level > 10 {
            return .unhandledType
        }

        if let isEventstream = json["eventstream"].bool, isEventstream {
            throw AWSServiceError.eventStreamingCodeGenerationsAreUnsupported
        }

        let type: ShapeType
        switch json["type"].stringValue {
        case "string":
            if let enumValues = json["enum"].arrayObject as? [String] {
                type = .enum(enumValues)
            } else {
                let max = json["max"].int
                let min = json["min"].int
                let pattern = json["pattern"].string
                type = .string(max: max, min: min, pattern: pattern)
            }

        case "integer":
            let max = json["max"].int
            let min = json["min"].int
            type = .integer(max: max, min: min)

        case "blob":
            let max = json["max"].int
            let min = json["min"].int
            type = .blob(max: max, min: min)


        case "list":
            let shapeJSON = apiJSON["shapes"][json["member"]["shape"].stringValue]
            /*if let locationName = json["member"]["locationName"].string, json["flattened"].bool != true {
                let _type = try shapeType(from: shapeJSON, level: level+1)
                let repeats = Shape(name: json["member"]["shape"].stringValue, type: _type)
                let shape = Shape(name: json["member"]["shape"].stringValue, type: .list(repeats))
                let member = Member(
                    name: locationName,
                    required: false,
                    shape: shape,
                    location: nil,
                    locationName: locationName,
                    collectionEncoding: nil,
                    xmlNamespace: nil,
                    isStreaming: false
                )
                type = .structure(StructureShape(members: [member], payload: nil))
            } else {*/
                let _type = try shapeType(from: shapeJSON, level: level+1)
                let shape = Shape(name: json["member"]["shape"].stringValue, type: _type)
                type = .list(shape)
            //}

        case "structure":
            var structure: [String: ShapeType] = [:]
            for (_, _struct) in json["members"].dictionaryValue {
                let shapeJSON = apiJSON["shapes"][_struct["shape"].stringValue]
                structure[_struct["shape"].stringValue] = try shapeType(from: shapeJSON, level: level+1)
            }

            let members: [Member] = try json["members"].dictionaryValue.map { name, memberJSON in
                let name = name
                let memberDict = try JSONSerialization.jsonObject(with: memberJSON.rawData(), options: []) as? [String: Any] ?? [:]
                let shapeName = memberJSON["shape"].stringValue
                let requireds = json["required"].arrayValue.map({ $0.stringValue })
                let dict = structure.filter({ $0.key == shapeName }).first!
                let shape = Shape(name: dict.key, type: dict.value)
                var location = Location(json: memberJSON)
                var locationName = memberJSON["locationName"].string
                // if member shape was flattened and has a location name then use that as the location name
                let shapeJSON = apiJSON["shapes"][shapeName]
                var encoding : CollectionEncoding? = nil
                // xml and query encoding needs collection encoding information
                switch serviceProtocol.type {
                case .query, .restxml:
                    encoding = CollectionEncoding(json: shapeJSON)
                default:
                    break
                }
                // if list is flattened
                if let flattenedLocationName = shapeJSON["member"]["locationName"].string, shapeJSON["flattened"].bool == true {
                    location = Location(json: shapeJSON["member"])
                    locationName = flattenedLocationName
                }
                // if this instance of the collection change the encoding to the equivalent flattened version
                if memberJSON["flattened"].bool == true, let encoding2 = encoding {
                    switch encoding2 {
                    case .list(_):
                        encoding = .flatList
                    case .map(_, let key, let value):
                        encoding = .flatMap(key: key, value: value)
                    default:
                        break;
                    }
                }
                return Member(
                    name: name,
                    required: requireds.contains(name),
                    shape: shape,
                    location: location,
                    locationName: locationName,
                    collectionEncoding: encoding,
                    xmlNamespace: XMLNamespace(dictionary: memberDict),
                    isStreaming: memberJSON["streaming"].bool ?? false
                )
            }.sorted{ $0.name < $1.name }

            let shape = StructureShape(members: members, payload: json["payload"].string)
            type = .structure(shape)
<commit_msg>Comment changes and removed commented out code<commit_after>//
//  AWSService.swift
//  AWSSDKSwift
//
//  Created by <NAME> on 2017/03/28.
//
//

import Foundation
import SwiftyJSON
import AWSSDKSwiftCore

enum AWSServiceError: Error {
    case eventStreamingCodeGenerationsAreUnsupported
}

struct AWSService {
    let apiJSON: JSON
    let docJSON: JSON
    let endpointJSON: JSON
    var shapes = [Shape]()
    var operations: [Operation] = []
    var errorShapeNames = [String]()
    var shapeDoc: [String: [String: String]] = [:]

    var version: String {
        return apiJSON["metadata"]["apiVersion"].stringValue
    }

    var jsonVersion: ServiceProtocol.Version? {
        if let version = apiJSON["metadata"]["jsonVersion"].string {
            let componets = version.components(separatedBy: ".")
            return ServiceProtocol.Version(major: Int(componets[0])!, minor: Int(componets[1])!)
        }

        return nil
    }

    var serviceProtocol: ServiceProtocol {
        if let version = jsonVersion {
            return ServiceProtocol(name: apiJSON["metadata"]["protocol"].stringValue, version: version)
        }
        return ServiceProtocol(name: apiJSON["metadata"]["protocol"].stringValue)
    }

    var endpointPrefix: String {
        return apiJSON["metadata"]["endpointPrefix"].stringValue
    }

    var serviceName: String {
        return apiJSON["serviceName"].stringValue.toSwiftClassCase()
    }

    var serviceErrorName: String {
        return serviceName+"ErrorType"
    }

    var serviceDescription: String {
        return docJSON["service"].stringValue.tagStriped()
    }

    var endpoint: JSON {
        return endpointJSON["partitions"].arrayValue[0]["services"][endpointPrefix]
    }

    var serviceEndpoints: [String: String] {
        var endpointMap: [String: String] = [:]
        endpoint["endpoints"].dictionaryValue.forEach {
            if let hostname = $0.value["hostname"].string {
                endpointMap[$0.key] = hostname
            }
        }
        return endpointMap
    }

    var partitionEndpoint: String? {
        return endpoint.dictionaryValue["partitionEndpoint"]?.string
    }

    init(fromAPIJSON apiJSON: JSON, docJSON: JSON, endpointJSON: JSON) throws {
        self.apiJSON = patch(apiJSON)
        self.docJSON = docJSON
        self.endpointJSON = endpointJSON
        self.shapes = try parseShapes()
        (self.operations, self.errorShapeNames) = try parseOperation(shapes: shapes)
        self.shapeDoc = parseDoc()
    }

    private func shapeType(from json: JSON, level: Int = 0) throws -> ShapeType {
        // TODO 10 is unspecific number to break
        if level > 10 {
            return .unhandledType
        }

        if let isEventstream = json["eventstream"].bool, isEventstream {
            throw AWSServiceError.eventStreamingCodeGenerationsAreUnsupported
        }

        let type: ShapeType
        switch json["type"].stringValue {
        case "string":
            if let enumValues = json["enum"].arrayObject as? [String] {
                type = .enum(enumValues)
            } else {
                let max = json["max"].int
                let min = json["min"].int
                let pattern = json["pattern"].string
                type = .string(max: max, min: min, pattern: pattern)
            }

        case "integer":
            let max = json["max"].int
            let min = json["min"].int
            type = .integer(max: max, min: min)

        case "blob":
            let max = json["max"].int
            let min = json["min"].int
            type = .blob(max: max, min: min)


        case "list":
            let shapeJSON = apiJSON["shapes"][json["member"]["shape"].stringValue]
            let _type = try shapeType(from: shapeJSON, level: level+1)
            let shape = Shape(name: json["member"]["shape"].stringValue, type: _type)
            type = .list(shape)

        case "structure":
            // Note that we need to do some extra preprocessing to clean up the formatting of the structure. Sometimes we have a "flattened" object which does not have extra fields (typically named `members`) wrapping the contents. Other times we do, and need to clear that out.
            var structure: [String: ShapeType] = [:]
            for (_, _struct) in json["members"].dictionaryValue {
                let shapeJSON = apiJSON["shapes"][_struct["shape"].stringValue]
                structure[_struct["shape"].stringValue] = try shapeType(from: shapeJSON, level: level+1)
            }

            let members: [Member] = try json["members"].dictionaryValue.map { name, memberJSON in
                let name = name
                let memberDict = try JSONSerialization.jsonObject(with: memberJSON.rawData(), options: []) as? [String: Any] ?? [:]
                let shapeName = memberJSON["shape"].stringValue
                let requireds = json["required"].arrayValue.map({ $0.stringValue })
                let dict = structure.filter({ $0.key == shapeName }).first!
                let shape = Shape(name: dict.key, type: dict.value)
                var location = Location(json: memberJSON)
                var locationName = memberJSON["locationName"].string
                // if member shape was flattened and has a location name then use that as the location name
                let shapeJSON = apiJSON["shapes"][shapeName]
                var encoding : CollectionEncoding? = nil
                // xml and query encoding needs collection encoding information
                switch serviceProtocol.type {
                case .query, .restxml:
                    encoding = CollectionEncoding(json: shapeJSON)
                default:
                    break
                }
                // If the list is flattened, then we need to pull out the right location name
                if let flattenedLocationName = shapeJSON["member"]["locationName"].string, shapeJSON["flattened"].bool == true {
                    location = Location(json: shapeJSON["member"])
                    locationName = flattenedLocationName
                }
                // If this struct should be flattened, then convert from the original format to the equivalent flattened version.
                if memberJSON["flattened"].bool == true, let encoding2 = encoding {
                    switch encoding2 {
                    case .list(_):
                        encoding = .flatList
                    case .map(_, let key, let value):
                        encoding = .flatMap(key: key, value: value)
                    default:
                        break;
                    }
                }
                return Member(
                    name: name,
                    required: requireds.contains(name),
                    shape: shape,
                    location: location,
                    locationName: locationName,
                    collectionEncoding: encoding,
                    xmlNamespace: XMLNamespace(dictionary: memberDict),
                    isStreaming: memberJSON["streaming"].bool ?? false
                )
            }.sorted{ $0.name < $1.name }

            let shape = StructureShape(members: members, payload: json["payload"].string)
            type = .structure(shape)

{'fraction_non_alphanumeric': 0.10351225977468523, 'fraction_numerical': 0.003512259774685222, 'mean_word_length': 1.8871245456284675, 'pattern_counts': {'":': 10, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 7}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5921488', 'n_tokens_mistral': 3878, 'n_tokens_neox': 3650, 'n_words': 1017}