<commit_before>from datetime import datetime
from typing import Optional, Sequence
from enum import Enum

from vnpy.event import EventEngine
from vnpy.trader.gateway import BaseGateway
from vnpy.trader.constant import (
    Exchange,
    Product,
    Offset,
    OrderType,
    Direction,
    Status
)
from vnpy.trader.object import (
    SubscribeRequest,
    CancelRequest,
    OrderRequest,
    ContractData,
    TickData,
    OrderData,
    TradeData,
    LogData
)

from .comstar_api import TdApi

VN_ENUMS = {
    "Exchange": Exchange,
    "Product": Product,
    "Offset": Offset,
    "OrderType": OrderType,
    "Direction": Direction,
    "Status": Status
}


class ComstarGateway(BaseGateway):
    """
    VN Trader Gateway for Comstar service.
    """

    default_setting = {
        "交易服务器": "",
        "用户名": "",
        "密码": "",
        "Key": ""
    }

    exchanges = [Exchange.CFETS]

    def __init__(self, event_engine: EventEngine):
        """Constructor"""
        super().__init__(event_engine, "COMSTAR")
        self.api = UserApi(self)

    def connect(self, setting: dict):
        """"""
        td_address = setting["交易服务器"]
        username = setting["用户名"]
        password = setting["密码"]
        key = setting["Key"]

        self.api.connect(username, password, key, td_address)

    def subscribe(self, req: SubscribeRequest):
        """"""
        # Symbol format: 180406_T0 or 180406_T1
        symbol, settle_type, *_ = req.symbol.split("_") + [""]
        if settle_type not in {"T0", "T1"}:
            self.write_log("请输入清算速度T0或T1")
            return ""

        data = vn_encode(req)
        data["symbol"] = symbol
        data["settle_type"] = settle_type
        self.api.subscribe(data, self.gateway_name)

    def send_order(self, req: OrderRequest):
        """"""
        # Offset is not supported for Comstar gateawy
        req.offset = Offset.NONE

        if req.type not in {OrderType.LIMIT, OrderType.FAK}:
            self.write_log("仅支持限价单和FAK单")
            return ""

        symbol, settle_type, *_ = req.symbol.split("_") + [""]
        if settle_type not in {"T0", "T1"}:
            self.write_log("请输入清算速度T0或T1")
            return ""

        data = vn_encode(req)
        data["symbol"] = symbol
        data["settle_type"] = settle_type
        data["strategy_name"] = data.pop("reference")

        order_id = self.api.send_order(data, self.gateway_name)
        # convert to vt_orderid
        return f"{self.gateway_name}.{order_id}"

    def cancel_order(self, req: CancelRequest):
        """"""
        data = vn_encode(req)
        symbol, settle_type, *_ = req.symbol.split("_") + [""]
        data["symbol"] = symbol
        data["settle_type"] = settle_type
        self.api.cancel_order(data, self.gateway_name)

    def query_account(self):
        """"""
        pass

    def query_position(self):
        """"""
        pass

    def query_all(self):
        """"""
        self.api.get_all_contracts()
        self.api.get_all_orders()
        self.api.get_all_trades()

    def close(self):
        """"""
        self.api.close()


class UserApi(TdApi):
    """
    Implements Comstar API.
    """

    def __init__(self, gateway: ComstarGateway):
        """Constructor"""
        super().__init__()

        self.gateway = gateway
        self.gateway_name = gateway.gateway_name

    def on_tick(self, tick: dict):
        """"""
        data = parse_tick(tick)
        self.gateway.on_tick(data)

    def on_order(self, order: dict):
        """"""
        data = parse_order(order)
        self.gateway.on_order(data)

    def on_trade(self, trade: dict):
        """"""
        data = parse_trade(trade)
        self.gateway.on_trade(data)

    def on_log(self, log: dict):
        data = parse_log(log)
        self.gateway.on_log(data)

    def on_login(self, data: dict):
        """"""
        if data["status"]:
            self.gateway.query_all()
            self.gateway.write_log("服务器登录成功")
        else:
            self.gateway.write_log("服务器登录失败")

    def on_disconnected(self, reason: str):
        """"""
        self.gateway.write_log(reason)

    def on_all_contracts(self, contracts: Sequence[dict]):
        """"""
        for data in contracts:
            for settle_type in ("T0", "T1"):
                contract = parse_contract(data, settle_type)
                contract.gateway_name = self.gateway_name
                self.gateway.on_contract(contract)

        self.gateway.write_log("合约信息查询成功")
<commit_msg>[Mod] filter duplicate data push after reconnect<commit_after>from datetime import datetime
from typing import Optional, Sequence, Dict
from enum import Enum

from vnpy.event import EventEngine
from vnpy.trader.gateway import BaseGateway
from vnpy.trader.constant import (
    Exchange,
    Product,
    Offset,
    OrderType,
    Direction,
    Status
)
from vnpy.trader.object import (
    SubscribeRequest,
    CancelRequest,
    OrderRequest,
    ContractData,
    TickData,
    OrderData,
    TradeData,
    LogData
)

from .comstar_api import TdApi

VN_ENUMS = {
    "Exchange": Exchange,
    "Product": Product,
    "Offset": Offset,
    "OrderType": OrderType,
    "Direction": Direction,
    "Status": Status
}


class ComstarGateway(BaseGateway):
    """
    VN Trader Gateway for Comstar service.
    """

    default_setting = {
        "交易服务器": "",
        "用户名": "",
        "密码": "",
        "Key": ""
    }

    exchanges = [Exchange.CFETS]

    def __init__(self, event_engine: EventEngine):
        """Constructor"""
        super().__init__(event_engine, "COMSTAR")
        self.api = UserApi(self)

    def connect(self, setting: dict):
        """"""
        td_address = setting["交易服务器"]
        username = setting["用户名"]
        password = setting["密码"]
        key = setting["Key"]

        self.api.connect(username, password, key, td_address)

    def subscribe(self, req: SubscribeRequest):
        """"""
        # Symbol format: 180406_T0 or 180406_T1
        symbol, settle_type, *_ = req.symbol.split("_") + [""]
        if settle_type not in {"T0", "T1"}:
            self.write_log("请输入清算速度T0或T1")
            return ""

        data = vn_encode(req)
        data["symbol"] = symbol
        data["settle_type"] = settle_type
        self.api.subscribe(data, self.gateway_name)

    def send_order(self, req: OrderRequest):
        """"""
        # Offset is not supported for Comstar gateawy
        req.offset = Offset.NONE

        if req.type not in {OrderType.LIMIT, OrderType.FAK}:
            self.write_log("仅支持限价单和FAK单")
            return ""

        symbol, settle_type, *_ = req.symbol.split("_") + [""]
        if settle_type not in {"T0", "T1"}:
            self.write_log("请输入清算速度T0或T1")
            return ""

        data = vn_encode(req)
        data["symbol"] = symbol
        data["settle_type"] = settle_type
        data["strategy_name"] = data.pop("reference")

        order_id = self.api.send_order(data, self.gateway_name)
        # convert to vt_orderid
        return f"{self.gateway_name}.{order_id}"

    def cancel_order(self, req: CancelRequest):
        """"""
        data = vn_encode(req)
        symbol, settle_type, *_ = req.symbol.split("_") + [""]
        data["symbol"] = symbol
        data["settle_type"] = settle_type
        self.api.cancel_order(data, self.gateway_name)

    def query_account(self):
        """"""
        pass

    def query_position(self):
        """"""
        pass

    def query_all(self):
        """"""
        self.api.get_all_contracts()
        self.api.get_all_orders()
        self.api.get_all_trades()

    def close(self):
        """"""
        self.api.close()


class UserApi(TdApi):
    """
    Implements Comstar API.
    """

    def __init__(self, gateway: ComstarGateway):
        """Constructor"""
        super().__init__()

        self.gateway = gateway
        self.gateway_name = gateway.gateway_name

        self.trades: Dict[str, TradeData] = {}
        self.orders: Dict[str, OrderData] = {}

    def on_tick(self, tick: dict):
        """"""
        data = parse_tick(tick)
        self.gateway.on_tick(data)

    def on_order(self, order: dict):
        """"""
        data = parse_order(order)

        # Filter duplicated order data push after reconnect
        last_order = self.orders.get(data.vt_orderid, None)
        if (
            last_order
            and data.traded == last_order.traded
            and data.status == last_order.status
        ):
            return
        self.orders[data.vt_orderid] = data

        self.gateway.on_order(data)

    def on_trade(self, trade: dict):
        """"""
        data = parse_trade(trade)

        # Filter duplicated trade data push after reconnect
        if data.vt_tradeid in self.trades:
            return
        self.trades[data.vt_tradeid] = data

        self.gateway.on_trade(data)

    def on_log(self, log: dict):
        data = parse_log(log)
        self.gateway.on_log(data)

    def on_login(self, data: dict):
        """"""
        if data["status"]:
            self.gateway.query_all()
            self.gateway.write_log("服务器登录成功")
        else:
            self.gateway.write_log("服务器登录失败")

    def on_disconnected(self, reason: str):
        """"""
        self.gateway.write_log(reason)

    def on_all_contracts(self, contracts: Sequence[dict]):
        """"""
        for data in contracts:
            for settle_type in ("T0", "T1"):
                contract = parse_contract(data, settle_type)
                contract.gateway_name = self.gateway_name
                self.gateway.on_contract(contract)

        self.gateway.write_log("合约信息查询成功")

{'fraction_non_alphanumeric': 0.12827387090121675, 'fraction_numerical': 0.004949474118374922, 'mean_word_length': 2.20522141440846, 'pattern_counts': {'":': 20, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3856041', 'n_tokens_mistral': 3202, 'n_tokens_neox': 3135, 'n_words': 610}