<commit_before>import requests
from bs4 import BeautifulSoup
from Crawling.mk_data import mk_data
from Crawling.mk_meta import mk_meta
import time
import datetime
from multiprocessing import Pool
import json
import git
import os
import socket
from uuid import getnode
import psutil

GET_URL = 'https://robin-api.oneprice.co.kr/tasks?agent='
POST_URL = 'https://robin-api.oneprice.co.kr/items'
DEVICE_URL = 'https://robin-api.oneprice.co.kr/agents/enroll'
MSG_URL = 'https://robin-api.oneprice.co.kr/agents/msg'
URL_F = 'http://shopping.naver.com/detail/detail.nhn?nv_mid='
URL_M = '&pkey='
URL_T = '&withFee='

def getNowDate():
    now = datetime.datetime.now()
    year = now.year
    month = now.month
    day = now.day
    hour = now.hour
    minute = now.minute
    nowDate = str(year) + "-" + str(month) + "-" + str(day) + "," + str(hour) + ":" + str(minute)
    return nowDate


def chk_ver(version):
    path = os.path.dirname(os.path.abspath('Run.py'))
    data = {}
    try:
        with open(path + '/version.json', "r") as f:
            json_data = f.read()
            data = json.loads(json_data)
        if version != data['version']:
            g = git.cmd.Git(path)
            print(g.pull())
        with open(path + '/version.json', "w") as f:
            data['version'] = version
            data['date'] = getNowDate()
            data['success'] = True
            json.dump(data, f, ensure_ascii=False, indent="\t")
    except:
        with open(path + '/version.json', "w") as f:
            data['date'] = getNowDate()
            data['success'] = False
            data['version'] = ''
            json.dump(data, f, ensure_ascii=False, indent="\t")


def get_MID():
    while 1:
        try:
            req = requests.get(GET_URL + str(socket.gethostname()))
            r_json = req.json()[0]
            data = {
                'name': str(socket.gethostname()),
                'uuid': str(getnode()),
            }
            requests.post(DEVICE_URL, json=data)
            chk_ver(r_json['clientVersion'])
            return r_json
        except:
            print('Server is Down')
            time.sleep(10)


def post(info, data_list):
    data = {
        'id': 'nv_' + info['id'],
        'mid': info['mid'],
        'data': data_list,
    }
    while 1:
        try:
            requests.post(POST_URL, json=data)
            return
        except:
            print('Server is Down')
            time.sleep(10)


def get_pkey(mid):
    work_list = []
    req = requests.get(URL_F + mid)
    html = req.text
    soup = BeautifulSoup(html, 'html.parser')
    try:
        valid_txt = soup.find('h3', class_='release').find(text=True)
        if '판매중단' in valid_txt:
            tmp_list = []
            tmp_list.append(mid)
            tmp_list.append(True)<commit_msg>Add agent<commit_after>import requests
from bs4 import BeautifulSoup
from Crawling.mk_data import mk_data
from Crawling.mk_meta import mk_meta
import time
import datetime
from multiprocessing import Pool
import json
import git
import os
import socket
from uuid import getnode
import psutil

GET_URL = 'https://robin-api.oneprice.co.kr/tasks?agent='
POST_URL = 'https://robin-api.oneprice.co.kr/items'
DEVICE_URL = 'https://robin-api.oneprice.co.kr/agents/enroll'
MSG_URL = 'https://robin-api.oneprice.co.kr/agents/msg'
URL_F = 'http://shopping.naver.com/detail/detail.nhn?nv_mid='
URL_M = '&pkey='
URL_T = '&withFee='

def getNowDate():
    now = datetime.datetime.now()
    year = now.year
    month = now.month
    day = now.day
    hour = now.hour
    minute = now.minute
    nowDate = str(year) + "-" + str(month) + "-" + str(day) + "," + str(hour) + ":" + str(minute)
    return nowDate


def chk_ver(version):
    path = os.path.dirname(os.path.abspath('Run.py'))
    data = {}
    try:
        with open(path + '/version.json', "r") as f:
            json_data = f.read()
            data = json.loads(json_data)
        if version != data['version']:
            g = git.cmd.Git(path)
            print(g.pull())
        with open(path + '/version.json', "w") as f:
            data['version'] = version
            data['date'] = getNowDate()
            data['success'] = True
            json.dump(data, f, ensure_ascii=False, indent="\t")
    except:
        with open(path + '/version.json', "w") as f:
            data['date'] = getNowDate()
            data['success'] = False
            data['version'] = ''
            json.dump(data, f, ensure_ascii=False, indent="\t")


def get_MID():
    while 1:
        try:
            req = requests.get(GET_URL + str(socket.gethostname()))
            r_json = req.json()[0]
            data = {
                'name': str(socket.gethostname()),
                'uuid': str(getnode()),
            }
            requests.post(DEVICE_URL, json=data)
            chk_ver(r_json['clientVersion'])
            return r_json
        except:
            print('Server is Down')
            time.sleep(10)


def post(info, data_list):
    data = {
        'agent': str(socket.gethostname()),
        'id': info['id'],
        'mid': info['mid'],
        'data': data_list
    }
    while 1:
        try:
            requests.post(POST_URL, json=data)
            return
        except:
            print('Server is Down')
            time.sleep(10)


def get_pkey(mid):
    work_list = []
    req = requests.get(URL_F + mid)
    html = req.text
    soup = BeautifulSoup(html, 'html.parser')
    try:
        valid_txt = soup.find('h3', class_='release').find(text=True)
        if '판매중단' in valid_txt:
            tmp_list = []
            tmp_list.append(mid)
            tmp_list.append(True)
{'fraction_non_alphanumeric': 0.14669020300088262, 'fraction_numerical': 0.003177405119152692, 'mean_word_length': 2.1114772103239976, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 8, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '24677', 'n_tokens_mistral': 1874, 'n_tokens_neox': 1866, 'n_words': 391}