<commit_before># -*- coding: utf-8 -*-
import json
import requests
from django.conf import settings
from main.models import Enterprise, Comuna

def downloadInfo():
    url = u'http://api.recursos.datos.gob.cl/datastreams/invoke/TURIS-' +\
        u'AVENT?auth_key=&(api_key)s&output=json_array' % \
        {'api_key': settings.API_KEY}

    response = requests.get(url)
    data = json.loads(response)
    table = data['result']
    for row in table:
        e = Enterprise()
        try:
            d_comuna = row[2]
            comuna = Comuna.objects.filter(name=d_comuna)
            if comuna.exists():
                comuna = comuna.first()
            e.comuna = comuna
        except:
            pass
        e.origin = row[3]
        e.destiny = row[4]
        e.name = row[5]
        e.address_name = row[6]
        e.address_number = row[7]
        e.address_extras = row[8]
        e.phone = row[9]
        e.web = row[10]

        found_tags = False
        #TODO: Acá debe parsear la info y buscar los tags

        if found_tags:
            e.save()<commit_msg>Agregada funcion metadata<commit_after># -*- coding: utf-8 -*-
import json
import requests
from django.conf import settings
from main.models import Enterprise, Comuna
from robobrowser import RoboBrowser

def downloadInfo():
    url = u'http://api.recursos.datos.gob.cl/datastreams/invoke/TURIS-' +\
        u'AVENT?auth_key=&(api_key)s&output=json_array' % \
        {'api_key': settings.API_KEY}

    response = requests.get(url)
    data = json.loads(response)
    table = data['result']
    for row in table:
        e = Enterprise()
        try:
            d_comuna = row[2]
            comuna = Comuna.objects.filter(name=d_comuna)
            if comuna.exists():
                comuna = comuna.first()
            e.comuna = comuna
        except:
            pass
        e.origin = row[3]
        e.destiny = row[4]
        e.name = row[5]
        e.address_name = row[6]
        e.address_number = row[7]
        e.address_extras = row[8]
        e.phone = row[9]
        e.web = row[10]

        found_tags = False
        #TODO: Acá debe parsear la info y buscar los tags

        if found_tags:
            e.save()

# [Utils: Crawler]
# @Return: List of Str or None
def metaData(url):
    # Variables:
    crawler     = RoboBrowser(history=True)
    endpoint    = "http://"+url 
    endpointSSL = "https://"+url 
    connFlag    = True
    metadata    = list()

    # Procedimientos:
    try:
        crawler.open(endpoint, verify=False)
    except:
        connFlag = False

    if not connFlag:
        try:
            crawler.open(endpointSSL, verify=False)
        except:
            return None
    
    metadata.append(crawler.select("meta"))
    return metadata
{'fraction_non_alphanumeric': 0.11424439624005785, 'fraction_numerical': 0.007953723788864787, 'mean_word_length': 1.9249471458773784, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6089600', 'n_tokens_mistral': 909, 'n_tokens_neox': 886, 'n_words': 203}