<commit_before>"""
"""
from __future__ import print_function

import urllib.parse
import asyncio
import time
import argparse

from xudd.tools import join_id
from xudd.hive import Hive
from xudd.actor import Actor
from xudd.lib.multiprocess import MultiProcessAmbassador


# Taken from asyncio docs
@asyncio.coroutine
def print_http_headers(url):
    url = urllib.parse.urlsplit(url)
    reader, writer = yield from asyncio.open_connection(url.hostname, 80)
    query = (
        "HEAD {url.path} HTTP/1.0\r\n"
        "Host: {url.hostname}\r\n"
        "\r\n").format(url=url)
    writer.write(query.encode("latin-1"))
    while True:
        line = yield from reader.readline()
        if not line:
            break

        line = line.decode("latin1").rstrip()
        if line:
            print("HTTP header> %s" % line)


class WebReader(Actor):
    def __init__(self, hive, id):
        super(WebReader, self).__init__(hive, id)

        self.message_routing.update(
            {"read_webs": self.read_webs,
             "chuckle_end": self.chuckle_end})

    def _setup_chuckle_end(self, future):
        self.hive.send_message(to=self.id, directive="chuckle_end")

    def chuckle_end(self, message):
        print("Haha!  Guess that's the end of that...")
        self.hive.send_shutdown()

    def read_webs(self, message):
        # Okay, first let's try doing this the "wait on future" style hack..
        # This is like asyncio support where we get the result back as a
        # message.
        # Pretty weird.
        print("first run via future_async()...")
        future = asyncio.async(print_http_headers(message.body["url"]))
        response = yield self.wait_on_future(future)
        print("after first run")

        # Next... Okay... This is Real Bona Fide asyncio style
        # "yield from" calls.
        #
        # The difference... you have to do just "yield".  The XUDD
        # actor takes care of the rest.
        print("now doing intermediate evil_print_http_headers...")
        url = message.body["url"]

        url = urllib.parse.urlsplit(url)
        reader, writer = yield asyncio.open_connection(url.hostname, 80)
        query = ('HEAD {url.path} HTTP/1.0\r\n'
                 'Host: {url.hostname}\r\n'
                 '\r\n').format(url=url)
        writer.write(query.encode('latin-1'))
        while True:
            line = yield reader.readline()
            if not line:
                break

            line = line.decode("latin1").rstrip()
            if line:
                print("HTTP header> %s" % line)
        print("Ultimate evil complete.")

        # Lastly, use the asyncio loop pretty much just connecting back to the
        # XUDDiverse manually
        print("before second one")
        future = asyncio.async(print_http_headers(message.body["url"]))
        future.add_done_callback(self._setup_chuckle_end)
        print("after second one's call")<commit_msg>Yield and yield from both work in asyncio calls<commit_after>"""
"""
from __future__ import print_function

import urllib.parse
import asyncio
import time
import argparse

from xudd.tools import join_id
from xudd.hive import Hive
from xudd.actor import Actor
from xudd.lib.multiprocess import MultiProcessAmbassador


# Taken from asyncio docs
@asyncio.coroutine
def print_http_headers(url):
    url = urllib.parse.urlsplit(url)
    reader, writer = yield from asyncio.open_connection(url.hostname, 80)
    query = (
        "HEAD {url.path} HTTP/1.0\r\n"
        "Host: {url.hostname}\r\n"
        "\r\n").format(url=url)
    writer.write(query.encode("latin-1"))
    while True:
        line = yield from reader.readline()
        if not line:
            break

        line = line.decode("latin1").rstrip()
        if line:
            print("HTTP header> %s" % line)


class WebReader(Actor):
    def __init__(self, hive, id):
        super(WebReader, self).__init__(hive, id)

        self.message_routing.update(
            {"read_webs": self.read_webs,
             "chuckle_end": self.chuckle_end})

    def _setup_chuckle_end(self, future):
        self.hive.send_message(to=self.id, directive="chuckle_end")

    def chuckle_end(self, message):
        print("Haha!  Guess that's the end of that...")
        self.hive.send_shutdown()

    def read_webs(self, message):
        # Okay, first let's try doing this the "wait on future" style hack..
        # This is like asyncio support where we get the result back as a
        # message.
        # Pretty weird.
        print("first run via future_async()...")
        future = asyncio.async(print_http_headers(message.body["url"]))
        response = yield self.wait_on_future(future)
        print("after first run")

        # Next... Okay... This is Real Bona Fide asyncio style
        # "yield from" calls.
        #
        # It looks like yield *or* yield from both work :o
        print("now doing intermediate evil_print_http_headers...")
        url = message.body["url"]

        url = urllib.parse.urlsplit(url)
        reader, writer = yield from asyncio.open_connection(url.hostname, 80)
        query = ('HEAD {url.path} HTTP/1.0\r\n'
                 'Host: {url.hostname}\r\n'
                 '\r\n').format(url=url)
        writer.write(query.encode('latin-1'))
        while True:
            line = yield from reader.readline()
            if not line:
                break

            line = line.decode("latin1").rstrip()
            if line:
                print("HTTP header> %s" % line)
        print("Ultimate evil complete.")

        # Lastly, use the asyncio loop pretty much just connecting back to the
        # XUDDiverse manually
        print("before second one")
        future = asyncio.async(print_http_headers(message.body["url"]))
        future.add_done_callback(self._setup_chuckle_end)
        print("after second one's call")
{'fraction_non_alphanumeric': 0.11351259360108917, 'fraction_numerical': 0.0040844111640571815, 'mean_word_length': 2.3354143019296254, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1843399', 'n_tokens_mistral': 1821, 'n_tokens_neox': 1763, 'n_words': 475}