<commit_before>import base64
import io
import hashlib
import hmac
import random

from .stream import Stream
from ..buffers import Buffer
from ..compat import str, bytes, urlparse
from ..exceptions import StreamError
from ..utils import swfdecompress, urlget, urlopen

from ..packages.flashmedia import FLV, FLVError
from ..packages.flashmedia.tag import ScriptData

class TokenGenerator(object):
    def __init__(self, stream):
        self.stream = stream

    def generate(self):
        raise NotImplementedError

class Auth3TokenGenerator(TokenGenerator):
    def generate(self):
        if not self.stream.swf:
            raise StreamError("A SWF URL is required to create session token")

        res = urlget(self.stream.swf, exception=StreamError)
        data = swfdecompress(res.content)

        md5 = hashlib.md5()
        md5.update(data)

        data = bytes(self.stream.sessionid, "ascii") + md5.digest()
        sig = hmac.new(b"foo", data, hashlib.sha1)
        b64 = base64.encodestring(sig.digest())
        token = str(b64, "ascii").replace("\n", "")

        return token

def cache_bust_string(length):
    rval = ""

    for i in range(length):
        rval += chr(65 + int(round(random.random() * 25)))

    return rval

class AkamaiHDStreamIO(io.IOBase):
    Version = "2.5.8"
    FlashVersion = "LNX 11,1,102,63"

    StreamURLFormat = "{host}/{streamname}"
    ControlURLFormat = "{host}/control/{streamname}"
    ControlData = b":)"

    TokenGenerators = {
        "c11e59dea648d56e864fc07a19f717b9": Auth3TokenGenerator
    }

    StatusComplete = 3
    StatusError = 4

    Errors = {
        1: "Stream not found",
        2: "Track not found",
        3: "Seek out of bounds",
        4: "Authentication failed",
        5: "DVR disabled",
        6: "Invalid bitrate test"
    }

    def __init__(self, session, url, swf=None, seek=None):
        parsed = urlparse(url)

        self.session = session
        self.logger = self.session.logger.new_module("stream.akamaihd")
        self.host = ("{scheme}://{netloc}").format(scheme=parsed.scheme, netloc=parsed.netloc)
        self.streamname = parsed.path[1:]
        self.swf = swf
        self.seek = seek

    def open(self):
        self.guid = cache_bust_string(12)
        self.islive = None
        self.sessionid = None
        self.flv = None

        self.buffer = Buffer()
        self.completed_handshake = False

        url = self.StreamURLFormat.format(host=self.host, streamname=self.streamname)
        params = self._create_params(seek=self.seek)

        self.logger.debug("Opening host={host} streamname={streamname}", host=self.host, streamname=self.streamname)

        try:
            res = urlget(url, stream=True, params=params)
        except Exception as err:
            raise StreamError(str(err))

        self.handshake(res.raw)

        return self

    def handshake(self, fd):
        try:
            self.flv = FLV(fd)
        except FLVError as err:
            raise StreamError(str(err))

        self.buffer.write(self.flv.header.serialize())
        self.logger.debug("Attempting to handshake")

        for i, tag in enumerate(self.flv):
            if i == 10:
                raise StreamError("No OnEdge metadata in FLV after 10 tags, probably not a AkamaiHD stream")

            self.process_tag(tag, exception=StreamError)

            if self.completed_handshake:
                self.logger.debug("Handshake successful")
                break

    def process_tag(self, tag, exception=IOError):
        if isinstance(tag.data, ScriptData) and tag.data.name == "onEdge":
            self._on_edge(tag.data.value, exception=exception)

        self.buffer.write(tag.serialize())

    def send_token(self, token):
        headers = { "x-Akamai-Streaming-SessionToken": token }

        self.logger.debug("Sending new session token")
        self.send_control("sendingNewToken", headers=headers,
                          swf=self.swf)

    def send_control(self, cmd, headers=None, **params):
        if not headers:
            headers = {}

        url = self.ControlURLFormat.format(host=self.host,
                                           streamname=self.streamname)

        headers["x-Akamai-Streaming-SessionID"] = self.sessionid

        params = self._create_params(cmd=cmd, **params)

        return urlopen(url, headers=headers, params=params,
                       data=self.ControlData, exception=StreamError)

    def read(self, size=-1):
        if not self.flv:
            return b""

        while self.buffer.length < size and self.flv:
            try:
                tag = next(self.flv)
            except StopIteration:
                self.flv = None
                break

            self.process_tag(tag)

        return self.buffer.read(size)

    def _create_params(self, **extra):
        params = dict(v=self.Version, fp=self.FlashVersion,
                      r=cache_bust_string(5), g=self.guid)
        params.update(extra)

        return params

    def _generate_session_token(self, data64):
        swfdata = base64.decodestring(bytes(data64, "ascii"))
        md5 = hashlib.md5()
        md5.update(swfdata)
        hash = md5.hexdigest()<commit_msg>stream.akamaihd: No need to parse FLV tags after handshake.<commit_after>import base64
import io
import hashlib
import hmac
import random

from .stream import Stream
from ..buffers import Buffer
from ..compat import str, bytes, urlparse
from ..exceptions import StreamError
from ..utils import swfdecompress, urlget, urlopen

from ..packages.flashmedia import FLV, FLVError
from ..packages.flashmedia.tag import ScriptData

class TokenGenerator(object):
    def __init__(self, stream):
        self.stream = stream

    def generate(self):
        raise NotImplementedError

class Auth3TokenGenerator(TokenGenerator):
    def generate(self):
        if not self.stream.swf:
            raise StreamError("A SWF URL is required to create session token")

        res = urlget(self.stream.swf, exception=StreamError)
        data = swfdecompress(res.content)

        md5 = hashlib.md5()
        md5.update(data)

        data = bytes(self.stream.sessionid, "ascii") + md5.digest()
        sig = hmac.new(b"foo", data, hashlib.sha1)
        b64 = base64.encodestring(sig.digest())
        token = str(b64, "ascii").replace("\n", "")

        return token

def cache_bust_string(length):
    rval = ""

    for i in range(length):
        rval += chr(65 + int(round(random.random() * 25)))

    return rval

class AkamaiHDStreamIO(io.IOBase):
    Version = "2.5.8"
    FlashVersion = "LNX 11,1,102,63"

    StreamURLFormat = "{host}/{streamname}"
    ControlURLFormat = "{host}/control/{streamname}"
    ControlData = b":)"

    TokenGenerators = {
        "c11e59dea648d56e864fc07a19f717b9": Auth3TokenGenerator
    }

    StatusComplete = 3
    StatusError = 4

    Errors = {
        1: "Stream not found",
        2: "Track not found",
        3: "Seek out of bounds",
        4: "Authentication failed",
        5: "DVR disabled",
        6: "Invalid bitrate test"
    }

    def __init__(self, session, url, swf=None, seek=None):
        parsed = urlparse(url)

        self.session = session
        self.logger = self.session.logger.new_module("stream.akamaihd")
        self.host = ("{scheme}://{netloc}").format(scheme=parsed.scheme, netloc=parsed.netloc)
        self.streamname = parsed.path[1:]
        self.swf = swf
        self.seek = seek

    def open(self):
        self.guid = cache_bust_string(12)
        self.islive = None
        self.sessionid = None
        self.flv = None

        self.buffer = Buffer()
        self.completed_handshake = False

        url = self.StreamURLFormat.format(host=self.host, streamname=self.streamname)
        params = self._create_params(seek=self.seek)

        self.logger.debug("Opening host={host} streamname={streamname}", host=self.host, streamname=self.streamname)

        try:
            res = urlget(url, stream=True, params=params)
            self.fd = res.raw
        except Exception as err:
            raise StreamError(str(err))

        self.handshake(self.fd)

        return self

    def handshake(self, fd):
        try:
            self.flv = FLV(fd)
        except FLVError as err:
            raise StreamError(str(err))

        self.buffer.write(self.flv.header.serialize())
        self.logger.debug("Attempting to handshake")

        for i, tag in enumerate(self.flv):
            if i == 10:
                raise StreamError("No OnEdge metadata in FLV after 10 tags, probably not a AkamaiHD stream")

            self.process_tag(tag, exception=StreamError)

            if self.completed_handshake:
                self.logger.debug("Handshake successful")
                break

    def process_tag(self, tag, exception=IOError):
        if isinstance(tag.data, ScriptData) and tag.data.name == "onEdge":
            self._on_edge(tag.data.value, exception=exception)

        self.buffer.write(tag.serialize())

    def send_token(self, token):
        headers = { "x-Akamai-Streaming-SessionToken": token }

        self.logger.debug("Sending new session token")
        self.send_control("sendingNewToken", headers=headers,
                          swf=self.swf)

    def send_control(self, cmd, headers=None, **params):
        if not headers:
            headers = {}

        url = self.ControlURLFormat.format(host=self.host,
                                           streamname=self.streamname)

        headers["x-Akamai-Streaming-SessionID"] = self.sessionid

        params = self._create_params(cmd=cmd, **params)

        return urlopen(url, headers=headers, params=params,
                       data=self.ControlData, exception=StreamError)

    def read(self, size=-1):
        if not (self.flv and self.fd):
            return b""

        if self.buffer.length:
            return self.buffer.read(size)
        else:
            return self.fd.read(size)

    def _create_params(self, **extra):
        params = dict(v=self.Version, fp=self.FlashVersion,
                      r=cache_bust_string(5), g=self.guid)
        params.update(extra)

        return params

    def _generate_session_token(self, data64):
        swfdata = base64.decodestring(bytes(data64, "ascii"))
        md5 = hashlib.md5()
        md5.update(swfdata)
        hash = md5.hexdigest()
{'fraction_non_alphanumeric': 0.11006440449870229, 'fraction_numerical': 0.014803422089781793, 'mean_word_length': 2.3165444692381256, 'pattern_counts': {'":': 8, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5090561', 'n_tokens_mistral': 3247, 'n_tokens_neox': 3061, 'n_words': 702}