<commit_before>import os
import logging
import traceback
import subprocess
import httplib
from collections import defaultdict
from loader import Loader, LoadResult, Timeout, TimeoutError

ENV = '/usr/bin/env'
ZombieJS = 'node'
ZombieLOADER = '/home/ec2-user/node-http2/example/pageloader_client.js'

# TODO: when do we return FAILURE_NO_200?
# TODO: enable caching
# TODO: user agent
# TODO: disable network cache

class ZombieJSLoader(Loader):
    '''Subclass of :class:`Loader` that loads pages using ZombieJS.
    
    .. note:: The :class:`ZombieJSLoader` currently does not support local caching.
    .. note:: The :class:`ZombieJSLoader` currently does not support disabling network caching.
    .. note:: The :class:`ZombieJSLoader` currently does not support single-object loading (i.e., it always loads the full page).
    '''

    def __init__(self, **kwargs):
        super(ZombieJSLoader, self).__init__(**kwargs)
        if not self._http2:
            raise NotImplementedError('ZombieJSLoader does not support HTTP1.1')
        if not self._disable_local_cache:
            raise NotImplementedError('ZombieJSLoader does not support local caching')
        if not self._full_page:
            raise NotImplementedError('ZombieJSLoader does not support loading only an object')
        if self._disable_network_cache:
            raise NotImplementedError('ZombieJSLoader does not support disabling network caches.')        

    def _load_page(self, url, outdir, trial_num=-1):    
        # load the specified URL
        logging.info('Loading page: %s', url)
        try:
	    # Cause a restart of the proxy
	    conn = httplib.HTTPConnection(self._proxy.split(':')[0]+':5678') # Assume restart always listens on this port for now
	    conn.request("GET", "/")
	    resp = conn.getresponse() # Don't need to do anything with it. Just want to know that the request was acknowledge

            # Load the page
            Zombie_cmd = [ENV, ZombieJS, ZombieLOADER, url, str(self._timeout)]<commit_msg>fixed a missing IF statement<commit_after>import os
import logging
import traceback
import subprocess
import httplib
from collections import defaultdict
from loader import Loader, LoadResult, Timeout, TimeoutError

ENV = '/usr/bin/env'
ZombieJS = 'node'
ZombieLOADER = '/home/ec2-user/node-http2/example/pageloader_client.js'

# TODO: when do we return FAILURE_NO_200?
# TODO: enable caching
# TODO: user agent
# TODO: disable network cache

class ZombieJSLoader(Loader):
    '''Subclass of :class:`Loader` that loads pages using ZombieJS.
    
    .. note:: The :class:`ZombieJSLoader` currently does not support local caching.
    .. note:: The :class:`ZombieJSLoader` currently does not support disabling network caching.
    .. note:: The :class:`ZombieJSLoader` currently does not support single-object loading (i.e., it always loads the full page).
    '''

    def __init__(self, **kwargs):
        super(ZombieJSLoader, self).__init__(**kwargs)
        if not self._http2:
            raise NotImplementedError('ZombieJSLoader does not support HTTP1.1')
        if not self._disable_local_cache:
            raise NotImplementedError('ZombieJSLoader does not support local caching')
        if not self._full_page:
            raise NotImplementedError('ZombieJSLoader does not support loading only an object')
        if self._disable_network_cache:
            raise NotImplementedError('ZombieJSLoader does not support disabling network caches.')        

    def _load_page(self, url, outdir, trial_num=-1):    
        # load the specified URL
        logging.info('Loading page: %s', url)
        try:	    
	    # Cause a restart of the proxy
	    if self._proxy:
	    	conn = httplib.HTTPConnection(self._proxy.split(':')[0]+':5678') # Assume restart always listens on this port for now
	    	conn.request("GET", "/")
	    	resp = conn.getresponse() # Don't need to do anything with it. Just want to know that the request was acknowledge

            # Load the page
            Zombie_cmd = [ENV, ZombieJS, ZombieLOADER, url, str(self._timeout)]
{'fraction_non_alphanumeric': 0.09171162334477685, 'fraction_numerical': 0.006866110838646395, 'mean_word_length': 3.334750265674814, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6060724', 'n_tokens_mistral': 1202, 'n_tokens_neox': 1196, 'n_words': 391}