<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os


class Shredder:
    def __init__(self, filepath, chunk_size):
        """For shreding and merging larger files to be uploaded."""
        self.filepath = filepath
        self.filename = os.path.split(filepath)[1]

        self.chunk_names = []
        self.chunk_size = chunk_size
        self.num_chunks = 0

    def shred_chunks(self):
        """ Split the file into smaller chunks.
        http://bdurblg.blogspot.com/2011/06/python-split-any-file-binary-to.html
        """
        # read the contents of the file
        f = open(self.filepath, 'rb')
        data = f.read() # read the entire content of the file(MEMORY!)
        f.close()

        # get the length of data, ie size of the input file in bytes
        bytes = len(data)

        # calculate the number of chunks to be created
        self.num_chunks = bytes/self.chunk_size
        if(bytes%self.chunk_size):
            self.num_chunks+=1

        # create chunks
        for i in range(0, bytes+1, self.chunk_size):
            fn1 = "upload/chunk%s" % i
            print(fn1)
            self.chunk_names.append(fn1)
            f = open(fn1, 'wb')
            f.write(data[i:i+ self.chunk_size])
            f.close()

        return self.chunk_names

    def merge_chunks(self):
        """ Join the chunks of files into a single file.
        http://bdurblg.blogspot.com/2011/06/python-split-any-file-binary-to.html
        """
        dataList = []

        for i in range(0,int(self.num_chunks),1):
            chunkNum=i * self.chunk_size
            chunkName = os.path.abspath('download/chunk%s'%chunkNum)
            print(chunkNum)
            print(chunkName)
            f = open(chunkName, 'rb')
            dataList.append(f.read())
            f.close()
            os.remove(chunkName)

        if os.path.exists("files/" + self.filename):
            os.remove("files/" + self.filename)
        for data in dataList:
            f = open("files/" + self.filename, 'ab')
            f.write(data)
            f.close()<commit_msg>Comment out unneeded shredder module<commit_after># #!/usr/bin/env python
# # -*- coding: utf-8 -*-
#
# import os
#
#
# class Shredder(object):
#     def __init__(self, filepath, chunk_size):
#         """For shreding and merging larger files to be uploaded."""
#         self.filepath = filepath
#         self.filename = os.path.split(filepath)[1]
#
#         self.chunk_names = []
#         self.chunk_size = chunk_size
#         self.num_chunks = 0
#
#     def shred_chunks(self):
#         """ Split the file into smaller chunks.
#         http://bdurblg.blogspot.com/2011/06/python-split-any-file-binary-to.html
#         """
#         # read the contents of the file
#         f = open(self.filepath, 'rb')
#         data = f.read()  # read the entire content of the file(MEMORY!)
#         f.close()
#
#         # get the length of data, ie size of the input file in bytes
#         bytes = len(data)
#
#         # calculate the number of chunks to be created
#         self.num_chunks = bytes/self.chunk_size
#         if bytes % self.chunk_size:
#             self.num_chunks += 1
#
#         # create chunks
#         for i in range(0, bytes+1, self.chunk_size):
#             fn1 = "upload/chunk%s" % i
#             print(fn1)
#             self.chunk_names.append(fn1)
#             f = open(fn1, 'wb')
#             f.write(data[i:i + self.chunk_size])
#             f.close()
#
#         return self.chunk_names
#
#     def merge_chunks(self):
#         """ Join the chunks of files into a single file.
#         http://bdurblg.blogspot.com/2011/06/python-split-any-file-binary-to.html
#         """
#         dataList = []
#
#         for i in range(0,int(self.num_chunks),1):
#             chunkNum=i * self.chunk_size
#             chunkName = os.path.abspath('download/chunk%s' % chunkNum)
#             print(chunkNum)
#             print(chunkName)
#             f = open(chunkName, 'rb')
#             dataList.append(f.read())
#             f.close()
#             os.remove(chunkName)
#
#         if os.path.exists("files/" + self.filename):
#             os.remove("files/" + self.filename)
#         for data in dataList:
#             f = open("files/" + self.filename, 'ab')
#             f.write(data)
#             f.close()
{'fraction_non_alphanumeric': 0.13143382352941177, 'fraction_numerical': 0.011029411764705883, 'mean_word_length': 1.815653298835705, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1031102', 'n_tokens_mistral': 1400, 'n_tokens_neox': 1354, 'n_words': 308}