<commit_before>import subprocess

from pydub import AudioSegment

class StreamSegment(object):
    def __init__(self, name, position, initial_time=None, end_time=None):
        self.name = name
        self.position = position
        self.initial_time = initial_time
        self.end_time = end_time
        self.orig_tmp_file = None

def time_to_mil(time):
    splitted_time = time.split(":")
    if len(splitted_time) == 2:
        return int(splitted_time[0])*60000 + int(splitted_time[1])*1000
    elif len(splitted_time) == 3:
        return int(splitted_time[0])*360000 + int(splitted_time[1])*60000 \
            + int(splitted_time[2])*1000

def build_time_range(stream_len, streamSegments):
    """
    Sets initial time/end time for segments
    """
    t_separator = 500 #separator between track, milseconds
    sortedStreamSegments = sort(streamSegments, key=lambda ss: ss.position)
    for idx, stream_segment in enumerate(sortedStreamSegments):
        if stream_segment.position == 0:
            stream_segment.initial_time = 0
            if stream_segment.end_time is None:
                stream_segment.end_time = time_to_mil(nxt_stream_init_t) - t_separator
        elif stream_segment.position != len(stream_segment) - 1:
            if stream_segment.end_time is None:
                nxt_stream_init_t = sortedStreamSegments[idx + 1].initial_time
                stream_segment.end_time = time_to_mil(nxt_stream_init_t) - t_separator
        else:
            if stream_segment.end_time is None:
                stream_segment.end_time = stream_len - time_separator
    return sortedStreamSegments

def split_audio(src_file, dest_path, audio_segments, audio_format='webm'):
    """
    Splits audio file given the time range in audio segments
    """
    audio_orig = AudioSegment.from_file(src_file, audio_format)
    for as_ in audio_segments:
        tmp_seg = audio_orig[as_.initial_time:as_.end_time]
        as_.orig_tmp_file = '/'.join([dest_path, as_.name + '.' + audio_format])
        tmp_seg.export(_as.orig_tmp_file, format=audio_format)
    return audio_segments

def export(src_file, dest_file, track_metadata=None, format='mp3'):
    """
    Exports webm file to mp3 or ogg format file, replaces pydub export
    due to issues with ffmpeg
    """
    ffmpeg_cmd = ['ffmpeg', '-i', src_file, '-vn', '-ab', '320k', 'ar', '44100',
                  '-y', dest_file ]
    if track_metadata is not None:
        ffmpeg_cmd = ffmpeg_cmd + track_metadata
    p = subprocess.Popen(ffmpeg_cmd,  stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE)
    out, err = p.communicate()
    if p.returncode != 0:
        raise Exception("Could not encode file")
    return dest_file

def create_tracks(dest_dir, audio_segments, artist, album, format='mp3'):
    dir_name = '_'.join([artist, album])
    dest_dir = '/'.join([dest_dir, dir_name])
    dest_file = '/'.join([dest_dir ,as_.name + '.' + format])
    for as_ in audio_segments:
        track_meta = _create_track_metadata(album, artist,
                                            as_.name, as._position)
        export(as_.orig_tmp_file, dest_file,
               track_metada=track_metada, format=format)

def _create_track_metadata(album, artist, name, track_number):
    quote_param = lambda str_: '"' + str_ + '"'
    return ['-metadata', '='.join(['title', quote_param(name)]),
            '-metadata', '='.join(['author', quote_param(artist)]),
            '-metadata', '='.join(['album', quote_param(album)]),
            '-metadata', '='.join(['track', str(track_number)])]
<commit_msg>Adds source_media and source_type params to create tracks function<commit_after>import subprocess

from pydub import AudioSegment
from filemanager import get_workingdir

class StreamSegment(object):
    def __init__(self, name, position, initial_time=None, end_time=None):
        self.name = name
        self.position = position
        self.initial_time = initial_time
        self.end_time = end_time
        self.orig_tmp_file = None

def time_to_mil(time):
    splitted_time = time.split(":")
    if len(splitted_time) == 2:
        return int(splitted_time[0])*60000 + int(splitted_time[1])*1000
    elif len(splitted_time) == 3:
        return int(splitted_time[0])*360000 + int(splitted_time[1])*60000 \
            + int(splitted_time[2])*1000

def build_time_range(stream_len, streamSegments):
    """
    Sets initial time/end time for segments
    """
    t_separator = 500 #separator between track, milseconds
    sortedStreamSegments = sort(streamSegments, key=lambda ss: ss.position)
    for idx, stream_segment in enumerate(sortedStreamSegments):
        if stream_segment.position == 0:
            stream_segment.initial_time = 0
            if stream_segment.end_time is None:
                stream_segment.end_time = time_to_mil(nxt_stream_init_t) - t_separator
        elif stream_segment.position != len(stream_segment) - 1:
            if stream_segment.end_time is None:
                nxt_stream_init_t = sortedStreamSegments[idx + 1].initial_time
                stream_segment.end_time = time_to_mil(nxt_stream_init_t) - t_separator
        else:
            if stream_segment.end_time is None:
                stream_segment.end_time = stream_len - time_separator
    return sortedStreamSegments

def split_audio(src_file, dest_path, audio_segments, audio_format='webm'):
    """
    Splits audio file given the time range in audio segments
    """
    audio_orig = AudioSegment.from_file(src_file, 'webm')
    for as_ in audio_segments:
        print(as_.initial_time)
        print(type(as_.initial_time))
        print(as_.end_time)
        print(type(as_.end_time))
        tmp_seg = audio_orig[int(as_.initial_time):int(as_.end_time)]
        as_.orig_tmp_file = '/'.join([dest_path, as_.name + '.' + audio_format])
        tmp_seg.export(_as.orig_tmp_file, format=audio_format)
    return audio_segments

def export(src_file, dest_file, track_metadata=None, format='mp3'):
    """
    Exports webm file to mp3 or ogg format file, replaces pydub export
    due to issues with ffmpeg
    """
    ffmpeg_cmd = ['ffmpeg', '-i', src_file, '-vn', '-ab', '320k', 'ar', '44100',
                  '-y', dest_file ]
    if track_metadata is not None:
        ffmpeg_cmd = ffmpeg_cmd + track_metadata
    p = subprocess.Popen(ffmpeg_cmd,  stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE)
    out, err = p.communicate()
    if p.returncode != 0:
        raise Exception("Could not encode file")
    return dest_file

def create_tracks(source_media, dest_dir, audio_segments, artist, album,
                  source_type, format='mp3'):

    src_format = format
    if source_type == 'youtube':
        src_format = 'webm'

    dir_name = '_'.join([artist.replace(' ', '_'), album.replace(' ', '_')])
    dest_dir = '/'.join([dest_dir, dir_name])
    dest_file = lambda name: '/'.join([dest_dir ,name.replace(' ', '_') + '.' +\
                                       format])
    splitted_segments = split_audio(source_media, get_workingdir(),
                                    audio_segments, audio_format=src_format)
    for as_ in splitted_segments:
        track_meta = _create_track_metadata(album, artist,
                                            as_.name, as_.position + 1)
        export(as_.orig_tmp_file, dest_file(as_.name),
               track_metada=track_meta, format=format)

def _create_track_metadata(album, artist, name, track_number):
    quote_param = lambda str_: '"' + str_ + '"'
    return ['-metadata', '='.join(['title', quote_param(name)]),
            '-metadata', '='.join(['author', quote_param(artist)]),
            '-metadata', '='.join(['album', quote_param(album)]),
            '-metadata', '='.join(['track', str(track_number)])]

{'fraction_non_alphanumeric': 0.11472055250031973, 'fraction_numerical': 0.012917252845632434, 'mean_word_length': 2.913913913913914, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1727108', 'n_tokens_mistral': 2526, 'n_tokens_neox': 2423, 'n_words': 519}