<commit_before># Copyright 2013 <NAME> <<EMAIL>>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import re
import yaml
import json
import urllib
import optparse
import logging
import gdata.youtube
import gdata.youtube.service

yt_service = gdata.youtube.service.YouTubeService()

# Turn on HTTPS/SSL access.
# Note: SSL is not available at this time for uploads.
yt_service.ssl = True

# Parse the yaml config file
with open('config.yaml', 'r') as config_file:
    config = yaml.load(config_file.read())

# A complete client login request
yt_service.email = config['user_email']
yt_service.password = config['<PASSWORD>']
yt_service.source = config['source']
yt_service.developer_key = config['developer_key']
yt_service.client_id = config['client_id']
yt_service.ProgrammaticLogin()

# Regex to extract video ID from uploaded video object. (Why there's
# no useful "ID" field is beyond me.)
video_id_regex = re.compile('http://gdata.youtube.com/feeds/api/videos/(\w+)</ns0:id>')

# Upload a video to YouTube and get back a YouTubeVideoEntry object
def upload_video(filename, metadata):
    # Create a container object for video metadata
    media_group = gdata.media.Group(
        title=gdata.media.Title(text=metadata['title']),
        description=gdata.media.Description(description_type='plain',
                                            text=metadata['description']),
        keywords=gdata.media.Keywords(text=metadata['keywords']),
        category=gdata.media.Category(
                text=metadata['category_term'],
                scheme='http://gdata.youtube.com/schemas/2007/categories.cat',
                label=metadata['category_label']),
        player=None,
        private=gdata.media.Private()
        )

    # Create a YouTubeVideoEntry with the metadata associated
    video_entry = gdata.youtube.YouTubeVideoEntry(media=media_group)

    # Upload the video at `filename` and associate it with the new
    # YouTubeVideoEntry
    new_entry = yt_service.InsertVideoEntry(video_entry, filename)
    
    return new_entry


# Download the video at a given URL
# TODO: handle missing or unavailable videos
def get_video_from_url(url):
    (filename, headers) = urllib.urlretrieve(url)
    return filename


# Extract a video's YT ID from its full URL
def get_entry_id(entry_url):
    m = video_id_regex.search(str(entry_url))
    if m:
        parsed_id = m.group(1)
        return parsed_id

    return False


# Map provided metadata to dict to be passed with new video (ensures
# empty strings are passed instead of None values)
def parse_metadata(metadata):
    all_metadata = {
        'local_id': '',
        'title': '',
        'description': '',
        'keywords': '',
        'category_term': '',
        'category_label': ''
        }

    for k in all_metadata:
        all_metadata[k] = metadata.get(k, '')

    return all_metadata


if __name__ == "__main__":
    # Set up the command line argument parser
    # TODO: check for required -i and -o parameters, exit if missing
    parser = optparse.OptionParser()

    parser.add_option('-i', '--input-file', action="store", dest="input_file",
                      help="""Input json file""", default="")

    parser.add_option('-o', '--output-file', action="store", dest="output_file",
                      help="""Output json file (with YouTube IDs)""",
                      default="")

    parser.add_option('-q', '--quiet', action='store_true', dest='quiet',
                      help="""Don't display informational logs""", 
                      default=False)

    options, args = parser.parse_args()

    # Set logging level
    if options.quiet:
        log_level = logging.ERROR
    else:
        log_level = logging.INFO

    logging.basicConfig(format='%(levelname)s: %(message)s', level=log_level)

    # Parse the json videos file<commit_msg>Display progress indicator when downloading videos<commit_after># Copyright 2013 <NAME> <<EMAIL>>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import re
import sys
import yaml
import json
import urllib
import optparse
import logging
import gdata.youtube
import gdata.youtube.service

yt_service = gdata.youtube.service.YouTubeService()

# Turn on HTTPS/SSL access.
# Note: SSL is not available at this time for uploads.
yt_service.ssl = True

# Parse the yaml config file
with open('config.yaml', 'r') as config_file:
    config = yaml.load(config_file.read())

# A complete client login request
yt_service.email = config['user_email']
yt_service.password = config['<PASSWORD>']
yt_service.source = config['source']
yt_service.developer_key = config['developer_key']
yt_service.client_id = config['client_id']
yt_service.ProgrammaticLogin()

# Regex to extract video ID from uploaded video object. (Why there's
# no useful "ID" field is beyond me.)
video_id_regex = re.compile('http://gdata.youtube.com/feeds/api/videos/(\w+)</ns0:id>')

# Initialize options global
options = None

# Upload a video to YouTube and get back a YouTubeVideoEntry object
def upload_video(filename, metadata):
    # Create a container object for video metadata
    media_group = gdata.media.Group(
        title=gdata.media.Title(text=metadata['title']),
        description=gdata.media.Description(description_type='plain',
                                            text=metadata['description']),
        keywords=gdata.media.Keywords(text=metadata['keywords']),
        category=gdata.media.Category(
                text=metadata['category_term'],
                scheme='http://gdata.youtube.com/schemas/2007/categories.cat',
                label=metadata['category_label']),
        player=None,
        private=gdata.media.Private()
        )

    # Create a YouTubeVideoEntry with the metadata associated
    video_entry = gdata.youtube.YouTubeVideoEntry(media=media_group)

    # Upload the video at `filename` and associate it with the new
    # YouTubeVideoEntry
    new_entry = yt_service.InsertVideoEntry(video_entry, filename)
    
    return new_entry


# Download the video at a given URL
# TODO: handle missing or unavailable videos
def get_video_from_url(url):
    if options.quiet: 
        reporthook = None
    else:
        reporthook = download_progress

    (filename, headers) = urllib.urlretrieve(url, reporthook=reporthook)

    if not options.quiet: sys.stdout.write("\n")

    return filename


# Extract a video's YT ID from its full URL
def get_entry_id(entry_url):
    m = video_id_regex.search(str(entry_url))
    if m:
        parsed_id = m.group(1)
        return parsed_id

    return False


# Map provided metadata to dict to be passed with new video (ensures
# empty strings are passed instead of None values)
def parse_metadata(metadata):
    all_metadata = {
        'local_id': '',
        'title': '',
        'description': '',
        'keywords': '',
        'category_term': '',
        'category_label': ''
        }

    for k in all_metadata:
        all_metadata[k] = metadata.get(k, '')

    return all_metadata


# Print file download progress to stdout
def download_progress(count, blockSize, totalSize):
    percent = int(count*blockSize*100/totalSize)
    sys.stdout.write("\rProgress: %d%%" % percent)
    sys.stdout.flush()


if __name__ == "__main__":
    # Set up the command line argument parser
    # TODO: check for required -i and -o parameters, exit if missing
    parser = optparse.OptionParser()

    parser.add_option('-i', '--input-file', action="store", dest="input_file",
                      help="""Input json file""", default="")

    parser.add_option('-o', '--output-file', action="store", dest="output_file",
                      help="""Output json file (with YouTube IDs)""",
                      default="")

    parser.add_option('-q', '--quiet', action='store_true', dest='quiet',
                      help="""Don't display informational logs""", 
                      default=False)

    options, args = parser.parse_args()

    # Set logging level
    if options.quiet:
        log_level = logging.ERROR
    else:
        log_level = logging.INFO

    logging.basicConfig(format='%(levelname)s: %(message)s', level=log_level)

    # Parse the json videos file
{'fraction_non_alphanumeric': 0.10788471995649809, 'fraction_numerical': 0.0033713974986405654, 'mean_word_length': 3.2106227106227108, 'pattern_counts': {'":': 2, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3551842', 'n_tokens_mistral': 2665, 'n_tokens_neox': 2554, 'n_words': 791}