<commit_before>#! /usr/bin/env python

""" module to synchronize Sailthru Content Library with edX data in catalog """
import argparse
import csv
import logging
import json
import os
import sys

from services.catalog_api_service import CatalogApiService
from services.sailthru_api_service import SailthruApiService
from services.sailthru_translation_service import SailthruTranslationService


"""
A script intended to be run periodically (e.g. nightly) which uses the Course Discovery API to populate/update
the Sailthru content library for programs.  Useful when Sailthru is used for email marketing.
The script invocation syntax is as follows:

 usage: refresh_programs.py [-h] [--access_token ACCESS_TOKEN]
                           [--oauth_host OAUTH_HOST] [--oauth_key OAUTH_KEY]
                           [--oauth_secret OAUTH_SECRET]
                           [--sailthru_key SAILTHRU_KEY]
                           [--sailthru_secret SAILTHRU_SECRET]
                           [--content_api_url CONTENT_API_URL]
                           [--lms_url LMS_URL]
                           [--email_report EMAIL_REPORT]
                           [--type [programs, courses, all]]
                           {list,upload,clear,preview}

The 'clear' command deletes all the entries currently in the Sailthru content library.  It should generally only be
used during testing before deployment.  The 'list' command displays, in JSON, up to 1000 entries currently in the
Sailthru content library.  The 'upload' command sends the desired programs and/or courses to Sailthru as a batch job.
The result is sent as a brief report to the address specified in --email_report.

The following options are available:

+--------------------------------+-------------------------------------------------------+
| Switch                         | Purpose                                               |
+================================+=======================================================+
| --access_token                 | An access token for the Course Discovery API          |
+--------------------------------+-------------------------------------------------------+
| --oauth_host                   | The host used to obtain Course Discovery access token |
+--------------------------------+-------------------------------------------------------+
| --oauth_key                    | Key used to obtain Course Discovery access token      |
+--------------------------------+-------------------------------------------------------+
| --oauth_secret                 | Secret used to obtain Course Discovery access token   |
+--------------------------------+-------------------------------------------------------+
| --sailthru_key                 | Access key for Sailthru api                           |
+--------------------------------+-------------------------------------------------------+
| --sailthru_secret              | Access secret for Sailthru api                        |
+--------------------------------+-------------------------------------------------------+
| --content_api_url              | Url of Course Discovery API                           |
+--------------------------------+-------------------------------------------------------+
| --lms_url                      | Url of LMS (default http://courses.edx.org            |
+--------------------------------+-------------------------------------------------------+
| --email_report                 | Email address to sent batch report to                 |
+--------------------------------+-------------------------------------------------------+
| --type                         | A choice of "programs", "courses" or "all"            |
+--------------------------------+-------------------------------------------------------+

To get access to the Course Discovery API, you need either an existing access token, or you can specify the
oauth_host, oauth_key and oauth_secret.

Sample command to load course content in batch:

sync_library.py --oauth_key=<api key> --oauth_secret=<api secret>
   --oauth_host=https://api.edx.org/oauth2/v1
   --sailthru_key=<sailthru key> --sailthru_secret=<sailthru secret>
   --content_api_url=https://prod-edx-discovery.edx.org/api/v1/
   --lms_url=https://courses.edx.org
   --fixups=fixups.csv
   --type=courses
   --email_report=<EMAIL>
   upload

"""


def setup_logging():
    """ Setup logging """
    logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s', level=logging.DEBUG)


def load_fixups(filename):
    """
    Read list of fixups.

    The fixups file should be a three column csv with any fields that should be overriden

    Each line should have:
        course_run,field_name,value
    """

    if not filename:
        return []

    with open(filename, 'rb') as f:
        reader = csv.reader(f)
        return list(reader)


def main(argv):

    args = get_args(argv)

    sailthru_service = SailthruApiService(args.sailthru_key, args.sailthru_secret, args.lms_url)

    if args.command == 'list':
        sailthru_service.list()
    elif args.command in ('upload', 'preview'):
        fixups_collection = None<commit_msg>Fix the loading of the fixup file<commit_after>#! /usr/bin/env python

""" module to synchronize Sailthru Content Library with edX data in catalog """
import argparse
import csv
import logging
import json
import os
import sys

from services.catalog_api_service import CatalogApiService
from services.sailthru_api_service import SailthruApiService
from services.sailthru_translation_service import SailthruTranslationService


"""
A script intended to be run periodically (e.g. nightly) which uses the Course Discovery API to populate/update
the Sailthru content library for programs.  Useful when Sailthru is used for email marketing.
The script invocation syntax is as follows:

 usage: refresh_programs.py [-h] [--access_token ACCESS_TOKEN]
                           [--oauth_host OAUTH_HOST] [--oauth_key OAUTH_KEY]
                           [--oauth_secret OAUTH_SECRET]
                           [--sailthru_key SAILTHRU_KEY]
                           [--sailthru_secret SAILTHRU_SECRET]
                           [--content_api_url CONTENT_API_URL]
                           [--lms_url LMS_URL]
                           [--email_report EMAIL_REPORT]
                           [--type [programs, courses, all]]
                           {list,upload,clear,preview}

The 'clear' command deletes all the entries currently in the Sailthru content library.  It should generally only be
used during testing before deployment.  The 'list' command displays, in JSON, up to 1000 entries currently in the
Sailthru content library.  The 'upload' command sends the desired programs and/or courses to Sailthru as a batch job.
The result is sent as a brief report to the address specified in --email_report.

The following options are available:

+--------------------------------+-------------------------------------------------------+
| Switch                         | Purpose                                               |
+================================+=======================================================+
| --access_token                 | An access token for the Course Discovery API          |
+--------------------------------+-------------------------------------------------------+
| --oauth_host                   | The host used to obtain Course Discovery access token |
+--------------------------------+-------------------------------------------------------+
| --oauth_key                    | Key used to obtain Course Discovery access token      |
+--------------------------------+-------------------------------------------------------+
| --oauth_secret                 | Secret used to obtain Course Discovery access token   |
+--------------------------------+-------------------------------------------------------+
| --sailthru_key                 | Access key for Sailthru api                           |
+--------------------------------+-------------------------------------------------------+
| --sailthru_secret              | Access secret for Sailthru api                        |
+--------------------------------+-------------------------------------------------------+
| --content_api_url              | Url of Course Discovery API                           |
+--------------------------------+-------------------------------------------------------+
| --lms_url                      | Url of LMS (default http://courses.edx.org            |
+--------------------------------+-------------------------------------------------------+
| --email_report                 | Email address to sent batch report to                 |
+--------------------------------+-------------------------------------------------------+
| --type                         | A choice of "programs", "courses" or "all"            |
+--------------------------------+-------------------------------------------------------+

To get access to the Course Discovery API, you need either an existing access token, or you can specify the
oauth_host, oauth_key and oauth_secret.

Sample command to load course content in batch:

sync_library.py --oauth_key=<api key> --oauth_secret=<api secret>
   --oauth_host=https://api.edx.org/oauth2/v1
   --sailthru_key=<sailthru key> --sailthru_secret=<sailthru secret>
   --content_api_url=https://prod-edx-discovery.edx.org/api/v1/
   --lms_url=https://courses.edx.org
   --fixups=fixups.csv
   --type=courses
   --email_report=<EMAIL>
   upload

"""


def setup_logging():
    """ Setup logging """
    logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s', level=logging.DEBUG)


def load_fixups(filename):
    """
    Read list of fixups.

    The fixups file should be a three column csv with any fields that should be overriden

    Each line should have:
        course_run,field_name,value
    """

    if not filename:
        return []

    with open(filename, 'rt') as f:
        reader = csv.reader(f)
        return list(reader)


def main(argv):

    args = get_args(argv)

    sailthru_service = SailthruApiService(args.sailthru_key, args.sailthru_secret, args.lms_url)

    if args.command == 'list':
        sailthru_service.list()
    elif args.command in ('upload', 'preview'):
        fixups_collection = None
{'fraction_non_alphanumeric': 0.27115051069570245, 'fraction_numerical': 0.0013490075158990172, 'mean_word_length': 1.2795958708543818, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1075619', 'n_tokens_mistral': 2458, 'n_tokens_neox': 2351, 'n_words': 743}