# Path: opendiamond/dataretriever/util.py
# def read_file_list(path):
#     sys.stdout.flush()
#     if not os.path.exists(path):
#         sys.exit('Error: Path {} does not exist'.format(path))
# 
#     with open(path,'r') as f:
#         data = f.read().splitlines()
#     return [d.strip() for d in data]
# 
# def write_data(path, lists_, seed):
#     _log.info("Writing data for path {}".format(path))
#     uid = pwd.getpwnam("dataretriever").pw_uid
#     gid = grp.getgrnam("dataretriever").gr_gid
#     data = []
#     for l in lists_:
#         data.extend(l)
#     random.Random(seed).shuffle(data)
#     with open(path,'w') as f:
#         for d in data:
#             print(d, file=f)
#     os.chown(path, uid, gid)
#     return data


from builtins import next
from builtins import range
from xml.sax.saxutils import quoteattr
from itertools import cycle
from flask import Blueprint, url_for, Response, stream_with_context, send_file, \
    jsonify
from werkzeug.datastructures import Headers
from opendiamond.dataretriever.util import read_file_list, write_data
import os
import datetime
import sys
import logging
import random
import glob
    headers = Headers()
    # With add_etags=True, conditional=True
    # Flask should be smart enough to do 304 Not Modified
    response = send_file(path,
                         cache_timeout=datetime.timedelta(
                             days=365).total_seconds(),
                         add_etags=True,
                         conditional=True)
    response.headers.extend(headers)
    return response

def create_index(base_dir, base_rate=0.05, seed=42, rank=0, total_servers=1):
    """
    Creates Index List File:
    Assuming name of files NEGATIVE (e.g:subset YFCC), POSITIVE
    """

    filepath_split = ['STREAM', "{:.2f}".format(base_rate), str(rank), str(total_servers), str(seed)]
    filepath = '_'.join(filepath_split)
    filepath = os.path.join(base_dir, filepath)
    positive_path = os.path.join(base_dir, 'POSITIVE')
    negative_path = os.path.join(base_dir, 'NEGATIVE')
    positive_firstline = open(positive_path).readline().rstrip()
    keyword = positive_firstline.split('/')[-2] # Assuming all positives are in the same parent dir

    _log.info("Dir {} BR: {} Seed:{} FP{}".format(base_dir, base_rate, seed, filepath))
    sys.stdout.flush()

    if not os.path.exists(filepath):
        positive_data = read_file_list(positive_path) # same across servers

# Path: opendiamond/dataretriever/util.py
# def read_file_list(path):
#     sys.stdout.flush()
#     if not os.path.exists(path):
#         sys.exit('Error: Path {} does not exist'.format(path))
# 
#     with open(path,'r') as f:
#         data = f.read().splitlines()
#     return [d.strip() for d in data]
# 
# def write_data(path, lists_, seed):
#     _log.info("Writing data for path {}".format(path))
#     uid = pwd.getpwnam("dataretriever").pw_uid
#     gid = grp.getgrnam("dataretriever").gr_gid
#     data = []
#     for l in lists_:
#         data.extend(l)
#     random.Random(seed).shuffle(data)
#     with open(path,'w') as f:
#         for d in data:
#             print(d, file=f)
#     os.chown(path, uid, gid)
#     return data


# Path: opendiamond/dataretriever/augment_store.py
from builtins import next
from builtins import range
from xml.sax.saxutils import quoteattr
from itertools import cycle
from flask import Blueprint, url_for, Response, stream_with_context, send_file, \
    jsonify
from werkzeug.datastructures import Headers
from opendiamond.dataretriever.util import read_file_list, write_data
import os
import datetime
import sys
import logging
import random
import glob

    headers = Headers()
    # With add_etags=True, conditional=True
    # Flask should be smart enough to do 304 Not Modified
    response = send_file(path,
                         cache_timeout=datetime.timedelta(
                             days=365).total_seconds(),
                         add_etags=True,
                         conditional=True)
    response.headers.extend(headers)
    return response

def create_index(base_dir, base_rate=0.05, seed=42, rank=0, total_servers=1):
    """
    Creates Index List File:
    Assuming name of files NEGATIVE (e.g:subset YFCC), POSITIVE
    """

    filepath_split = ['STREAM', "{:.2f}".format(base_rate), str(rank), str(total_servers), str(seed)]
    filepath = '_'.join(filepath_split)
    filepath = os.path.join(base_dir, filepath)
    positive_path = os.path.join(base_dir, 'POSITIVE')
    negative_path = os.path.join(base_dir, 'NEGATIVE')
    positive_firstline = open(positive_path).readline().rstrip()
    keyword = positive_firstline.split('/')[-2] # Assuming all positives are in the same parent dir

    _log.info("Dir {} BR: {} Seed:{} FP{}".format(base_dir, base_rate, seed, filepath))
    sys.stdout.flush()

    if not os.path.exists(filepath):
        positive_data = read_file_list(positive_path) # same across servers

        negative_data = read_file_list(negative_path) # different across servers