<commit_before>/*
 * Copyright 2009-2010 by The Regents of the University of California
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * you may obtain a copy of the License from
 * 
 *     http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package edu.uci.ics.hyracks.control.cc.dataset;

import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;

import edu.uci.ics.hyracks.api.comm.NetworkAddress;
import edu.uci.ics.hyracks.api.dataset.IDatasetDirectoryService;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.job.JobId;

public class DatasetDirectoryService implements IDatasetDirectoryService {
    private final Map<JobId, NetworkAddress[]> jobPartitionLocationsMap;

    public DatasetDirectoryService() {
        jobPartitionLocationsMap = new HashMap<JobId, NetworkAddress[]>();<commit_msg>Add an additional TODO comment for the global dataset directory information.<commit_after>/*
 * Copyright 2009-2010 by The Regents of the University of California
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * you may obtain a copy of the License from
 * 
 *     http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package edu.uci.ics.hyracks.control.cc.dataset;

import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;

import edu.uci.ics.hyracks.api.comm.NetworkAddress;
import edu.uci.ics.hyracks.api.dataset.IDatasetDirectoryService;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.job.JobId;

/**
 * TODO: The potential perils of this global dataset directory service implementation is that, the jobs location information
 * is never evicted from the memory and the memory useage grows as the number of jobs in the system grows. What we should
 * possibly do is, add an API call for the client to say that it received everything it has to for the job (after it receives
 * all the results) completely. Then we can just get rid of the location information for that job.
 */
public class DatasetDirectoryService implements IDatasetDirectoryService {
    private final Map<JobId, NetworkAddress[]> jobPartitionLocationsMap;

    public DatasetDirectoryService() {
        jobPartitionLocationsMap = new HashMap<JobId, NetworkAddress[]>();
{'fraction_non_alphanumeric': 0.07515387107223842, 'fraction_numerical': 0.007774538386783284, 'mean_word_length': 4.494661921708185, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6726753', 'n_tokens_mistral': 844, 'n_tokens_neox': 825, 'n_words': 333}