<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.accumulo.core.client.mock;

import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertFalse;
import static junit.framework.Assert.assertTrue;

import java.util.Collections;
import java.util.Iterator;
import java.util.Map.Entry;
import java.util.Random;

import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.BatchDeleter;
import org.apache.accumulo.core.client.BatchScanner;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.IteratorSetting;
import org.apache.accumulo.core.client.MultiTableBatchWriter;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.iterators.Combiner;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.iterators.user.SummingCombiner;
import org.apache.hadoop.io.Text;
import org.junit.Test;

public class MockConnectorTest {
  Random random = new Random();
  
  static Text asText(int i) {
    return new Text(Integer.toHexString(i));
  }
  
  @Test
  public void testSunnyDay() throws Exception {
    Connector c = new MockConnector("root");
    c.tableOperations().create("test");
    BatchWriter bw = c.createBatchWriter("test", 10000L, 1000L, 4);
    for (int i = 0; i < 100; i++) {
      int r = random.nextInt();
      Mutation m = new Mutation(asText(r));
      m.put(asText(random.nextInt()), asText(random.nextInt()), new Value(Integer.toHexString(r).getBytes()));
      bw.addMutation(m);
    }
    bw.close();
    BatchScanner s = c.createBatchScanner("test", Constants.NO_AUTHS, 2);
    s.setRanges(Collections.singletonList(new Range()));
    Key key = null;
    int count = 0;
    for (Entry<Key,Value> entry : s) {
      if (key != null)
        assertTrue(key.compareTo(entry.getKey()) < 0);
      assertEquals(entry.getKey().getRow(), new Text(entry.getValue().get()));
      key = entry.getKey();
      count++;
    }
    assertEquals(100, count);
  }
  
  @Test
  public void testAggregation() throws Exception {
    MockInstance mockInstance = new MockInstance();
    Connector c = mockInstance.getConnector("root", new byte[] {});
    String table = "perDayCounts";
    c.tableOperations().create(table);
    Class<? extends SortedKeyValueIterator<Key,Value>> clazz = SummingCombiner.class;
    IteratorSetting is = new IteratorSetting(10, "String Summation", clazz);
    Combiner.setColumns(is, Collections.singletonList(new Combiner.Column("day")));
    is.addOption(SummingCombiner.TYPE, SummingCombiner.Type.STRING.name());
    c.tableOperations().attachIterator(table, is);
    String keys[][] = { {"foo", "day", "20080101"}, {"foo", "day", "20080101"}, {"foo", "day", "20080103"}, {"bar", "day", "20080101"},
        {"bar", "day", "20080101"},};
    BatchWriter bw = c.createBatchWriter("perDayCounts", 1000L, 1000L, 1);
    for (String elt[] : keys) {
      Mutation m = new Mutation(new Text(elt[0]));
      m.put(new Text(elt[1]), new Text(elt[2]), new Value("1".getBytes()));
      bw.addMutation(m);
    }
    bw.close();
    
    Scanner s = c.createScanner("perDayCounts", Constants.NO_AUTHS);
    Iterator<Entry<Key,Value>> iterator = s.iterator();
    assertTrue(iterator.hasNext());
    checkEntry(iterator.next(), "bar", "day", "20080101", "2");
    assertTrue(iterator.hasNext());
    checkEntry(iterator.next(), "foo", "day", "20080101", "2");<commit_msg>ACCUMULO-172 merge from 1.3 branch<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.accumulo.core.client.mock;

import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertFalse;
import static junit.framework.Assert.assertTrue;

import java.util.Collections;
import java.util.Iterator;
import java.util.Map.Entry;
import java.util.Random;

import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.BatchDeleter;
import org.apache.accumulo.core.client.BatchScanner;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.IteratorSetting;
import org.apache.accumulo.core.client.MultiTableBatchWriter;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.iterators.Combiner;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.iterators.user.SummingCombiner;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.hadoop.io.Text;
import org.junit.Test;

public class MockConnectorTest {
  Random random = new Random();
  
  static Text asText(int i) {
    return new Text(Integer.toHexString(i));
  }
  
  @Test
  public void testSunnyDay() throws Exception {
    Connector c = new MockConnector("root");
    c.tableOperations().create("test");
    BatchWriter bw = c.createBatchWriter("test", 10000L, 1000L, 4);
    for (int i = 0; i < 100; i++) {
      int r = random.nextInt();
      Mutation m = new Mutation(asText(r));
      m.put(asText(random.nextInt()), asText(random.nextInt()), new Value(Integer.toHexString(r).getBytes()));
      bw.addMutation(m);
    }
    bw.close();
    BatchScanner s = c.createBatchScanner("test", Constants.NO_AUTHS, 2);
    s.setRanges(Collections.singletonList(new Range()));
    Key key = null;
    int count = 0;
    for (Entry<Key,Value> entry : s) {
      if (key != null)
        assertTrue(key.compareTo(entry.getKey()) < 0);
      assertEquals(entry.getKey().getRow(), new Text(entry.getValue().get()));
      key = entry.getKey();
      count++;
    }
    assertEquals(100, count);
  }
  
  @Test
  public void testChangeAuths() throws Exception {
    Connector c = new MockConnector("root");
    c.securityOperations().createUser("greg", new byte[] {}, new Authorizations("A", "B", "C"));
    assertTrue(c.securityOperations().getUserAuthorizations("greg").contains("A".getBytes()));
    c.securityOperations().changeUserAuthorizations("greg", new Authorizations("X", "Y", "Z"));
    assertTrue(c.securityOperations().getUserAuthorizations("greg").contains("X".getBytes()));
    assertFalse(c.securityOperations().getUserAuthorizations("greg").contains("A".getBytes()));
  }
  
  @Test
  public void testAggregation() throws Exception {
    MockInstance mockInstance = new MockInstance();
    Connector c = mockInstance.getConnector("root", new byte[] {});
    String table = "perDayCounts";
    c.tableOperations().create(table);
    Class<? extends SortedKeyValueIterator<Key,Value>> clazz = SummingCombiner.class;
    IteratorSetting is = new IteratorSetting(10, "String Summation", clazz);
    Combiner.setColumns(is, Collections.singletonList(new Combiner.Column("day")));
    is.addOption(SummingCombiner.TYPE, SummingCombiner.Type.STRING.name());
    c.tableOperations().attachIterator(table, is);
    String keys[][] = { {"foo", "day", "20080101"}, {"foo", "day", "20080101"}, {"foo", "day", "20080103"}, {"bar", "day", "20080101"},
        {"bar", "day", "20080101"},};
    BatchWriter bw = c.createBatchWriter("perDayCounts", 1000L, 1000L, 1);
    for (String elt[] : keys) {
      Mutation m = new Mutation(new Text(elt[0]));
      m.put(new Text(elt[1]), new Text(elt[2]), new Value("1".getBytes()));
      bw.addMutation(m);
    }
    bw.close();
    
    Scanner s = c.createScanner("perDayCounts", Constants.NO_AUTHS);
    Iterator<Entry<Key,Value>> iterator = s.iterator();
    assertTrue(iterator.hasNext());
    checkEntry(iterator.next(), "bar", "day", "20080101", "2");
    assertTrue(iterator.hasNext());
    checkEntry(iterator.next(), "foo", "day", "20080101", "2");
{'fraction_non_alphanumeric': 0.14078782826177574, 'fraction_numerical': 0.020737807419758232, 'mean_word_length': 4.422033898305084, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3232114', 'n_tokens_mistral': 3061, 'n_tokens_neox': 2918, 'n_words': 718}