<commit_before>/*
 * Druid - a distributed column store.
 * Copyright (C) 2012  Metamarkets Group Inc.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version 2
 * of the License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 */

package com.metamx.druid.coordination;

import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Throwables;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.metamx.common.ISE;
import com.metamx.common.logger.Logger;
import com.metamx.druid.client.DataSegment;
import com.metamx.druid.curator.announcement.Announcer;
import com.metamx.druid.initialization.ZkDataSegmentAnnouncerConfig;
import org.apache.curator.utils.ZKPaths;
import org.joda.time.DateTime;

import java.io.IOException;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentSkipListSet;

/**
 */
public class BatchDataSegmentAnnouncer extends AbstractDataSegmentAnnouncer
{
  private static final Logger log = new Logger(BatchDataSegmentAnnouncer.class);

  private final ZkDataSegmentAnnouncerConfig config;
  private final Announcer announcer;
  private final ObjectMapper jsonMapper;
  private final String liveSegmentLocation;

  private final Object lock = new Object();

  private final Set<SegmentZNode> availableZNodes = new ConcurrentSkipListSet<SegmentZNode>();
  private final Map<DataSegment, SegmentZNode> segmentLookup = Maps.newConcurrentMap();

  public BatchDataSegmentAnnouncer(
      DruidServerMetadata server,
      ZkDataSegmentAnnouncerConfig config,
      Announcer announcer,
      ObjectMapper jsonMapper
  )
  {
    super(server, config, announcer, jsonMapper);

    this.config = config;
    this.announcer = announcer;
    this.jsonMapper = jsonMapper;
    this.liveSegmentLocation = ZKPaths.makePath(config.getLiveSegmentsPath(), server.getName());
  }

  @Override
  public void announceSegment(DataSegment segment) throws IOException
  {
    int newBytesLen = jsonMapper.writeValueAsBytes(segment).length;
    if (newBytesLen > config.getMaxNumBytes()) {
      throw new ISE("byte size %,d exceeds %,d", newBytesLen, config.getMaxNumBytes());
    }

    synchronized (lock) {
      // create new batch
      if (availableZNodes.isEmpty()) {
        SegmentZNode availableZNode = new SegmentZNode(makeServedSegmentPath(new DateTime().toString()));
        availableZNode.addSegment(segment);

        log.info("Announcing segment[%s] at path[%s]", segment.getIdentifier(), availableZNode.getPath());
        announcer.announce(availableZNode.getPath(), availableZNode.getBytes());
        segmentLookup.put(segment, availableZNode);
        availableZNodes.add(availableZNode);
      } else { // update existing batch
        Iterator<SegmentZNode> iter = availableZNodes.iterator();
        boolean done = false;
        while (iter.hasNext() && !done) {
          SegmentZNode availableZNode = iter.next();
          if (availableZNode.getBytes().length + newBytesLen < config.getMaxNumBytes()) {
            availableZNode.addSegment(segment);

            log.info("Announcing segment[%s] at path[%s]", segment.getIdentifier(), availableZNode.getPath());
            announcer.update(availableZNode.getPath(), availableZNode.getBytes());
            segmentLookup.put(segment, availableZNode);

            if (availableZNode.getCount() >= config.getSegmentsPerNode()) {
              availableZNodes.remove(availableZNode);
            }

            done = true;
          }
        }
      }
    }
  }

  @Override
  public void unannounceSegment(DataSegment segment) throws IOException
  {
    final SegmentZNode segmentZNode = segmentLookup.remove(segment);
    if (segmentZNode == null) {
      return;
    }

    segmentZNode.removeSegment(segment);

    log.info("Unannouncing segment[%s] at path[%s]", segment.getIdentifier(), segmentZNode.getPath());
    if (segmentZNode.getCount() == 0) {
      availableZNodes.remove(segmentZNode);
      announcer.unannounce(segmentZNode.getPath());
    } else {
      announcer.update(segmentZNode.getPath(), segmentZNode.getBytes());
      availableZNodes.add(segmentZNode);
    }
  }

  @Override
  public void announceSegments(Iterable<DataSegment> segments) throws IOException
  {
    SegmentZNode segmentZNode = new SegmentZNode(makeServedSegmentPath(new DateTime().toString()));
    Set<DataSegment> batch = Sets.newHashSet();
    int byteSize = 0;
    int count = 0;

    for (DataSegment segment : segments) {
      int newBytesLen = jsonMapper.writeValueAsBytes(segment).length;

      if (newBytesLen > config.getMaxNumBytes()) {
        throw new ISE("byte size %,d exceeds %,d", newBytesLen, config.getMaxNumBytes());
      }

      if (count >= config.getSegmentsPerNode() || byteSize + newBytesLen > config.getMaxNumBytes()) {
        segmentZNode.addSegments(batch);
        announcer.announce(segmentZNode.getPath(), segmentZNode.getBytes());

        segmentZNode = new SegmentZNode(makeServedSegmentPath(new DateTime().toString()));
        batch = Sets.newHashSet();
        count = 0;
        byteSize = 0;
      }

      log.info("Announcing segment[%s] at path[%s]", segment.getIdentifier(), segmentZNode.getPath());
      segmentLookup.put(segment, segmentZNode);
      batch.add(segment);
      count++;
      byteSize += newBytesLen;
    }

    segmentZNode.addSegments(batch);
    announcer.announce(segmentZNode.getPath(), segmentZNode.getBytes());
  }

  @Override
  public void unannounceSegments(Iterable<DataSegment> segments) throws IOException
  {
    for (DataSegment segment : segments) {
      unannounceSegment(segment);
    }
  }

  private String makeServedSegmentPath(String zNode)
  {
    return ZKPaths.makePath(liveSegmentLocation, zNode);
  }

  private class SegmentZNode implements Comparable<SegmentZNode>
  {
    private final String path;

    private byte[] bytes = new byte[]{};
    private int count = 0;

    public SegmentZNode(String path)
    {
      this.path = path;<commit_msg>fix previous fix because multithread is not correct<commit_after>/*
 * Druid - a distributed column store.
 * Copyright (C) 2012  Metamarkets Group Inc.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version 2
 * of the License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 */

package com.metamx.druid.coordination;

import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Throwables;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.metamx.common.ISE;
import com.metamx.common.logger.Logger;
import com.metamx.druid.client.DataSegment;
import com.metamx.druid.curator.announcement.Announcer;
import com.metamx.druid.initialization.ZkDataSegmentAnnouncerConfig;
import org.apache.curator.utils.ZKPaths;
import org.joda.time.DateTime;

import java.io.IOException;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentSkipListSet;
import java.util.concurrent.atomic.AtomicLong;

/**
 */
public class BatchDataSegmentAnnouncer extends AbstractDataSegmentAnnouncer
{
  private static final Logger log = new Logger(BatchDataSegmentAnnouncer.class);

  private final ZkDataSegmentAnnouncerConfig config;
  private final Announcer announcer;
  private final ObjectMapper jsonMapper;
  private final String liveSegmentLocation;

  private final Object lock = new Object();
  private final AtomicLong counter = new AtomicLong(0);

  private final Set<SegmentZNode> availableZNodes = new ConcurrentSkipListSet<SegmentZNode>();
  private final Map<DataSegment, SegmentZNode> segmentLookup = Maps.newConcurrentMap();

  public BatchDataSegmentAnnouncer(
      DruidServerMetadata server,
      ZkDataSegmentAnnouncerConfig config,
      Announcer announcer,
      ObjectMapper jsonMapper
  )
  {
    super(server, config, announcer, jsonMapper);

    this.config = config;
    this.announcer = announcer;
    this.jsonMapper = jsonMapper;
    this.liveSegmentLocation = ZKPaths.makePath(config.getLiveSegmentsPath(), server.getName());
  }

  @Override
  public void announceSegment(DataSegment segment) throws IOException
  {
    int newBytesLen = jsonMapper.writeValueAsBytes(segment).length;
    if (newBytesLen > config.getMaxNumBytes()) {
      throw new ISE("byte size %,d exceeds %,d", newBytesLen, config.getMaxNumBytes());
    }

    synchronized (lock) {
      // create new batch
      if (availableZNodes.isEmpty()) {
        SegmentZNode availableZNode = new SegmentZNode(makeServedSegmentPath(new DateTime().toString()));
        availableZNode.addSegment(segment);

        log.info("Announcing segment[%s] at path[%s]", segment.getIdentifier(), availableZNode.getPath());
        announcer.announce(availableZNode.getPath(), availableZNode.getBytes());
        segmentLookup.put(segment, availableZNode);
        availableZNodes.add(availableZNode);
      } else { // update existing batch
        Iterator<SegmentZNode> iter = availableZNodes.iterator();
        boolean done = false;
        while (iter.hasNext() && !done) {
          SegmentZNode availableZNode = iter.next();
          if (availableZNode.getBytes().length + newBytesLen < config.getMaxNumBytes()) {
            availableZNode.addSegment(segment);

            log.info("Announcing segment[%s] at path[%s]", segment.getIdentifier(), availableZNode.getPath());
            announcer.update(availableZNode.getPath(), availableZNode.getBytes());
            segmentLookup.put(segment, availableZNode);

            if (availableZNode.getCount() >= config.getSegmentsPerNode()) {
              availableZNodes.remove(availableZNode);
            }

            done = true;
          }
        }
      }
    }
  }

  @Override
  public void unannounceSegment(DataSegment segment) throws IOException
  {
    final SegmentZNode segmentZNode = segmentLookup.remove(segment);
    if (segmentZNode == null) {
      return;
    }

    synchronized (lock) {
      segmentZNode.removeSegment(segment);

      log.info("Unannouncing segment[%s] at path[%s]", segment.getIdentifier(), segmentZNode.getPath());
      if (segmentZNode.getCount() == 0) {
        availableZNodes.remove(segmentZNode);
        announcer.unannounce(segmentZNode.getPath());
      } else {
        announcer.update(segmentZNode.getPath(), segmentZNode.getBytes());
        availableZNodes.add(segmentZNode);
      }
    }
  }

  @Override
  public void announceSegments(Iterable<DataSegment> segments) throws IOException
  {
    SegmentZNode segmentZNode = new SegmentZNode(makeServedSegmentPath(new DateTime().toString()));
    Set<DataSegment> batch = Sets.newHashSet();
    int byteSize = 0;
    int count = 0;

    for (DataSegment segment : segments) {
      int newBytesLen = jsonMapper.writeValueAsBytes(segment).length;

      if (newBytesLen > config.getMaxNumBytes()) {
        throw new ISE("byte size %,d exceeds %,d", newBytesLen, config.getMaxNumBytes());
      }

      if (count >= config.getSegmentsPerNode() || byteSize + newBytesLen > config.getMaxNumBytes()) {
        segmentZNode.addSegments(batch);
        announcer.announce(segmentZNode.getPath(), segmentZNode.getBytes());

        segmentZNode = new SegmentZNode(makeServedSegmentPath(new DateTime().toString()));
        batch = Sets.newHashSet();
        count = 0;
        byteSize = 0;
      }

      log.info("Announcing segment[%s] at path[%s]", segment.getIdentifier(), segmentZNode.getPath());
      segmentLookup.put(segment, segmentZNode);
      batch.add(segment);
      count++;
      byteSize += newBytesLen;
    }

    segmentZNode.addSegments(batch);
    announcer.announce(segmentZNode.getPath(), segmentZNode.getBytes());
  }

  @Override
  public void unannounceSegments(Iterable<DataSegment> segments) throws IOException
  {
    for (DataSegment segment : segments) {
      unannounceSegment(segment);
    }
  }

  private String makeServedSegmentPath(String zNode)
  {
    return ZKPaths.makePath(liveSegmentLocation, String.format("%s%s", zNode, counter.getAndIncrement()));
  }

  private class SegmentZNode implements Comparable<SegmentZNode>
  {
    private final String path;

    private byte[] bytes = new byte[]{};
    private int count = 0;

    public SegmentZNode(String path)
    {
      this.path = path;
{'fraction_non_alphanumeric': 0.10241538800381764, 'fraction_numerical': 0.0033037221936715365, 'mean_word_length': 3.771278458844133, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6003281', 'n_tokens_mistral': 3941, 'n_tokens_neox': 3843, 'n_words': 938}