<commit_before>package com.segment.analytics;

import android.content.Context;
import android.os.Handler;
import android.os.HandlerThread;
import android.os.Looper;
import android.os.Message;
import android.util.JsonWriter;
import android.util.Log;
import java.io.BufferedWriter;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.nio.charset.Charset;
import java.util.Date;
import java.util.Map;

import static android.os.Process.THREAD_PRIORITY_BACKGROUND;
import static com.segment.analytics.Logger.OWNER_SEGMENT;
import static com.segment.analytics.Logger.VERB_ENQUEUE;
import static com.segment.analytics.Utils.isConnected;
import static com.segment.analytics.Utils.isNullOrEmpty;
import static com.segment.analytics.Utils.panic;
import static com.segment.analytics.Utils.quitThread;
import static com.segment.analytics.Utils.toISO8601Date;

/**
 * The actual service that posts data to Segment's servers.
 *
 * @since 2.3
 */
class Segment {
  private static final Charset UTF_8 = Charset.forName("UTF-8");
  private static final String SEGMENT_THREAD_NAME = Utils.THREAD_PREFIX + "Segment";
  private static final String PAYLOAD_QUEUE_FILE_SUFFIX = "-payload-v1";
  // we can probably relax/adjust these limits after getting more feedback
  private static final int MAX_FLUSH_BATCH_SIZE = 50; // only flush 50 payloads in a single request
  private static final int MAX_QUEUE_SIZE = 1000; // reject any payloads if queue grows to over this

  final Context context;
  final QueueFile payloadQueueFile;
  final SegmentHTTPApi segmentHTTPApi;
  final int flushQueueSize;
  final int flushInterval;
  final Stats stats;
  final Handler handler;
  final HandlerThread segmentThread;
  final Logger logger;
  final Map<String, Boolean> integrations;

  static synchronized Segment create(Context context, int flushQueueSize, int flushInterval,
      SegmentHTTPApi segmentHTTPApi, Map<String, Boolean> integrations, String tag, Stats stats,
      Logger logger) {
    File parent = context.getFilesDir();
    String filePrefix = tag.replaceAll("[^A-Za-z0-9]", ""); // sanitize input
    QueueFile payloadQueueFile;
    try {
      if (parent.exists() || parent.mkdirs() || parent.isDirectory()) {
        payloadQueueFile = new QueueFile(new File(parent, filePrefix + PAYLOAD_QUEUE_FILE_SUFFIX));
      } else {
        throw new IOException();
      }
    } catch (IOException e) {
      throw new RuntimeException("Could not create disk queue " + filePrefix + " in " + parent, e);
    }
    flushQueueSize = Math.min(flushQueueSize, MAX_FLUSH_BATCH_SIZE);
    return new Segment(context, flushQueueSize, flushInterval, segmentHTTPApi, payloadQueueFile,
        integrations, stats, logger);
  }

  Segment(Context context, int flushQueueSize, int flushInterval, SegmentHTTPApi segmentHTTPApi,
      QueueFile payloadQueueFile, Map<String, Boolean> integrations, Stats stats, Logger logger) {
    this.context = context;
    this.flushQueueSize = flushQueueSize;
    this.segmentHTTPApi = segmentHTTPApi;
    this.payloadQueueFile = payloadQueueFile;
    this.stats = stats;
    this.logger = logger;
    this.integrations = integrations;
    this.flushInterval = flushInterval * 1000;
    segmentThread = new HandlerThread(SEGMENT_THREAD_NAME, THREAD_PRIORITY_BACKGROUND);
    segmentThread.start();
    handler = new SegmentHandler(segmentThread.getLooper(), this);<commit_msg>Respect flushQueueSize as passed in. If it is greater than MAX_BATCH_SIZE, then each flush triggered by the disk queue hitting flushQueueSize will make multiple requests, breaking each into MAX_BATCH_SIZE events at a time<commit_after>package com.segment.analytics;

import android.content.Context;
import android.os.Handler;
import android.os.HandlerThread;
import android.os.Looper;
import android.os.Message;
import android.util.JsonWriter;
import android.util.Log;
import java.io.BufferedWriter;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.nio.charset.Charset;
import java.util.Date;
import java.util.Map;

import static android.os.Process.THREAD_PRIORITY_BACKGROUND;
import static com.segment.analytics.Logger.OWNER_SEGMENT;
import static com.segment.analytics.Logger.VERB_ENQUEUE;
import static com.segment.analytics.Utils.isConnected;
import static com.segment.analytics.Utils.isNullOrEmpty;
import static com.segment.analytics.Utils.panic;
import static com.segment.analytics.Utils.quitThread;
import static com.segment.analytics.Utils.toISO8601Date;

/**
 * The actual service that posts data to Segment's servers.
 *
 * @since 2.3
 */
class Segment {
  private static final Charset UTF_8 = Charset.forName("UTF-8");
  private static final String SEGMENT_THREAD_NAME = Utils.THREAD_PREFIX + "Segment";
  private static final String PAYLOAD_QUEUE_FILE_SUFFIX = "-payload-v1";
  // we can probably relax/adjust these limits after getting more feedback
  private static final int MAX_FLUSH_BATCH_SIZE = 50; // only flush 50 payloads in a single request
  private static final int MAX_QUEUE_SIZE = 1000; // reject any payloads if queue grows to over this

  final Context context;
  final QueueFile payloadQueueFile;
  final SegmentHTTPApi segmentHTTPApi;
  final int flushQueueSize;
  final int flushInterval;
  final Stats stats;
  final Handler handler;
  final HandlerThread segmentThread;
  final Logger logger;
  final Map<String, Boolean> integrations;

  static synchronized Segment create(Context context, int flushQueueSize, int flushInterval,
      SegmentHTTPApi segmentHTTPApi, Map<String, Boolean> integrations, String tag, Stats stats,
      Logger logger) {
    File parent = context.getFilesDir();
    String filePrefix = tag.replaceAll("[^A-Za-z0-9]", ""); // sanitize input
    QueueFile payloadQueueFile;
    try {
      if (parent.exists() || parent.mkdirs() || parent.isDirectory()) {
        payloadQueueFile = new QueueFile(new File(parent, filePrefix + PAYLOAD_QUEUE_FILE_SUFFIX));
      } else {
        throw new IOException();
      }
    } catch (IOException e) {
      throw new RuntimeException("Could not create disk queue " + filePrefix + " in " + parent, e);
    }
    return new Segment(context, flushQueueSize, flushInterval, segmentHTTPApi, payloadQueueFile,
        integrations, stats, logger);
  }

  Segment(Context context, int flushQueueSize, int flushInterval, SegmentHTTPApi segmentHTTPApi,
      QueueFile payloadQueueFile, Map<String, Boolean> integrations, Stats stats, Logger logger) {
    this.context = context;
    this.flushQueueSize = flushQueueSize;
    this.segmentHTTPApi = segmentHTTPApi;
    this.payloadQueueFile = payloadQueueFile;
    this.stats = stats;
    this.logger = logger;
    this.integrations = integrations;
    this.flushInterval = flushInterval * 1000;
    segmentThread = new HandlerThread(SEGMENT_THREAD_NAME, THREAD_PRIORITY_BACKGROUND);
    segmentThread.start();
    handler = new SegmentHandler(segmentThread.getLooper(), this);
{'fraction_non_alphanumeric': 0.08597157731813705, 'fraction_numerical': 0.006472491909385114, 'mean_word_length': 4.627870150435471, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3950640', 'n_tokens_mistral': 2077, 'n_tokens_neox': 2044, 'n_words': 578}