<commit_before>'use strict';

const BigNumber = require('bignumber.js');
const _ = require('lodash');
const logger = require('winston');

const twitter = require('./twitter');
const archive = require('./archive');
const utils = require('./utils');

const getUrlEntities = tweet => {
  if (!tweet.entities.urls) return [];
  return tweet.entities.urls.map(e => {
    return {
      url: e.url,
      display_url: e.display_url,
      expanded_url: e.expanded_url
    };
  });
};

const getMediaEntities = tweet => {
  if (!tweet.entities.media) return [];
  return tweet.entities.media.map(e => {
    return {
      url: e.url,
      display_url: e.display_url,
      media_url: e.media_url
    };
  });
};

const buildTree = (root_id, tweets) => {
  logger.debug(`Building tree from ${tweets.length} tweets...`);

  const tree = [];

  // Determine the tweet ID start next search from (this is an optimization)
  const max_tweet_id = BigNumber.max(tweets.map(t => t.id_str)).toString();
  const start_from_id = (new BigNumber(max_tweet_id)).plus(1).toString();
  const meta = {
    start_from_id: start_from_id
  };

  const add = (tweet, parent) => {
    const range = tweet.display_text_range;
    const text = tweet.full_text.substr(range[0], range[1])<commit_msg>Fix truncating of URLs<commit_after>'use strict';

const BigNumber = require('bignumber.js');
const _ = require('lodash');
const logger = require('winston');

const twitter = require('./twitter');
const archive = require('./archive');
const utils = require('./utils');

const getUrlEntities = tweet => {
  if (!tweet.entities.urls) return [];
  return tweet.entities.urls.map(e => {
    return {
      url: e.url,
      display_url: e.display_url,
      expanded_url: e.expanded_url
    };
  });
};

const getMediaEntities = tweet => {
  if (!tweet.entities.media) return [];
  return tweet.entities.media.map(e => {
    return {
      url: e.url,
      display_url: e.display_url,
      media_url: e.media_url
    };
  });
};

const buildTree = (root_id, tweets) => {
  logger.debug(`Building tree from ${tweets.length} tweets...`);

  const tree = [];

  // Determine the tweet ID start next search from (this is an optimization)
  const max_tweet_id = BigNumber.max(tweets.map(t => t.id_str)).toString();
  const start_from_id = (new BigNumber(max_tweet_id)).plus(1).toString();
  const meta = {
    start_from_id: start_from_id
  };

  const add = (tweet, parent) => {
    const range = tweet.display_text_range;
    const text = tweet.full_text.substr(range[0])
{'fraction_non_alphanumeric': 0.1551519936833794, 'fraction_numerical': 0.0019739439399921043, 'mean_word_length': 3.5493716337522443, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2297161', 'n_tokens_mistral': 862, 'n_tokens_neox': 848, 'n_words': 191}