<commit_before><template>
  <span
    v-if="hasEntities && !deleted"
    class="apply-pre">
    <template v-if="spoiler && !showSpoiler">
      <button
        class="btn btn-link mr-3 btn-primary"
        type="button"
        @click="toggleSpoiler">
        <span class="d-sm-inline ml-2">Show Spoiler:
          <span v-emojify>{{ spoiler.topic }}</span>
        </span>
      </button>
    </template>
    <template v-else-if="!longpost || !showLongpost">
      <template v-for="(entity, i) in entities">
        <nuxt-link
          v-emojify
          v-if="entity.type === 'mentions'"
          :to="`/@${entity.text}`"
          :key="`mention-${i}`">@{{ entity.text }}</nuxt-link>
        <nuxt-link
          v-emojify
          v-else-if="entity.type === 'tags'"
          :to="`/tags/${entity.text}`"
          :key="`tags-${i}`">#{{ entity.text }}</nuxt-link>
        <a
          v-emojify
          v-else-if="entity.type === 'links'"
          :href="entity.link"
          :key="`links-${i}`"
          target="_new">{{ unicodeSubstring(entity.text, 0, entity.len) }}</a>
        <span
          v-emojify
          v-else
          :key="`text-${i}`">{{ entity.text }}</span>
      </template>
      <template v-if="longpost">
        <button
          class="btn btn-link mr-3 btn-primary"
          style="margin-top:.8em;display:block"
          type="button"
          @click="toggleLongpost">
          <font-awesome-icon
            icon="plus"
            aria-hidden="true"/> Expand Post
        </button>
      </template>
    </template>
    <template v-else>
      <h5
        v-emojify
        v-if="longpost.title">{{ longpost.title }}</h5>
      <span v-emojify>{{ longpost.body }}</span>
      <div style="margin-top:.8em">
        <button
          class="btn btn-link mr-3 btn-primary"
          type="button"
          @click="toggleLongpost">
          <font-awesome-icon
            icon="minus"
            aria-hidden="true"/> Collapse Post
        </button>
      </div>
    </template>
  </span>
  <span v-else>
    <slot />
  </span>
</template>
<script>
import unicodeSubstring from 'unicode-substring'
import stringLength from 'string-length'
export default {
  props: {
    content: {
      type: Object,
      default: () => ({})
    },
    deleted: {
      type: Boolean,
      default: false
    },
    spoiler: {
      type: Object,
      default: null
    },
    longpost: {
      type: Object,
      default: null
    }
  },
  data() {
    return {
      showSpoiler: false,
      showLongpost: false
    }
  },
  computed: {
    hasEntities() {
      return 'entities' in this.content && 'text' in this.content
    },
    // text and entities to html
    entities() {
      const { text: orig, entities } = this.content
      const { links, mentions, tags } = entities
      addTypeKey(links, 'links')
      addTypeKey(mentions, 'mentions')
      addTypeKey(tags, 'tags')
      return [{ text: '', type: 'text', pos: 0, len: 0 }]
        .concat(links, mentions, tags, {
          text: '',
          type: 'text',
          pos: stringLength(orig),
          len: 0
        })
        .sort((a, b) => a.pos - b.pos)
        .reduce((res, cur, i, ary) => {
          if (i === 0) return res
          const prev = ary[i - 1]
          const pos = prev.pos + prev.len
          const len = cur.pos
          const text = unicodeSubstring(orig, pos, len)
          res.push({
            text,
            type: 'text',
            pos,
            len: stringLength(text)
          })
          res.push(cur)
          return res
        }, [])
        .filter(entity => !(entity.type === 'text' && entity.text === ''))
    }
  },
  methods: {
    unicodeSubstring,
    toggleSpoiler() {
      this.showSpoiler = !this.showSpoiler
    },
    toggleLongpost() {
      this.showLongpost = !this.showLongpost
    }
  }
}

function addTypeKey(entities, value) {
  return entities.map(entity => {
    entity.type = value
    return entity
  })
}
</script>
<style scoped>
.apply-pre {
  white-space: pre-wrap;
}
</style><commit_msg>Replace patter link with own domain<commit_after><template>
  <span
    v-if="hasEntities && !deleted"
    class="apply-pre">
    <template v-if="spoiler && !showSpoiler">
      <button
        class="btn btn-link mr-3 btn-primary"
        type="button"
        @click="toggleSpoiler">
        <span class="d-sm-inline ml-2">Show Spoiler:
          <span v-emojify>{{ spoiler.topic }}</span>
        </span>
      </button>
    </template>
    <template v-else-if="!longpost || !showLongpost">
      <template v-for="(entity, i) in entities">
        <nuxt-link
          v-emojify
          v-if="entity.type === 'mentions'"
          :to="`/@${entity.text}`"
          :key="`mention-${i}`">@{{ entity.text }}</nuxt-link>
        <nuxt-link
          v-emojify
          v-else-if="entity.type === 'tags'"
          :to="`/tags/${entity.text}`"
          :key="`tags-${i}`">#{{ entity.text }}</nuxt-link>
        <a
          v-emojify
          v-else-if="entity.type === 'links'"
          :href="entity.link"
          :key="`links-${i}`"
          target="_new">{{ unicodeSubstring(entity.text, 0, entity.len) }}</a>
        <span
          v-emojify
          v-else
          :key="`text-${i}`">{{ entity.text }}</span>
      </template>
      <template v-if="longpost">
        <button
          class="btn btn-link mr-3 btn-primary"
          style="margin-top:.8em;display:block"
          type="button"
          @click="toggleLongpost">
          <font-awesome-icon
            icon="plus"
            aria-hidden="true"/> Expand Post
        </button>
      </template>
    </template>
    <template v-else>
      <h5
        v-emojify
        v-if="longpost.title">{{ longpost.title }}</h5>
      <span v-emojify>{{ longpost.body }}</span>
      <div style="margin-top:.8em">
        <button
          class="btn btn-link mr-3 btn-primary"
          type="button"
          @click="toggleLongpost">
          <font-awesome-icon
            icon="minus"
            aria-hidden="true"/> Collapse Post
        </button>
      </div>
    </template>
  </span>
  <span v-else>
    <slot />
  </span>
</template>
<script>
import unicodeSubstring from 'unicode-substring'
import stringLength from 'string-length'
export default {
  props: {
    content: {
      type: Object,
      default: () => ({})
    },
    deleted: {
      type: Boolean,
      default: false
    },
    spoiler: {
      type: Object,
      default: null
    },
    longpost: {
      type: Object,
      default: null
    }
  },
  data() {
    return {
      showSpoiler: false,
      showLongpost: false
    }
  },
  computed: {
    hasEntities() {
      return 'entities' in this.content && 'text' in this.content
    },
    // text and entities to html
    entities() {
      const { text: orig, entities } = this.content
      const {
        links: originalLinks,
        mentions: originalMentions,
        tags: originalTags
      } = entities
      const links = addTypeKey(originalLinks, 'links')
      const mentions = addTypeKey(originalMentions, 'mentions')
      const tags = addTypeKey(originalTags, 'tags')
      return [{ text: '', type: 'text', pos: 0, len: 0 }]
        .concat(links, mentions, tags, {
          text: '',
          type: 'text',
          pos: stringLength(orig),
          len: 0
        })
        .sort((a, b) => a.pos - b.pos)
        .reduce((res, cur, i, ary) => {
          if (i === 0) return res
          const prev = ary[i - 1]
          const pos = prev.pos + prev.len
          const len = cur.pos
          const text = unicodeSubstring(orig, pos, len)
          res.push({
            text,
            type: 'text',
            pos,
            len: stringLength(text)
          })
          res.push(cur)
          return res
        }, [])
        .filter(entity => !(entity.type === 'text' && entity.text === ''))
    }
  },
  methods: {
    unicodeSubstring,
    toggleSpoiler() {
      this.showSpoiler = !this.showSpoiler
    },
    toggleLongpost() {
      this.showLongpost = !this.showLongpost
    }
  }
}

function addTypeKey(entities, value) {
  return entities.map(entity => {
    entity.type = value
    if (value !== 'links') return entity
    return modifyURL(entity)
  })
}

const ReplaceUrls = [
  {
    test: /^https?:\/\/patter.chat\/room\/(\d+)/,
    replace: 'https://beta.pnut.io/messages/$1'
  }
]

function modifyURL(entity) {
  const itemToReplace = ReplaceUrls.find(({ test }) => test.test(entity.link))
  if (!itemToReplace) return entity
  const { test, replace } = itemToReplace
  const link = entity.link.replace(test, replace)
  const text = entity.text.replace(test, replace)
  const len = test.test(entity.text) ? link.length : entity.len
  return {
    ...entity,
    link,
    text,
    len
  }
}
</script>
<style scoped>
.apply-pre {
  white-space: pre-wrap;
}
</style>
{'fraction_non_alphanumeric': 0.1627228888639677, 'fraction_numerical': 0.0032522148704721317, 'mean_word_length': 1.855587576048671, 'pattern_counts': {'":': 0, '<': 97, '<?xml version=': 0, '>': 108, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4407471', 'n_tokens_mistral': 2925, 'n_tokens_neox': 2919, 'n_words': 603}