<commit_before>import * as _ from 'lodash';

export const defaultPageSize = 20
export type AggCallback = (aggName:string,key:string,isCrowd?:boolean) => void;
export interface AggBucket {
  key: string
  docCount: number
}
export interface AggItem {
  name: string
  buckets: AggBucket[]
}
export interface AggBucketMap { [key:string] : AggBucket[] }
export type AggFilterMap = { [key:string] : Set<string> }

export interface AggFilterListItem { 
  // These field names are used in graphql query
  field:string,
  values: string[]
}

export interface SortItem {
  id: string
  desc?: boolean
}

export interface SearchParams {
  q: string
  page: number
  pageSize: number
  aggFilters: AggFilterListItem[]
  crowdAggFilters: AggFilterListItem[]
  sorts: SortItem[]
}

export function flattenAggs(aggs : AggFilterMap) : AggFilterListItem[] {
  if (!aggs) return []
  const result =
    Object.keys(aggs)
    .filter(k => aggs[k].size > 0)
    .map(k => ({field: k, values: [...aggs[k].values()]}))
  if (result.length == 0) return []
  return result;
}

export function expandAggs(aggs: AggFilterListItem[]) {
  if (aggs) {
    return aggs.reduce( (acc,agg) => {
      acc[agg.field] = new Set(agg.values)
      return acc
    }, {} as AggFilterMap );
  }
  return null
}

const version_marker = '!'
interface CompactSearchParams {
  // page
  p: number
  // page size
  z: number
  // aggFilters
  a: AggFilterListItem[]
  // crowdAggFilters
  c: AggFilterListItem[]
  s: SortItem[]
}

function compact_search(p : SearchParams) : CompactSearchParams {
  var res = {
    p: p.page,
    z : p.pageSize,
    a : p.aggFilters,
    c : p.crowdAggFilters,
    s : p.sorts
  }
  // Erase default values
  if (res.p == 0) delete res.p
  if (res.z === defaultPageSize) delete res.z
  Object.keys(res).forEach(k => {
    if (Array.isArray(res[k]) && res[k].length === 0) delete res[k]
  })
  if (Object.keys(res).length == 0) return null;
  return res
}
function expand(q : string, p : CompactSearchParams) : SearchParams {
  return {
    q,
    page: p.p||0,
    pageSize: p.z||defaultPageSize,
    aggFilters: p.a||[],
    crowdAggFilters: p.c||[],
    sorts: p.s||[]
  }
}

export function encodeSearchParams(params : SearchParams) : string {
  let shortNames = compact_search(params)
  if (shortNames == null) return ""
  let temp = JSON.stringify(shortNames)
  return version_marker + btoa(temp)
}

export function getSearchParamsFromURL() : SearchParams {
  const u = new URL(window.location.href)
  const encodedQuery = u.pathname.startsWith("/search/") ? _.last(u.pathname.split('/')) : ""
  const query = decodeURIComponent(encodedQuery)
  const encodedParams = u.searchParams.get('p')
  if (encodedParams && encodedParams[0] == version_marker) {
      let decoded = atob(encodedParams.substr(1))
      const temp = JSON.parse(decoded)
      return expand(query, temp);
  }
  return  expand(query, <CompactSearchParams>{})
}<commit_msg>Protect against invalid/corrupt url query encoding<commit_after>import * as _ from 'lodash';

export const defaultPageSize = 20
export type AggCallback = (aggName:string,key:string,isCrowd?:boolean) => void;
export interface AggBucket {
  key: string
  docCount: number
}
export interface AggItem {
  name: string
  buckets: AggBucket[]
}
export interface AggBucketMap { [key:string] : AggBucket[] }
export type AggFilterMap = { [key:string] : Set<string> }

export interface AggFilterListItem { 
  // These field names are used in graphql query
  field:string,
  values: string[]
}

export interface SortItem {
  id: string
  desc?: boolean
}

export interface SearchParams {
  q: string
  page: number
  pageSize: number
  aggFilters: AggFilterListItem[]
  crowdAggFilters: AggFilterListItem[]
  sorts: SortItem[]
}

export function flattenAggs(aggs : AggFilterMap) : AggFilterListItem[] {
  if (!aggs) return []
  const result =
    Object.keys(aggs)
    .filter(k => aggs[k].size > 0)
    .map(k => ({field: k, values: [...aggs[k].values()]}))
  if (result.length == 0) return []
  return result;
}

export function expandAggs(aggs: AggFilterListItem[]) {
  if (aggs) {
    return aggs.reduce( (acc,agg) => {
      acc[agg.field] = new Set(agg.values)
      return acc
    }, {} as AggFilterMap );
  }
  return null
}

const version_marker = '!'
interface CompactSearchParams {
  // page
  p: number
  // page size
  z: number
  // aggFilters
  a: AggFilterListItem[]
  // crowdAggFilters
  c: AggFilterListItem[]
  s: SortItem[]
}

function compact_search(p : SearchParams) : CompactSearchParams {
  var res = {
    p: p.page,
    z : p.pageSize,
    a : p.aggFilters,
    c : p.crowdAggFilters,
    s : p.sorts
  }
  // Erase default values
  if (res.p == 0) delete res.p
  if (res.z === defaultPageSize) delete res.z
  Object.keys(res).forEach(k => {
    if (Array.isArray(res[k]) && res[k].length === 0) delete res[k]
  })
  if (Object.keys(res).length == 0) return null;
  return res
}
function expand(q : string, p : CompactSearchParams) : SearchParams {
  return {
    q,
    page: p.p||0,
    pageSize: p.z||defaultPageSize,
    aggFilters: p.a||[],
    crowdAggFilters: p.c||[],
    sorts: p.s||[]
  }
}

export function encodeSearchParams(params : SearchParams) : string {
  let shortNames = compact_search(params)
  if (shortNames == null) return ""
  let temp = JSON.stringify(shortNames)
  return version_marker + btoa(temp)
}

export function getSearchParamsFromURL() : SearchParams {
  try {
    const u = new URL(window.location.href)
    const encodedQuery = u.pathname.startsWith("/search/") ? _.last(u.pathname.split('/')) : ""
    const query = decodeURIComponent(encodedQuery)
    const encodedParams = u.searchParams.get('p')
    if (encodedParams && encodedParams[0] == version_marker) {
        let decoded = atob(encodedParams.substr(1))
        const temp = JSON.parse(decoded)
        return expand(query, temp);
    }
    return  expand(query, <CompactSearchParams>{})
  }
  catch(e) {
    console.log(`Error decoding search params ${e}`)
    return expand("", <any>{})
  }
}
{'fraction_non_alphanumeric': 0.13600264725347452, 'fraction_numerical': 0.0033090668431502318, 'mean_word_length': 3.614503816793893, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4876751', 'n_tokens_mistral': 2076, 'n_tokens_neox': 2039, 'n_words': 511}