<commit_before>// Copyright 2017, 2018 <NAME>
// Published at https://github.com/networkimprov/mnm
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/

package qlib

import (
   "sync/atomic"
   "encoding/base32"
   "hash/crc32"
   "fmt"
   "io"
   "encoding/json"
   "net"
   "os"
   "crypto/rand"
   "crypto/sha1"
   "crypto/sha256"
   "sort"
   "strconv"
   "strings"
   "sync"
   "time"
   "unicode/utf8"
)

const kLoginTimeout time.Duration =  5 * time.Second
const kPulseTimeout time.Duration = 2 * time.Minute
const kQueueAckTimeout time.Duration = 30 * time.Second
const kQueueIdleMax time.Duration = 28 * time.Hour
const kStoreIdIncr = 1000
const kMsgHeaderMinLen = int64(len(`{"op":1}`))
const kMsgHeaderMaxLen = int64(1 << 16)
const kMsgPingDataMax = 140
const kNodeIdLen = 25
const kAliasMinLen = 8
const kPostDateFormat = "2006-01-02T15:04:05.000Z07:00"

var UDb UserDatabase // set by caller

// encoding without vowels to avoid words
var sBase32 = base32.NewEncoding("%+123456789BCDFGHJKLMNPQRSTVWXYZ")

var sCrc32c = crc32.MakeTable(crc32.Castagnoli)
var sSendDoor, sRecvDoor sync.RWMutex
var sOhi = tOhi{from: tOhiMap{}}
var sNode = tNodes{list: tNodeMap{}}
var sStore = tStore{}


type tHeader struct {
   Op uint8
   DataLen, DataHead int64
   DataSum uint64
   Uid, Gid string
   Id string
   Node, NewNode string
   NewAlias, From, To string // alias
   Type string
   Act string
   For []tHeaderFor
}

const (
   eOpTmtpRev = iota
   eOpRegister; eOpLogin
   eOpUserEdit; eOpOhiEdit;
   eOpGroupInvite; eOpGroupEdit
   eOpPost; eOpPing
   eOpAck
   eOpPulse; eOpQuit
   eOpEnd
)

type tHeaderFor struct { Id string; Type int8 }

const ( _=iota; eForUser; eForGroupAll; eForGroupExcl; eForSelf )

var sHeaderDefs = [...]tHeader{
   eOpTmtpRev    : { Id:"1" },
   eOpRegister   : { NewNode:"1", NewAlias:"1" },
   eOpLogin      : { Uid:"1", Node:"1" },
   eOpUserEdit   : { Id:"1" },
   eOpOhiEdit    : { Id:"1", For:[]tHeaderFor{{}}, Type:"1" },
   eOpGroupInvite: { Id:"1", DataLen:1, Gid:"1", From:"1", To:"1" },
   eOpGroupEdit  : { Id:"1", Act:"1", Gid:"1" },
   eOpPost       : { Id:"1", DataLen:1, For:[]tHeaderFor{} },
   eOpPing       : { Id:"1", DataLen:1, To:"1" },
   eOpAck        : { Id:"1", Type:"1" },
   eOpPulse      : {  },
   eOpQuit       : {  },
}

func (o *tHeader) check() bool {
   if o.Op >= eOpEnd { return false }
   aDef := &sHeaderDefs[o.Op]
   aFail :=
      o.DataLen | o.DataHead < 0                     ||
      o.DataLen < o.DataHead                         ||
      (aDef.DataLen == 0)    != (o.DataLen == 0)     ||
      aDef.DataSum       > 0 && o.DataSum       == 0 ||
      len(aDef.Uid)      > 0 && len(o.Uid)      == 0 ||<commit_msg>qlib: Drop tHeader.check() test for .DataSum<commit_after>// Copyright 2017, 2018 <NAME>
// Published at https://github.com/networkimprov/mnm
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/

package qlib

import (
   "sync/atomic"
   "encoding/base32"
   "hash/crc32"
   "fmt"
   "io"
   "encoding/json"
   "net"
   "os"
   "crypto/rand"
   "crypto/sha1"
   "crypto/sha256"
   "sort"
   "strconv"
   "strings"
   "sync"
   "time"
   "unicode/utf8"
)

const kLoginTimeout time.Duration =  5 * time.Second
const kPulseTimeout time.Duration = 2 * time.Minute
const kQueueAckTimeout time.Duration = 30 * time.Second
const kQueueIdleMax time.Duration = 28 * time.Hour
const kStoreIdIncr = 1000
const kMsgHeaderMinLen = int64(len(`{"op":1}`))
const kMsgHeaderMaxLen = int64(1 << 16)
const kMsgPingDataMax = 140
const kNodeIdLen = 25
const kAliasMinLen = 8
const kPostDateFormat = "2006-01-02T15:04:05.000Z07:00"

var UDb UserDatabase // set by caller

// encoding without vowels to avoid words
var sBase32 = base32.NewEncoding("%+123456789BCDFGHJKLMNPQRSTVWXYZ")

var sCrc32c = crc32.MakeTable(crc32.Castagnoli)
var sSendDoor, sRecvDoor sync.RWMutex
var sOhi = tOhi{from: tOhiMap{}}
var sNode = tNodes{list: tNodeMap{}}
var sStore = tStore{}


type tHeader struct {
   Op uint8
   DataLen, DataHead int64
   DataSum uint64
   Uid, Gid string
   Id string
   Node, NewNode string
   NewAlias, From, To string // alias
   Type string
   Act string
   For []tHeaderFor
}

const (
   eOpTmtpRev = iota
   eOpRegister; eOpLogin
   eOpUserEdit; eOpOhiEdit;
   eOpGroupInvite; eOpGroupEdit
   eOpPost; eOpPing
   eOpAck
   eOpPulse; eOpQuit
   eOpEnd
)

type tHeaderFor struct { Id string; Type int8 }

const ( _=iota; eForUser; eForGroupAll; eForGroupExcl; eForSelf )

var sHeaderDefs = [...]tHeader{
   eOpTmtpRev    : { Id:"1" },
   eOpRegister   : { NewNode:"1", NewAlias:"1" },
   eOpLogin      : { Uid:"1", Node:"1" },
   eOpUserEdit   : { Id:"1" },
   eOpOhiEdit    : { Id:"1", For:[]tHeaderFor{{}}, Type:"1" },
   eOpGroupInvite: { Id:"1", DataLen:1, Gid:"1", From:"1", To:"1" },
   eOpGroupEdit  : { Id:"1", Act:"1", Gid:"1" },
   eOpPost       : { Id:"1", DataLen:1, For:[]tHeaderFor{} },
   eOpPing       : { Id:"1", DataLen:1, To:"1" },
   eOpAck        : { Id:"1", Type:"1" },
   eOpPulse      : {  },
   eOpQuit       : {  },
}

func (o *tHeader) check() bool {
   if o.Op >= eOpEnd { return false }
   aDef := &sHeaderDefs[o.Op]
   aFail :=
      o.DataLen | o.DataHead < 0                     ||
      o.DataLen < o.DataHead                         ||
      (aDef.DataLen == 0)    != (o.DataLen == 0)     ||
      len(aDef.Uid)      > 0 && len(o.Uid)      == 0 ||
{'fraction_non_alphanumeric': 0.14147115893455636, 'fraction_numerical': 0.04233550890809667, 'mean_word_length': 2.8362652232746957, 'pattern_counts': {'":': 2, '<': 13, '<?xml version=': 0, '>': 10, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3654017', 'n_tokens_mistral': 2335, 'n_tokens_neox': 2188, 'n_words': 529}