<commit_before>#maybe this should be split into more smaller threads. One can focus on finding out peer's heights, one can download blocks, etc.

defmodule Talker do
  #this module creates and maintains connections with multiple peers. It downloads blocks from them.
  #grabs the list of peers from peers thread.
  #has ability to black-list peers who are bad?
  #don't talke to same peers too frequently
  #don't ignore any peer much longer than the others.
  #download blocks and peers.
  use GenServer
  @name __MODULE__
  def start_link() do
    GenServer.start_link(__MODULE__, :ok, [name: @name])
  end
  def still_on(blocks) do blocks == :ok or blocks == [] or (is_tuple(hd(blocks)) and :error in Dict.keys(blocks)) end
  def add_peers(x) do Enum.map(x, fn(x) -> Peers.add_peer(x) end) end
	def flip(x) do flip(x, []) end
	def flip([], x) do x end
	def flip([head|tail], out \\ []) do flip(tail, [head|out]) end
  def download_blocks(i, u, p) do
		blocks = Cli.blocks(i, min(i+50, u), p) |> flip
		if blocks != [] do
			parent = hd(blocks).data.hash |> KV.get
			Task.start(fn() -> BlockAbsorber.absorb(blocks) end)
			if parent == nil do download_blocks(i-5, i, p) end 
		end
  end
  def trade_peers(p) do
		keys = fn(z) -> Enum.map(z, fn(x) -> Peers.peer_key(x) end) end
		my_peers = Cli.all_peers
    peers = Cli.all_peers(p)
		my_keys = keys.(my_peers)
		peers_keys = keys.(peers)
    if my_peers == :ok or peers == :ok do
      IO.puts("peer died 1")
    else
      not_yours = Enum.filter(my_peers, &(not Peers.peer_key(&1) in peers_keys))
      not_mine = Enum.filter(peers, &(not Peers.peer_key(&1) in my_keys))
      Enum.map(not_yours,&(Cli.add_peer(&1,p)))
      Enum.map(not_mine, &(Peers.add_peer(&1)))
    end
  end
  def check_peer(p) do
		#first we should check if the peer has been connected to before. If we never connected before, then only connect to them with probability 10%.
		#IO.puts("check peer #{inspect p}")
		:random.seed(:erlang.now)
		r = trunc(10*:random.uniform)
		cond do
			p.height == 0 and r > 0 -> nil
			true -> check_peer_1(p)
		end
	end
	def check_peer_1(p) do
    status = Cli.status(p)
    cond do
			not is_map(status) -> status
      status.height > 0 and is_number(status.height) ->
        x = Peers.get(p)
        |> Map.put(:height, status.height)
        |> Map.put(:hash, status.hash)
				x |> Peers.add_peer
        check_peer_2(x, status)
			true -> IO.puts("nothing to do")
    end
  end
  def check_peer_2(p, status) do
		#IO.puts("check peer 2 #{inspect p}")
    trade_peers(p)
    txs = Cli.txs(p)
    u = status.height
    i = KV.get("height")
		hash = Blockchain.blockhash(Blockchain.get_block(u))
    cond do
      txs == :ok -> IO.puts("txs shouldn't be :ok")
      (txs != []) and length(txs)>0 and is_tuple(hd(txs)) ->
        IO.puts("tx error #{inspect txs}")
      u > i ->
				IO.puts("download blocks")
				download_blocks(i, u, p)
      u == i ->
				Enum.map(txs, &(Mempool.add_tx(&1)))
				Cli.txs |> Enum.filter(&(not &1 in txs)) |> Enum.map(&(Cli.pushtx(&1, p)))
			status.hash == hash ->
				Enum.map((u+1)..min((u+10), i), &(Blockchain.get_block(&1)))
				|> Cli.add_blocks(p)
      true ->
				#blocks = Enum.map((u-10)..min((u+2), i), &(Blockchain.get_block(&1)))
				#IO.puts("send block #{inspect blocks}")
				Enum.map((u-10)..min((u+2), i), &(Blockchain.get_block(&1)))
				|> Cli.add_blocks(p)
        true
    end
  end
  def check_peers do
    Cli.all_peers
    |> Enum.map(&(Task.start_link(fn -> check_peer(&1) end)))
   end
  def init(_) do
		Task.start_link(fn() -> timer end)
    #Enum.map(0..Constants.max_nodes, &(%Peer{ip: "localhost", port: Constants.tcp_port+&1})) 
    #|> Enum.map(&(Peers.add_peer(&1)))
		Constants.server |> Peers.add_peer
		#%Peer{ip: "172.16.31.10", port: Constants.tcp_port} |> Enum.map(&(Peers.add_peer(&1)))
		#Peers.add_peer(%Peer{ip: "172.16.17.32", port: 6669})
    {:ok, []}
  end
  def doit do GenServer.cast(@name, :doit) end
  def handle_cast(:doit, state) do
    check_peers
    {:noreply, state}
  end
  def timer do
    :timer.sleep(3000)
    doit
    timer
  end
end<commit_msg>how many blocks at once tests<commit_after>#maybe this should be split into more smaller threads. One can focus on finding out peer's heights, one can download blocks, etc.

defmodule Talker do
  #this module creates and maintains connections with multiple peers. It downloads blocks from them.
  #grabs the list of peers from peers thread.
  #has ability to black-list peers who are bad?
  #don't talke to same peers too frequently
  #don't ignore any peer much longer than the others.
  #download blocks and peers.
  use GenServer
  @name __MODULE__
  def start_link() do
    GenServer.start_link(__MODULE__, :ok, [name: @name])
  end
  def still_on(blocks) do blocks == :ok or blocks == [] or (is_tuple(hd(blocks)) and :error in Dict.keys(blocks)) end
  def add_peers(x) do Enum.map(x, fn(x) -> Peers.add_peer(x) end) end
	def flip(x) do flip(x, []) end
	def flip([], x) do x end
	def flip([head|tail], out \\ []) do flip(tail, [head|out]) end
  def download_blocks(i, u, p) do
		blocks = Cli.blocks(i, min(i+50, u), p) |> flip
		if blocks != [] do
			parent = hd(blocks).data.hash |> KV.get
			Task.start(fn() -> BlockAbsorber.absorb(blocks) end)
			if parent == nil do download_blocks(i-5, i, p) end 
		end
  end
  def trade_peers(p) do
		keys = fn(z) -> Enum.map(z, fn(x) -> Peers.peer_key(x) end) end
		my_peers = Cli.all_peers
    peers = Cli.all_peers(p)
		my_keys = keys.(my_peers)
		peers_keys = keys.(peers)
    if my_peers == :ok or peers == :ok do
      IO.puts("peer died 1")
    else
      not_yours = Enum.filter(my_peers, &(not Peers.peer_key(&1) in peers_keys))
      not_mine = Enum.filter(peers, &(not Peers.peer_key(&1) in my_keys))
      Enum.map(not_yours,&(Cli.add_peer(&1,p)))
      Enum.map(not_mine, &(Peers.add_peer(&1)))
    end
  end
  def check_peer(p) do
		#first we should check if the peer has been connected to before. If we never connected before, then only connect to them with probability 10%.
		#IO.puts("check peer #{inspect p}")
		:random.seed(:erlang.now)
		r = trunc(10*:random.uniform)
		cond do
			p.height == 0 and r > 0 -> nil
			true -> check_peer_1(p)
		end
	end
	def check_peer_1(p) do
    status = Cli.status(p)
    cond do
			not is_map(status) -> status
      status.height > 0 and is_number(status.height) ->
        x = Peers.get(p)
        |> Map.put(:height, status.height)
        |> Map.put(:hash, status.hash)
				x |> Peers.add_peer
        check_peer_2(x, status)
			true -> IO.puts("nothing to do")
    end
  end
  def check_peer_2(p, status) do
		#IO.puts("check peer 2 #{inspect p}")
    trade_peers(p)
    txs = Cli.txs(p)
    u = status.height
    i = KV.get("height")
		hash = Blockchain.blockhash(Blockchain.get_block(u))
    cond do
      txs == :ok -> IO.puts("txs shouldn't be :ok")
      (txs != []) and length(txs)>0 and is_tuple(hd(txs)) ->
        IO.puts("tx error #{inspect txs}")
      u > i ->
				IO.puts("download blocks")
				download_blocks(i, u, p)
      u == i ->
				Enum.map(txs, &(Mempool.add_tx(&1)))
				Cli.txs |> Enum.filter(&(not &1 in txs)) |> Enum.map(&(Cli.pushtx(&1, p)))
			status.hash == hash ->
				Enum.map((u+1)..min((u+100), i), &(Blockchain.get_block(&1)))
				|> Cli.add_blocks(p)
      true ->
				#blocks = Enum.map((u-10)..min((u+2), i), &(Blockchain.get_block(&1)))
				#IO.puts("send block #{inspect blocks}")
				Enum.map((u-10)..min((u+2), i), &(Blockchain.get_block(&1)))
				|> Cli.add_blocks(p)
        true
    end
  end
  def check_peers do
    Cli.all_peers
    |> Enum.map(&(Task.start_link(fn -> check_peer(&1) end)))
   end
  def init(_) do
		Task.start_link(fn() -> timer end)
		Constants.server |> Peers.add_peer
    {:ok, []}
  end
  def doit do GenServer.cast(@name, :doit) end
  def handle_cast(:doit, state) do
    check_peers
    {:noreply, state}
  end
  def timer do
    :timer.sleep(3000)
    doit
    timer
  end
end
{'fraction_non_alphanumeric': 0.15655994978028875, 'fraction_numerical': 0.013684871311989956, 'mean_word_length': 3.1167958656330748, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 69, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6425994', 'n_tokens_mistral': 3230, 'n_tokens_neox': 3019, 'n_words': 809}