# Evaluating MixTCP
```elixir
Mix.install([
{:kino_vega_lite, "~> 0.1.7"},
{:jason, "~> 1.4"},
{:statistex, "~> 1.0"},
{:explorer, "~> 0.7.1"}
])
alias Explorer.{DataFrame, Series}
```
## Setup
```elixir
test_runs_folder = "/Users/arnold/Projects/elixir/mix_tcp/test_runs/ash_paper_trail"
```
```elixir
server = :tcp@localhost
Node.set_cookie(:"1234")
Node.connect(server)
```
## Helper Functions
```elixir
parse_test_run = fn test_run ->
:erpc.call(server, MixTcp.TestRunParser, :parse, [test_run])
end
tcp = fn how_many_cycles ->
:erpc.call(server, MixTcp.Server, :tcp, [test_runs_folder, how_many_cycles])
end
napfd = fn testcases, faults, fault_exposed_by_testcase_fn ->
n = length(testcases)
faults_with_index =
faults
|> Stream.map(fn fault ->
testcases
|> Enum.find_index(&fault_exposed_by_testcase_fn.(fault, &1))
|> then(&{fault, &1})
end)
|> Stream.filter(fn {_fault, idx} -> idx != nil end)
|> Enum.into(%{})
m = Enum.count(faults)
p = map_size(faults_with_index) / m
if m != 0 and n != 0 do
sum =
faults_with_index
|> Map.values()
|> Enum.sum()
p - sum / (m * n) + p / (2 * n)
else
0.0
end
end
```
## Preparing data
```elixir
test_runs_per_cycles_stream =
test_runs_folder
|> File.ls!()
|> Enum.sort()
|> Stream.map(&Path.join(test_runs_folder, &1))
|> Stream.map(&File.read!/1)
|> Stream.map(parse_test_run)
|> Stream.with_index()
|> Stream.flat_map(fn {test_runs, cycle} ->
Enum.map(test_runs, &Map.put(&1, :cycle, cycle))
end)
```
```elixir
Enum.frequencies_by(test_runs_per_cycles_stream, & &1.cycle)
```
```elixir
id_by_testcase =
test_runs_per_cycles_stream
|> Stream.map(&{{&1.test_file, &1.line}, &1.id})
|> Stream.uniq()
|> Enum.into(%{})
faults =
test_runs_per_cycles_stream
|> Stream.filter(& &1.fail?)
|> Stream.map(& &1.id)
|> Stream.uniq()
|> Enum.to_list()
```
```elixir
cycles_to_evaluate = 0..7
tests_count_for_napfd = 30
```
```elixir
napfd_per_run =
cycles_to_evaluate
|> Enum.map(fn cycles_count_to_predict_from ->
testcases =
test_runs_per_cycles_stream
|> Stream.filter(&(&1.cycle < cycles_count_to_predict_from))
|> Stream.map(& &1.id)
|> Stream.uniq()
|> Enum.to_list()
verdict_per_id =
test_runs_per_cycles_stream
|> Stream.filter(&(&1.cycle == cycles_count_to_predict_from))
|> Enum.group_by(& &1.id, & &1.fail?)
|> Enum.into(%{}, fn {key, fails} ->
{key, Enum.any?(fails, & &1)}
end)
neutron_napfd =
cycles_count_to_predict_from
|> tcp.()
|> Enum.take(tests_count_for_napfd)
|> Enum.map(&id_by_testcase[&1])
|> napfd.(faults, fn
id, id -> verdict_per_id[id]
_, _ -> false
end)
random_napfd =
1..100
|> Enum.map(fn _ ->
testcases
|> Enum.shuffle()
|> Enum.take(tests_count_for_napfd)
|> napfd.(faults, fn
id, id -> verdict_per_id[id]
_, _ -> false
end)
end)
|> Statistex.average()
%{
cycle: cycles_count_to_predict_from,
neutron: neutron_napfd,
random: random_napfd
}
end)
```
## Visualization
```elixir
data =
napfd_per_run
|> Enum.drop(1)
|> Enum.flat_map(fn %{cycle: cycle, random: random_napfd, neutron: neutron_napfd} ->
[
%{"Cycle" => cycle, "NAPFD" => random_napfd, "Method" => "Random", "position" => 1},
%{"Cycle" => cycle, "NAPFD" => neutron_napfd, "Method" => "NEUTRON", "position" => 2}
]
end)
```
```elixir
VegaLite.new()
|> VegaLite.data_from_values(data)
|> VegaLite.mark(:bar)
|> VegaLite.encode_field(:x, "Cycle", type: :ordinal)
|> VegaLite.encode_field(:y, "NAPFD", type: :quantitative)
|> VegaLite.encode_field(:color, "Method")
|> VegaLite.encode_field(:x_offset, "position")
```
