Skip to content
GitLab
Menu
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
PMC-SOG
experiments
hybrid
Commits
0ce14882
Commit
0ce14882
authored
Dec 28, 2020
by
Jaime Arias
Browse files
update plot scripts
parent
ccbae29f
Changes
2
Hide whitespace changes
Inline
Side-by-side
scripts/plot-results.ipynb
View file @
0ce14882
This source diff could not be displayed because it is too large. You can
view the blob
instead.
scripts/plot-results.py
View file @
0ce14882
...
...
@@ -59,7 +59,7 @@ LAYOUT_FIGURES = dict(
# # Auxiliary Functions
# In[
4
]:
# In[
3
]:
def
create_folder
(
path
):
...
...
@@ -79,7 +79,7 @@ def create_folder(path):
os
.
makedirs
(
path
)
# In[
5
]:
# In[
4
]:
def
create_figure
(
df
,
model
):
...
...
@@ -118,7 +118,7 @@ def create_figure(df, model):
return
figure
# In[
6
]:
# In[
5
]:
def
get_axis_title
(
experiment
,
show_strategy
=
True
):
...
...
@@ -170,38 +170,75 @@ def get_axis_title(experiment, show_strategy=True):
return
title
# In[
7
]:
# In[
6
]:
def
get_info
(
df
,
model
,
experiment
):
"""Get some statistics from a table for a specific model and experiment"""
info
=
df
.
loc
[
model
][
experiment
]
def
filter_errors
(
df_exp1
,
df_exp2
):
"""Returns dataframes of specific experiments without errors"""
time_limit
=
info
[
info
==
"TIME LIMIT"
].
count
()
error
=
info
[(
info
!=
"TIME LIMIT"
)
&
(
info
!=
"OK"
)].
count
()
ok
=
info
[
info
==
"OK"
].
count
()
nan_1
=
df_exp1
[
df_exp1
.
isna
().
any
(
axis
=
1
)].
index
nan_2
=
df_exp2
[
df_exp2
.
isna
().
any
(
axis
=
1
)].
index
df_exp1
=
df_exp1
.
drop
(
nan_2
)
df_exp2
=
df_exp2
.
drop
(
nan_1
)
df_exp1
=
df_exp1
.
dropna
()
df_exp2
=
df_exp2
.
dropna
()
if
((
time_limit
+
error
+
ok
)
!=
info
.
count
()):
raise
Exception
(
"Some information is missing in the table"
)
return
df_exp1
,
df_exp2
def
get_info
(
info
):
"""Get some statistics from a table for a specific model and experiment"""
time_limit
=
len
(
info
[(
info
.
error
==
"TIME LIMIT"
)
|
(
info
.
error
==
"TIMEOUT"
)])
error
=
len
(
info
[(
info
.
error
!=
"TIME LIMIT"
)
&
(
info
.
error
!=
"TIMEOUT"
)
&
(
info
.
error
!=
"OK"
)
&
(
info
.
error
!=
'MDD'
)
&
(
info
.
error
!=
'TABLE FULL'
)])
memory
=
len
(
info
[(
info
.
error
==
'MDD'
)
|
(
info
.
error
==
'TABLE FULL'
)])
ok
=
len
(
info
[
info
.
error
==
"OK"
])
if
((
time_limit
+
error
+
ok
+
memory
)
!=
len
(
info
)):
raise
Exception
(
"Some information is missing in the table"
)
return
{
"time limit"
:
time_limit
,
"error"
:
error
,
"memory"
:
memory
,
"OK"
:
ok
}
def
get_table
(
df
,
model
,
experiments
):
def
get_best_times
(
table_time
,
table_error
,
model
,
exp1
,
exp2
):
exp1
=
pd
.
DataFrame
({
"times_exp1"
:
table_time
.
loc
[
model
][
exp1
],
"errors_exp1"
:
table_error
.
loc
[
model
][
exp1
]})
exp2
=
pd
.
DataFrame
({
"times_exp2"
:
table_time
.
loc
[
model
][
exp2
],
"errors_exp2"
:
table_error
.
loc
[
model
][
exp2
]})
exp1
,
exp2
=
filter_errors
(
exp1
,
exp2
)
df_
=
pd
.
concat
([
exp1
,
exp2
],
axis
=
1
,
sort
=
False
)
df_
=
df_
[
df_
[
"times_exp1"
]
!=
df_
[
"times_exp2"
]]
df_
[
'best'
]
=
np
.
where
((
df_
[
"times_exp1"
]
<
df_
[
"times_exp2"
]),
"exp1"
,
"exp2"
)
count
=
df_
.
groupby
([
"best"
]).
size
()
return
count
.
get
(
"exp1"
,
0
),
count
.
get
(
"exp2"
,
0
)
def
get_table
(
df_time
,
df_errors
,
model
,
exp1
,
exp2
):
"""Creates a table with some statistics from a dataframe for a model and experiments"""
rows
=
[[
"<b>Experiment</b>"
,
"<b>Time Limit</b>"
,
"<b>Error</b>"
,
"<b>OK</b>"
]]
rows
=
[[
"<b>Experiment</b>"
,
"<b>Time Limit</b>"
,
"<b>Memory</b>"
,
"<b>Unknown Error</b>"
,
"<b>OK</b>"
,
"<b>Faster</b>"
]]
df_exp1
=
pd
.
DataFrame
({
"error"
:
df_errors
.
loc
[
model
][
exp1
]})
df_exp2
=
pd
.
DataFrame
({
"error"
:
df_errors
.
loc
[
model
][
exp2
]})
df_exp1
,
df_exp2
=
filter_errors
(
df_exp1
,
df_exp2
)
info1
=
get_info
(
df_exp1
)
info2
=
get_info
(
df_exp2
)
for
experiment
in
experiments
:
info
=
get_info
(
df
,
model
,
experiment
)
rows
.
append
([
experiment
,
info
[
"time limit"
],
info
[
"error"
],
info
[
"OK"
]])
best1
,
best2
=
get_best_times
(
df_time
,
df_errors
,
model
,
exp1
,
exp2
)
for
(
experiment
,
info
,
best
)
in
[(
exp1
,
info1
,
best1
),
(
exp2
,
info2
,
best2
)]:
rows
.
append
([
experiment
,
info
[
"time limit"
],
info
[
"memory"
],
info
[
"error"
],
info
[
"OK"
],
best
])
return
ff
.
create_table
(
rows
)
# In[
8
]:
# In[
7
]:
import
webbrowser
...
...
@@ -316,7 +353,7 @@ myPlot.on('plotly_click', function(data){
"""
# In[
9
]:
# In[
8
]:
def
create_figure_explored_states
(
table_explored_states
,
model
):
...
...
@@ -388,7 +425,7 @@ def create_figure_explored_states(table_explored_states, model):
return
fig
# In[
10
]:
# In[
9
]:
def
create_log_figure
(
table
,
table_errors
,
model
,
tool_x
,
tool_y
,
show_strategy
=
True
,
callback
=
None
):
...
...
@@ -440,6 +477,8 @@ def create_log_figure(table, table_errors, model, tool_x, tool_y, show_strategy=
full_table_y
=
pd
.
concat
([
table_model
[
tool_y
],
table_model
[
'property'
],
table_errors_model
[
tool_y
]],
axis
=
1
)
full_table_y
.
columns
=
[
'time'
,
'property'
,
'error'
]
full_table_x
,
full_table_y
=
filter_errors
(
full_table_x
,
full_table_y
)
traces
=
[
{
"property"
:
'T'
,
"color"
:
"green"
},
...
...
@@ -509,7 +548,7 @@ def create_log_figure(table, table_errors, model, tool_x, tool_y, show_strategy=
print
(
e
)
# In[1
1
]:
# In[1
0
]:
# Experiment filters
...
...
@@ -632,7 +671,7 @@ plots = {
# # Load Data
# In[1
2
]:
# In[1
1
]:
# Root folder
...
...
@@ -649,7 +688,7 @@ OUTPUT_FOLDER = os.path.join(PROJECT_FOLDER,"results", "figures")
create_folder
(
OUTPUT_FOLDER
)
# In[1
3
]:
# In[1
2
]:
# read data
...
...
@@ -659,12 +698,40 @@ df = pd.read_csv(csv_file)
df
[
'tool'
]
=
df
[[
'tool'
,
'strategy'
,
'num_nodes'
,
'num_threads'
]].
astype
(
str
).
apply
(
'_'
.
join
,
axis
=
1
)
df
=
df
.
drop
(
columns
=
[
'strategy'
,
'num_nodes'
,
'num_threads'
])
df
.
head
()
# filtering philo20 experiments
df
=
df
[
df
.
model
!=
"philo20"
]
df
# In[13]:
# Found errors
errors_found
=
df
.
error
.
unique
()
errors_found
# In[14]:
# filtering errors
df
=
df
[(
df
.
error
!=
"SEGMENTATION FAULT"
)
&
(
df
.
error
!=
"ABORTED"
)
&
(
df
.
error
!=
"TERMINATE"
)
&
(
df
.
error
!=
"MDD"
)]
df
=
df
.
reset_index
(
drop
=
True
)
df
# In[15]:
# Found errors
errors_found
=
df
.
error
.
unique
()
errors_found
# In[16]:
# ground truth for properties
frames
=
[]
...
...
@@ -681,21 +748,22 @@ p_df = p_df.reindex(columns=["model", "formula", "property"])
p_df
=
p_df
[
p_df
[
'model'
].
isin
(
df
.
model
.
unique
())]
p_df
[
'property'
]
=
p_df
[
'property'
].
replace
([
True
,
False
],
[
"T"
,
"F"
])
p_df
=
p_df
.
set_index
([
"model"
,
"formula"
])
p_df
.
sort_index
(
inplace
=
True
)
p_df
# In[1
5
]:
# In[1
7
]:
# table with times, verification output and error for each experiment
table
=
df
.
set_index
([
'model'
,
'formula'
,
'tool'
],
drop
=
True
).
unstack
(
'tool'
)
table
.
head
()
table
# # Preprocessing of data
# In[1
6
]:
# In[1
8
]:
# table with times for each experiment
...
...
@@ -710,10 +778,10 @@ table_time.replace(0.0, ZERO, inplace=True)
# add verification output to the table
table_time
=
pd
.
concat
([
table_time
,
p_df
],
axis
=
1
)
table_time
.
head
()
table_time
# In[1
7
]:
# In[1
9
]:
# table with verification output for each experiment
...
...
@@ -728,7 +796,7 @@ table_property = pd.concat([table_property, p_df], axis=1)
table_property
.
head
()
# In[
18
]:
# In[
20
]:
# table with error for each experiment
...
...
@@ -737,7 +805,7 @@ table_error = table['error'].copy()
table_error
.
head
()
# In[1
9
]:
# In[
2
1]:
# table with explored states for each experiment using ltsmin
...
...
@@ -755,7 +823,7 @@ table_explored_states = table_explored_states.reset_index()
table_explored_states
.
head
()
# In[2
0
]:
# In[2
2
]:
# calculate the stats of the number of explored states
...
...
@@ -767,35 +835,35 @@ table_explored_states_stats.head()
# # Examples
# In[2
1
]:
# In[2
3
]:
create_figure_explored_states
(
table_explored_states
,
'spool5'
)
# In[2
2
]:
# In[2
4
]:
create_figure
(
df
,
"
spool5
"
)
create_figure
(
df
,
"
robot50
"
)
# In[2
3
]:
# In[2
5
]:
log_figure
=
create_log_figure
(
table_time
,
table_error
,
"
spool5
"
,
"pmc-sog_otf_couv99-default_2_16"
,
"pnml2lts-mc_dfs_1_16"
,
True
,
open_logs_callback
)
log_figure
=
create_log_figure
(
table_time
,
table_error
,
"
robot50
"
,
"pmc-sog_otf_couv99-default_2_16"
,
"pnml2lts-mc_dfs_1_16"
,
True
,
open_logs_callback
)
log_figure
# In[2
4
]:
# In[2
6
]:
table
=
get_table
(
table_error
,
"
spool5
"
,
[
"pmc-sog_otf_couv99-default_2_16"
,
"pnml2lts-mc_dfs_1_16"
]
)
table
=
get_table
(
table_time
,
table_error
,
"
robot50
"
,
"pmc-sog_otf_couv99-default_2_16"
,
"pnml2lts-mc_dfs_1_16"
)
table
# # Generate Figures
# In[2
5
]:
# In[2
7
]:
# models
...
...
@@ -805,7 +873,7 @@ models = df.model.unique()
tools
=
df
.
tool
.
unique
()
# In[2
6
]:
# In[2
9
]:
# create all the figures of explored states
...
...
@@ -824,7 +892,7 @@ for model in models:
print
(
"Error: {} was not plotted"
.
format
(
model
))
# In[
27
]:
# In[
30
]:
# create all the figures formula vs time
...
...
@@ -843,7 +911,7 @@ for model in models:
print
(
"Error: {} was not plotted"
.
format
(
model
))
# In[
28
]:
# In[
31
]:
# create all the log figures
...
...
@@ -861,7 +929,7 @@ for plot, filter_method in plots.items():
try
:
show_strategy
=
plot
==
'compare_couvreur_algorithm'
fig
=
create_log_figure
(
table_time
,
table_error
,
model
,
axe
[
0
],
axe
[
1
],
show_strategy
)
table
=
get_table
(
table_error
,
model
,
axe
)
table
=
get_table
(
table_time
,
table_error
,
model
,
axe
[
0
],
axe
[
1
]
)
# save figures in html and pdf
figure_name
=
os
.
path
.
join
(
folder
,
'{}-{}-VS-{}-log'
.
format
(
model
,
axe
[
0
],
axe
[
1
]))
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment