Skip to content
GitLab
Menu
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
PMC-SOG
experiments
hybrid
Commits
4d915b1a
Commit
4d915b1a
authored
Dec 02, 2020
by
Jaime Arias
Browse files
update plot scripts
parent
33413d08
Changes
2
Expand all
Hide whitespace changes
Inline
Side-by-side
scripts/plot-results.ipynb
View file @
4d915b1a
This diff is collapsed.
Click to expand it.
scripts/plot-results.py
View file @
4d915b1a
...
...
@@ -12,12 +12,15 @@ TIMEOUT = 10 * 60 # 10 minutes = 600 seconds
import
os
import
glob
import
re
import
pandas
as
pd
import
numpy
as
np
import
plotly.io
as
pio
import
plotly.express
as
px
import
plotly.graph_objs
as
go
from
itertools
import
combinations
import
plotly.figure_factory
as
ff
from
plotly.subplots
import
make_subplots
# render figures in notebook
...
...
@@ -56,7 +59,7 @@ LAYOUT_FIGURES = dict(
# # Auxiliary Functions
# In[
3
]:
# In[
4
]:
def
create_folder
(
path
):
...
...
@@ -76,7 +79,7 @@ def create_folder(path):
os
.
makedirs
(
path
)
# In[
4
]:
# In[
5
]:
def
create_figure
(
df
,
model
):
...
...
@@ -115,7 +118,7 @@ def create_figure(df, model):
return
figure
# In[
5
]:
# In[
6
]:
def
get_axis_title
(
experiment
,
show_strategy
=
True
):
...
...
@@ -167,128 +170,38 @@ def get_axis_title(experiment, show_strategy=True):
return
title
# In[
6
]:
# In[
7
]:
def
create_log_figure
(
table
,
table_errors
,
model
,
tool_x
,
tool_y
,
show_strategy
=
True
,
callback
=
None
):
"""Creates a Scatter figure in logarithmic scale comparing the performance of two tools
def
get_info
(
df
,
model
,
experiment
):
"""Get some statistics from a table for a specific model and experiment"""
info
=
df
.
loc
[
model
][
experiment
]
Parameters
----------
table : pandas.Dataframe
Dataframe with the times of each experiment
table_errors : pandas.Dataframe
Dataframe with the errors of each experiment
model : string
Model to be analyzed
tool_x : string
Tool to be compared and plotted on the x-axis
tool_y : string
Tool to be compared and plotted on the y-axis
show_strategy : bool
Flag to show the stretagy used by the tools
callback : function
Function to be called when clicking on a point
Returns
-------
plotly.graph_objects.Figure
Scatter figure
Examples
--------
>>> import os
>>> import pandas as pd
>>> csv_file = os.path.join("results", "output.csv")
>>> df = pd.read_csv(csv_file)
>>> table = df.set_index(['model', 'formula', 'tool'], drop=True).unstack('tool')
>>> fig = create_log_figure(table['time'], table['error'], 'philo10', 'pmc-sog_otfL_couv99-default_1_8', 'pmc-sog_otfP_couv99-default_1_8')
"""
try
:
min_value
=
ZERO
max_value
=
TIMEOUT
min_value_log
=
np
.
log10
(
min_value
)
max_value_log
=
np
.
log10
(
max_value
)
table_model
=
table
.
loc
[
model
]
table_errors_model
=
table_error
.
loc
[
model
]
full_table_x
=
pd
.
concat
([
table_model
[
tool_x
],
table_model
[
'property'
],
table_errors_model
[
tool_x
]],
axis
=
1
)
full_table_x
.
columns
=
[
'time'
,
'property'
,
'error'
]
full_table_y
=
pd
.
concat
([
table_model
[
tool_y
],
table_model
[
'property'
],
table_errors_model
[
tool_y
]],
axis
=
1
)
full_table_y
.
columns
=
[
'time'
,
'property'
,
'error'
]
traces
=
[
{
"property"
:
'T'
,
"color"
:
"green"
},
{
"property"
:
'F'
,
"color"
:
"red"
},
{
"property"
:
'U'
,
"color"
:
"black"
}
]
figures
=
[]
for
t
in
traces
:
# filter by verification output
table_x
=
full_table_x
[
full_table_x
.
property
==
t
[
'property'
]]
table_y
=
full_table_y
[
full_table_y
.
property
==
t
[
'property'
]]
# custom data
custom_data
=
list
(
zip
(
table_x
.
index
,
table_x
.
error
,
table_y
.
error
))
# tools
metainfo
=
{
'model'
:
model
,
'tools'
:
{
'x'
:
tool_x
,
'y'
:
tool_y
},
'folder'
:
os
.
path
.
join
(
os
.
path
.
abspath
(
os
.
pardir
),
"results"
)
}
figures
.
append
(
go
.
Scatter
(
x
=
table_x
.
time
,
y
=
table_y
.
time
,
name
=
t
[
'property'
],
mode
=
'markers'
,
marker_symbol
=
'circle-open'
,
marker_color
=
t
[
'color'
],
meta
=
metainfo
,
customdata
=
custom_data
,
hovertemplate
=
'<b>Formula # %{customdata[0]}</b><br>'
+
'<br><b>Times:</b><br>'
+
'<b>x:</b> %{x} s'
+
'<br><b>y:</b> %{y} s<br>'
+
'<br><b>Errors:</b><br>'
+
'<b>x:</b> %{customdata[1]}<br>'
+
'<b>y:</b> %{customdata[2]}'
,
))
time_limit
=
info
[
info
==
"TIME LIMIT"
].
count
()
error
=
info
[(
info
!=
"TIME LIMIT"
)
&
(
info
!=
"OK"
)].
count
()
ok
=
info
[
info
==
"OK"
].
count
()
if
((
time_limit
+
error
+
ok
)
!=
info
.
count
()):
raise
Exception
(
"Some information is missing in the table"
)
return
{
"time limit"
:
time_limit
,
"error"
:
error
,
"OK"
:
ok
}
# Line
figures
.
append
(
go
.
Scatter
(
x
=
[
min_value
,
max_value
],
y
=
[
min_value
,
max_value
],
mode
=
'lines'
,
showlegend
=
False
,
line
=
dict
(
color
=
'black'
,
width
=
1
)))
# Create figure
figure
=
go
.
FigureWidget
(
figures
)
figure
.
update_layout
(
LAYOUT_FIGURES
,
title_text
=
model
,
hoverlabel
=
dict
(
bgcolor
=
"white"
,
align
=
'auto'
),
legend_title_text
=
'property'
,
xaxis
=
dict
(
type
=
'log'
,
autorange
=
False
,
range
=
[
min_value_log
,
max_value_log
]),
yaxis
=
dict
(
type
=
'log'
,
autorange
=
False
,
range
=
[
min_value_log
,
max_value_log
]),
xaxis_title
=
get_axis_title
(
tool_x
,
show_strategy
),
yaxis_title
=
get_axis_title
(
tool_y
,
show_strategy
))
# Add event
if
callback
is
not
None
:
for
i
in
range
(
len
(
figure
.
data
)):
figure
.
data
[
i
].
on_click
(
callback
)
def
get_table
(
df
,
model
,
experiments
):
"""Creates a table with some statistics from a dataframe for a model and experiments"""
rows
=
[[
"<b>Experiment</b>"
,
"<b>Time Limit</b>"
,
"<b>Error</b>"
,
"<b>OK</b>"
]]
return
figure
except
Exception
as
e
:
print
(
"Error when ploting model: {} - tool_x: {} - tool_y: {}"
.
format
(
model
,
tool_x
,
tool_y
))
print
(
e
)
for
experiment
in
experiments
:
info
=
get_info
(
df
,
model
,
experiment
)
rows
.
append
([
experiment
,
info
[
"time limit"
],
info
[
"error"
],
info
[
"OK"
]])
return
ff
.
create_table
(
rows
)
# In[
7
]:
# In[
8
]:
import
webbrowser
...
...
@@ -403,7 +316,7 @@ myPlot.on('plotly_click', function(data){
"""
# In[
8
]:
# In[
9
]:
def
create_figure_explored_states
(
table_explored_states
,
model
):
...
...
@@ -475,7 +388,128 @@ def create_figure_explored_states(table_explored_states, model):
return
fig
# In[9]:
# In[10]:
def
create_log_figure
(
table
,
table_errors
,
model
,
tool_x
,
tool_y
,
show_strategy
=
True
,
callback
=
None
):
"""Creates a Scatter figure in logarithmic scale comparing the performance of two tools
Parameters
----------
table : pandas.Dataframe
Dataframe with the times of each experiment
table_errors : pandas.Dataframe
Dataframe with the errors of each experiment
model : string
Model to be analyzed
tool_x : string
Tool to be compared and plotted on the x-axis
tool_y : string
Tool to be compared and plotted on the y-axis
show_strategy : bool
Flag to show the stretagy used by the tools
callback : function
Function to be called when clicking on a point
Returns
-------
plotly.graph_objects.Figure
Scatter figure
Examples
--------
>>> import os
>>> import pandas as pd
>>> csv_file = os.path.join("results", "output.csv")
>>> df = pd.read_csv(csv_file)
>>> table = df.set_index(['model', 'formula', 'tool'], drop=True).unstack('tool')
>>> fig = create_log_figure(table['time'], table['error'], 'philo10', 'pmc-sog_otfL_couv99-default_1_8', 'pmc-sog_otfP_couv99-default_1_8')
"""
try
:
min_value
=
ZERO
max_value
=
TIMEOUT
min_value_log
=
np
.
log10
(
min_value
)
max_value_log
=
np
.
log10
(
max_value
)
table_model
=
table
.
loc
[
model
]
table_errors_model
=
table_error
.
loc
[
model
]
full_table_x
=
pd
.
concat
([
table_model
[
tool_x
],
table_model
[
'property'
],
table_errors_model
[
tool_x
]],
axis
=
1
)
full_table_x
.
columns
=
[
'time'
,
'property'
,
'error'
]
full_table_y
=
pd
.
concat
([
table_model
[
tool_y
],
table_model
[
'property'
],
table_errors_model
[
tool_y
]],
axis
=
1
)
full_table_y
.
columns
=
[
'time'
,
'property'
,
'error'
]
traces
=
[
{
"property"
:
'T'
,
"color"
:
"green"
},
{
"property"
:
'F'
,
"color"
:
"red"
},
{
"property"
:
'U'
,
"color"
:
"black"
}
]
figures
=
[]
for
t
in
traces
:
# filter by verification output
table_x
=
full_table_x
[
full_table_x
.
property
==
t
[
'property'
]]
table_y
=
full_table_y
[
full_table_y
.
property
==
t
[
'property'
]]
# custom data
custom_data
=
list
(
zip
(
table_x
.
index
,
table_x
.
error
,
table_y
.
error
))
# tools
metainfo
=
{
'model'
:
model
,
'tools'
:
{
'x'
:
tool_x
,
'y'
:
tool_y
},
'folder'
:
os
.
path
.
join
(
os
.
path
.
abspath
(
os
.
pardir
),
"results"
)
}
figures
.
append
(
go
.
Scatter
(
x
=
table_x
.
time
,
y
=
table_y
.
time
,
name
=
t
[
'property'
],
mode
=
'markers'
,
marker_symbol
=
'circle-open'
,
marker_color
=
t
[
'color'
],
meta
=
metainfo
,
customdata
=
custom_data
,
hovertemplate
=
'<b>Formula # %{customdata[0]}</b><br>'
+
'<br><b>Times:</b><br>'
+
'<b>x:</b> %{x} s'
+
'<br><b>y:</b> %{y} s<br>'
+
'<br><b>Errors:</b><br>'
+
'<b>x:</b> %{customdata[1]}<br>'
+
'<b>y:</b> %{customdata[2]}'
,
))
# Line
figures
.
append
(
go
.
Scatter
(
x
=
[
min_value
,
max_value
],
y
=
[
min_value
,
max_value
],
mode
=
'lines'
,
showlegend
=
False
,
line
=
dict
(
color
=
'black'
,
width
=
1
)))
# Create figure
figure
=
go
.
FigureWidget
(
figures
)
figure
.
update_layout
(
LAYOUT_FIGURES
,
title_text
=
model
,
hoverlabel
=
dict
(
bgcolor
=
"white"
,
align
=
'auto'
),
legend_title_text
=
'property'
,
xaxis
=
dict
(
type
=
'log'
,
autorange
=
False
,
range
=
[
min_value_log
,
max_value_log
]),
yaxis
=
dict
(
type
=
'log'
,
autorange
=
False
,
range
=
[
min_value_log
,
max_value_log
]),
xaxis_title
=
get_axis_title
(
tool_x
,
show_strategy
),
yaxis_title
=
get_axis_title
(
tool_y
,
show_strategy
))
# Add event
if
callback
is
not
None
:
for
i
in
range
(
len
(
figure
.
data
)):
figure
.
data
[
i
].
on_click
(
callback
)
return
figure
except
Exception
as
e
:
print
(
"Error when ploting model: {} - tool_x: {} - tool_y: {}"
.
format
(
model
,
tool_x
,
tool_y
))
print
(
e
)
# In[11]:
# Experiment filters
...
...
@@ -598,7 +632,7 @@ plots = {
# # Load Data
# In[1
0
]:
# In[1
2
]:
# Root folder
...
...
@@ -607,12 +641,15 @@ PROJECT_FOLDER = os.path.abspath(os.pardir)
# csv file with the output
csv_file
=
os
.
path
.
join
(
PROJECT_FOLDER
,
"results"
,
"output.csv"
)
# formulas folder
FORMULAS_FOLDER
=
os
.
path
.
join
(
PROJECT_FOLDER
,
"formulas"
)
# Output folder
OUTPUT_FOLDER
=
os
.
path
.
join
(
PROJECT_FOLDER
,
"results"
,
"figures"
)
create_folder
(
OUTPUT_FOLDER
)
# In[1
1
]:
# In[1
3
]:
# read data
...
...
@@ -625,27 +662,30 @@ df = df.drop(columns=['strategy', 'num_nodes', 'num_threads'])
df
.
head
()
# In[1
2
]:
# In[1
4
]:
# ground truth for properties
p_df
=
pd
.
read_csv
(
csv_file
)
p_df
=
p_df
[
(
p_df
.
tool
==
'pnml2lts-mc'
)
&
(
p_df
.
strategy
==
'ndfs'
)
&
(
p_df
.
num_nodes
==
1
)
&
(
p_df
.
num_threads
==
16
)]
frames
=
[]
# only property column is needed
p_df
=
p_df
.
drop
(
columns
=
[
'tool'
,
'strategy'
,
'num_nodes'
,
'num_threads'
,
'time'
,
'explored_states'
,
'error'
])
p_df
.
fillna
(
'U'
,
inplace
=
True
)
p_df
.
set_index
([
'model'
,
'formula'
],
inplace
=
True
)
p_df
.
sort_index
(
inplace
=
True
)
formula_results
=
glob
.
glob
(
os
.
path
.
join
(
FORMULAS_FOLDER
,
"**/formula_results"
),
recursive
=
True
)
for
f
in
formula_results
:
model
,
out_file
=
f
.
split
(
'/'
)[
-
2
:]
tmp_df
=
pd
.
read_csv
(
f
,
sep
=
";"
,
header
=
None
,
names
=
[
"formula"
,
"property"
])
tmp_df
[
"model"
]
=
model
frames
.
append
(
tmp_df
)
p_df
=
pd
.
concat
(
frames
)
p_df
=
p_df
.
reindex
(
columns
=
[
"model"
,
"formula"
,
"property"
])
p_df
=
p_df
[
p_df
[
'model'
].
isin
(
df
.
model
.
unique
())]
p_df
[
'property'
]
=
p_df
[
'property'
].
replace
([
True
,
False
],
[
"T"
,
"F"
])
p_df
=
p_df
.
set_index
([
"model"
,
"formula"
])
p_df
.
head
()
p_df
# In[1
3
]:
# In[1
5
]:
# table with times, verification output and error for each experiment
...
...
@@ -655,7 +695,7 @@ table.head()
# # Preprocessing of data
# In[1
4
]:
# In[1
6
]:
# table with times for each experiment
...
...
@@ -673,7 +713,7 @@ table_time = pd.concat([table_time, p_df], axis=1)
table_time
.
head
()
# In[1
5
]:
# In[1
7
]:
# table with verification output for each experiment
...
...
@@ -688,7 +728,7 @@ table_property = pd.concat([table_property, p_df], axis=1)
table_property
.
head
()
# In[1
6
]:
# In[1
8
]:
# table with error for each experiment
...
...
@@ -697,7 +737,7 @@ table_error = table['error'].copy()
table_error
.
head
()
# In[1
7
]:
# In[1
9
]:
# table with explored states for each experiment using ltsmin
...
...
@@ -715,7 +755,7 @@ table_explored_states = table_explored_states.reset_index()
table_explored_states
.
head
()
# In[
18
]:
# In[
20
]:
# calculate the stats of the number of explored states
...
...
@@ -727,27 +767,35 @@ table_explored_states_stats.head()
# # Examples
# In[1
9
]:
# In[
2
1]:
create_figure_explored_states
(
table_explored_states
,
'spool
1
'
)
create_figure_explored_states
(
table_explored_states
,
'spool
5
'
)
# In[2
0
]:
# In[2
2
]:
create_figure
(
df
,
"spool
1
"
)
create_figure
(
df
,
"spool
5
"
)
# In[ ]:
# In[23]:
log_figure
=
create_log_figure
(
table_time
,
table_error
,
"spool5"
,
"pmc-sog_otf_couv99-default_2_16"
,
"pnml2lts-mc_dfs_1_16"
,
True
,
open_logs_callback
)
log_figure
create_log_figure
(
table_time
,
table_error
,
"spool1"
,
"pmc-sog_otf_couv99-default_2_16"
,
"pnml2lts-mc_dfs_1_16"
,
True
,
open_logs_callback
)
# In[24]:
table
=
get_table
(
table_error
,
"spool5"
,
[
"pmc-sog_otf_couv99-default_2_16"
,
"pnml2lts-mc_dfs_1_16"
])
table
# # Generate Figures
# In[
]:
# In[
25
]:
# models
...
...
@@ -757,7 +805,7 @@ models = df.model.unique()
tools
=
df
.
tool
.
unique
()
# In[
]:
# In[
26
]:
# create all the figures of explored states
...
...
@@ -776,7 +824,7 @@ for model in models:
print
(
"Error: {} was not plotted"
.
format
(
model
))
# In[
]:
# In[
27
]:
# create all the figures formula vs time
...
...
@@ -795,7 +843,7 @@ for model in models:
print
(
"Error: {} was not plotted"
.
format
(
model
))
# In[
]:
# In[
28
]:
# create all the log figures
...
...
@@ -813,10 +861,15 @@ for plot, filter_method in plots.items():
try
:
show_strategy
=
plot
==
'compare_couvreur_algorithm'
fig
=
create_log_figure
(
table_time
,
table_error
,
model
,
axe
[
0
],
axe
[
1
],
show_strategy
)
table
=
get_table
(
table_error
,
model
,
axe
)
# save figures in html and pdf
figure_name
=
os
.
path
.
join
(
folder
,
'{}-{}-VS-{}-log'
.
format
(
model
,
axe
[
0
],
axe
[
1
]))
fig
.
write_html
(
figure_name
+
'.html'
,
include_plotlyjs
=
'cdn'
,
post_script
=
OPEN_LOGS_CALLBACK_JS
)
with
open
(
figure_name
+
'.html'
,
'w'
)
as
f
:
f
.
write
(
fig
.
to_html
(
full_html
=
False
,
include_plotlyjs
=
'cdn'
,
post_script
=
OPEN_LOGS_CALLBACK_JS
))
f
.
write
(
table
.
to_html
(
full_html
=
False
,
include_plotlyjs
=
'cdn'
,
post_script
=
OPEN_LOGS_CALLBACK_JS
))
fig
.
write_image
(
figure_name
+
'.pdf'
)
except
KeyError
:
print
(
"Error: {} was not plotted"
.
format
(
model
))
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment