Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
PMC-SOG
experiments
hybrid
Commits
23b1ecd7
Commit
23b1ecd7
authored
May 28, 2020
by
Jaime Arias
Browse files
add figure handler to open log files
parent
9f384d41
Changes
2
Expand all
Hide whitespace changes
Inline
Side-by-side
scripts/plot-results.ipynb
View file @
23b1ecd7
This diff is collapsed.
Click to expand it.
scripts/plot-results.py
View file @
23b1ecd7
#!/usr/bin/env python
# coding: utf-8
# In[1
9
]:
# In[1]:
import
os
...
...
@@ -164,7 +164,7 @@ def get_axis_title(experiment, show_strategy=True):
# In[5]:
def
create_log_figure
(
table
,
table_errors
,
model
,
tool_x
,
tool_y
,
show_strategy
=
True
):
def
create_log_figure
(
table
,
table_errors
,
model
,
tool_x
,
tool_y
,
callback
=
None
,
show_strategy
=
True
):
"""Creates a Scatter figure in logarithmic scale comparing the performance of two tools
Parameters
...
...
@@ -179,6 +179,8 @@ def create_log_figure(table, table_errors, model, tool_x, tool_y, show_strategy=
Tool to be compared and plotted on the x-axis
tool_y : string
Tool to be compared and plotted on the y-axis
callback : function
Function to be called when clicking on a point
show_strategy : bool
Flag to show the stretagy used by the tools
...
...
@@ -201,32 +203,81 @@ def create_log_figure(table, table_errors, model, tool_x, tool_y, show_strategy=
max_values
=
table
.
loc
[
model
].
max
()
min_value
=
min
(
min_values
[
tool_x
],
min_values
[
tool_y
])
/
2.
min_value_log
=
np
.
log10
(
min_value
)
max_value
=
max
(
max_values
[
tool_x
],
max_values
[
tool_y
])
max_value_log
=
np
.
log10
(
max_value
)
figure
=
px
.
scatter
(
table
.
loc
[
model
],
title
=
model
,
x
=
tool_x
,
y
=
tool_y
,
log_x
=
True
,
log_y
=
True
,
range_x
=
[
min_value
,
max_value
],
range_y
=
[
min_value
,
max_value
],
color
=
"property"
,
hover_data
=
[
[
'formula #{}'
.
format
(
i
)
for
i
in
table
.
loc
[
model
].
index
],
table_errors
.
loc
[
model
,
tool_x
],
table_errors
.
loc
[
model
,
tool_y
]
],
color_discrete_map
=
{
"T"
:
"green"
,
"F"
:
"red"
,
"U"
:
"black"
},
symbol_sequence
=
[
"circle-open"
])
line
=
go
.
Scatter
(
x
=
[
min_value
,
max_value
],
y
=
[
min_value
,
max_value
],
mode
=
'lines'
,
showlegend
=
False
,
line
=
dict
(
color
=
'black'
,
width
=
1
))
figure
.
add_traces
(
line
)
figure
.
update_layout
(
LAYOUT_FIGURES
,
xaxis_title
=
get_axis_title
(
tool_x
,
show_strategy
),
yaxis_title
=
get_axis_title
(
tool_y
,
show_strategy
))
table_model
=
table
.
loc
[
model
]
table_errors_model
=
table_error
.
loc
[
model
]
full_table_x
=
pd
.
concat
([
table_model
[
tool_x
],
table_model
[
'property'
],
table_errors_model
[
tool_x
]],
axis
=
1
)
full_table_x
.
columns
=
[
'time'
,
'property'
,
'error'
]
full_table_y
=
pd
.
concat
([
table_model
[
tool_y
],
table_model
[
'property'
],
table_errors_model
[
tool_y
]],
axis
=
1
)
full_table_y
.
columns
=
[
'time'
,
'property'
,
'error'
]
traces
=
[
{
"property"
:
'T'
,
"color"
:
"green"
},
{
"property"
:
'F'
,
"color"
:
"red"
},
{
"property"
:
'U'
,
"color"
:
"black"
}
]
figures
=
[]
for
t
in
traces
:
# filter by verification output
table_x
=
full_table_x
[
full_table_x
.
property
==
t
[
'property'
]]
table_y
=
full_table_y
[
full_table_y
.
property
==
t
[
'property'
]]
# custom data
custom_data
=
list
(
zip
(
table_x
.
index
,
table_x
.
error
,
table_y
.
error
))
# tools
metainfo
=
{
'model'
:
model
,
'tools'
:
{
'x'
:
tool_x
,
'y'
:
tool_y
},
'folder'
:
os
.
path
.
join
(
os
.
path
.
abspath
(
os
.
pardir
),
"results"
)
}
figures
.
append
(
go
.
Scatter
(
x
=
table_x
.
time
,
y
=
table_y
.
time
,
name
=
t
[
'property'
],
mode
=
'markers'
,
marker_symbol
=
'circle-open'
,
marker_color
=
t
[
'color'
],
meta
=
metainfo
,
customdata
=
custom_data
,
hovertemplate
=
'<b>Formula # %{customdata[0]}</b><br>'
+
'<br><b>Times:</b><br>'
+
'<b>x:</b> %{x} s'
+
'<br><b>y:</b> %{y} s<br>'
+
'<br><b>Errors:</b><br>'
+
'<b>x:</b> %{customdata[1]}<br>'
+
'<b>y:</b> %{customdata[2]}'
,
))
# Line
figures
.
append
(
go
.
Scatter
(
x
=
[
min_value
,
max_value
],
y
=
[
min_value
,
max_value
],
mode
=
'lines'
,
showlegend
=
False
,
line
=
dict
(
color
=
'black'
,
width
=
1
)))
# Create figure
figure
=
go
.
FigureWidget
(
figures
)
figure
.
update_layout
(
LAYOUT_FIGURES
,
title_text
=
model
,
hoverlabel
=
dict
(
bgcolor
=
"white"
,
align
=
'auto'
),
legend_title_text
=
'property'
,
xaxis
=
dict
(
type
=
'log'
,
autorange
=
False
,
range
=
[
min_value_log
,
max_value_log
]),
yaxis
=
dict
(
type
=
'log'
,
autorange
=
False
,
range
=
[
min_value_log
,
max_value_log
]),
xaxis_title
=
get_axis_title
(
tool_x
,
show_strategy
),
yaxis_title
=
get_axis_title
(
tool_y
,
show_strategy
))
# Add event
if
callback
is
not
None
:
for
i
in
range
(
len
(
figure
.
data
)):
figure
.
data
[
i
].
on_click
(
callback
)
return
figure
except
Exception
as
e
:
...
...
@@ -237,6 +288,76 @@ def create_log_figure(table, table_errors, model, tool_x, tool_y, show_strategy=
# In[6]:
import
webbrowser
def
get_filename
(
base_path
,
tool
,
model
,
model_instance
,
formula
):
"""Returns the absolute path of the experiment log
Parameters
----------
base_path : string
Path of the folder where logs are saved
tool : string
Tool name
model : string
Model name
model_instance : string
Name of the model instance
formula : string
Identifier of the formula
Returns
-------
string
Absolute path of the log file
"""
information
=
tool
.
split
(
'_'
)
tool_name
=
information
[
0
]
tool_configuration
=
'_'
.
join
(
information
[:
-
2
])
nb_nodes
=
information
[
-
2
]
nb_cores
=
information
[
-
1
]
experiment_folder
=
os
.
path
.
join
(
base_path
,
tool_name
,
tool_configuration
,
model
,
model_instance
)
filename
=
f
'
{
tool_name
}
_
{
model_instance
}
-n
{
nb_nodes
}
-th
{
nb_cores
}
-f
{
formula
}
'
absolute_path
=
os
.
path
.
join
(
experiment_folder
,
filename
)
return
absolute_path
def
open_logs_callback
(
trace
,
points
,
selector
):
"""Callback that open the log files when clicking on a point of the figure
Parameters
----------
trace : plotly.graph_objects.Figure
the figure to attach the callback
points : plotly.callbacks.Points
points of the figure selected
selector: plotly.callbacks.InputDeviceState
Device information
"""
inds
=
points
.
point_inds
if
(
inds
):
index
=
inds
[
0
]
formula
,
error_x
,
error_y
=
trace
[
'customdata'
][
index
]
model_instance
=
trace
[
'meta'
][
'model'
]
model
=
''
.
join
(
c
for
c
in
model_instance
if
not
c
.
isdigit
())
tools
=
trace
[
'meta'
][
'tools'
]
logs_folder
=
trace
[
'meta'
][
'folder'
]
filename_x
=
get_filename
(
logs_folder
,
tools
[
'x'
],
model
,
model_instance
,
formula
)
filename_y
=
get_filename
(
logs_folder
,
tools
[
'y'
],
model
,
model_instance
,
formula
)
for
f
in
[
filename_x
,
filename_y
]:
webbrowser
.
open
(
f
'file://
{
f
}
.err'
)
webbrowser
.
open
(
f
'file://
{
f
}
.out'
)
# In[7]:
def
create_figure_explored_states
(
table_explored_states
,
model
):
"""Creates figure showing the number of explorated states during the verification
...
...
@@ -304,7 +425,7 @@ def create_figure_explored_states(table_explored_states, model):
return
fig
# In[
7
]:
# In[
8
]:
# Experiment filters
...
...
@@ -423,7 +544,7 @@ plots = {
# # Load Data
# In[
8
]:
# In[
9
]:
# Root folder
...
...
@@ -437,7 +558,7 @@ OUTPUT_FOLDER = os.path.join(PROJECT_FOLDER,"results", "figures")
create_folder
(
OUTPUT_FOLDER
)
# In[
9
]:
# In[
10
]:
# read data
...
...
@@ -450,7 +571,7 @@ df = df.drop(columns=['strategy', 'num_nodes', 'num_threads'])
df
.
head
()
# In[1
0
]:
# In[1
1
]:
# ground truth for properties
...
...
@@ -470,7 +591,7 @@ p_df.sort_index(inplace=True)
p_df
.
head
()
# In[1
1
]:
# In[1
2
]:
# table with times, verification output and error for each experiment
...
...
@@ -480,14 +601,14 @@ table.head()
# # Preprocessing of data
# In[1
2
]:
# In[1
3
]:
ZERO
=
10e-5
TIMEOUT
=
10
*
60
# 10 minutes = 600 seconds
# In[1
3
]:
# In[1
4
]:
# table with times for each experiment
...
...
@@ -505,7 +626,7 @@ table_time = pd.concat([table_time, p_df], axis=1)
table_time
.
head
()
# In[1
4
]:
# In[1
5
]:
# table with verification output for each experiment
...
...
@@ -520,7 +641,7 @@ table_property = pd.concat([table_property, p_df], axis=1)
table_property
.
head
()
# In[1
5
]:
# In[1
6
]:
# table with error for each experiment
...
...
@@ -529,7 +650,7 @@ table_error = table['error'].copy()
table_error
.
head
()
# In[1
6
]:
# In[1
7
]:
# table with explored states for each experiment using ltsmin
...
...
@@ -547,7 +668,7 @@ table_explored_states = table_explored_states.reset_index()
table_explored_states
.
head
()
# In[1
7
]:
# In[1
8
]:
# calculate the stats of the number of explored states
...
...
@@ -559,27 +680,27 @@ table_explored_states_stats.head()
# # Examples
# In[
20
]:
# In[
19
]:
create_figure_explored_states
(
table_explored_states
,
'robot20'
)
# In[2
1
]:
# In[2
0
]:
create_figure
(
df
,
"philo10"
)
# In[2
2
]:
# In[2
1
]:
create_log_figure
(
table_time
,
table_error
,
"philo10"
,
"pmc-sog_otf_couv99-default_2_8"
,
"pnml2lts-mc_dfs_1_16"
)
create_log_figure
(
table_time
,
table_error
,
"philo10"
,
"pmc-sog_otf_couv99-default_2_8"
,
"pnml2lts-mc_dfs_1_16"
,
open_logs_callback
)
# # Generate Figures
# In[
23
]:
# In[
]:
# models
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment