Commit 3da23b16 authored by Jaime Arias's avatar Jaime Arias
Browse files

fix notebook

parent 121075cd
%% Cell type:code id: tags:
``` python
import os
import pandas as pd
import numpy as np
import plotly.io as pio
import plotly.express as px
import plotly.graph_objs as go
from itertools import combinations
from plotly.subplots import make_subplots
# render figures in notebook
pio.renderers.default = "notebook_connected"
# templates figures
px.defaults.template = "simple_white"
pio.templates.default = "simple_white"
# layout for all figures
LAYOUT_FIGURES = dict(
autosize=False,
width = 500,
height = 500,
xaxis = dict(
constrain="domain",
mirror=True,
showexponent="all",
exponentformat="power"
),
yaxis = dict(
scaleanchor = "x",
scaleratio = 1,
mirror=True,
showexponent="all",
exponentformat="power"
),
title = dict(
y = 0.9,
x = 0.5,
xanchor = 'center',
yanchor = 'top'
)
)
```
%% Cell type:markdown id: tags:
# Auxiliary Functions
%% Cell type:code id: tags:
``` python
def create_folder(path):
"""Creates a folder if it does not exist
Parameters
----------
path : str
Path of the new folder
Examples
--------
>>> create_folder('./results')
"""
if not os.path.exists(path):
os.makedirs(path)
```
%% Cell type:code id: tags:
``` python
def create_figure(df, model):
"""Creates a scatter figure showing the time taken by each tool to verify each property of a model
Parameters
----------
df : pandas.Dataframe
Dataframe containing the results of the experiments
model : string
model to be plotted
Returns
-------
plotly.graph_objects.Figure
Scatter figure
Examples
--------
>>> import os
>>> import pandas as pd
>>> csv_file = os.path.join("results", "output.csv")
>>> df = pd.read_csv(csv_file)
>>> fig = create_figure(df, 'philo10')
"""
model_df = df[df.model == model]
figure = px.scatter(model_df,
x="formula", y="time",
title=model,
color="tool",
symbol_sequence=['x'])
figure.update_layout(yaxis_title="time (s)", title=LAYOUT_FIGURES['title'])
return figure
```
%% Cell type:code id: tags:
``` python
def get_axis_title(experiment, show_strategy=True):
"""Get the axis title of a figure depending on the experiment being plotted
Parameters
----------
experiment : str
String with the experiment information
show_strategy : bool, optional
Flag to show the information related to the strategy used by the tool
Returns
-------
str
axis title
Examples
--------
>>> get_axis_title('pmc-sog_otfL_couv99-default_1_1', True)
pmc-sog (Lace, strategy: couv99-default, # cores: 1)
"""
information = experiment.split('_')
tool_name = information[0]
info = []
library_dic = {
'otfL': 'Lace',
'otfP': 'Pthreads',
'otfC': 'Cthreads',
'otf': 'Hybrid'
}
if (len(information) == 5):
info.append(library_dic[information[1]])
if (show_strategy):
info.append('strategy: {}'.format(information[-3]))
nb_nodes = int(information[-2])
if (nb_nodes > 1):
info.append('# nodes: {}'.format(nb_nodes))
info.append('# cores: {}'.format(information[-1]))
title = '{} ({})'.format(tool_name, ', '.join(info))
return title
```
%% Cell type:code id: tags:
``` python
def create_log_figure(table, table_errors, model, tool_x, tool_y, show_strategy=True, callback=None):
"""Creates a Scatter figure in logarithmic scale comparing the performance of two tools
Parameters
----------
table : pandas.Dataframe
Dataframe with the times of each experiment
table_errors : pandas.Dataframe
Dataframe with the errors of each experiment
model : string
Model to be analyzed
tool_x : string
Tool to be compared and plotted on the x-axis
tool_y : string
Tool to be compared and plotted on the y-axis
show_strategy : bool
Flag to show the stretagy used by the tools
callback : function
Function to be called when clicking on a point
Returns
-------
plotly.graph_objects.Figure
Scatter figure
Examples
--------
>>> import os
>>> import pandas as pd
>>> csv_file = os.path.join("results", "output.csv")
>>> df = pd.read_csv(csv_file)
>>> table = df.set_index(['model', 'formula', 'tool'], drop=True).unstack('tool')
>>> fig = create_log_figure(table['time'], table['error'], 'philo10', 'pmc-sog_otfL_couv99-default_1_8', 'pmc-sog_otfP_couv99-default_1_8')
"""
try:
min_values = table.loc[model].min()
max_values = table.loc[model].max()
min_value = min(min_values[tool_x], min_values[tool_y])/2.
min_value_log = np.log10(min_value)
max_value = max(max_values[tool_x], max_values[tool_y])
max_value_log = np.log10(max_value)
table_model = table.loc[model]
table_errors_model = table_error.loc[model]
full_table_x = pd.concat([table_model[tool_x],table_model['property'], table_errors_model[tool_x]], axis=1)
full_table_x.columns = ['time', 'property', 'error']
full_table_y = pd.concat([table_model[tool_y],table_model['property'], table_errors_model[tool_y]], axis=1)
full_table_y.columns = ['time', 'property', 'error']
traces = [
{"property": 'T', "color":"green"},
{"property": 'F', "color":"red"},
{"property": 'U', "color":"black"}
]
figures = []
for t in traces:
# filter by verification output
table_x = full_table_x[full_table_x.property == t['property']]
table_y = full_table_y[full_table_y.property == t['property']]
# custom data
custom_data = list(zip(table_x.index, table_x.error,table_y.error))
# tools
metainfo = {
'model': model,
'tools': {'x': tool_x, 'y': tool_y},
'folder': os.path.join(os.path.abspath(os.pardir), "results")
}
figures.append(go.Scatter(x=table_x.time,
y=table_y.time,
name=t['property'],
mode='markers',
marker_symbol='circle-open',
marker_color=t['color'],
meta = metainfo,
customdata=custom_data,
hovertemplate =
'<b>Formula # %{customdata[0]}</b><br>' +
'<br><b>Times:</b><br>' +
'<b>x:</b> %{x} s' +
'<br><b>y:</b> %{y} s<br>' +
'<br><b>Errors:</b><br>' +
'<b>x:</b> %{customdata[1]}<br>' +
'<b>y:</b> %{customdata[2]}',
))
# Line
figures.append(go.Scatter(x=[min_value, max_value],
y=[min_value, max_value],
mode='lines', showlegend=False,
line=dict(color='black', width=1)))
# Create figure
figure = go.FigureWidget(figures)
figure.update_layout(LAYOUT_FIGURES,
title_text=model,
hoverlabel=dict(bgcolor="white", align='auto'),
legend_title_text='property',
xaxis=dict(type='log', autorange=False, range=[min_value_log, max_value_log]),
yaxis=dict(type='log', autorange=False, range=[min_value_log, max_value_log]),
xaxis_title=get_axis_title(tool_x, show_strategy),
yaxis_title=get_axis_title(tool_y, show_strategy))
# Add event
if callback is not None:
for i in range(len(figure.data)):
figure.data[i].on_click(callback)
return figure
except Exception as e:
print("Error when ploting model: {} - tool_x: {} - tool_y: {}".format(model, tool_x, tool_y))
print(e)
```
%% Cell type:code id: tags:
``` python
import webbrowser
def get_filename(base_path, tool, model, model_instance, formula):
"""Returns the absolute path of the experiment log
Parameters
----------
base_path : string
Path of the folder where logs are saved
tool : string
Tool name
model : string
Model name
model_instance : string
Name of the model instance
formula : string
Identifier of the formula
Returns
-------
string
Absolute path of the log file
"""
information = tool.split('_')
tool_name = information[0]
tool_configuration = '_'.join(information[:-2])
nb_nodes = information[-2]
nb_cores = information[-1]
experiment_folder = os.path.join(base_path, tool_name, tool_configuration, model, model_instance)
filename = f'{tool_name}_{model_instance}-n{nb_nodes}-th{nb_cores}-f{formula}'
absolute_path = os.path.join(experiment_folder, filename)
return absolute_path
def open_logs_callback(trace, points, selector):
"""Callback that open the log files when clicking on a point of the figure
Parameters
----------
trace : plotly.graph_objects.Figure
the figure to attach the callback
points : plotly.callbacks.Points
points of the figure selected
selector: plotly.callbacks.InputDeviceState
Device information
"""
inds = points.point_inds
if (inds):
index = inds[0]
formula, error_x, error_y = trace['customdata'][index]
model_instance = trace['meta']['model']
model = ''.join(c for c in model_instance if not c.isdigit())
tools = trace['meta']['tools']
logs_folder = trace['meta']['folder']
filename_x = get_filename(logs_folder, tools['x'], model, model_instance, formula)
filename_y = get_filename(logs_folder, tools['y'], model, model_instance, formula)
for f in [filename_x, filename_y]:
webbrowser.open(f'file://{f}.err')
webbrowser.open(f'file://{f}.out')
OPEN_LOGS_CALLBACK_JS = """
function get_filename (base_path, tool, model_instance, formula) {
const information = tool.split('_');
const size = information.length;
const tool_name = information[0];
const tool_configuration = information.slice(0, size - 2).join('_');
const nb_nodes = information[size - 2];
const nb_cores = information[size - 1];
const model = model_instance.replace(/[0-9]/g, '');
const experiment_folder = `${base_path}/${tool_name}/${tool_configuration}/${model}/${model_instance}`;
const filename = `${tool_name}_${model_instance}-n${nb_nodes}-th${nb_cores}-f${formula}`;
return `${experiment_folder}/${filename}`;
}
const plots = document.getElementsByClassName("plotly-graph-div js-plotly-plot");
const myPlot = plots[0];
myPlot.on('plotly_click', function(data){
const points = data.points;
if (points.length != 1) {return ;}
const myPoint = points[0];
const formula = myPoint.customdata[0];
const meta = myPoint.data.meta;
const href = window.location.href.split('/');
const base_path = href.splice(0,href.length-2).join('/');
const results_path = `${base_path}/results`;
const filename_x = get_filename(results_path, meta.tools.x, meta.model, formula);
const filename_y = get_filename(results_path, meta.tools.y, meta.model, formula);
console.log('x: ' + filename_x);
window.open(`${filename_x}.err`);
window.open(`${filename_x}.out`);
console.log('y: ' + filename_y);
window.open(`${filename_y}.err`);
window.open(`${filename_y}.out`);
});
"""
```
%% Cell type:code id: tags:
``` python
def create_figure_explored_states(table_explored_states, model):
"""Creates figure showing the number of explorated states during the verification
Parameters
----------
table_explored_states : pandas.Dataframe
Dataframe with the explorated states of each experiment
model : string
Model to be analyzed
Returns
-------
plotly.graph_objects.Figure
Scatter figure
"""
colors={'T': 'green', 'F': 'red'}
float_formatter = "{:.2E}".format
table_model = table_explored_states[table_explored_states.property != 'U']
table_model = table_model[table_model.model == model]
table_stats = table_model.groupby(['property']).agg(['mean'])
fig = go.Figure()
for p in table_stats.index:
data = table_model[table_model.property==p]
stats = table_stats.loc[p]
x_axis = np.arange(1, data['formula'].count()+1, 1)
mean = stats['explored_states','mean']
figure = px.scatter(data,
x=x_axis,
y="explored_states",
title=model,
color='property',
color_discrete_map=colors,
symbol_sequence=["circle"])
line = go.Scatter(x=[x_axis[0], x_axis[-1]],
y=[mean, mean],
mode='lines', showlegend=False,
line=dict(color=colors[p], width=1.5))
fig.add_trace(figure['data'][0])
fig.add_trace(line)
fig.add_annotation(x=1,
y=mean,
font=dict(color=colors[p]),
text=f"mean = {float_formatter(mean)}")
fig.update_layout(title_text=model, title=LAYOUT_FIGURES['title'],
width = 500, height = 500, margin=dict(r=110))
fig.update_xaxes(title="formula", range=[x_axis[0]-1, x_axis[-1]+1])
fig.update_yaxes(title="# explored states")
fig.update_annotations(dict(
showarrow=False,
xanchor="left",
yanchor="middle",
xref='paper'))
return fig
```
%% Cell type:code id: tags:
``` python
# Experiment filters
def versus_dfs(experiments):
"""Selects only experiments using DFS strategy"""
exp1, exp2 = experiments
strategy_exp1= exp1.split('_')[1]
strategy_exp2= exp2.split('_')[1]
return strategy_exp1 == 'dfs' or strategy_exp2 == 'dfs'
def versus_sequential(experiments):
"""Selects only experiments run sequentially """
exp1, exp2 = experiments
nodes_exp1, threads_exp1 = exp1.split('_')[-2:]
nodes_exp2, threads_exp2 = exp2.split('_')[-2:]
return (nodes_exp1 == '1' and nodes_exp2 == '1') and \
(threads_exp1 == '1' or threads_exp2 == '1')
def same_tool(experiments, tool):
"""Selects only experiments comparing the same tool"""
exp1, exp2 = experiments
tool_exp1= exp1.split('_')[0]
tool_exp2= exp2.split('_')[0]
return tool_exp1.startswith(tool) and tool_exp2.startswith(tool)
def same_number_threads(experiments):
"""Selects only experiments comparing the same number of processes and cores"""
exp1, exp2 = experiments
nodes_exp1, threads_exp1 = exp1.split('_')[-2:]
nodes_exp2, threads_exp2 = exp2.split('_')[-2:]
return (nodes_exp1 == nodes_exp2) and (threads_exp1 == threads_exp2)
def same_thread_library(experiments):
"""Selects only experiments comparing the same parallelization library"""
exp1, exp2 = experiments
library_exp1 = exp1.split('_')[1]
library_exp2 = exp2.split('_')[1]
return library_exp1 == library_exp2
def same_strategy(experiments):
"""Selects only experiments comparing the same strategy"""
exp1, exp2 = experiments
strategy_exp1 = exp1.split('_')[2]
strategy_exp2 = exp2.split('_')[2]
return strategy_exp1 == strategy_exp2
def only_couvreur_strategy(experiments):
"""Selects only experiments comparing couvreur emptiness check algorithm"""
exp1, exp2 = experiments
strategy_exp1 = exp1.split('_')[2]
strategy_exp2 = exp2.split('_')[2]
return strategy_exp1.startswith('couv99') and strategy_exp2.startswith('couv99')
def compare_threads_library(experiments):
"""Compares parallization libraries used in pmc-sog.
It selects experiments where the tool is only pmc-sog and the strategy, number of threads,
number of processus are the same.
"""
return same_tool(experiments, 'pmc-sog') and \
same_strategy(experiments) and \
same_number_threads(experiments) and \
not same_thread_library(experiments)
def compare_couvreur_strategies(experiments):
"""Compares couvreurs strategies used in pmc-sog.
It selects experiments where the tool is only pmc-sog, the strategy is couvreur, and
the parallelization library, number of threads, number of processus are the same.
"""
return only_couvreur_strategy(experiments) and \
same_thread_library(experiments) and \
same_number_threads(experiments)
def same_distributed_number_threads(experiments):
"""Selects only experiments where the multiplication of theirs nodes with cores are the same."""
exp1, exp2 = experiments
nodes_exp1, threads_exp1 = exp1.split('_')[-2:]
nodes_exp2, threads_exp2 = exp2.split('_')[-2:]
return (int(nodes_exp1) * int(threads_exp1)) == (int(nodes_exp2) * int(threads_exp2))
def compare_tools(experiments):
"""Compares pmc-sog and pnml2lts-mc using the DFS algorithm.
It selects experiments where the tools are not the same, the exploration algorithm is DFS and
the number of processus and cores are the same.
"""
return not (same_tool(experiments, 'pmc-sog') or same_tool(experiments,'pnml2lts-mc')) and \
versus_dfs(experiments)
def compare_multithreading(experiments):
"""Compares the sequential and multi-core version of pmc-sog.
It selects experiments where the tools is pmc-sog, the parallelization library, the emptiness check
strategy are the same. Here the number of processus and cores are different.
"""
return same_tool(experiments, 'pmc-sog') and \
same_thread_library(experiments) and \
same_strategy(experiments) and \
versus_sequential(experiments)
def against_hybrid(experiments):
"""Selects only experiments comparing with hybrid mode"""
exp1, exp2 = experiments
library_exp1 = exp1.split('_')[1]
library_exp2 = exp2.split('_')[1]
return (library_exp1 == 'otf') or (library_exp2 == 'otf')
def compare_distributed(experiments):
"""Compares the hybrid version of pmc-sog"""
return same_tool(experiments, 'pmc-sog') and \
same_strategy(experiments) and \
same_distributed_number_threads(experiments) and \
against_hybrid(experiments)
# Plots to be created
plots = {
'compare_thread_library': compare_threads_library,
'compare_couvreur_algorithm': compare_couvreur_strategies,
'compare_tools': compare_tools,
'compare_multicore': compare_multithreading,
'compare_distributed': compare_distributed
}
```
%% Cell type:markdown id: tags:
# Load Data
%% Cell type:code id: tags:
``` python
# Root folder
PROJECT_FOLDER = os.path.abspath(os.pardir)
# csv file with the output
csv_file = os.path.join(PROJECT_FOLDER, "results", "output.csv")
# Output folder
OUTPUT_FOLDER = os.path.join(PROJECT_FOLDER,"results", "figures")
create_folder(OUTPUT_FOLDER)
```
%% Cell type:code id: tags:
``` python
# read data
df = pd.read_csv(csv_file)
# merge the information related to the experiment (# nodes, # threads, strategy) to the tool column
df['tool'] = df[['tool', 'strategy', 'num_nodes', 'num_threads']].astype(str).apply('_'.join, axis=1)
df = df.drop(columns=['strategy', 'num_nodes', 'num_threads'])
df.head()
```
%% Output
model formula tool time property \
0 philo10 1 pmc-sog_otfL_couv99-default_1_16 6.572 F
1 philo10 1 pmc-sog_otfL_couv99-shy_1_16 5.926 F
2 philo10 1 pmc-sog_otfP_couv99-default_1_16 6.811 F
3 philo10 1 pmc-sog_otfP_couv99-shy_1_16 6.862 F
4 philo10 1 pmc-sog_otf_couv99-default_2_8 5.705 F
explored_states error
0 NaN OK
1 NaN OK
2 NaN OK
3 NaN OK
4 NaN OK
%% Cell type:code id: tags:
``` python
# ground truth for properties
p_df = pd.read_csv(csv_file)
p_df =p_df[
(p_df.tool=='pnml2lts-mc') &
(p_df.strategy == 'ndfs') &
(p_df.num_nodes == 1) &
(p_df.num_threads == 16)]
# only property column is needed
p_df = p_df.drop(columns=['tool', 'strategy', 'num_nodes', 'num_threads', 'time', 'explored_states', 'error'])
p_df.fillna('U', inplace=True)
p_df.set_index(['model', 'formula'], inplace=True)
p_df.sort_index(inplace=True)
p_df.head()
```
%% Output
property
model formula
philo10 1 F
2 F
3 F
4 F
5 F
%% Cell type:code id: tags:
``` python
# table with times, verification output and error for each experiment
table = df.set_index(['model', 'formula', 'tool'], drop=True).unstack('tool')
table.head()
```
%% Output
time \
tool pmc-sog_otfL_couv99-default_1_1
model formula
philo10 1 NaN
2 NaN
3 NaN
4 NaN
5 NaN
\
tool pmc-sog_otfL_couv99-default_1_16 pmc-sog_otfL_couv99-shy_1_1
model formula
philo10 1 6.572 NaN
2 5.179 NaN
3 2.620 NaN
4 2.379 NaN
5 4.260 NaN
\
tool pmc-sog_otfL_couv99-shy_1_16 pmc-sog_otfP_couv99-default_1_1
model formula
philo10 1 5.926 NaN
2 5.090 NaN
3 4.383 NaN
4 4.183 NaN
5 7.320 NaN
\
tool pmc-sog_otfP_couv99-default_1_16 pmc-sog_otfP_couv99-shy_1_1
model formula
philo10 1 6.811 NaN
2 4.575 NaN
3 3.489 NaN
4 4.322 NaN
5 7.995 NaN
\
tool pmc-sog_otfP_couv99-shy_1_16 pmc-sog_otf_couv99-default_2_16
model formula
philo10 1 6.862 5.688
2 4.350 5.206
3 4.143 3.801
4 4.149 4.647
5 7.629 8.440
... \
tool pmc-sog_otf_couv99-default_2_8 ...
model formula ...
philo10 1 5.705 ...
2 6.592 ...
3 6.219 ...
4 5.837 ...
5 8.405 ...
error \
tool pmc-sog_otf_couv99-default_8_2 pmc-sog_otf_couv99-shy_2_16
model formula
philo10 1 OK TIME LIMIT
2 OK OK
3 OK OK
4 OK OK
5 OK OK
\
tool pmc-sog_otf_couv99-shy_2_8 pmc-sog_otf_couv99-shy_4_16
model formula
philo10 1 OK OK
2 OK OK
3 OK OK
4 OK OK
5 OK OK
\
tool pmc-sog_otf_couv99-shy_4_4 pmc-sog_otf_couv99-shy_8_2
model formula
philo10 1 OK OK
2 OK OK
3 OK OK
4 OK OK
5 OK OK
\
tool pnml2lts-mc_dfs_1_1 pnml2lts-mc_dfs_1_16 pnml2lts-mc_ndfs_1_1
model formula
philo10 1 NaN OK NaN
2 NaN OK NaN
3 NaN OK NaN
4 NaN OK NaN
5 NaN OK NaN
tool pnml2lts-mc_ndfs_1_16
model formula
philo10 1 OK
2 OK
3 OK
4 OK
5 OK
[5 rows x 88 columns]
%% Cell type:markdown id: tags:
# Preprocessing of data
%% Cell type:code id: tags:
``` python
ZERO = 10e-5
TIMEOUT = 10 * 60 # 10 minutes = 600 seconds
```
%% Cell type:code id: tags:
``` python
# table with times for each experiment
table_time = table['time'].copy()
# replace non finished experiments with a dummy value, e.g. timeout
table_time.fillna(TIMEOUT, inplace=True)
# replace 0.00 time for 10^(-5), we cannot plot log(0)
table_time.replace(0.0, ZERO, inplace=True)
# add verification output to the table
table_time = pd.concat([table_time, p_df], axis=1)
table_time.head()
```
%% Output
pmc-sog_otfL_couv99-default_1_1 \
model formula
philo10 1 600.0
2 600.0
3 600.0
4 600.0
5 600.0
pmc-sog_otfL_couv99-default_1_16 \
model formula
philo10 1 6.572
2 5.179
3 2.620
4 2.379
5 4.260
pmc-sog_otfL_couv99-shy_1_1 pmc-sog_otfL_couv99-shy_1_16 \
model formula
philo10 1 600.0 5.926
2 600.0 5.090
3 600.0 4.383
4 600.0 4.183
5 600.0 7.320
pmc-sog_otfP_couv99-default_1_1 \
model formula
philo10 1 600.0
2 600.0
3 600.0
4 600.0
5 600.0
pmc-sog_otfP_couv99-default_1_16 \
model formula
philo10 1 6.811
2 4.575
3 3.489
4 4.322
5 7.995
pmc-sog_otfP_couv99-shy_1_1 pmc-sog_otfP_couv99-shy_1_16 \
model formula
philo10 1 600.0 6.862
2 600.0 4.350
3 600.0 4.143
4 600.0 4.149
5 600.0 7.629
pmc-sog_otf_couv99-default_2_16 \
model formula
philo10 1 5.688
2 5.206
3 3.801
4 4.647
5 8.440
pmc-sog_otf_couv99-default_2_8 ... \
model formula ...
philo10 1 5.705 ...
2 6.592 ...
3 6.219 ...
4 5.837 ...
5 8.405 ...
pmc-sog_otf_couv99-shy_2_16 pmc-sog_otf_couv99-shy_2_8 \
model formula
philo10 1 600.000 6.921
2 5.091 7.795
3 4.822 6.958
4 4.932 5.782
5 9.307 8.542
pmc-sog_otf_couv99-shy_4_16 pmc-sog_otf_couv99-shy_4_4 \
model formula
philo10 1 12.902 13.706
2 6.936 9.216
3 6.129 8.621
4 8.317 8.562
5 16.590 13.513
pmc-sog_otf_couv99-shy_8_2 pnml2lts-mc_dfs_1_1 \
model formula
philo10 1 16.728 600.0
2 13.315 600.0
3 10.305 600.0
4 11.722 600.0
5 16.733 600.0
pnml2lts-mc_dfs_1_16 pnml2lts-mc_ndfs_1_1 \
model formula
philo10 1 0.81 600.0
2 0.43 600.0
3 2.36 600.0
4 0.54 600.0
5 0.67 600.0
pnml2lts-mc_ndfs_1_16 property
model formula
philo10 1 0.19 F
2 0.12 F
3 0.16 F
4 0.18 F
5 0.08 F
[5 rows x 23 columns]
%% Cell type:code id: tags:
``` python
# table with verification output for each experiment
table_property = table['property'].copy()
# replace non finished experiments with a dummy value
table_property.fillna('U', inplace=True)
# add ground truth to the table
table_property = pd.concat([table_property, p_df], axis=1)
table_property.head()
```
%% Output
pmc-sog_otfL_couv99-default_1_1 \
model formula
philo10 1 U
2 U
3 U
4 U
5 U
pmc-sog_otfL_couv99-default_1_16 pmc-sog_otfL_couv99-shy_1_1 \
model formula
philo10 1 F U
2 F U
3 F U
4 F U
5 F U
pmc-sog_otfL_couv99-shy_1_16 pmc-sog_otfP_couv99-default_1_1 \
model formula
philo10 1 F U
2 F U
3 F U
4 F U
5 F U
pmc-sog_otfP_couv99-default_1_16 pmc-sog_otfP_couv99-shy_1_1 \
model formula
philo10 1 F U
2 F U
3 F U
4 F U
5 F U
pmc-sog_otfP_couv99-shy_1_16 pmc-sog_otf_couv99-default_2_16 \
model formula
philo10 1 F F
2 F F
3 F F
4 F F
5 F F
pmc-sog_otf_couv99-default_2_8 ... \
model formula ...
philo10 1 F ...
2 F ...
3 F ...
4 F ...
5 F ...
pmc-sog_otf_couv99-shy_2_16 pmc-sog_otf_couv99-shy_2_8 \
model formula
philo10 1 U F
2 F F
3 F F
4 F F
5 F F
pmc-sog_otf_couv99-shy_4_16 pmc-sog_otf_couv99-shy_4_4 \
model formula
philo10 1 F F
2 F F
3 F F
4 F F
5 F F
pmc-sog_otf_couv99-shy_8_2 pnml2lts-mc_dfs_1_1 \
model formula
philo10 1 F U
2 F U
3 F U
4 F U
5 F U
pnml2lts-mc_dfs_1_16 pnml2lts-mc_ndfs_1_1 \
model formula
philo10 1 U U
2 U U
3 U U
4 U U
5 U U
pnml2lts-mc_ndfs_1_16 property
model formula
philo10 1 F F
2 F F
3 F F
4 F F
5 F F
[5 rows x 23 columns]
%% Cell type:code id: tags:
``` python
# table with error for each experiment
table_error = table['error'].copy()
table_error.head()
```
%% Output
tool pmc-sog_otfL_couv99-default_1_1 \
model formula
philo10 1 NaN
2 NaN
3 NaN
4 NaN
5 NaN
tool pmc-sog_otfL_couv99-default_1_16 pmc-sog_otfL_couv99-shy_1_1 \
model formula
philo10 1 OK NaN
2 OK NaN
3 OK NaN
4 OK NaN
5 OK NaN
tool pmc-sog_otfL_couv99-shy_1_16 pmc-sog_otfP_couv99-default_1_1 \
model formula
philo10 1 OK NaN
2 OK NaN
3 OK NaN
4 OK NaN
5 OK NaN
tool pmc-sog_otfP_couv99-default_1_16 pmc-sog_otfP_couv99-shy_1_1 \
model formula
philo10 1 OK NaN
2 OK NaN
3 OK NaN
4 OK NaN
5 OK NaN
tool pmc-sog_otfP_couv99-shy_1_16 pmc-sog_otf_couv99-default_2_16 \
model formula
philo10 1 OK OK
2 OK OK
3 OK OK
4 OK OK
5 OK OK
tool pmc-sog_otf_couv99-default_2_8 ... \
model formula ...
philo10 1 OK ...
2 OK ...
3 OK ...
4 OK ...
5 OK ...
tool pmc-sog_otf_couv99-default_8_2 pmc-sog_otf_couv99-shy_2_16 \
model formula
philo10 1 OK TIME LIMIT
2 OK OK
3 OK OK
4 OK OK
5 OK OK
tool pmc-sog_otf_couv99-shy_2_8 pmc-sog_otf_couv99-shy_4_16 \
model formula
philo10 1 OK OK
2 OK OK
3 OK OK
4 OK OK
5 OK OK
tool pmc-sog_otf_couv99-shy_4_4 pmc-sog_otf_couv99-shy_8_2 \
model formula
philo10 1 OK OK
2 OK OK
3 OK OK
4 OK OK
5 OK OK
tool pnml2lts-mc_dfs_1_1 pnml2lts-mc_dfs_1_16 pnml2lts-mc_ndfs_1_1 \
model formula
philo10 1 NaN OK NaN
2 NaN OK NaN
3 NaN OK NaN
4 NaN OK NaN
5 NaN OK NaN
tool pnml2lts-mc_ndfs_1_16
model formula
philo10 1 OK
2 OK
3 OK
4 OK
5 OK
[5 rows x 22 columns]
%% Cell type:code id: tags:
``` python
# table with explored states for each experiment using ltsmin
table_explored_states = table.copy()
table_explored_states = table_explored_states['explored_states']
table_explored_states = table_explored_states[['pnml2lts-mc_dfs_1_16']]
table_explored_states = table_explored_states.rename(columns={"pnml2lts-mc_dfs_1_16": "explored_states"})
# add verification output to the table
table_explored_states = pd.concat([table_explored_states, p_df], axis=1)
# reshape
table_explored_states = table_explored_states.reset_index()
table_explored_states.head()
```
%% Output
model formula explored_states property
0 philo10 1 373978.0 F
1 philo10 2 51192.0 F
2 philo10 3 1877419.0 F
3 philo10 4 177147.0 F
4 philo10 5 293788.0 F
%% Cell type:code id: tags:
``` python
# calculate the stats of the number of explored states
table_explored_states_stats = table_explored_states.groupby(['model', 'property']).agg(['mean', 'min', 'max'])
table_explored_states_stats = table_explored_states_stats['explored_states']
table_explored_states_stats.head()
```
%% Output
mean min max
model property
philo10 F 4.124072e+05 51192.0 6910975.0
T 7.566208e+04 34993.0 196831.0
philo5 F 6.228700e+03 1378.0 30625.0
T 2.247610e+03 1018.0 9046.0
robot10 F 1.025143e+08 77925.0 724200973.0
%% Cell type:markdown id: tags:
# Examples
%% Cell type:code id: tags:
``` python
create_figure_explored_states(table_explored_states, 'robot20')
```
%% Output
%% Cell type:code id: tags:
``` python
create_figure(df, "philo10")
```
%% Output
%% Cell type:code id: tags:
``` python
create_log_figure(table_time, table_error, "philo10", "pmc-sog_otf_couv99-default_2_8", "pnml2lts-mc_dfs_1_16", True, open_logs_callback)
```
%% Output
%% Cell type:markdown id: tags:
# Generate Figures
%% Cell type:code id: tags:
``` python
# models
models = df.model.unique()
# tools
tools = df.tool.unique()
```
%% Cell type:code id: tags:
``` python
# create all the figures of explored states
folder = os.path.join(OUTPUT_FOLDER, 'explored-states')
create_folder(folder)
for model in models:
try:
fig = create_figure_explored_states(table_explored_states, model)
# save figures in html and pdf
fig.write_html(os.path.join(folder, model + '.html'), include_plotlyjs='cdn')
fig.write_image(os.path.join(folder, model + '.pdf'))
except KeyError:
print("Error: {} was not plotted".format(model))
```
%% Cell type:code id: tags:
``` python
# create all the figures formula vs time
folder = os.path.join(OUTPUT_FOLDER, 'time-plots')
create_folder(folder)
for model in models:
try:
fig = create_figure(df, model)
# save figures in html and pdf
fig.write_html(os.path.join(folder, model + '.html'), , include_plotlyjs='cdn')
fig.write_html(os.path.join(folder, model + '.html'), include_plotlyjs='cdn')
fig.write_image(os.path.join(folder, model + '.pdf'))
except KeyError:
print("Error: {} was not plotted".format(model))
```
%% Cell type:code id: tags:
``` python
# create all the log figures
tools_pairs = [sorted(t) for t in (combinations(tools, 2))]
for plot, filter_method in plots.items():
axes = list(filter(filter_method, tools_pairs))
for model in models:
folder = os.path.join(OUTPUT_FOLDER, plot, model)
create_folder(folder)
for axe in axes:
try:
show_strategy = plot == 'compare_couvreur_algorithm'
fig = create_log_figure(table_time, table_error, model, axe[0], axe[1], show_strategy)
# save figures in html and pdf
figure_name = os.path.join(folder, '{}-{}-VS-{}-log'.format(model, axe[0], axe[1]))
fig.write_html(figure_name + '.html', , include_plotlyjs='cdn', post_script=OPEN_LOGS_CALLBACK_JS)
fig.write_html(figure_name + '.html', include_plotlyjs='cdn', post_script=OPEN_LOGS_CALLBACK_JS)
fig.write_image(figure_name + '.pdf')
except KeyError:
print("Error: {} was not plotted".format(model))
```
%% Cell type:code id: tags:
``` python
```
......
......@@ -787,7 +787,7 @@ for model in models:
fig = create_figure(df, model)
# save figures in html and pdf
fig.write_html(os.path.join(folder, model + '.html'), , include_plotlyjs='cdn')
fig.write_html(os.path.join(folder, model + '.html'), include_plotlyjs='cdn')
fig.write_image(os.path.join(folder, model + '.pdf'))
except KeyError:
print("Error: {} was not plotted".format(model))
......@@ -814,7 +814,7 @@ for plot, filter_method in plots.items():
# save figures in html and pdf
figure_name = os.path.join(folder, '{}-{}-VS-{}-log'.format(model, axe[0], axe[1]))
fig.write_html(figure_name + '.html', , include_plotlyjs='cdn', post_script=OPEN_LOGS_CALLBACK_JS)
fig.write_html(figure_name + '.html', include_plotlyjs='cdn', post_script=OPEN_LOGS_CALLBACK_JS)
fig.write_image(figure_name + '.pdf')
except KeyError:
print("Error: {} was not plotted".format(model))
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment