Skip to content

GitLab

Commit 4d915b1a authored by Jaime Arias's avatar Jaime Arias
Browse files

update plot scripts

parent 33413d08
Expand all Hide whitespace changes
Inline Side-by-side
scripts/plot-results.ipynb
View file @ 4d915b1a
This diff is collapsed.
scripts/plot-results.py
View file @ 4d915b1a
......@@ -12,12 +12,15 @@ TIMEOUT = 10 * 60 # 10 minutes = 600 seconds
import os
import glob
import re
import pandas as pd
import numpy as np
import plotly.io as pio
import plotly.express as px
import plotly.graph_objs as go
from itertools import combinations
import plotly.figure_factory as ff
from plotly.subplots import make_subplots
# render figures in notebook
......@@ -56,7 +59,7 @@ LAYOUT_FIGURES = dict(
# # Auxiliary Functions
# In[3]:
# In[4]:
def create_folder(path):
......@@ -76,7 +79,7 @@ def create_folder(path):
os.makedirs(path)
# In[4]:
# In[5]:
def create_figure(df, model):
......@@ -115,7 +118,7 @@ def create_figure(df, model):
return figure
# In[5]:
# In[6]:
def get_axis_title(experiment, show_strategy=True):
......@@ -167,128 +170,38 @@ def get_axis_title(experiment, show_strategy=True):
return title
# In[6]:
# In[7]:
def create_log_figure(table, table_errors, model, tool_x, tool_y, show_strategy=True, callback=None):
"""Creates a Scatter figure in logarithmic scale comparing the performance of two tools
def get_info(df, model, experiment):
"""Get some statistics from a table for a specific model and experiment"""
info = df.loc[model][experiment]
Parameters
----------
table : pandas.Dataframe
Dataframe with the times of each experiment
table_errors : pandas.Dataframe
Dataframe with the errors of each experiment
model : string
Model to be analyzed
tool_x : string
Tool to be compared and plotted on the x-axis
tool_y : string
Tool to be compared and plotted on the y-axis
show_strategy : bool
Flag to show the stretagy used by the tools
callback : function
Function to be called when clicking on a point
Returns
-------
plotly.graph_objects.Figure
Scatter figure
Examples
--------
>>> import os
>>> import pandas as pd
>>> csv_file = os.path.join("results", "output.csv")
>>> df = pd.read_csv(csv_file)
>>> table = df.set_index(['model', 'formula', 'tool'], drop=True).unstack('tool')
>>> fig = create_log_figure(table['time'], table['error'], 'philo10', 'pmc-sog_otfL_couv99-default_1_8', 'pmc-sog_otfP_couv99-default_1_8')
"""
try:
min_value = ZERO
max_value = TIMEOUT
min_value_log = np.log10(min_value)
max_value_log = np.log10(max_value)
table_model = table.loc[model]
table_errors_model = table_error.loc[model]
full_table_x = pd.concat([table_model[tool_x],table_model['property'], table_errors_model[tool_x]], axis=1)
full_table_x.columns = ['time', 'property', 'error']
full_table_y = pd.concat([table_model[tool_y],table_model['property'], table_errors_model[tool_y]], axis=1)
full_table_y.columns = ['time', 'property', 'error']
traces = [
{"property": 'T', "color":"green"},
{"property": 'F', "color":"red"},
{"property": 'U', "color":"black"}
]
figures = []
for t in traces:
# filter by verification output
table_x = full_table_x[full_table_x.property == t['property']]
table_y = full_table_y[full_table_y.property == t['property']]
# custom data
custom_data = list(zip(table_x.index, table_x.error,table_y.error))
# tools
metainfo = {
'model': model,
'tools': {'x': tool_x, 'y': tool_y},
'folder': os.path.join(os.path.abspath(os.pardir), "results")
}
figures.append(go.Scatter(x=table_x.time,
y=table_y.time,
name=t['property'],
mode='markers',
marker_symbol='circle-open',
marker_color=t['color'],
meta = metainfo,
customdata=custom_data,
hovertemplate =
'<b>Formula # %{customdata[0]}</b><br>' +
'<br><b>Times:</b><br>' +
'<b>x:</b> %{x} s' +
'<br><b>y:</b> %{y} s<br>' +
'<br><b>Errors:</b><br>' +
'<b>x:</b> %{customdata[1]}<br>' +
'<b>y:</b> %{customdata[2]}',
))
time_limit = info[info=="TIME LIMIT"].count()
error = info[(info!="TIME LIMIT") & (info!="OK")].count()
ok = info[info=="OK"].count()
if ((time_limit + error + ok) != info.count()): raise Exception("Some information is missing in the table")
return {
"time limit": time_limit,
"error": error,
"OK": ok
}
# Line
figures.append(go.Scatter(x=[min_value, max_value],
y=[min_value, max_value],
mode='lines', showlegend=False,
line=dict(color='black', width=1)))
# Create figure
figure = go.FigureWidget(figures)
figure.update_layout(LAYOUT_FIGURES,
title_text=model,
hoverlabel=dict(bgcolor="white", align='auto'),
legend_title_text='property',
xaxis=dict(type='log', autorange=False, range=[min_value_log, max_value_log]),
yaxis=dict(type='log', autorange=False, range=[min_value_log, max_value_log]),
xaxis_title=get_axis_title(tool_x, show_strategy),
yaxis_title=get_axis_title(tool_y, show_strategy))
# Add event
if callback is not None:
for i in range(len(figure.data)):
figure.data[i].on_click(callback)
def get_table(df, model, experiments):
"""Creates a table with some statistics from a dataframe for a model and experiments"""
rows=[["<b>Experiment</b>", "<b>Time Limit</b>", "<b>Error</b>", "<b>OK</b>"]]
return figure
except Exception as e:
print("Error when ploting model: {} - tool_x: {} - tool_y: {}".format(model, tool_x, tool_y))
print(e)
for experiment in experiments:
info = get_info(df, model, experiment)
rows.append([experiment, info["time limit"], info["error"], info["OK"]])
return ff.create_table(rows)
# In[7]:
# In[8]:
import webbrowser
......@@ -403,7 +316,7 @@ myPlot.on('plotly_click', function(data){
"""
# In[8]:
# In[9]:
def create_figure_explored_states(table_explored_states, model):
......@@ -475,7 +388,128 @@ def create_figure_explored_states(table_explored_states, model):
return fig
# In[9]:
# In[10]:
def create_log_figure(table, table_errors, model, tool_x, tool_y, show_strategy=True, callback=None):
"""Creates a Scatter figure in logarithmic scale comparing the performance of two tools
Parameters
----------
table : pandas.Dataframe
Dataframe with the times of each experiment
table_errors : pandas.Dataframe
Dataframe with the errors of each experiment
model : string
Model to be analyzed
tool_x : string
Tool to be compared and plotted on the x-axis
tool_y : string
Tool to be compared and plotted on the y-axis
show_strategy : bool
Flag to show the stretagy used by the tools
callback : function
Function to be called when clicking on a point
Returns
-------
plotly.graph_objects.Figure
Scatter figure
Examples
--------
>>> import os
>>> import pandas as pd
>>> csv_file = os.path.join("results", "output.csv")
>>> df = pd.read_csv(csv_file)
>>> table = df.set_index(['model', 'formula', 'tool'], drop=True).unstack('tool')
>>> fig = create_log_figure(table['time'], table['error'], 'philo10', 'pmc-sog_otfL_couv99-default_1_8', 'pmc-sog_otfP_couv99-default_1_8')
"""
try:
min_value = ZERO
max_value = TIMEOUT
min_value_log = np.log10(min_value)
max_value_log = np.log10(max_value)
table_model = table.loc[model]
table_errors_model = table_error.loc[model]
full_table_x = pd.concat([table_model[tool_x],table_model['property'], table_errors_model[tool_x]], axis=1)
full_table_x.columns = ['time', 'property', 'error']
full_table_y = pd.concat([table_model[tool_y],table_model['property'], table_errors_model[tool_y]], axis=1)
full_table_y.columns = ['time', 'property', 'error']
traces = [
{"property": 'T', "color":"green"},
{"property": 'F', "color":"red"},
{"property": 'U', "color":"black"}
]
figures = []
for t in traces:
# filter by verification output
table_x = full_table_x[full_table_x.property == t['property']]
table_y = full_table_y[full_table_y.property == t['property']]
# custom data
custom_data = list(zip(table_x.index, table_x.error,table_y.error))
# tools
metainfo = {
'model': model,
'tools': {'x': tool_x, 'y': tool_y},
'folder': os.path.join(os.path.abspath(os.pardir), "results")
}
figures.append(go.Scatter(x=table_x.time,
y=table_y.time,
name=t['property'],
mode='markers',
marker_symbol='circle-open',
marker_color=t['color'],
meta = metainfo,
customdata=custom_data,
hovertemplate =
'<b>Formula # %{customdata[0]}</b><br>' +
'<br><b>Times:</b><br>' +
'<b>x:</b> %{x} s' +
'<br><b>y:</b> %{y} s<br>' +
'<br><b>Errors:</b><br>' +
'<b>x:</b> %{customdata[1]}<br>' +
'<b>y:</b> %{customdata[2]}',
))
# Line
figures.append(go.Scatter(x=[min_value, max_value],
y=[min_value, max_value],
mode='lines', showlegend=False,
line=dict(color='black', width=1)))
# Create figure
figure = go.FigureWidget(figures)
figure.update_layout(LAYOUT_FIGURES,
title_text=model,
hoverlabel=dict(bgcolor="white", align='auto'),
legend_title_text='property',
xaxis=dict(type='log', autorange=False, range=[min_value_log, max_value_log]),
yaxis=dict(type='log', autorange=False, range=[min_value_log, max_value_log]),
xaxis_title=get_axis_title(tool_x, show_strategy),
yaxis_title=get_axis_title(tool_y, show_strategy))
# Add event
if callback is not None:
for i in range(len(figure.data)):
figure.data[i].on_click(callback)
return figure
except Exception as e:
print("Error when ploting model: {} - tool_x: {} - tool_y: {}".format(model, tool_x, tool_y))
print(e)
# In[11]:
# Experiment filters
......@@ -598,7 +632,7 @@ plots = {
# # Load Data
# In[10]:
# In[12]:
# Root folder
......@@ -607,12 +641,15 @@ PROJECT_FOLDER = os.path.abspath(os.pardir)
# csv file with the output
csv_file = os.path.join(PROJECT_FOLDER, "results", "output.csv")
# formulas folder
FORMULAS_FOLDER = os.path.join(PROJECT_FOLDER, "formulas")
# Output folder
OUTPUT_FOLDER = os.path.join(PROJECT_FOLDER,"results", "figures")
create_folder(OUTPUT_FOLDER)
# In[11]:
# In[13]:
# read data
......@@ -625,27 +662,30 @@ df = df.drop(columns=['strategy', 'num_nodes', 'num_threads'])
df.head()
# In[12]:
# In[14]:
# ground truth for properties
p_df = pd.read_csv(csv_file)
p_df =p_df[
(p_df.tool=='pnml2lts-mc') &
(p_df.strategy == 'ndfs') &
(p_df.num_nodes == 1) &
(p_df.num_threads == 16)]
frames = []
# only property column is needed
p_df = p_df.drop(columns=['tool', 'strategy', 'num_nodes', 'num_threads', 'time', 'explored_states', 'error'])
p_df.fillna('U', inplace=True)
p_df.set_index(['model', 'formula'], inplace=True)
p_df.sort_index(inplace=True)
formula_results = glob.glob(os.path.join(FORMULAS_FOLDER, "**/formula_results"), recursive=True)
for f in formula_results:
model, out_file = f.split('/')[-2:]
tmp_df = pd.read_csv(f, sep=";", header=None, names=["formula", "property"])
tmp_df["model"] = model
frames.append(tmp_df)
p_df = pd.concat(frames)
p_df = p_df.reindex(columns=["model", "formula", "property"])
p_df = p_df[p_df['model'].isin(df.model.unique())]
p_df['property'] = p_df['property'].replace([True, False], ["T", "F"])
p_df = p_df.set_index(["model", "formula"])
p_df.head()
p_df
# In[13]:
# In[15]:
# table with times, verification output and error for each experiment
......@@ -655,7 +695,7 @@ table.head()
# # Preprocessing of data
# In[14]:
# In[16]:
# table with times for each experiment
......@@ -673,7 +713,7 @@ table_time = pd.concat([table_time, p_df], axis=1)
table_time.head()
# In[15]:
# In[17]:
# table with verification output for each experiment
......@@ -688,7 +728,7 @@ table_property = pd.concat([table_property, p_df], axis=1)
table_property.head()
# In[16]:
# In[18]:
# table with error for each experiment
......@@ -697,7 +737,7 @@ table_error = table['error'].copy()
table_error.head()
# In[17]:
# In[19]:
# table with explored states for each experiment using ltsmin
......@@ -715,7 +755,7 @@ table_explored_states = table_explored_states.reset_index()
table_explored_states.head()
# In[18]:
# In[20]:
# calculate the stats of the number of explored states
......@@ -727,27 +767,35 @@ table_explored_states_stats.head()
# # Examples
# In[19]:
# In[21]:
create_figure_explored_states(table_explored_states, 'spool1')
create_figure_explored_states(table_explored_states, 'spool5')
# In[20]:
# In[22]:
create_figure(df, "spool1")
create_figure(df, "spool5")
# In[ ]:
# In[23]:
log_figure = create_log_figure(table_time, table_error, "spool5", "pmc-sog_otf_couv99-default_2_16", "pnml2lts-mc_dfs_1_16", True, open_logs_callback)
log_figure
create_log_figure(table_time, table_error, "spool1", "pmc-sog_otf_couv99-default_2_16", "pnml2lts-mc_dfs_1_16", True, open_logs_callback)
# In[24]:
table = get_table(table_error, "spool5", ["pmc-sog_otf_couv99-default_2_16", "pnml2lts-mc_dfs_1_16"])
table
# # Generate Figures
# In[ ]:
# In[25]:
# models
......@@ -757,7 +805,7 @@ models = df.model.unique()
tools = df.tool.unique()
# In[ ]:
# In[26]:
# create all the figures of explored states
......@@ -776,7 +824,7 @@ for model in models:
print("Error: {} was not plotted".format(model))
# In[ ]:
# In[27]:
# create all the figures formula vs time
......@@ -795,7 +843,7 @@ for model in models:
print("Error: {} was not plotted".format(model))
# In[ ]:
# In[28]:
# create all the log figures
......@@ -813,10 +861,15 @@ for plot, filter_method in plots.items():
try:
show_strategy = plot == 'compare_couvreur_algorithm'
fig = create_log_figure(table_time, table_error, model, axe[0], axe[1], show_strategy)
table = get_table(table_error, model, axe)
# save figures in html and pdf
figure_name = os.path.join(folder, '{}-{}-VS-{}-log'.format(model, axe[0], axe[1]))
fig.write_html(figure_name + '.html', include_plotlyjs='cdn', post_script=OPEN_LOGS_CALLBACK_JS)
with open(figure_name + '.html', 'w') as f:
f.write(fig.to_html(full_html=False, include_plotlyjs='cdn', post_script=OPEN_LOGS_CALLBACK_JS))
f.write(table.to_html(full_html=False, include_plotlyjs='cdn', post_script=OPEN_LOGS_CALLBACK_JS))
fig.write_image(figure_name + '.pdf')
except KeyError:
print("Error: {} was not plotted".format(model))
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment