Commit 3da23b16 authored by Jaime Arias's avatar Jaime Arias
Browse files

fix notebook

parent 121075cd
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
import os import os
import pandas as pd import pandas as pd
import numpy as np import numpy as np
import plotly.io as pio import plotly.io as pio
import plotly.express as px import plotly.express as px
import plotly.graph_objs as go import plotly.graph_objs as go
from itertools import combinations from itertools import combinations
from plotly.subplots import make_subplots from plotly.subplots import make_subplots
# render figures in notebook # render figures in notebook
pio.renderers.default = "notebook_connected" pio.renderers.default = "notebook_connected"
# templates figures # templates figures
px.defaults.template = "simple_white" px.defaults.template = "simple_white"
pio.templates.default = "simple_white" pio.templates.default = "simple_white"
# layout for all figures # layout for all figures
LAYOUT_FIGURES = dict( LAYOUT_FIGURES = dict(
autosize=False, autosize=False,
width = 500, width = 500,
height = 500, height = 500,
xaxis = dict( xaxis = dict(
constrain="domain", constrain="domain",
mirror=True, mirror=True,
showexponent="all", showexponent="all",
exponentformat="power" exponentformat="power"
), ),
yaxis = dict( yaxis = dict(
scaleanchor = "x", scaleanchor = "x",
scaleratio = 1, scaleratio = 1,
mirror=True, mirror=True,
showexponent="all", showexponent="all",
exponentformat="power" exponentformat="power"
), ),
title = dict( title = dict(
y = 0.9, y = 0.9,
x = 0.5, x = 0.5,
xanchor = 'center', xanchor = 'center',
yanchor = 'top' yanchor = 'top'
) )
) )
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
# Auxiliary Functions # Auxiliary Functions
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
def create_folder(path): def create_folder(path):
"""Creates a folder if it does not exist """Creates a folder if it does not exist
Parameters Parameters
---------- ----------
path : str path : str
Path of the new folder Path of the new folder
Examples Examples
-------- --------
>>> create_folder('./results') >>> create_folder('./results')
""" """
if not os.path.exists(path): if not os.path.exists(path):
os.makedirs(path) os.makedirs(path)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
def create_figure(df, model): def create_figure(df, model):
"""Creates a scatter figure showing the time taken by each tool to verify each property of a model """Creates a scatter figure showing the time taken by each tool to verify each property of a model
Parameters Parameters
---------- ----------
df : pandas.Dataframe df : pandas.Dataframe
Dataframe containing the results of the experiments Dataframe containing the results of the experiments
model : string model : string
model to be plotted model to be plotted
Returns Returns
------- -------
plotly.graph_objects.Figure plotly.graph_objects.Figure
Scatter figure Scatter figure
Examples Examples
-------- --------
>>> import os >>> import os
>>> import pandas as pd >>> import pandas as pd
>>> csv_file = os.path.join("results", "output.csv") >>> csv_file = os.path.join("results", "output.csv")
>>> df = pd.read_csv(csv_file) >>> df = pd.read_csv(csv_file)
>>> fig = create_figure(df, 'philo10') >>> fig = create_figure(df, 'philo10')
""" """
model_df = df[df.model == model] model_df = df[df.model == model]
figure = px.scatter(model_df, figure = px.scatter(model_df,
x="formula", y="time", x="formula", y="time",
title=model, title=model,
color="tool", color="tool",
symbol_sequence=['x']) symbol_sequence=['x'])
figure.update_layout(yaxis_title="time (s)", title=LAYOUT_FIGURES['title']) figure.update_layout(yaxis_title="time (s)", title=LAYOUT_FIGURES['title'])
return figure return figure
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
def get_axis_title(experiment, show_strategy=True): def get_axis_title(experiment, show_strategy=True):
"""Get the axis title of a figure depending on the experiment being plotted """Get the axis title of a figure depending on the experiment being plotted
Parameters Parameters
---------- ----------
experiment : str experiment : str
String with the experiment information String with the experiment information
show_strategy : bool, optional show_strategy : bool, optional
Flag to show the information related to the strategy used by the tool Flag to show the information related to the strategy used by the tool
Returns Returns
------- -------
str str
axis title axis title
Examples Examples
-------- --------
>>> get_axis_title('pmc-sog_otfL_couv99-default_1_1', True) >>> get_axis_title('pmc-sog_otfL_couv99-default_1_1', True)
pmc-sog (Lace, strategy: couv99-default, # cores: 1) pmc-sog (Lace, strategy: couv99-default, # cores: 1)
""" """
information = experiment.split('_') information = experiment.split('_')
tool_name = information[0] tool_name = information[0]
info = [] info = []
library_dic = { library_dic = {
'otfL': 'Lace', 'otfL': 'Lace',
'otfP': 'Pthreads', 'otfP': 'Pthreads',
'otfC': 'Cthreads', 'otfC': 'Cthreads',
'otf': 'Hybrid' 'otf': 'Hybrid'
} }
if (len(information) == 5): if (len(information) == 5):
info.append(library_dic[information[1]]) info.append(library_dic[information[1]])
if (show_strategy): if (show_strategy):
info.append('strategy: {}'.format(information[-3])) info.append('strategy: {}'.format(information[-3]))
nb_nodes = int(information[-2]) nb_nodes = int(information[-2])
if (nb_nodes > 1): if (nb_nodes > 1):
info.append('# nodes: {}'.format(nb_nodes)) info.append('# nodes: {}'.format(nb_nodes))
info.append('# cores: {}'.format(information[-1])) info.append('# cores: {}'.format(information[-1]))
title = '{} ({})'.format(tool_name, ', '.join(info)) title = '{} ({})'.format(tool_name, ', '.join(info))
return title return title
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
def create_log_figure(table, table_errors, model, tool_x, tool_y, show_strategy=True, callback=None): def create_log_figure(table, table_errors, model, tool_x, tool_y, show_strategy=True, callback=None):
"""Creates a Scatter figure in logarithmic scale comparing the performance of two tools """Creates a Scatter figure in logarithmic scale comparing the performance of two tools
Parameters Parameters
---------- ----------
table : pandas.Dataframe table : pandas.Dataframe
Dataframe with the times of each experiment Dataframe with the times of each experiment
table_errors : pandas.Dataframe table_errors : pandas.Dataframe
Dataframe with the errors of each experiment Dataframe with the errors of each experiment
model : string model : string
Model to be analyzed Model to be analyzed
tool_x : string tool_x : string
Tool to be compared and plotted on the x-axis Tool to be compared and plotted on the x-axis
tool_y : string tool_y : string
Tool to be compared and plotted on the y-axis Tool to be compared and plotted on the y-axis
show_strategy : bool show_strategy : bool
Flag to show the stretagy used by the tools Flag to show the stretagy used by the tools
callback : function callback : function
Function to be called when clicking on a point Function to be called when clicking on a point
Returns Returns
------- -------
plotly.graph_objects.Figure plotly.graph_objects.Figure
Scatter figure Scatter figure
Examples Examples
-------- --------
>>> import os >>> import os
>>> import pandas as pd >>> import pandas as pd
>>> csv_file = os.path.join("results", "output.csv") >>> csv_file = os.path.join("results", "output.csv")
>>> df = pd.read_csv(csv_file) >>> df = pd.read_csv(csv_file)
>>> table = df.set_index(['model', 'formula', 'tool'], drop=True).unstack('tool') >>> table = df.set_index(['model', 'formula', 'tool'], drop=True).unstack('tool')
>>> fig = create_log_figure(table['time'], table['error'], 'philo10', 'pmc-sog_otfL_couv99-default_1_8', 'pmc-sog_otfP_couv99-default_1_8') >>> fig = create_log_figure(table['time'], table['error'], 'philo10', 'pmc-sog_otfL_couv99-default_1_8', 'pmc-sog_otfP_couv99-default_1_8')
""" """
try: try:
min_values = table.loc[model].min() min_values = table.loc[model].min()
max_values = table.loc[model].max() max_values = table.loc[model].max()
min_value = min(min_values[tool_x], min_values[tool_y])/2. min_value = min(min_values[tool_x], min_values[tool_y])/2.
min_value_log = np.log10(min_value) min_value_log = np.log10(min_value)
max_value = max(max_values[tool_x], max_values[tool_y]) max_value = max(max_values[tool_x], max_values[tool_y])
max_value_log = np.log10(max_value) max_value_log = np.log10(max_value)
table_model = table.loc[model] table_model = table.loc[model]
table_errors_model = table_error.loc[model] table_errors_model = table_error.loc[model]
full_table_x = pd.concat([table_model[tool_x],table_model['property'], table_errors_model[tool_x]], axis=1) full_table_x = pd.concat([table_model[tool_x],table_model['property'], table_errors_model[tool_x]], axis=1)
full_table_x.columns = ['time', 'property', 'error'] full_table_x.columns = ['time', 'property', 'error']
full_table_y = pd.concat([table_model[tool_y],table_model['property'], table_errors_model[tool_y]], axis=1) full_table_y = pd.concat([table_model[tool_y],table_model['property'], table_errors_model[tool_y]], axis=1)
full_table_y.columns = ['time', 'property', 'error'] full_table_y.columns = ['time', 'property', 'error']
traces = [ traces = [
{"property": 'T', "color":"green"}, {"property": 'T', "color":"green"},
{"property": 'F', "color":"red"}, {"property": 'F', "color":"red"},
{"property": 'U', "color":"black"} {"property": 'U', "color":"black"}
] ]
figures = [] figures = []
for t in traces: for t in traces:
# filter by verification output # filter by verification output
table_x = full_table_x[full_table_x.property == t['property']] table_x = full_table_x[full_table_x.property == t['property']]
table_y = full_table_y[full_table_y.property == t['property']] table_y = full_table_y[full_table_y.property == t['property']]
# custom data # custom data
custom_data = list(zip(table_x.index, table_x.error,table_y.error)) custom_data = list(zip(table_x.index, table_x.error,table_y.error))
# tools # tools
metainfo = { metainfo = {
'model': model, 'model': model,
'tools': {'x': tool_x, 'y': tool_y}, 'tools': {'x': tool_x, 'y': tool_y},
'folder': os.path.join(os.path.abspath(os.pardir), "results") 'folder': os.path.join(os.path.abspath(os.pardir), "results")
} }
figures.append(go.Scatter(x=table_x.time, figures.append(go.Scatter(x=table_x.time,
y=table_y.time, y=table_y.time,
name=t['property'], name=t['property'],
mode='markers', mode='markers',
marker_symbol='circle-open', marker_symbol='circle-open',
marker_color=t['color'], marker_color=t['color'],
meta = metainfo, meta = metainfo,
customdata=custom_data, customdata=custom_data,
hovertemplate = hovertemplate =
'<b>Formula # %{customdata[0]}</b><br>' + '<b>Formula # %{customdata[0]}</b><br>' +
'<br><b>Times:</b><br>' + '<br><b>Times:</b><br>' +
'<b>x:</b> %{x} s' + '<b>x:</b> %{x} s' +
'<br><b>y:</b> %{y} s<br>' + '<br><b>y:</b> %{y} s<br>' +
'<br><b>Errors:</b><br>' + '<br><b>Errors:</b><br>' +
'<b>x:</b> %{customdata[1]}<br>' + '<b>x:</b> %{customdata[1]}<br>' +
'<b>y:</b> %{customdata[2]}', '<b>y:</b> %{customdata[2]}',
)) ))
# Line # Line
figures.append(go.Scatter(x=[min_value, max_value], figures.append(go.Scatter(x=[min_value, max_value],
y=[min_value, max_value], y=[min_value, max_value],
mode='lines', showlegend=False, mode='lines', showlegend=False,
line=dict(color='black', width=1))) line=dict(color='black', width=1)))
# Create figure # Create figure
figure = go.FigureWidget(figures) figure = go.FigureWidget(figures)
figure.update_layout(LAYOUT_FIGURES, figure.update_layout(LAYOUT_FIGURES,
title_text=model, title_text=model,
hoverlabel=dict(bgcolor="white", align='auto'), hoverlabel=dict(bgcolor="white", align='auto'),
legend_title_text='property', legend_title_text='property',
xaxis=dict(type='log', autorange=False, range=[min_value_log, max_value_log]), xaxis=dict(type='log', autorange=False, range=[min_value_log, max_value_log]),
yaxis=dict(type='log', autorange=False, range=[min_value_log, max_value_log]), yaxis=dict(type='log', autorange=False, range=[min_value_log, max_value_log]),
xaxis_title=get_axis_title(tool_x, show_strategy), xaxis_title=get_axis_title(tool_x, show_strategy),
yaxis_title=get_axis_title(tool_y, show_strategy)) yaxis_title=get_axis_title(tool_y, show_strategy))
# Add event # Add event
if callback is not None: if callback is not None:
for i in range(len(figure.data)): for i in range(len(figure.data)):
figure.data[i].on_click(callback) figure.data[i].on_click(callback)
return figure return figure
except Exception as e: except Exception as e:
print("Error when ploting model: {} - tool_x: {} - tool_y: {}".format(model, tool_x, tool_y)) print("Error when ploting model: {} - tool_x: {} - tool_y: {}".format(model, tool_x, tool_y))
print(e) print(e)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
import webbrowser import webbrowser
def get_filename(base_path, tool, model, model_instance, formula): def get_filename(base_path, tool, model, model_instance, formula):
"""Returns the absolute path of the experiment log """Returns the absolute path of the experiment log
Parameters Parameters
---------- ----------
base_path : string base_path : string
Path of the folder where logs are saved Path of the folder where logs are saved
tool : string tool : string
Tool name Tool name
model : string model : string
Model name Model name
model_instance : string model_instance : string
Name of the model instance Name of the model instance
formula : string formula : string
Identifier of the formula Identifier of the formula
Returns Returns
------- -------
string string
Absolute path of the log file Absolute path of the log file
""" """
information = tool.split('_') information = tool.split('_')
tool_name = information[0] tool_name = information[0]
tool_configuration = '_'.join(information[:-2]) tool_configuration = '_'.join(information[:-2])
nb_nodes = information[-2] nb_nodes = information[-2]
nb_cores = information[-1] nb_cores = information[-1]
experiment_folder = os.path.join(base_path, tool_name, tool_configuration, model, model_instance) experiment_folder = os.path.join(base_path, tool_name, tool_configuration, model, model_instance)
filename = f'{tool_name}_{model_instance}-n{nb_nodes}-th{nb_cores}-f{formula}' filename = f'{tool_name}_{model_instance}-n{nb_nodes}-th{nb_cores}-f{formula}'
absolute_path = os.path.join(experiment_folder, filename) absolute_path = os.path.join(experiment_folder, filename)
return absolute_path return absolute_path
def open_logs_callback(trace, points, selector): def open_logs_callback(trace, points, selector):
"""Callback that open the log files when clicking on a point of the figure """Callback that open the log files when clicking on a point of the figure
Parameters Parameters
---------- ----------
trace : plotly.graph_objects.Figure trace : plotly.graph_objects.Figure
the figure to attach the callback the figure to attach the callback
points : plotly.callbacks.Points points : plotly.callbacks.Points
points of the figure selected points of the figure selected
selector: plotly.callbacks.InputDeviceState selector: plotly.callbacks.InputDeviceState
Device information Device information
""" """
inds = points.point_inds inds = points.point_inds
if (inds): if (inds):
index = inds[0] index = inds[0]
formula, error_x, error_y = trace['customdata'][index] formula, error_x, error_y = trace['customdata'][index]
model_instance = trace['meta']['model'] model_instance = trace['meta']['model']
model = ''.join(c for c in model_instance if not c.isdigit()) model = ''.join(c for c in model_instance if not c.isdigit())
tools = trace['meta']['tools']