Commit 0ce14882 authored by Jaime Arias's avatar Jaime Arias
Browse files

update plot scripts

parent ccbae29f
This source diff could not be displayed because it is too large. You can view the blob instead.
......@@ -59,7 +59,7 @@ LAYOUT_FIGURES = dict(
# # Auxiliary Functions
# In[4]:
# In[3]:
def create_folder(path):
......@@ -79,7 +79,7 @@ def create_folder(path):
os.makedirs(path)
# In[5]:
# In[4]:
def create_figure(df, model):
......@@ -118,7 +118,7 @@ def create_figure(df, model):
return figure
# In[6]:
# In[5]:
def get_axis_title(experiment, show_strategy=True):
......@@ -170,38 +170,75 @@ def get_axis_title(experiment, show_strategy=True):
return title
# In[7]:
# In[6]:
def get_info(df, model, experiment):
"""Get some statistics from a table for a specific model and experiment"""
info = df.loc[model][experiment]
def filter_errors(df_exp1, df_exp2):
"""Returns dataframes of specific experiments without errors"""
time_limit = info[info=="TIME LIMIT"].count()
error = info[(info!="TIME LIMIT") & (info!="OK")].count()
ok = info[info=="OK"].count()
nan_1 = df_exp1[df_exp1.isna().any(axis=1)].index
nan_2 = df_exp2[df_exp2.isna().any(axis=1)].index
df_exp1 = df_exp1.drop(nan_2)
df_exp2 = df_exp2.drop(nan_1)
df_exp1 = df_exp1.dropna()
df_exp2 = df_exp2.dropna()
if ((time_limit + error + ok) != info.count()): raise Exception("Some information is missing in the table")
return df_exp1, df_exp2
def get_info(info):
"""Get some statistics from a table for a specific model and experiment"""
time_limit = len(info[(info.error == "TIME LIMIT") | (info.error == "TIMEOUT")])
error = len(info[(info.error != "TIME LIMIT") & (info.error != "TIMEOUT") & (info.error != "OK") & (info.error != 'MDD') & (info.error != 'TABLE FULL')])
memory = len(info[(info.error == 'MDD') | (info.error == 'TABLE FULL')])
ok = len(info[info.error == "OK"])
if ((time_limit + error + ok + memory) != len(info)): raise Exception("Some information is missing in the table")
return {
"time limit": time_limit,
"error": error,
"memory": memory,
"OK": ok
}
def get_table(df, model, experiments):
def get_best_times(table_time, table_error, model, exp1, exp2):
exp1 = pd.DataFrame({"times_exp1": table_time.loc[model][exp1], "errors_exp1": table_error.loc[model][exp1]})
exp2 = pd.DataFrame({"times_exp2": table_time.loc[model][exp2], "errors_exp2": table_error.loc[model][exp2]})
exp1, exp2 = filter_errors(exp1, exp2)
df_ = pd.concat([exp1, exp2], axis=1, sort=False)
df_ = df_[df_["times_exp1"] != df_["times_exp2"]]
df_['best'] = np.where((df_["times_exp1"] < df_["times_exp2"]), "exp1", "exp2")
count = df_.groupby(["best"]).size()
return count.get("exp1",0), count.get("exp2",0)
def get_table(df_time, df_errors, model, exp1, exp2):
"""Creates a table with some statistics from a dataframe for a model and experiments"""
rows=[["<b>Experiment</b>", "<b>Time Limit</b>", "<b>Error</b>", "<b>OK</b>"]]
rows=[["<b>Experiment</b>", "<b>Time Limit</b>", "<b>Memory</b>", "<b>Unknown Error</b>", "<b>OK</b>", "<b>Faster</b>"]]
df_exp1 = pd.DataFrame({"error": df_errors.loc[model][exp1]})
df_exp2 = pd.DataFrame({"error": df_errors.loc[model][exp2]})
df_exp1, df_exp2 = filter_errors(df_exp1, df_exp2)
info1 = get_info(df_exp1)
info2 = get_info(df_exp2)
for experiment in experiments:
info = get_info(df, model, experiment)
rows.append([experiment, info["time limit"], info["error"], info["OK"]])
best1, best2 = get_best_times(df_time, df_errors, model, exp1, exp2)
for (experiment, info, best) in [(exp1, info1, best1), (exp2, info2, best2)]:
rows.append([experiment, info["time limit"], info["memory"], info["error"], info["OK"], best])
return ff.create_table(rows)
# In[8]:
# In[7]:
import webbrowser
......@@ -316,7 +353,7 @@ myPlot.on('plotly_click', function(data){
"""
# In[9]:
# In[8]:
def create_figure_explored_states(table_explored_states, model):
......@@ -388,7 +425,7 @@ def create_figure_explored_states(table_explored_states, model):
return fig
# In[10]:
# In[9]:
def create_log_figure(table, table_errors, model, tool_x, tool_y, show_strategy=True, callback=None):
......@@ -440,6 +477,8 @@ def create_log_figure(table, table_errors, model, tool_x, tool_y, show_strategy=
full_table_y = pd.concat([table_model[tool_y],table_model['property'], table_errors_model[tool_y]], axis=1)
full_table_y.columns = ['time', 'property', 'error']
full_table_x, full_table_y = filter_errors(full_table_x, full_table_y)
traces = [
{"property": 'T', "color":"green"},
......@@ -509,7 +548,7 @@ def create_log_figure(table, table_errors, model, tool_x, tool_y, show_strategy=
print(e)
# In[11]:
# In[10]:
# Experiment filters
......@@ -632,7 +671,7 @@ plots = {
# # Load Data
# In[12]:
# In[11]:
# Root folder
......@@ -649,7 +688,7 @@ OUTPUT_FOLDER = os.path.join(PROJECT_FOLDER,"results", "figures")
create_folder(OUTPUT_FOLDER)
# In[13]:
# In[12]:
# read data
......@@ -659,12 +698,40 @@ df = pd.read_csv(csv_file)
df['tool'] = df[['tool', 'strategy', 'num_nodes', 'num_threads']].astype(str).apply('_'.join, axis=1)
df = df.drop(columns=['strategy', 'num_nodes', 'num_threads'])
df.head()
# filtering philo20 experiments
df = df[df.model != "philo20"]
df
# In[13]:
# Found errors
errors_found = df.error.unique()
errors_found
# In[14]:
# filtering errors
df = df[(df.error != "SEGMENTATION FAULT") & (df.error != "ABORTED") & (df.error != "TERMINATE") & (df.error != "MDD")]
df = df.reset_index(drop=True)
df
# In[15]:
# Found errors
errors_found = df.error.unique()
errors_found
# In[16]:
# ground truth for properties
frames = []
......@@ -681,21 +748,22 @@ p_df = p_df.reindex(columns=["model", "formula", "property"])
p_df = p_df[p_df['model'].isin(df.model.unique())]
p_df['property'] = p_df['property'].replace([True, False], ["T", "F"])
p_df = p_df.set_index(["model", "formula"])
p_df.sort_index(inplace=True)
p_df
# In[15]:
# In[17]:
# table with times, verification output and error for each experiment
table = df.set_index(['model', 'formula', 'tool'], drop=True).unstack('tool')
table.head()
table
# # Preprocessing of data
# In[16]:
# In[18]:
# table with times for each experiment
......@@ -710,10 +778,10 @@ table_time.replace(0.0, ZERO, inplace=True)
# add verification output to the table
table_time = pd.concat([table_time, p_df], axis=1)
table_time.head()
table_time
# In[17]:
# In[19]:
# table with verification output for each experiment
......@@ -728,7 +796,7 @@ table_property = pd.concat([table_property, p_df], axis=1)
table_property.head()
# In[18]:
# In[20]:
# table with error for each experiment
......@@ -737,7 +805,7 @@ table_error = table['error'].copy()
table_error.head()
# In[19]:
# In[21]:
# table with explored states for each experiment using ltsmin
......@@ -755,7 +823,7 @@ table_explored_states = table_explored_states.reset_index()
table_explored_states.head()
# In[20]:
# In[22]:
# calculate the stats of the number of explored states
......@@ -767,35 +835,35 @@ table_explored_states_stats.head()
# # Examples
# In[21]:
# In[23]:
create_figure_explored_states(table_explored_states, 'spool5')
# In[22]:
# In[24]:
create_figure(df, "spool5")
create_figure(df, "robot50")
# In[23]:
# In[25]:
log_figure = create_log_figure(table_time, table_error, "spool5", "pmc-sog_otf_couv99-default_2_16", "pnml2lts-mc_dfs_1_16", True, open_logs_callback)
log_figure = create_log_figure(table_time, table_error, "robot50", "pmc-sog_otf_couv99-default_2_16", "pnml2lts-mc_dfs_1_16", True, open_logs_callback)
log_figure
# In[24]:
# In[26]:
table = get_table(table_error, "spool5", ["pmc-sog_otf_couv99-default_2_16", "pnml2lts-mc_dfs_1_16"])
table = get_table(table_time, table_error, "robot50", "pmc-sog_otf_couv99-default_2_16", "pnml2lts-mc_dfs_1_16")
table
# # Generate Figures
# In[25]:
# In[27]:
# models
......@@ -805,7 +873,7 @@ models = df.model.unique()
tools = df.tool.unique()
# In[26]:
# In[29]:
# create all the figures of explored states
......@@ -824,7 +892,7 @@ for model in models:
print("Error: {} was not plotted".format(model))
# In[27]:
# In[30]:
# create all the figures formula vs time
......@@ -843,7 +911,7 @@ for model in models:
print("Error: {} was not plotted".format(model))
# In[28]:
# In[31]:
# create all the log figures
......@@ -861,7 +929,7 @@ for plot, filter_method in plots.items():
try:
show_strategy = plot == 'compare_couvreur_algorithm'
fig = create_log_figure(table_time, table_error, model, axe[0], axe[1], show_strategy)
table = get_table(table_error, model, axe)
table = get_table(table_time, table_error, model, axe[0], axe[1])
# save figures in html and pdf
figure_name = os.path.join(folder, '{}-{}-VS-{}-log'.format(model, axe[0], axe[1]))
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment