Commit 0ce14882 authored by Jaime Arias's avatar Jaime Arias
Browse files

update plot scripts

parent ccbae29f
This diff is collapsed.
......@@ -59,7 +59,7 @@ LAYOUT_FIGURES = dict(
# # Auxiliary Functions
# In[4]:
# In[3]:
def create_folder(path):
......@@ -79,7 +79,7 @@ def create_folder(path):
os.makedirs(path)
# In[5]:
# In[4]:
def create_figure(df, model):
......@@ -118,7 +118,7 @@ def create_figure(df, model):
return figure
# In[6]:
# In[5]:
def get_axis_title(experiment, show_strategy=True):
......@@ -170,38 +170,75 @@ def get_axis_title(experiment, show_strategy=True):
return title
# In[7]:
# In[6]:
def get_info(df, model, experiment):
"""Get some statistics from a table for a specific model and experiment"""
info = df.loc[model][experiment]
def filter_errors(df_exp1, df_exp2):
"""Returns dataframes of specific experiments without errors"""
time_limit = info[info=="TIME LIMIT"].count()
error = info[(info!="TIME LIMIT") & (info!="OK")].count()
ok = info[info=="OK"].count()
nan_1 = df_exp1[df_exp1.isna().any(axis=1)].index
nan_2 = df_exp2[df_exp2.isna().any(axis=1)].index
df_exp1 = df_exp1.drop(nan_2)
df_exp2 = df_exp2.drop(nan_1)
df_exp1 = df_exp1.dropna()
df_exp2 = df_exp2.dropna()
if ((time_limit + error + ok) != info.count()): raise Exception("Some information is missing in the table")
return df_exp1, df_exp2
def get_info(info):
"""Get some statistics from a table for a specific model and experiment"""
time_limit = len(info[(info.error == "TIME LIMIT") | (info.error == "TIMEOUT")])
error = len(info[(info.error != "TIME LIMIT") & (info.error != "TIMEOUT") & (info.error != "OK") & (info.error != 'MDD') & (info.error != 'TABLE FULL')])
memory = len(info[(info.error == 'MDD') | (info.error == 'TABLE FULL')])
ok = len(info[info.error == "OK"])
if ((time_limit + error + ok + memory) != len(info)): raise Exception("Some information is missing in the table")
return {
"time limit": time_limit,
"error": error,
"memory": memory,
"OK": ok
}
def get_table(df, model, experiments):
def get_best_times(table_time, table_error, model, exp1, exp2):
exp1 = pd.DataFrame({"times_exp1": table_time.loc[model][exp1], "errors_exp1": table_error.loc[model][exp1]})
exp2 = pd.DataFrame({"times_exp2": table_time.loc[model][exp2], "errors_exp2": table_error.loc[model][exp2]})
exp1, exp2 = filter_errors(exp1, exp2)
df_ = pd.concat([exp1, exp2], axis=1, sort=False)
df_ = df_[df_["times_exp1"] != df_["times_exp2"]]
df_['best'] = np.where((df_["times_exp1"] < df_["times_exp2"]), "exp1", "exp2")
count = df_.groupby(["best"]).size()
return count.get("exp1",0), count.get("exp2",0)
def get_table(df_time, df_errors, model, exp1, exp2):
"""Creates a table with some statistics from a dataframe for a model and experiments"""
rows=[["<b>Experiment</b>", "<b>Time Limit</b>", "<b>Error</b>", "<b>OK</b>"]]
rows=[["<b>Experiment</b>", "<b>Time Limit</b>", "<b>Memory</b>", "<b>Unknown Error</b>", "<b>OK</b>", "<b>Faster</b>"]]
df_exp1 = pd.DataFrame({"error": df_errors.loc[model][exp1]})
df_exp2 = pd.DataFrame({"error": df_errors.loc[model][exp2]})
df_exp1, df_exp2 = filter_errors(df_exp1, df_exp2)
info1 = get_info(df_exp1)
info2 = get_info(df_exp2)
for experiment in experiments:
info = get_info(df, model, experiment)
rows.append([experiment, info["time limit"], info["error"], info["OK"]])
best1, best2 = get_best_times(df_time, df_errors, model, exp1, exp2)
for (experiment, info, best) in [(exp1, info1, best1), (exp2, info2, best2)]:
rows.append([experiment, info["time limit"], info["memory"], info["error"], info["OK"], best])
return ff.create_table(rows)
# In[8]:
# In[7]:
import webbrowser
......@@ -316,7 +353,7 @@ myPlot.on('plotly_click', function(data){
"""
# In[9]:
# In[8]:
def create_figure_explored_states(table_explored_states, model):
......@@ -388,7 +425,7 @@ def create_figure_explored_states(table_explored_states, model):
return fig
# In[10]:
# In[9]:
def create_log_figure(table, table_errors, model, tool_x, tool_y, show_strategy=True, callback=None):
......@@ -440,6 +477,8 @@ def create_log_figure(table, table_errors, model, tool_x, tool_y, show_strategy=
full_table_y = pd.concat([table_model[tool_y],table_model['property'], table_errors_model[tool_y]], axis=1)
full_table_y.columns = ['time', 'property', 'error']
full_table_x, full_table_y = filter_errors(full_table_x, full_table_y)
traces = [
{"property": 'T', "color":"green"},
......@@ -509,7 +548,7 @@ def create_log_figure(table, table_errors, model, tool_x, tool_y, show_strategy=
print(e)
# In[11]:
# In[10]:
# Experiment filters
......@@ -632,7 +671,7 @@ plots = {
# # Load Data
# In[12]:
# In[11]:
# Root folder
......@@ -649,7 +688,7 @@ OUTPUT_FOLDER = os.path.join(PROJECT_FOLDER,"results", "figures")
create_folder(OUTPUT_FOLDER)
# In[13]:
# In[12]:
# read data
......@@ -659,12 +698,40 @@ df = pd.read_csv(csv_file)
df['tool'] = df[['tool', 'strategy', 'num_nodes', 'num_threads']].astype(str).apply('_'.join, axis=1)
df = df.drop(columns=['strategy', 'num_nodes', 'num_threads'])
df.head()
# filtering philo20 experiments
df = df[df.model != "philo20"]
df
# In[13]:
# Found errors
errors_found = df.error.unique()
errors_found
# In[14]:
# filtering errors
df = df[(df.error != "SEGMENTATION FAULT") & (df.error != "ABORTED") & (df.error != "TERMINATE") & (df.error != "MDD")]
df = df.reset_index(drop=True)
df
# In[15]:
# Found errors
errors_found = df.error.unique()
errors_found
# In[16]:
# ground truth for properties
frames = []
......@@ -681,21 +748,22 @@ p_df = p_df.reindex(columns=["model", "formula", "property"])
p_df = p_df[p_df['model'].isin(df.model.unique())]
p_df['property'] = p_df['property'].replace([True, False], ["T", "F"])
p_df = p_df.set_index(["model", "formula"])
p_df.sort_index(inplace=True)
p_df
# In[15]:
# In[17]:
# table with times, verification output and error for each experiment
table = df.set_index(['model', 'formula', 'tool'], drop=True).unstack('tool')
table.head()
table
# # Preprocessing of data
# In[16]:
# In[18]:
# table with times for each experiment
......@@ -710,10 +778,10 @@ table_time.replace(0.0, ZERO, inplace=True)
# add verification output to the table
table_time = pd.concat([table_time, p_df], axis=1)
table_time.head()
table_time
# In[17]:
# In[19]:
# table with verification output for each experiment
......@@ -728,7 +796,7 @@ table_property = pd.concat([table_property, p_df], axis=1)
table_property.head()
# In[18]:
# In[20]:
# table with error for each experiment
......@@ -737,7 +805,7 @@ table_error = table['error'].copy()
table_error.head()
# In[19]:
# In[21]:
# table with explored states for each experiment using ltsmin
......@@ -755,7 +823,7 @@ table_explored_states = table_explored_states.reset_index()
table_explored_states.head()
# In[20]:
# In[22]:
# calculate the stats of the number of explored states
......@@ -767,35 +835,35 @@ table_explored_states_stats.head()
# # Examples
# In[21]:
# In[23]:
create_figure_explored_states(table_explored_states, 'spool5')
# In[22]:
# In[24]:
create_figure(df, "spool5")
create_figure(df, "robot50")
# In[23]:
# In[25]:
log_figure = create_log_figure(table_time, table_error, "spool5", "pmc-sog_otf_couv99-default_2_16", "pnml2lts-mc_dfs_1_16", True, open_logs_callback)
log_figure = create_log_figure(table_time, table_error, "robot50", "pmc-sog_otf_couv99-default_2_16", "pnml2lts-mc_dfs_1_16", True, open_logs_callback)
log_figure
# In[24]:
# In[26]:
table = get_table(table_error, "spool5", ["pmc-sog_otf_couv99-default_2_16", "pnml2lts-mc_dfs_1_16"])
table = get_table(table_time, table_error, "robot50", "pmc-sog_otf_couv99-default_2_16", "pnml2lts-mc_dfs_1_16")
table
# # Generate Figures
# In[25]:
# In[27]:
# models
......@@ -805,7 +873,7 @@ models = df.model.unique()
tools = df.tool.unique()
# In[26]:
# In[29]:
# create all the figures of explored states
......@@ -824,7 +892,7 @@ for model in models:
print("Error: {} was not plotted".format(model))
# In[27]:
# In[30]:
# create all the figures formula vs time
......@@ -843,7 +911,7 @@ for model in models:
print("Error: {} was not plotted".format(model))
# In[28]:
# In[31]:
# create all the log figures
......@@ -861,7 +929,7 @@ for plot, filter_method in plots.items():
try:
show_strategy = plot == 'compare_couvreur_algorithm'
fig = create_log_figure(table_time, table_error, model, axe[0], axe[1], show_strategy)
table = get_table(table_error, model, axe)
table = get_table(table_time, table_error, model, axe[0], axe[1])
# save figures in html and pdf
figure_name = os.path.join(folder, '{}-{}-VS-{}-log'.format(model, axe[0], axe[1]))
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment