update plot scripts

scripts/plot-results.py

@@ -59,7 +59,7 @@ LAYOUT_FIGURES = dict(
 
 # # Auxiliary Functions
 
-# In[4]:
+# In[3]:
 
 
 def create_folder(path):
@@ -79,7 +79,7 @@ def create_folder(path):
         os.makedirs(path)
 
 
-# In[5]:
+# In[4]:
 
 
 def create_figure(df, model):
@@ -118,7 +118,7 @@ def create_figure(df, model):
     return figure
 
 
-# In[6]:
+# In[5]:
 
 
 def get_axis_title(experiment, show_strategy=True):
@@ -170,38 +170,75 @@ def get_axis_title(experiment, show_strategy=True):
     return title
 
 
-# In[7]:
+# In[6]:
 
 
-def get_info(df, model, experiment):
-    """Get some statistics from a table for a specific model and experiment"""
-    info = df.loc[model][experiment]
+def filter_errors(df_exp1, df_exp2):
+    """Returns dataframes of specific experiments without errors"""
     
-    time_limit = info[info=="TIME LIMIT"].count()
-    error = info[(info!="TIME LIMIT") & (info!="OK")].count()
-    ok = info[info=="OK"].count()
+    nan_1 = df_exp1[df_exp1.isna().any(axis=1)].index
+    nan_2 = df_exp2[df_exp2.isna().any(axis=1)].index
+
+    df_exp1 = df_exp1.drop(nan_2)
+    df_exp2 = df_exp2.drop(nan_1)
+
+    df_exp1 = df_exp1.dropna()
+    df_exp2 = df_exp2.dropna()
     
-    if ((time_limit + error + ok) != info.count()): raise Exception("Some information is missing in the table")
+    return df_exp1, df_exp2
+
+
+def get_info(info):
+    """Get some statistics from a table for a specific model and experiment"""    
+    time_limit = len(info[(info.error == "TIME LIMIT") | (info.error == "TIMEOUT")])
+    error = len(info[(info.error != "TIME LIMIT") & (info.error != "TIMEOUT") & (info.error != "OK") & (info.error != 'MDD') & (info.error != 'TABLE FULL')])
+    memory = len(info[(info.error == 'MDD') | (info.error == 'TABLE FULL')])
+    ok = len(info[info.error == "OK"])
+    
+    if ((time_limit + error + ok + memory) != len(info)): raise Exception("Some information is missing in the table")
     
     return {
         "time limit": time_limit,
         "error": error,
+        "memory": memory,
         "OK": ok
     }
 
 
-def get_table(df, model, experiments):
+def get_best_times(table_time, table_error, model, exp1, exp2):
+    exp1 = pd.DataFrame({"times_exp1": table_time.loc[model][exp1], "errors_exp1": table_error.loc[model][exp1]})
+    exp2 = pd.DataFrame({"times_exp2": table_time.loc[model][exp2], "errors_exp2": table_error.loc[model][exp2]})
+
+    exp1, exp2 = filter_errors(exp1, exp2)
+    df_ = pd.concat([exp1, exp2], axis=1, sort=False)
+
+    df_ = df_[df_["times_exp1"] != df_["times_exp2"]]
+    df_['best'] = np.where((df_["times_exp1"] < df_["times_exp2"]), "exp1", "exp2")
+    count = df_.groupby(["best"]).size()
+
+    return count.get("exp1",0), count.get("exp2",0)
+
+
+def get_table(df_time, df_errors, model, exp1, exp2):
     """Creates a table with some statistics from a dataframe for a model and experiments"""
-    rows=[["<b>Experiment</b>", "<b>Time Limit</b>", "<b>Error</b>", "<b>OK</b>"]]
+    rows=[["<b>Experiment</b>", "<b>Time Limit</b>", "<b>Memory</b>", "<b>Unknown Error</b>", "<b>OK</b>", "<b>Faster</b>"]]
+    
+    df_exp1 = pd.DataFrame({"error": df_errors.loc[model][exp1]})
+    df_exp2 = pd.DataFrame({"error": df_errors.loc[model][exp2]})
+    df_exp1, df_exp2 = filter_errors(df_exp1, df_exp2)
+    
+    info1 = get_info(df_exp1)
+    info2 = get_info(df_exp2)
     
-    for experiment in experiments:
-        info = get_info(df, model, experiment)
-        rows.append([experiment, info["time limit"], info["error"], info["OK"]])
+    best1, best2 = get_best_times(df_time, df_errors, model, exp1, exp2)
+    
+    for (experiment, info, best) in [(exp1, info1, best1), (exp2, info2, best2)]:
+        rows.append([experiment, info["time limit"], info["memory"], info["error"], info["OK"], best])
         
     return ff.create_table(rows)
 
 
-# In[8]:
+# In[7]:
 
 
 import webbrowser
@@ -316,7 +353,7 @@ myPlot.on('plotly_click', function(data){
 """
 
 
-# In[9]:
+# In[8]:
 
 
 def create_figure_explored_states(table_explored_states, model):
@@ -388,7 +425,7 @@ def create_figure_explored_states(table_explored_states, model):
     return fig
 
 
-# In[10]:
+# In[9]:
 
 
 def create_log_figure(table, table_errors, model, tool_x, tool_y, show_strategy=True, callback=None):
@@ -440,6 +477,8 @@ def create_log_figure(table, table_errors, model, tool_x, tool_y, show_strategy=
 
         full_table_y = pd.concat([table_model[tool_y],table_model['property'], table_errors_model[tool_y]], axis=1)
         full_table_y.columns = ['time', 'property', 'error']
+        
+        full_table_x, full_table_y = filter_errors(full_table_x, full_table_y)
 
         traces = [
             {"property": 'T', "color":"green"},
@@ -509,7 +548,7 @@ def create_log_figure(table, table_errors, model, tool_x, tool_y, show_strategy=
         print(e)
 
 
-# In[11]:
+# In[10]:
 
 
 # Experiment filters
@@ -632,7 +671,7 @@ plots = {
 
 # # Load Data
 
-# In[12]:
+# In[11]:
 
 
 # Root folder
@@ -649,7 +688,7 @@ OUTPUT_FOLDER = os.path.join(PROJECT_FOLDER,"results", "figures")
 create_folder(OUTPUT_FOLDER)
 
 
-# In[13]:
+# In[12]:
 
 
 # read data
@@ -659,12 +698,40 @@ df = pd.read_csv(csv_file)
 df['tool'] = df[['tool', 'strategy', 'num_nodes', 'num_threads']].astype(str).apply('_'.join, axis=1)
 df = df.drop(columns=['strategy', 'num_nodes', 'num_threads'])
 
-df.head()
+# filtering philo20 experiments
+df = df[df.model != "philo20"]
+df
+
+
+# In[13]:
+
+
+# Found errors
+errors_found = df.error.unique()
+errors_found
 
 
 # In[14]:
 
 
+# filtering errors
+df = df[(df.error != "SEGMENTATION FAULT") & (df.error != "ABORTED") & (df.error != "TERMINATE") & (df.error != "MDD")]
+
+df = df.reset_index(drop=True)
+df
+
+
+# In[15]:
+
+
+# Found errors
+errors_found = df.error.unique()
+errors_found
+
+
+# In[16]:
+
+
 # ground truth for properties
 frames = []
 
@@ -681,21 +748,22 @@ p_df = p_df.reindex(columns=["model", "formula", "property"])
 p_df = p_df[p_df['model'].isin(df.model.unique())]
 p_df['property'] = p_df['property'].replace([True, False], ["T", "F"])
 p_df = p_df.set_index(["model", "formula"])
+p_df.sort_index(inplace=True)
 
 p_df
 
 
-# In[15]:
+# In[17]:
 
 
 # table with times, verification output and error for each experiment
 table = df.set_index(['model', 'formula', 'tool'], drop=True).unstack('tool')
-table.head()
+table
 
 
 # # Preprocessing of data
 
-# In[16]:
+# In[18]:
 
 
 # table with times for each experiment
@@ -710,10 +778,10 @@ table_time.replace(0.0, ZERO, inplace=True)
 # add verification output to the table
 table_time = pd.concat([table_time, p_df], axis=1)
 
-table_time.head()
+table_time
 
 
-# In[17]:
+# In[19]:
 
 
 # table with verification output for each experiment
@@ -728,7 +796,7 @@ table_property = pd.concat([table_property, p_df], axis=1)
 table_property.head()
 
 
-# In[18]:
+# In[20]:
 
 
 # table with error for each experiment
@@ -737,7 +805,7 @@ table_error = table['error'].copy()
 table_error.head()
 
 
-# In[19]:
+# In[21]:
 
 
 # table with explored states for each experiment using ltsmin
@@ -755,7 +823,7 @@ table_explored_states = table_explored_states.reset_index()
 table_explored_states.head()
 
 
-# In[20]:
+# In[22]:
 
 
 # calculate the stats of the number of explored states
@@ -767,35 +835,35 @@ table_explored_states_stats.head()
 
 # # Examples
 
-# In[21]:
+# In[23]:
 
 
 create_figure_explored_states(table_explored_states, 'spool5')
 
 
-# In[22]:
+# In[24]:
 
 
-create_figure(df, "spool5")
+create_figure(df, "robot50")
 
 
-# In[23]:
+# In[25]:
 
 
-log_figure = create_log_figure(table_time, table_error, "spool5", "pmc-sog_otf_couv99-default_2_16", "pnml2lts-mc_dfs_1_16", True, open_logs_callback)
+log_figure = create_log_figure(table_time, table_error, "robot50", "pmc-sog_otf_couv99-default_2_16", "pnml2lts-mc_dfs_1_16", True, open_logs_callback)
 log_figure
 
 
-# In[24]:
+# In[26]:
 
 
-table = get_table(table_error, "spool5", ["pmc-sog_otf_couv99-default_2_16", "pnml2lts-mc_dfs_1_16"])
+table = get_table(table_time, table_error, "robot50", "pmc-sog_otf_couv99-default_2_16", "pnml2lts-mc_dfs_1_16")
 table
 
 
 # # Generate Figures
 
-# In[25]:
+# In[27]:
 
 
 # models
@@ -805,7 +873,7 @@ models = df.model.unique()
 tools = df.tool.unique()
 
 
-# In[26]:
+# In[29]:
 
 
 # create all the figures of explored states
@@ -824,7 +892,7 @@ for model in models:
         print("Error: {} was not plotted".format(model))
 
 
-# In[27]:
+# In[30]:
 
 
 # create all the figures formula vs time
@@ -843,7 +911,7 @@ for model in models:
         print("Error: {} was not plotted".format(model))
 
 
-# In[28]:
+# In[31]:
 
 
 # create all the log figures
@@ -861,7 +929,7 @@ for plot, filter_method in plots.items():
             try:
                 show_strategy = plot == 'compare_couvreur_algorithm'
                 fig = create_log_figure(table_time, table_error, model, axe[0], axe[1], show_strategy)
-                table = get_table(table_error, model, axe)
+                table = get_table(table_time, table_error, model, axe[0], axe[1])
                 
                 # save figures in html and pdf
                 figure_name = os.path.join(folder, '{}-{}-VS-{}-log'.format(model, axe[0], axe[1]))