plot-results.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "ZERO = 10e-5\n",
    "TIMEOUT = 10 * 60 # 10 minutes = 6000 seconds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import glob\n",
    "import re\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import plotly.io as pio\n",
    "import plotly.express as px\n",
    "import plotly.graph_objs as go\n",
    "from itertools import combinations \n",
    "import plotly.figure_factory as ff\n",
    "from plotly.subplots import make_subplots\n",
    "\n",
    "# render figures in notebook\n",
    "pio.renderers.default = \"notebook_connected\"\n",
    "\n",
    "# templates figures\n",
    "px.defaults.template = \"simple_white\"\n",
    "pio.templates.default = \"simple_white\"\n",
    "\n",
    "# layout for all figures\n",
    "LAYOUT_FIGURES = dict(\n",
    "    autosize=False,\n",
    "    width = 500,\n",
    "    height = 500,\n",
    "    xaxis = dict(\n",
    "      constrain=\"domain\",\n",
    "      mirror=True,\n",
    "      showexponent=\"all\",\n",
    "      exponentformat=\"power\"\n",
    "    ),\n",
    "    yaxis = dict(\n",
    "      scaleanchor = \"x\",\n",
    "      scaleratio = 1,\n",
    "      mirror=True,\n",
    "      showexponent=\"all\",\n",
    "      exponentformat=\"power\"\n",
    "    ),\n",
    "    title = dict(\n",
    "      y = 0.9,\n",
    "      x = 0.5,\n",
    "      xanchor = 'center',\n",
    "      yanchor = 'top'\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Auxiliary Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_folder(path):\n",
    "    \"\"\"Creates a folder if it does not exist\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    path : str\n",
    "        Path of the new folder\n",
    "    \n",
    "    Examples\n",
    "    --------\n",
    "    \n",
    "    >>> create_folder('./results')\n",
    "    \"\"\"\n",
    "    if not os.path.exists(path):\n",
    "        os.makedirs(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_figure(df, model):\n",
    "    \"\"\"Creates a scatter figure showing the time taken by each tool to verify each property of a model\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    df : pandas.Dataframe\n",
    "        Dataframe containing the results of the experiments\n",
    "    model : string\n",
    "        model to be plotted\n",
    "    \n",
    "    Returns\n",
    "    -------\n",
    "    plotly.graph_objects.Figure\n",
    "        Scatter figure\n",
    "    \n",
    "    Examples\n",
    "    --------\n",
    "    \n",
    "    >>> import os\n",
    "    >>> import pandas as pd\n",
    "    >>> csv_file = os.path.join(\"results\", \"output.csv\")\n",
    "    >>> df = pd.read_csv(csv_file)\n",
    "    >>> fig = create_figure(df, 'philo10')\n",
    "    \"\"\"\n",
    "    model_df = df[df.model == model]\n",
    "\n",
    "    figure = px.scatter(model_df, \n",
    "                        x=\"formula\", y=\"time\",\n",
    "                        title=model, \n",
    "                        color=\"tool\", \n",
    "                        symbol_sequence=['x'])\n",
    "\n",
    "    figure.update_layout(yaxis_title=\"time (s)\", title=LAYOUT_FIGURES['title'])\n",
    "    return figure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_axis_title(experiment, show_strategy=True):\n",
    "    \"\"\"Get the axis title of a figure depending on the experiment being plotted\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    experiment : str\n",
    "        String with the experiment information\n",
    "    show_strategy : bool, optional\n",
    "        Flag to show the information related to the strategy used by the tool\n",
    "    \n",
    "    Returns\n",
    "    -------\n",
    "    str\n",
    "        axis title\n",
    "        \n",
    "    Examples\n",
    "    --------\n",
    "    \n",
    "    >>> get_axis_title('pmc-sog_otfL_couv99-default_1_1', True)\n",
    "    pmc-sog (Lace, strategy: couv99-default, # cores: 1)\n",
    "    \"\"\"\n",
    "    information = experiment.split('_')\n",
    "    tool_name = information[0]\n",
    "    \n",
    "    info = []\n",
    "    library_dic = {\n",
    "        'otfL': 'Lace',\n",
    "        'otfP': 'Pthreads',\n",
    "        'otfC': 'Cthreads',\n",
    "        'otfPR': 'otfPR',\n",
    "        'otf': 'Hybrid'\n",
    "    }\n",
    "    \n",
    "    #if (len(information) == 5):\n",
    "    #    info.append(\"mode \" + library_dic[information[1]])\n",
    "\n",
    "    #if (show_strategy):\n",
    "    #    info.append('strategy: {}'.format(information[-3]))\n",
    "\n",
    "    nb_nodes = int(information[-2])\n",
    "    if (nb_nodes):\n",
    "        info.append('# nodes: {}'.format(nb_nodes))\n",
    "\n",
    "    info.append('# cores: {}'.format(information[-1]))\n",
    "\n",
    "    title = 'Verification time (seconds)<br>{} ({})'.format(tool_name, ', '.join(info))\n",
    "    \n",
    "    return title"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def filter_errors(df_exp1, df_exp2):\n",
    "    \"\"\"Returns dataframes of specific experiments without errors\"\"\"\n",
    "    \n",
    "    nan_1 = df_exp1[df_exp1.isna().any(axis=1)].index\n",
    "    nan_2 = df_exp2[df_exp2.isna().any(axis=1)].index\n",
    "\n",
    "    df_exp1 = df_exp1.drop(nan_2)\n",
    "    df_exp2 = df_exp2.drop(nan_1)\n",
    "\n",
    "    df_exp1 = df_exp1.dropna()\n",
    "    df_exp2 = df_exp2.dropna()\n",
    "    \n",
    "    return df_exp1, df_exp2\n",
    "\n",
    "\n",
    "def get_info(info):\n",
    "    \"\"\"Get some statistics from a table for a specific model and experiment\"\"\"    \n",
    "    time_limit = len(info[(info.error == \"TIME LIMIT\") | (info.error == \"TIMEOUT\")])\n",
    "    error = len(info[(info.error != \"TIME LIMIT\") & (info.error != \"TIMEOUT\") & (info.error != \"OK\") & (info.error != 'MDD') & (info.error != 'TABLE FULL')])\n",
    "    memory = len(info[(info.error == 'MDD') | (info.error == 'TABLE FULL')])\n",
    "    ok = len(info[info.error == \"OK\"])\n",
    "    \n",
    "    if ((time_limit + error + ok + memory) != len(info)): raise Exception(\"Some information is missing in the table\")\n",
    "    \n",
    "    return {\n",
    "        \"time limit\": time_limit,\n",
    "        \"error\": error,\n",
    "        \"memory\": memory,\n",
    "        \"OK\": ok\n",
    "    }\n",
    "\n",
    "\n",
    "def get_best_times(table_time, table_error, model, exp1, exp2):\n",
    "    exp1 = pd.DataFrame({\"times_exp1\": table_time.loc[model][exp1], \"errors_exp1\": table_error.loc[model][exp1]})\n",
    "    exp2 = pd.DataFrame({\"times_exp2\": table_time.loc[model][exp2], \"errors_exp2\": table_error.loc[model][exp2]})\n",
    "\n",
    "    exp1, exp2 = filter_errors(exp1, exp2)\n",
    "    df_ = pd.concat([exp1, exp2], axis=1, sort=False)\n",
    "\n",
    "    df_ = df_[df_[\"times_exp1\"] != df_[\"times_exp2\"]]\n",
    "    df_['best'] = np.where((df_[\"times_exp1\"] < df_[\"times_exp2\"]), \"exp1\", \"exp2\")\n",
    "    count = df_.groupby([\"best\"]).size()\n",
    "\n",
    "    return count.get(\"exp1\",0), count.get(\"exp2\",0)\n",
    "\n",
    "\n",
    "def get_table(df_time, df_errors, model, exp1, exp2):\n",
    "    \"\"\"Creates a table with some statistics from a dataframe for a model and experiments\"\"\"\n",
    "    rows=[[\"<b>Experiment</b>\", \"<b>Time Limit</b>\", \"<b>Memory</b>\", \"<b>Unknown Error</b>\", \"<b>OK</b>\", \"<b>Faster</b>\"]]\n",
    "    \n",
    "    df_exp1 = pd.DataFrame({\"error\": df_errors.loc[model][exp1]})\n",
    "    df_exp2 = pd.DataFrame({\"error\": df_errors.loc[model][exp2]})\n",
    "    df_exp1, df_exp2 = filter_errors(df_exp1, df_exp2)\n",
    "    \n",
    "    info1 = get_info(df_exp1)\n",
    "    info2 = get_info(df_exp2)\n",
    "    \n",
    "    best1, best2 = get_best_times(df_time, df_errors, model, exp1, exp2)\n",
    "    \n",
    "    for (experiment, info, best) in [(exp1, info1, best1), (exp2, info2, best2)]:\n",
    "        rows.append([experiment, info[\"time limit\"], info[\"memory\"], info[\"error\"], info[\"OK\"], best])\n",
    "        \n",
    "    return ff.create_table(rows)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import webbrowser\n",
    "\n",
    "def get_filename(base_path, tool, model, model_instance, formula):\n",
    "    \"\"\"Returns the absolute path of the experiment log\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    base_path : string\n",
    "        Path of the folder where logs are saved\n",
    "    tool : string\n",
    "        Tool name\n",
    "    model : string\n",
    "        Model name\n",
    "    model_instance : string\n",
    "        Name of the model instance\n",
    "    formula : string\n",
    "        Identifier of the formula\n",
    "    \n",
    "    Returns\n",
    "    -------\n",
    "    string\n",
    "        Absolute path of the log file\n",
    "    \n",
    "    \"\"\"\n",
    "    information = tool.split('_')\n",
    "    \n",
    "    tool_name = information[0]\n",
    "    tool_configuration = '_'.join(information[:-2])\n",
    "    nb_nodes = information[-2]\n",
    "    nb_cores = information[-1]\n",
    "    \n",
    "    experiment_folder = os.path.join(base_path, tool_name, tool_configuration, model, model_instance)\n",
    "    filename = f'{tool_name}_{model_instance}-n{nb_nodes}-th{nb_cores}-f{formula}'\n",
    "    absolute_path = os.path.join(experiment_folder, filename)\n",
    "    \n",
    "    return absolute_path\n",
    "\n",
    "def open_logs_callback(trace, points, selector):\n",
    "    \"\"\"Callback that open the log files when clicking on a point of the figure\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    trace : plotly.graph_objects.Figure\n",
    "        the figure to attach the callback\n",
    "    points : plotly.callbacks.Points \n",
    "        points of the figure selected\n",
    "    selector: plotly.callbacks.InputDeviceState \n",
    "        Device information \n",
    "    \"\"\"\n",
    "    inds = points.point_inds\n",
    "    if (inds):\n",
    "        index = inds[0]\n",
    "\n",
    "        formula, error_x, error_y = trace['customdata'][index]\n",
    "        model_instance = trace['meta']['model']\n",
    "        model = ''.join(c for c in model_instance if not c.isdigit())\n",
    "        tools = trace['meta']['tools']\n",
    "        logs_folder = trace['meta']['folder']\n",
    "\n",
    "        filename_x = get_filename(logs_folder, tools['x'], model, model_instance, formula)\n",
    "        filename_y = get_filename(logs_folder, tools['y'], model, model_instance, formula)\n",
    "\n",
    "        for f in [filename_x, filename_y]:\n",
    "            webbrowser.open(f'file://{f}.err')\n",
    "            webbrowser.open(f'file://{f}.out')\n",
    "\n",
    "OPEN_LOGS_CALLBACK_JS = \"\"\"\n",
    "function get_filename (base_path, tool, model_instance, formula) {\n",
    "  const information = tool.split('_');\n",
    "  const size = information.length;\n",
    "\n",
    "  const tool_name = information[0];\n",
    "  const tool_configuration = information.slice(0, size - 2).join('_');\n",
    "  const nb_nodes = information[size - 2];\n",
    "  const nb_cores = information[size - 1];\n",
    "\n",
    "  const model = model_instance.replace(/[0-9]/g, '');\n",
    "\n",
    "  const experiment_folder = `${base_path}/${tool_name}/${tool_configuration}/${model}/${model_instance}`;\n",
    "  const filename = `${tool_name}_${model_instance}-n${nb_nodes}-th${nb_cores}-f${formula}`;\n",
    "\n",
    "  return `${experiment_folder}/${filename}`;\n",
    "}\n",
    "\n",
    "const plots = document.getElementsByClassName(\"plotly-graph-div js-plotly-plot\");\n",
    "const myPlot = plots[0];\n",
    "\n",
    "myPlot.on('plotly_click', function(data){\n",
    "    const points = data.points;\n",
    "    if (points.length != 1) {return ;}\n",
    "    \n",
    "    const myPoint = points[0];\n",
    "    const formula = myPoint.customdata[0];\n",
    "    const meta = myPoint.data.meta;\n",
    "    \n",
    "    const href = window.location.href.split('/');\n",
    "    const base_path = href.splice(0,href.length-4).join('/');\n",
    "    \n",
    "    const filename_x = get_filename(base_path, meta.tools.x, meta.model, formula);\n",
    "    const filename_y = get_filename(base_path, meta.tools.y, meta.model, formula);\n",
    "    \n",
    "    console.log('x: ' + filename_x);\n",
    "    window.open(`${filename_x}.err`);\n",
    "    window.open(`${filename_x}.out`);\n",
    "    \n",
    "    console.log('y: ' + filename_y);\n",
    "    window.open(`${filename_y}.err`);\n",
    "    window.open(`${filename_y}.out`);\n",
    "});\n",
    "\"\"\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_figure_explored_states(table_explored_states, model):\n",
    "    \"\"\"Creates figure showing the number of explorated states during the verification \n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    table_explored_states : pandas.Dataframe\n",
    "        Dataframe with the explorated states of each experiment\n",
    "    model : string\n",
    "        Model to be analyzed\n",
    "        \n",
    "    Returns\n",
    "    -------\n",
    "    plotly.graph_objects.Figure\n",
    "        Scatter figure\n",
    "    \"\"\"\n",
    "    colors={'T': 'green', 'F': 'red'}\n",
    "    float_formatter = \"{:.2E}\".format\n",
    "\n",
    "    table_model = table_explored_states[table_explored_states.property != 'U']\n",
    "    table_model = table_model[table_model.model == model]\n",
    "\n",
    "    table_stats = table_model.groupby(['property']).agg(['mean']) \n",
    "\n",
    "    fig = go.Figure()\n",
    "    max_x = 0\n",
    "    for p in table_stats.index:\n",
    "        data = table_model[table_model.property==p]\n",
    "        stats = table_stats.loc[p]\n",
    "        \n",
    "        x_axis = np.arange(1, data['formula'].count()+1, 1)\n",
    "        max_x = max(max_x, x_axis[-1]+1)\n",
    "        mean = stats['explored_states','mean']\n",
    "        \n",
    "        figure = px.scatter(data, \n",
    "                            x=x_axis, \n",
    "                            y=\"explored_states\",\n",
    "                            title=model, \n",
    "                            color='property',\n",
    "                            color_discrete_map=colors,\n",
    "                            symbol_sequence=[\"circle\"])\n",
    "\n",
    "        line = go.Scatter(x=[x_axis[0], x_axis[-1]], \n",
    "                          y=[mean, mean],\n",
    "                          mode='lines', showlegend=False,                          \n",
    "                          line=dict(color=colors[p], width=1.5))\n",
    "\n",
    "        fig.add_trace(figure['data'][0])\n",
    "        fig.add_trace(line)\n",
    "        \n",
    "        fig.add_annotation(x=1, \n",
    "                           y=mean,\n",
    "                           font=dict(color=colors[p]),\n",
    "                           text=f\"mean = {float_formatter(mean)}\")\n",
    "\n",
    "    fig.update_layout(title_text=model, title=LAYOUT_FIGURES['title'], \n",
    "                      width = 500, height = 500, margin=dict(r=110))\n",
    "    \n",
    "    fig.update_xaxes(title=\"formula\", range=[0, max_x])\n",
    "    fig.update_yaxes(title=\"# explored states\")\n",
    "    \n",
    "    fig.update_annotations(dict(\n",
    "        showarrow=False,\n",
    "        xanchor=\"left\",\n",
    "        yanchor=\"middle\",\n",
    "        xref='paper'))\n",
    "    \n",
    "    return fig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_log_figure(table, table_errors, model, tool_x, tool_y, show_strategy=True, callback=None):\n",
    "    \"\"\"Creates a Scatter figure in logarithmic scale comparing the performance of two tools\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    table : pandas.Dataframe\n",
    "        Dataframe with the times of each experiment\n",
    "    table_errors : pandas.Dataframe\n",
    "        Dataframe with the errors of each experiment\n",
    "    model : string\n",
    "        Model to be analyzed\n",
    "    tool_x : string\n",
    "        Tool to be compared and plotted on the x-axis\n",
    "    tool_y : string\n",
    "        Tool to be compared and plotted on the y-axis\n",
    "    show_strategy : bool\n",
    "        Flag to show the stretagy used by the tools\n",
    "    callback : function\n",
    "        Function to be called when clicking on a point\n",
    "        \n",
    "    Returns\n",
    "    -------\n",
    "    plotly.graph_objects.Figure\n",
    "        Scatter figure\n",
    "        \n",
    "    Examples\n",
    "    --------\n",
    "    >>> import os\n",
    "    >>> import pandas as pd\n",
    "    >>> csv_file = os.path.join(\"results\", \"output.csv\")\n",
    "    >>> df = pd.read_csv(csv_file)\n",
    "    >>> table = df.set_index(['model', 'formula', 'tool'], drop=True).unstack('tool')\n",
    "    >>> fig = create_log_figure(table['time'], table['error'], 'philo10', 'pmc-sog_otfL_couv99-default_1_8', 'pmc-sog_otfP_couv99-default_1_8')\n",
    "    \"\"\"\n",
    "    try:\n",
    "        min_value = ZERO\n",
    "        max_value = TIMEOUT\n",
    "        \n",
    "        min_value_log = np.log10(min_value)\n",
    "        max_value_log = np.log10(max_value)\n",
    "\n",
    "        table_model = table.loc[model]\n",
    "        table_errors_model = table_error.loc[model]\n",
    "        \n",
    "        full_table_x = pd.concat([table_model[tool_x],table_model['property'], table_errors_model[tool_x]], axis=1)\n",
    "        full_table_x.columns = ['time', 'property', 'error']\n",
    "\n",
    "        full_table_y = pd.concat([table_model[tool_y],table_model['property'], table_errors_model[tool_y]], axis=1)\n",
    "        full_table_y.columns = ['time', 'property', 'error']\n",
    "        \n",
    "        full_table_x, full_table_y = filter_errors(full_table_x, full_table_y)\n",
    "\n",
    "        traces = [\n",
    "            {\"property\": 'T', \"color\":\"green\", \"label\": \"True\"},\n",
    "            {\"property\": 'F', \"color\":\"red\", \"label\": \"False\"},\n",
    "            {\"property\": 'U', \"color\":\"black\", \"label\": \"Unknown\"}\n",
    "        ]\n",
    "\n",
    "        figures = []\n",
    "        for t in traces:\n",
    "            # filter by verification output\n",
    "            table_x = full_table_x[full_table_x.property == t['property']]\n",
    "            table_y = full_table_y[full_table_y.property == t['property']]\n",
    "\n",
    "            # custom data\n",
    "            custom_data = list(zip(table_x.index, table_x.error,table_y.error))\n",
    "            \n",
    "            # tools\n",
    "            metainfo = {\n",
    "                'model': model, \n",
    "                'tools': {'x': tool_x, 'y': tool_y},\n",
    "                'folder': os.path.join(os.path.abspath(os.pardir), \"results\")\n",
    "            }\n",
    "\n",
    "            figures.append(go.Scatter(x=table_x.time,\n",
    "                                      y=table_y.time,\n",
    "                                      name=t['label'],\n",
    "                                      mode='markers',\n",
    "                                      marker_symbol='circle-open',\n",
    "                                      marker_color=t['color'],\n",
    "                                      meta = metainfo,\n",
    "                                      customdata=custom_data,\n",
    "                                      hovertemplate =\n",
    "                                        '<b>Formula # %{customdata[0]}</b><br>' +\n",
    "                                        '<br><b>Times:</b><br>' +\n",
    "                                        '<b>x:</b> %{x} s' +\n",
    "                                        '<br><b>y:</b> %{y} s<br>' +\n",
    "                                        '<br><b>Errors:</b><br>' +\n",
    "                                        '<b>x:</b> %{customdata[1]}<br>' +\n",
    "                                        '<b>y:</b> %{customdata[2]}',\n",
    "                                        ))    \n",
    "\n",
    "        # Line\n",
    "        figures.append(go.Scatter(x=[min_value, max_value], \n",
    "                                    y=[min_value, max_value],\n",
    "                                    mode='lines', showlegend=False,\n",
    "                                    line=dict(color='black', width=1)))\n",
    "\n",
    "        # Create figure\n",
    "        figure = go.FigureWidget(figures)\n",
    "        figure.update_layout(LAYOUT_FIGURES,\n",
    "                             title_text=model,\n",
    "                             hoverlabel=dict(bgcolor=\"white\", align='auto'),\n",
    "                             legend_title_text='property',\n",
    "                             xaxis=dict(type='log', autorange=False, range=[min_value_log, max_value_log]),\n",
    "                             yaxis=dict(type='log', autorange=False, range=[min_value_log, max_value_log]),\n",
    "                             xaxis_title=get_axis_title(tool_x, show_strategy),\n",
    "                             yaxis_title=get_axis_title(tool_y, show_strategy))\n",
    "\n",
    "        # Add event\n",
    "        if callback is not None:\n",
    "            for i in range(len(figure.data)):\n",
    "                figure.data[i].on_click(callback)\n",
    "    \n",
    "        return figure\n",
    "    except Exception as e:\n",
    "        print(\"Error when ploting model: {} - tool_x: {} - tool_y: {}\".format(model, tool_x, tool_y))\n",
    "        print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Experiment filters\n",
    "\n",
    "def versus_dfs(experiments):\n",
    "    \"\"\"Selects only experiments using DFS strategy\"\"\"\n",
    "    exp1, exp2 = experiments\n",
    "    strategy_exp1= exp1.split('_')[1]\n",
    "    strategy_exp2= exp2.split('_')[1]\n",
    "    \n",
    "    return strategy_exp1 == 'dfs' or strategy_exp2 == 'dfs'\n",
    "\n",
    "def versus_sequential(experiments):\n",
    "    \"\"\"Selects only experiments run sequentially \"\"\"\n",
    "    exp1, exp2 = experiments\n",
    "    nodes_exp1, threads_exp1 = exp1.split('_')[-2:]\n",
    "    nodes_exp2, threads_exp2 = exp2.split('_')[-2:]\n",
    "\n",
    "    return (nodes_exp1 == '1' and nodes_exp2 == '1') and \\\n",
    "            (threads_exp1 == '1' or threads_exp2 == '1')\n",
    "\n",
    "def same_tool(experiments, tool):\n",
    "    \"\"\"Selects only experiments comparing the same tool\"\"\"\n",
    "    exp1, exp2 = experiments\n",
    "    tool_exp1= exp1.split('_')[0]\n",
    "    tool_exp2= exp2.split('_')[0]\n",
    "    return tool_exp1.startswith(tool) and tool_exp2.startswith(tool)\n",
    "\n",
    "def same_number_threads(experiments):\n",
    "    \"\"\"Selects only experiments comparing the same number of processes and cores\"\"\"\n",
    "    exp1, exp2 = experiments\n",
    "    nodes_exp1, threads_exp1 = exp1.split('_')[-2:]\n",
    "    nodes_exp2, threads_exp2 = exp2.split('_')[-2:]\n",
    "    return (nodes_exp1 == nodes_exp2) and (threads_exp1 == threads_exp2) \n",
    "\n",
    "def same_thread_library(experiments):\n",
    "    \"\"\"Selects only experiments comparing the same parallelization library\"\"\"\n",
    "    exp1, exp2 = experiments\n",
    "    library_exp1 = exp1.split('_')[1]\n",
    "    library_exp2 = exp2.split('_')[1]\n",
    "    return library_exp1 == library_exp2\n",
    "\n",
    "def same_strategy(experiments):\n",
    "    \"\"\"Selects only experiments comparing the same strategy\"\"\"\n",
    "    exp1, exp2 = experiments\n",
    "    strategy_exp1 = exp1.split('_')[2]\n",
    "    strategy_exp2 = exp2.split('_')[2]\n",
    "    return strategy_exp1 == strategy_exp2\n",
    "\n",
    "def only_couvreur_strategy(experiments):\n",
    "    \"\"\"Selects only experiments comparing couvreur emptiness check algorithm\"\"\"\n",
    "    exp1, exp2 = experiments\n",
    "    strategy_exp1 = exp1.split('_')[2]\n",
    "    strategy_exp2 = exp2.split('_')[2]\n",
    "    return strategy_exp1.startswith('couv99') and strategy_exp2.startswith('couv99')\n",
    "\n",
    "def compare_threads_library(experiments):\n",
    "    \"\"\"Compares parallization libraries used in pmc-sog. \n",
    "    \n",
    "    It selects experiments where the tool is only pmc-sog and the strategy, number of threads, \n",
    "    number of processus are the same.\n",
    "    \"\"\"\n",
    "    return same_tool(experiments, 'pmc-sog') and \\\n",
    "            same_strategy(experiments) and \\\n",
    "            same_number_threads(experiments) and \\\n",
    "            not same_thread_library(experiments)\n",
    "\n",
    "def compare_couvreur_strategies(experiments):\n",
    "    \"\"\"Compares couvreurs strategies used in pmc-sog. \n",
    "    \n",
    "    It selects experiments where the tool is only pmc-sog, the strategy is couvreur, and \n",
    "    the parallelization library, number of threads, number of processus are the same.\n",
    "    \"\"\"\n",
    "    return only_couvreur_strategy(experiments) and \\\n",
    "            same_thread_library(experiments) and \\\n",
    "            same_number_threads(experiments)\n",
    "\n",
    "def same_distributed_number_threads(experiments):\n",
    "    \"\"\"Selects only experiments where the multiplication of theirs nodes with cores are the same.\"\"\"\n",
    "    exp1, exp2 = experiments\n",
    "    nodes_exp1, threads_exp1 = exp1.split('_')[-2:]\n",
    "    nodes_exp2, threads_exp2 = exp2.split('_')[-2:]\n",
    "    return (int(nodes_exp1) * int(threads_exp1)) == (int(nodes_exp2) * int(threads_exp2))\n",
    "\n",
    "def compare_tools(experiments):\n",
    "    \"\"\"Compares pmc-sog and pnml2lts-mc using the DFS algorithm. \n",
    "    \n",
    "    It selects experiments where the tools are not the same, the exploration algorithm is DFS and \n",
    "    the number of processus and cores are the same.\n",
    "    \"\"\"\n",
    "    return not (same_tool(experiments, 'pmc-sog') or same_tool(experiments,'pnml2lts-mc')) and \\\n",
    "            versus_dfs(experiments)\n",
    "\n",
    "def compare_multithreading(experiments):\n",
    "    \"\"\"Compares the sequential and multi-core version of pmc-sog. \n",
    "    \n",
    "    It selects experiments where the tools is pmc-sog, the parallelization library, the emptiness check \n",
    "    strategy are the same. Here the number of processus and cores are different.\n",
    "    \"\"\"\n",
    "    return same_tool(experiments, 'pmc-sog') and \\\n",
    "            same_thread_library(experiments) and \\\n",
    "            same_strategy(experiments) and \\\n",
    "            versus_sequential(experiments)\n",
    "\n",
    "def against_hybrid(experiments):\n",
    "    \"\"\"Selects only experiments comparing with hybrid mode\"\"\"\n",
    "    exp1, exp2 = experiments\n",
    "    library_exp1 = exp1.split('_')[1]\n",
    "    library_exp2 = exp2.split('_')[1]\n",
    "    return (library_exp1 == 'otf') or (library_exp2 == 'otf')\n",
    "\n",
    "\n",
    "def compare_distributed(experiments):\n",
    "    \"\"\"Compares the hybrid version of pmc-sog\"\"\"\n",
    "    return same_tool(experiments, 'pmc-sog') and \\\n",
    "        same_strategy(experiments) and \\\n",
    "        same_distributed_number_threads(experiments) and \\\n",
    "        against_hybrid(experiments)\n",
    "\n",
    "def compare_others(experiments):\n",
    "    return (not compare_threads_library(experiments)) and \\\n",
    "        (not compare_couvreur_strategies(experiments)) and \\\n",
    "        (not compare_tools(experiments)) and \\\n",
    "        (not compare_multithreading(experiments)) and \\\n",
    "        (not compare_distributed(experiments))\n",
    "\n",
    "# Plots to be created\n",
    "plots = {\n",
    "    'compare_thread_library': compare_threads_library,\n",
    "    'compare_couvreur_algorithm': compare_couvreur_strategies,\n",
    "    'compare_tools': compare_tools,\n",
    "    'compare_multicore': compare_multithreading,\n",
    "    'compare_distributed': compare_distributed,\n",
    "    'others' : compare_others\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Root folder\n",
    "PROJECT_FOLDER = os.path.abspath(os.pardir)\n",
    "\n",
    "# csv file with the output\n",
    "csv_file = os.path.join(PROJECT_FOLDER, \"results\", \"output.csv\")\n",
    "\n",
    "# formulas folder\n",
    "FORMULAS_FOLDER = os.path.join(PROJECT_FOLDER, \"formulas\")\n",
    "\n",
    "# Output folder\n",
    "OUTPUT_FOLDER = os.path.join(PROJECT_FOLDER,\"results\", \"figures\")\n",
    "create_folder(OUTPUT_FOLDER)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>formula</th>\n",
       "      <th>tool</th>\n",
       "      <th>time</th>\n",
       "      <th>property</th>\n",
       "      <th>explored_states</th>\n",
       "      <th>error</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>philo10</td>\n",
       "      <td>1</td>\n",
       "      <td>pmc-sog_otfC_couv99-default_1_16</td>\n",
       "      <td>6.894</td>\n",
       "      <td>F</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>philo10</td>\n",
       "      <td>1</td>\n",
       "      <td>pmc-sog_otfC_couv99-shy_1_16</td>\n",
       "      <td>7.070</td>\n",
       "      <td>F</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>philo10</td>\n",
       "      <td>1</td>\n",
       "      <td>pmc-sog_otfP_couv99-default_1_16</td>\n",
       "      <td>7.622</td>\n",
       "      <td>F</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>philo10</td>\n",
       "      <td>1</td>\n",
       "      <td>pmc-sog_otfP_couv99-shy_1_16</td>\n",
       "      <td>8.650</td>\n",
       "      <td>F</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>philo10</td>\n",
       "      <td>1</td>\n",
       "      <td>pmc-sog_otf_couv99-default_2_8</td>\n",
       "      <td>11.579</td>\n",
       "      <td>F</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27195</th>\n",
       "      <td>tring5</td>\n",
       "      <td>200</td>\n",
       "      <td>pmc-sog_otf_couv99-default_2_16</td>\n",
       "      <td>0.829</td>\n",
       "      <td>F</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27196</th>\n",
       "      <td>tring5</td>\n",
       "      <td>200</td>\n",
       "      <td>pmc-sog_otf_couv99-shy_2_8</td>\n",
       "      <td>0.807</td>\n",
       "      <td>F</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27197</th>\n",
       "      <td>tring5</td>\n",
       "      <td>200</td>\n",
       "      <td>pmc-sog_otf_couv99-shy_2_16</td>\n",
       "      <td>0.813</td>\n",
       "      <td>F</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27198</th>\n",
       "      <td>tring5</td>\n",
       "      <td>200</td>\n",
       "      <td>pnml2lts-mc_dfs_1_16</td>\n",
       "      <td>0.160</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1540.0</td>\n",
       "      <td>OK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27199</th>\n",
       "      <td>tring5</td>\n",
       "      <td>200</td>\n",
       "      <td>pnml2lts-mc_ndfs_1_16</td>\n",
       "      <td>0.270</td>\n",
       "      <td>F</td>\n",
       "      <td>9043.0</td>\n",
       "      <td>OK</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>24400 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         model  formula                              tool    time property  \\\n",
       "0      philo10        1  pmc-sog_otfC_couv99-default_1_16   6.894        F   \n",
       "1      philo10        1      pmc-sog_otfC_couv99-shy_1_16   7.070        F   \n",
       "2      philo10        1  pmc-sog_otfP_couv99-default_1_16   7.622        F   \n",
       "3      philo10        1      pmc-sog_otfP_couv99-shy_1_16   8.650        F   \n",
       "4      philo10        1    pmc-sog_otf_couv99-default_2_8  11.579        F   \n",
       "...        ...      ...                               ...     ...      ...   \n",
       "27195   tring5      200   pmc-sog_otf_couv99-default_2_16   0.829        F   \n",
       "27196   tring5      200        pmc-sog_otf_couv99-shy_2_8   0.807        F   \n",
       "27197   tring5      200       pmc-sog_otf_couv99-shy_2_16   0.813        F   \n",
       "27198   tring5      200              pnml2lts-mc_dfs_1_16   0.160      NaN   \n",
       "27199   tring5      200             pnml2lts-mc_ndfs_1_16   0.270        F   \n",
       "\n",
       "       explored_states error  \n",
       "0                  NaN    OK  \n",
       "1                  NaN    OK  \n",
       "2                  NaN    OK  \n",
       "3                  NaN    OK  \n",
       "4                  NaN    OK  \n",
       "...                ...   ...  \n",
       "27195              NaN    OK  \n",
       "27196              NaN    OK  \n",
       "27197              NaN    OK  \n",
       "27198           1540.0    OK  \n",
       "27199           9043.0    OK  \n",
       "\n",
       "[24400 rows x 7 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# read data\n",
    "df = pd.read_csv(csv_file)\n",
    "\n",
    "# merge the information related to the experiment (# nodes, # threads, strategy) to the tool column\n",
    "df['tool'] = df[['tool', 'strategy', 'num_nodes', 'num_threads']].astype(str).apply('_'.join, axis=1)\n",
    "df = df.drop(columns=['strategy', 'num_nodes', 'num_threads'])\n",
    "\n",
    "# FIX: filtering philo20 experiments because there is a problem with the generated formulas\n",
    "df = df[df.model != \"philo20\"]\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>formula</th>\n",
       "      <th>tool</th>\n",
       "      <th>time</th>\n",
       "      <th>property</th>\n",
       "      <th>explored_states</th>\n",
       "      <th>error</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>philo10</td>\n",
       "      <td>1</td>\n",
       "      <td>pmc-sog_otfC_couv99-default_1_16</td>\n",
       "      <td>6.894</td>\n",
       "      <td>F</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>philo10</td>\n",
       "      <td>1</td>\n",
       "      <td>pmc-sog_otfC_couv99-shy_1_16</td>\n",
       "      <td>7.070</td>\n",
       "      <td>F</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>philo10</td>\n",
       "      <td>1</td>\n",
       "      <td>pmc-sog_otfP_couv99-default_1_16</td>\n",
       "      <td>7.622</td>\n",
       "      <td>F</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>philo10</td>\n",
       "      <td>1</td>\n",
       "      <td>pmc-sog_otfP_couv99-shy_1_16</td>\n",
       "      <td>8.650</td>\n",
       "      <td>F</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>philo10</td>\n",
       "      <td>1</td>\n",
       "      <td>pmc-sog_otf_couv99-default_2_8</td>\n",
       "      <td>11.579</td>\n",
       "      <td>F</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21645</th>\n",
       "      <td>tring5</td>\n",
       "      <td>200</td>\n",
       "      <td>pmc-sog_otf_couv99-default_2_16</td>\n",
       "      <td>0.829</td>\n",
       "      <td>F</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21646</th>\n",
       "      <td>tring5</td>\n",
       "      <td>200</td>\n",
       "      <td>pmc-sog_otf_couv99-shy_2_8</td>\n",
       "      <td>0.807</td>\n",
       "      <td>F</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21647</th>\n",
       "      <td>tring5</td>\n",
       "      <td>200</td>\n",
       "      <td>pmc-sog_otf_couv99-shy_2_16</td>\n",
       "      <td>0.813</td>\n",
       "      <td>F</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21648</th>\n",
       "      <td>tring5</td>\n",
       "      <td>200</td>\n",
       "      <td>pnml2lts-mc_dfs_1_16</td>\n",
       "      <td>0.160</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1540.0</td>\n",
       "      <td>OK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21649</th>\n",
       "      <td>tring5</td>\n",
       "      <td>200</td>\n",
       "      <td>pnml2lts-mc_ndfs_1_16</td>\n",
       "      <td>0.270</td>\n",
       "      <td>F</td>\n",
       "      <td>9043.0</td>\n",
       "      <td>OK</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>21650 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         model  formula                              tool    time property  \\\n",
       "0      philo10        1  pmc-sog_otfC_couv99-default_1_16   6.894        F   \n",
       "1      philo10        1      pmc-sog_otfC_couv99-shy_1_16   7.070        F   \n",
       "2      philo10        1  pmc-sog_otfP_couv99-default_1_16   7.622        F   \n",
       "3      philo10        1      pmc-sog_otfP_couv99-shy_1_16   8.650        F   \n",
       "4      philo10        1    pmc-sog_otf_couv99-default_2_8  11.579        F   \n",
       "...        ...      ...                               ...     ...      ...   \n",
       "21645   tring5      200   pmc-sog_otf_couv99-default_2_16   0.829        F   \n",
       "21646   tring5      200        pmc-sog_otf_couv99-shy_2_8   0.807        F   \n",
       "21647   tring5      200       pmc-sog_otf_couv99-shy_2_16   0.813        F   \n",
       "21648   tring5      200              pnml2lts-mc_dfs_1_16   0.160      NaN   \n",
       "21649   tring5      200             pnml2lts-mc_ndfs_1_16   0.270        F   \n",
       "\n",
       "       explored_states error  \n",
       "0                  NaN    OK  \n",
       "1                  NaN    OK  \n",
       "2                  NaN    OK  \n",
       "3                  NaN    OK  \n",
       "4                  NaN    OK  \n",
       "...                ...   ...  \n",
       "21645              NaN    OK  \n",
       "21646              NaN    OK  \n",
       "21647              NaN    OK  \n",
       "21648           1540.0    OK  \n",
       "21649           9043.0    OK  \n",
       "\n",
       "[21650 rows x 7 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# filtering runtime errors\n",
    "df = df[(df.error != \"SEGMENTATION FAULT\") & \\\n",
    "        (df.error != \"ABORTED\") & \\\n",
    "        (df.error != \"TERMINATE\") & \\\n",
    "        (df.error != \"MDD\")]\n",
    "\n",
    "df = df.reset_index(drop=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>property</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model</th>\n",
       "      <th>formula</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">philo10</th>\n",
       "      <th>1</th>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">tring5</th>\n",
       "      <th>196</th>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>197</th>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>198</th>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199</th>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>200</th>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1800 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                property\n",
       "model   formula         \n",
       "philo10 1              F\n",
       "        2              F\n",
       "        3              F\n",
       "        4              F\n",
       "        5              F\n",
       "...                  ...\n",
       "tring5  196            F\n",
       "        197            F\n",
       "        198            F\n",
       "        199            F\n",
       "        200            F\n",
       "\n",
       "[1800 rows x 1 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# ground truth for properties\n",
    "frames = []\n",
    "\n",
    "formula_results = glob.glob(os.path.join(FORMULAS_FOLDER, \"**/formula_results\"), recursive=True)\n",
    "for f in formula_results:\n",
    "    model, out_file = f.split('/')[-2:]\n",
    "    \n",
    "    tmp_df = pd.read_csv(f, sep=\";\", header=None, names=[\"formula\", \"property\"])\n",
    "    tmp_df[\"model\"] = model\n",
    "    frames.append(tmp_df)\n",
    "    \n",
    "p_df = pd.concat(frames)\n",
    "p_df = p_df.reindex(columns=[\"model\", \"formula\", \"property\"])\n",
    "p_df = p_df[p_df['model'].isin(df.model.unique())]\n",
    "p_df['property'] = p_df['property'].replace([True, False], [\"T\", \"F\"])\n",
    "p_df = p_df.set_index([\"model\", \"formula\"])\n",
    "p_df.sort_index(inplace=True)\n",
    "\n",
    "p_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th colspan=\"10\" halign=\"left\">time</th>\n",
       "      <th>...</th>\n",
       "      <th colspan=\"10\" halign=\"left\">error</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>tool</th>\n",
       "      <th>pmc-sog_otfC_couv99-default_1_1</th>\n",
       "      <th>pmc-sog_otfC_couv99-default_1_16</th>\n",
       "      <th>pmc-sog_otfC_couv99-shy_1_1</th>\n",
       "      <th>pmc-sog_otfC_couv99-shy_1_16</th>\n",
       "      <th>pmc-sog_otfP_couv99-default_1_1</th>\n",
       "      <th>pmc-sog_otfP_couv99-default_1_16</th>\n",
       "      <th>pmc-sog_otfP_couv99-shy_1_1</th>\n",
       "      <th>pmc-sog_otfP_couv99-shy_1_16</th>\n",
       "      <th>pmc-sog_otf_couv99-default_2_16</th>\n",
       "      <th>pmc-sog_otf_couv99-default_2_8</th>\n",
       "      <th>...</th>\n",
       "      <th>pmc-sog_otf_couv99-default_4_16</th>\n",
       "      <th>pmc-sog_otf_couv99-default_4_8</th>\n",
       "      <th>pmc-sog_otf_couv99-shy_2_16</th>\n",
       "      <th>pmc-sog_otf_couv99-shy_2_8</th>\n",
       "      <th>pmc-sog_otf_couv99-shy_4_16</th>\n",
       "      <th>pmc-sog_otf_couv99-shy_4_8</th>\n",
       "      <th>pnml2lts-mc_dfs_1_1</th>\n",
       "      <th>pnml2lts-mc_dfs_1_16</th>\n",
       "      <th>pnml2lts-mc_ndfs_1_1</th>\n",
       "      <th>pnml2lts-mc_ndfs_1_16</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model</th>\n",
       "      <th>formula</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">philo10</th>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>6.894</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7.070</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7.622</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8.650</td>\n",
       "      <td>22.025</td>\n",
       "      <td>11.579</td>\n",
       "      <td>...</td>\n",
       "      <td>OK</td>\n",
       "      <td>OK</td>\n",
       "      <td>OK</td>\n",
       "      <td>OK</td>\n",
       "      <td>OK</td>\n",
       "      <td>OK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>4.048</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.976</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.092</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.480</td>\n",
       "      <td>13.093</td>\n",
       "      <td>10.419</td>\n",
       "      <td>...</td>\n",
       "      <td>OK</td>\n",
       "      <td>OK</td>\n",
       "      <td>OK</td>\n",
       "      <td>OK</td>\n",
       "      <td>OK</td>\n",
       "      <td>OK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>3.291</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.623</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.369</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.915</td>\n",
       "      <td>10.478</td>\n",
       "      <td>6.654</td>\n",
       "      <td>...</td>\n",
       "      <td>OK</td>\n",
       "      <td>OK</td>\n",
       "      <td>OK</td>\n",
       "      <td>OK</td>\n",
       "      <td>OK</td>\n",
       "      <td>OK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NaN</td>\n",
       "      <td>3.375</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.280</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.478</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.285</td>\n",
       "      <td>18.139</td>\n",
       "      <td>8.911</td>\n",
       "      <td>...</td>\n",
       "      <td>OK</td>\n",
       "      <td>OK</td>\n",
       "      <td>OK</td>\n",
       "      <td>OK</td>\n",
       "      <td>OK</td>\n",
       "      <td>OK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>NaN</td>\n",
       "      <td>5.874</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.811</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7.777</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7.032</td>\n",
       "      <td>24.453</td>\n",
       "      <td>11.391</td>\n",
       "      <td>...</td>\n",
       "      <td>OK</td>\n",
       "      <td>OK</td>\n",
       "      <td>OK</td>\n",
       "      <td>OK</td>\n",
       "      <td>OK</td>\n",
       "      <td>OK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">tring5</th>\n",
       "      <th>196</th>\n",
       "      <td>NaN</td>\n",
       "      <td>0.075</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.075</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.104</td>\n",
       "      <td>NaN</td>\n",