plot-results.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "ZERO = 20e-5\n",
    "TIMEOUT = 10 * 60 # 10 minutes = 600 seconds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import glob\n",
    "import re\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import plotly.io as pio\n",
    "import plotly.express as px\n",
    "import plotly.graph_objs as go\n",
    "from itertools import combinations \n",
    "import plotly.figure_factory as ff\n",
    "from plotly.subplots import make_subplots\n",
    "\n",
    "# render figures in notebook\n",
    "pio.renderers.default = \"notebook_connected\"\n",
    "\n",
    "# templates figures\n",
    "px.defaults.template = \"simple_white\"\n",
    "pio.templates.default = \"simple_white\"\n",
    "\n",
    "# layout for all figures\n",
    "LAYOUT_FIGURES = dict(\n",
    "    autosize=False,\n",
    "    width = 500,\n",
    "    height = 500,\n",
    "    xaxis = dict(\n",
    "      constrain=\"domain\",\n",
    "      mirror=True,\n",
    "      showexponent=\"all\",\n",
    "      exponentformat=\"power\"\n",
    "    ),\n",
    "    yaxis = dict(\n",
    "      scaleanchor = \"x\",\n",
    "      scaleratio = 1,\n",
    "      mirror=True,\n",
    "      showexponent=\"all\",\n",
    "      exponentformat=\"power\"\n",
    "    ),\n",
    "    title = dict(\n",
    "      y = 0.9,\n",
    "      x = 0.5,\n",
    "      xanchor = 'center',\n",
    "      yanchor = 'top'\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Auxiliary Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_folder(path):\n",
    "    \"\"\"Creates a folder if it does not exist\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    path : str\n",
    "        Path of the new folder\n",
    "    \n",
    "    Examples\n",
    "    --------\n",
    "    \n",
    "    >>> create_folder('./results')\n",
    "    \"\"\"\n",
    "    if not os.path.exists(path):\n",
    "        os.makedirs(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_figure(df, model):\n",
    "    \"\"\"Creates a scatter figure showing the time taken by each tool to verify each property of a model\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    df : pandas.Dataframe\n",
    "        Dataframe containing the results of the experiments\n",
    "    model : string\n",
    "        model to be plotted\n",
    "    \n",
    "    Returns\n",
    "    -------\n",
    "    plotly.graph_objects.Figure\n",
    "        Scatter figure\n",
    "    \n",
    "    Examples\n",
    "    --------\n",
    "    \n",
    "    >>> import os\n",
    "    >>> import pandas as pd\n",
    "    >>> csv_file = os.path.join(\"results\", \"output.csv\")\n",
    "    >>> df = pd.read_csv(csv_file)\n",
    "    >>> fig = create_figure(df, 'philo10')\n",
    "    \"\"\"\n",
    "    model_df = df[df.model == model]\n",
    "\n",
    "    figure = px.scatter(model_df, \n",
    "                        x=\"formula\", y=\"time\",\n",
    "                        title=model, \n",
    "                        color=\"tool\", \n",
    "                        symbol_sequence=['x'])\n",
    "\n",
    "    figure.update_layout(yaxis_title=\"time (s)\", title=LAYOUT_FIGURES['title'])\n",
    "    return figure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_axis_title(experiment, show_strategy=True):\n",
    "    \"\"\"Get the axis title of a figure depending on the experiment being plotted\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    experiment : str\n",
    "        String with the experiment information\n",
    "    show_strategy : bool, optional\n",
    "        Flag to show the information related to the strategy used by the tool\n",
    "    \n",
    "    Returns\n",
    "    -------\n",
    "    str\n",
    "        axis title\n",
    "        \n",
    "    Examples\n",
    "    --------\n",
    "    \n",
    "    >>> get_axis_title('pmc-sog_otfL_couv99-default_1_1', True)\n",
    "    pmc-sog (Lace, strategy: couv99-default, # cores: 1)\n",
    "    \"\"\"\n",
    "    information = experiment.split('_')\n",
    "    tool_name = information[0]\n",
    "    \n",
    "    info = []\n",
    "    library_dic = {\n",
    "        'otfL': 'Lace',\n",
    "        'otfP': 'Pthreads',\n",
    "        'otfC': 'Cthreads',\n",
    "        'otfPR': 'otfPR',\n",
    "        'otf': 'Hybrid'\n",
    "    }\n",
    "    \n",
    "    if (len(information) == 5):\n",
    "        info.append(library_dic[information[1]])\n",
    "\n",
    "    if (show_strategy):\n",
    "        info.append('strategy: {}'.format(information[-3]))\n",
    "\n",
    "    nb_nodes = int(information[-2])\n",
    "    if (nb_nodes > 1):\n",
    "        info.append('# nodes: {}'.format(nb_nodes))\n",
    "\n",
    "    info.append('# cores: {}'.format(information[-1]))\n",
    "\n",
    "    title = '{} ({})'.format(tool_name, ', '.join(info))\n",
    "    \n",
    "    return title"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def filter_errors(df_exp1, df_exp2):\n",
    "    \"\"\"Returns dataframes of specific experiments without errors\"\"\"\n",
    "    \n",
    "    nan_1 = df_exp1[df_exp1.isna().any(axis=1)].index\n",
    "    nan_2 = df_exp2[df_exp2.isna().any(axis=1)].index\n",
    "\n",
    "    df_exp1 = df_exp1.drop(nan_2)\n",
    "    df_exp2 = df_exp2.drop(nan_1)\n",
    "\n",
    "    df_exp1 = df_exp1.dropna()\n",
    "    df_exp2 = df_exp2.dropna()\n",
    "    \n",
    "    return df_exp1, df_exp2\n",
    "\n",
    "\n",
    "def get_info(info):\n",
    "    \"\"\"Get some statistics from a table for a specific model and experiment\"\"\"    \n",
    "    time_limit = len(info[(info.error == \"TIME LIMIT\") | (info.error == \"TIMEOUT\")])\n",
    "    error = len(info[(info.error != \"TIME LIMIT\") & (info.error != \"TIMEOUT\") & (info.error != \"OK\") & (info.error != 'MDD') & (info.error != 'TABLE FULL')])\n",
    "    memory = len(info[(info.error == 'MDD') | (info.error == 'TABLE FULL')])\n",
    "    ok = len(info[info.error == \"OK\"])\n",
    "    \n",
    "    if ((time_limit + error + ok + memory) != len(info)): raise Exception(\"Some information is missing in the table\")\n",
    "    \n",
    "    return {\n",
    "        \"time limit\": time_limit,\n",
    "        \"error\": error,\n",
    "        \"memory\": memory,\n",
    "        \"OK\": ok\n",
    "    }\n",
    "\n",
    "\n",
    "def get_best_times(table_time, table_error, model, exp1, exp2):\n",
    "    exp1 = pd.DataFrame({\"times_exp1\": table_time.loc[model][exp1], \"errors_exp1\": table_error.loc[model][exp1]})\n",
    "    exp2 = pd.DataFrame({\"times_exp2\": table_time.loc[model][exp2], \"errors_exp2\": table_error.loc[model][exp2]})\n",
    "\n",
    "    exp1, exp2 = filter_errors(exp1, exp2)\n",
    "    df_ = pd.concat([exp1, exp2], axis=1, sort=False)\n",
    "\n",
    "    df_ = df_[df_[\"times_exp1\"] != df_[\"times_exp2\"]]\n",
    "    df_['best'] = np.where((df_[\"times_exp1\"] < df_[\"times_exp2\"]), \"exp1\", \"exp2\")\n",
    "    count = df_.groupby([\"best\"]).size()\n",
    "\n",
    "    return count.get(\"exp1\",0), count.get(\"exp2\",0)\n",
    "\n",
    "\n",
    "def get_table(df_time, df_errors, model, exp1, exp2):\n",
    "    \"\"\"Creates a table with some statistics from a dataframe for a model and experiments\"\"\"\n",
    "    rows=[[\"<b>Experiment</b>\", \"<b>Time Limit</b>\", \"<b>Memory</b>\", \"<b>Unknown Error</b>\", \"<b>OK</b>\", \"<b>Faster</b>\"]]\n",
    "    \n",
    "    df_exp1 = pd.DataFrame({\"error\": df_errors.loc[model][exp1]})\n",
    "    df_exp2 = pd.DataFrame({\"error\": df_errors.loc[model][exp2]})\n",
    "    df_exp1, df_exp2 = filter_errors(df_exp1, df_exp2)\n",
    "    \n",
    "    info1 = get_info(df_exp1)\n",
    "    info2 = get_info(df_exp2)\n",
    "    \n",
    "    best1, best2 = get_best_times(df_time, df_errors, model, exp1, exp2)\n",
    "    \n",
    "    for (experiment, info, best) in [(exp1, info1, best1), (exp2, info2, best2)]:\n",
    "        rows.append([experiment, info[\"time limit\"], info[\"memory\"], info[\"error\"], info[\"OK\"], best])\n",
    "        \n",
    "    return ff.create_table(rows)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import webbrowser\n",
    "\n",
    "def get_filename(base_path, tool, model, model_instance, formula):\n",
    "    \"\"\"Returns the absolute path of the experiment log\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    base_path : string\n",
    "        Path of the folder where logs are saved\n",
    "    tool : string\n",
    "        Tool name\n",
    "    model : string\n",
    "        Model name\n",
    "    model_instance : string\n",
    "        Name of the model instance\n",
    "    formula : string\n",
    "        Identifier of the formula\n",
    "    \n",
    "    Returns\n",
    "    -------\n",
    "    string\n",
    "        Absolute path of the log file\n",
    "    \n",
    "    \"\"\"\n",
    "    information = tool.split('_')\n",
    "    \n",
    "    tool_name = information[0]\n",
    "    tool_configuration = '_'.join(information[:-2])\n",
    "    nb_nodes = information[-2]\n",
    "    nb_cores = information[-1]\n",
    "    \n",
    "    experiment_folder = os.path.join(base_path, tool_name, tool_configuration, model, model_instance)\n",
    "    filename = f'{tool_name}_{model_instance}-n{nb_nodes}-th{nb_cores}-f{formula}'\n",
    "    absolute_path = os.path.join(experiment_folder, filename)\n",
    "    \n",
    "    return absolute_path\n",
    "\n",
    "def open_logs_callback(trace, points, selector):\n",
    "    \"\"\"Callback that open the log files when clicking on a point of the figure\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    trace : plotly.graph_objects.Figure\n",
    "        the figure to attach the callback\n",
    "    points : plotly.callbacks.Points \n",
    "        points of the figure selected\n",
    "    selector: plotly.callbacks.InputDeviceState \n",
    "        Device information \n",
    "    \"\"\"\n",
    "    inds = points.point_inds\n",
    "    if (inds):\n",
    "        index = inds[0]\n",
    "\n",
    "        formula, error_x, error_y = trace['customdata'][index]\n",
    "        model_instance = trace['meta']['model']\n",
    "        model = ''.join(c for c in model_instance if not c.isdigit())\n",
    "        tools = trace['meta']['tools']\n",
    "        logs_folder = trace['meta']['folder']\n",
    "\n",
    "        filename_x = get_filename(logs_folder, tools['x'], model, model_instance, formula)\n",
    "        filename_y = get_filename(logs_folder, tools['y'], model, model_instance, formula)\n",
    "\n",
    "        for f in [filename_x, filename_y]:\n",
    "            webbrowser.open(f'file://{f}.err')\n",
    "            webbrowser.open(f'file://{f}.out')\n",
    "\n",
    "OPEN_LOGS_CALLBACK_JS = \"\"\"\n",
    "function get_filename (base_path, tool, model_instance, formula) {\n",
    "  const information = tool.split('_');\n",
    "  const size = information.length;\n",
    "\n",
    "  const tool_name = information[0];\n",
    "  const tool_configuration = information.slice(0, size - 2).join('_');\n",
    "  const nb_nodes = information[size - 2];\n",
    "  const nb_cores = information[size - 1];\n",
    "\n",
    "  const model = model_instance.replace(/[0-9]/g, '');\n",
    "\n",
    "  const experiment_folder = `${base_path}/${tool_name}/${tool_configuration}/${model}/${model_instance}`;\n",
    "  const filename = `${tool_name}_${model_instance}-n${nb_nodes}-th${nb_cores}-f${formula}`;\n",
    "\n",
    "  return `${experiment_folder}/${filename}`;\n",
    "}\n",
    "\n",
    "const plots = document.getElementsByClassName(\"plotly-graph-div js-plotly-plot\");\n",
    "const myPlot = plots[0];\n",
    "\n",
    "myPlot.on('plotly_click', function(data){\n",
    "    const points = data.points;\n",
    "    if (points.length != 1) {return ;}\n",
    "    \n",
    "    const myPoint = points[0];\n",
    "    const formula = myPoint.customdata[0];\n",
    "    const meta = myPoint.data.meta;\n",
    "    \n",
    "    const href = window.location.href.split('/');\n",
    "    const base_path = href.splice(0,href.length-4).join('/');\n",
    "    \n",
    "    const filename_x = get_filename(base_path, meta.tools.x, meta.model, formula);\n",
    "    const filename_y = get_filename(base_path, meta.tools.y, meta.model, formula);\n",
    "    \n",
    "    console.log('x: ' + filename_x);\n",
    "    window.open(`${filename_x}.err`);\n",
    "    window.open(`${filename_x}.out`);\n",
    "    \n",
    "    console.log('y: ' + filename_y);\n",
    "    window.open(`${filename_y}.err`);\n",
    "    window.open(`${filename_y}.out`);\n",
    "});\n",
    "\"\"\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_figure_explored_states(table_explored_states, model):\n",
    "    \"\"\"Creates figure showing the number of explorated states during the verification \n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    table_explored_states : pandas.Dataframe\n",
    "        Dataframe with the explorated states of each experiment\n",
    "    model : string\n",
    "        Model to be analyzed\n",
    "        \n",
    "    Returns\n",
    "    -------\n",
    "    plotly.graph_objects.Figure\n",
    "        Scatter figure\n",
    "    \"\"\"\n",
    "    colors={'T': 'green', 'F': 'red'}\n",
    "    float_formatter = \"{:.2E}\".format\n",
    "\n",
    "    table_model = table_explored_states[table_explored_states.property != 'U']\n",
    "    table_model = table_model[table_model.model == model]\n",
    "\n",
    "    table_stats = table_model.groupby(['property']).agg(['mean']) \n",
    "\n",
    "    fig = go.Figure()\n",
    "    max_x = 0\n",
    "    for p in table_stats.index:\n",
    "        data = table_model[table_model.property==p]\n",
    "        stats = table_stats.loc[p]\n",
    "        \n",
    "        x_axis = np.arange(1, data['formula'].count()+1, 1)\n",
    "        max_x = max(max_x, x_axis[-1]+1)\n",
    "        mean = stats['explored_states','mean']\n",
    "        \n",
    "        figure = px.scatter(data, \n",
    "                            x=x_axis, \n",
    "                            y=\"explored_states\",\n",
    "                            title=model, \n",
    "                            color='property',\n",
    "                            color_discrete_map=colors,\n",
    "                            symbol_sequence=[\"circle\"])\n",
    "\n",
    "        line = go.Scatter(x=[x_axis[0], x_axis[-1]], \n",
    "                          y=[mean, mean],\n",
    "                          mode='lines', showlegend=False,                          \n",
    "                          line=dict(color=colors[p], width=1.5))\n",
    "\n",
    "        fig.add_trace(figure['data'][0])\n",
    "        fig.add_trace(line)\n",
    "        \n",
    "        fig.add_annotation(x=1, \n",
    "                           y=mean,\n",
    "                           font=dict(color=colors[p]),\n",
    "                           text=f\"mean = {float_formatter(mean)}\")\n",
    "\n",
    "    fig.update_layout(title_text=model, title=LAYOUT_FIGURES['title'], \n",
    "                      width = 500, height = 500, margin=dict(r=110))\n",
    "    \n",
    "    fig.update_xaxes(title=\"formula\", range=[0, max_x])\n",
    "    fig.update_yaxes(title=\"# explored states\")\n",
    "    \n",
    "    fig.update_annotations(dict(\n",
    "        showarrow=False,\n",
    "        xanchor=\"left\",\n",
    "        yanchor=\"middle\",\n",
    "        xref='paper'))\n",
    "    \n",
    "    return fig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_log_figure(table, table_errors, model, tool_x, tool_y, show_strategy=True, callback=None):\n",
    "    \"\"\"Creates a Scatter figure in logarithmic scale comparing the performance of two tools\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    table : pandas.Dataframe\n",
    "        Dataframe with the times of each experiment\n",
    "    table_errors : pandas.Dataframe\n",
    "        Dataframe with the errors of each experiment\n",
    "    model : string\n",
    "        Model to be analyzed\n",
    "    tool_x : string\n",
    "        Tool to be compared and plotted on the x-axis\n",
    "    tool_y : string\n",
    "        Tool to be compared and plotted on the y-axis\n",
    "    show_strategy : bool\n",
    "        Flag to show the stretagy used by the tools\n",
    "    callback : function\n",
    "        Function to be called when clicking on a point\n",
    "        \n",
    "    Returns\n",
    "    -------\n",
    "    plotly.graph_objects.Figure\n",
    "        Scatter figure\n",
    "        \n",
    "    Examples\n",
    "    --------\n",
    "    >>> import os\n",
    "    >>> import pandas as pd\n",
    "    >>> csv_file = os.path.join(\"results\", \"output.csv\")\n",
    "    >>> df = pd.read_csv(csv_file)\n",
    "    >>> table = df.set_index(['model', 'formula', 'tool'], drop=True).unstack('tool')\n",
    "    >>> fig = create_log_figure(table['time'], table['error'], 'philo10', 'pmc-sog_otfL_couv99-default_1_8', 'pmc-sog_otfP_couv99-default_1_8')\n",
    "    \"\"\"\n",
    "    try:\n",
    "        min_value = ZERO\n",
    "        max_value = TIMEOUT\n",
    "        \n",
    "        min_value_log = np.log10(min_value)\n",
    "        max_value_log = np.log10(max_value)\n",
    "\n",
    "        table_model = table.loc[model]\n",
    "        table_errors_model = table_error.loc[model]\n",
    "        \n",
    "        full_table_x = pd.concat([table_model[tool_x],table_model['property'], table_errors_model[tool_x]], axis=1)\n",
    "        full_table_x.columns = ['time', 'property', 'error']\n",
    "\n",
    "        full_table_y = pd.concat([table_model[tool_y],table_model['property'], table_errors_model[tool_y]], axis=1)\n",
    "        full_table_y.columns = ['time', 'property', 'error']\n",
    "        \n",
    "        full_table_x, full_table_y = filter_errors(full_table_x, full_table_y)\n",
    "\n",
    "        traces = [\n",
    "            {\"property\": 'T', \"color\":\"green\"},\n",
    "            {\"property\": 'F', \"color\":\"red\"},\n",
    "            {\"property\": 'U', \"color\":\"black\"}\n",
    "        ]\n",
    "\n",
    "        figures = []\n",
    "        for t in traces:\n",
    "            # filter by verification output\n",
    "            table_x = full_table_x[full_table_x.property == t['property']]\n",
    "            table_y = full_table_y[full_table_y.property == t['property']]\n",
    "\n",
    "            # custom data\n",
    "            custom_data = list(zip(table_x.index, table_x.error,table_y.error))\n",
    "            \n",
    "            # tools\n",
    "            metainfo = {\n",
    "                'model': model, \n",
    "                'tools': {'x': tool_x, 'y': tool_y},\n",
    "                'folder': os.path.join(os.path.abspath(os.pardir), \"results\")\n",
    "            }\n",
    "\n",
    "            figures.append(go.Scatter(x=table_x.time,\n",
    "                                      y=table_y.time,\n",
    "                                      name=t['property'],\n",
    "                                      mode='markers',\n",
    "                                      marker_symbol='circle-open',\n",
    "                                      marker_color=t['color'],\n",
    "                                      meta = metainfo,\n",
    "                                      customdata=custom_data,\n",
    "                                      hovertemplate =\n",
    "                                        '<b>Formula # %{customdata[0]}</b><br>' +\n",
    "                                        '<br><b>Times:</b><br>' +\n",
    "                                        '<b>x:</b> %{x} s' +\n",
    "                                        '<br><b>y:</b> %{y} s<br>' +\n",
    "                                        '<br><b>Errors:</b><br>' +\n",
    "                                        '<b>x:</b> %{customdata[1]}<br>' +\n",
    "                                        '<b>y:</b> %{customdata[2]}',\n",
    "                                        ))    \n",
    "\n",
    "        # Line\n",
    "        figures.append(go.Scatter(x=[min_value, max_value], \n",
    "                                    y=[min_value, max_value],\n",
    "                                    mode='lines', showlegend=False,\n",
    "                                    line=dict(color='black', width=1)))\n",
    "\n",
    "        # Create figure\n",
    "        figure = go.FigureWidget(figures)\n",
    "        figure.update_layout(LAYOUT_FIGURES,\n",
    "                             title_text=model,\n",
    "                             hoverlabel=dict(bgcolor=\"white\", align='auto'),\n",
    "                             legend_title_text='property',\n",
    "                             xaxis=dict(type='log', autorange=False, range=[min_value_log, max_value_log]),\n",
    "                             yaxis=dict(type='log', autorange=False, range=[min_value_log, max_value_log]),\n",
    "                             xaxis_title=get_axis_title(tool_x, show_strategy),\n",
    "                             yaxis_title=get_axis_title(tool_y, show_strategy))\n",
    "\n",
    "        # Add event\n",
    "        if callback is not None:\n",
    "            for i in range(len(figure.data)):\n",
    "                figure.data[i].on_click(callback)\n",
    "    \n",
    "        return figure\n",
    "    except Exception as e:\n",
    "        print(\"Error when ploting model: {} - tool_x: {} - tool_y: {}\".format(model, tool_x, tool_y))\n",
    "        print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Experiment filters\n",
    "\n",
    "def versus_dfs(experiments):\n",
    "    \"\"\"Selects only experiments using DFS strategy\"\"\"\n",
    "    exp1, exp2 = experiments\n",
    "    strategy_exp1= exp1.split('_')[1]\n",
    "    strategy_exp2= exp2.split('_')[1]\n",
    "    \n",
    "    return strategy_exp1 == 'dfs' or strategy_exp2 == 'dfs'\n",
    "\n",
    "def versus_sequential(experiments):\n",
    "    \"\"\"Selects only experiments run sequentially \"\"\"\n",
    "    exp1, exp2 = experiments\n",
    "    nodes_exp1, threads_exp1 = exp1.split('_')[-2:]\n",
    "    nodes_exp2, threads_exp2 = exp2.split('_')[-2:]\n",
    "\n",
    "    return (nodes_exp1 == '1' and nodes_exp2 == '1') and \\\n",
    "            (threads_exp1 == '1' or threads_exp2 == '1')\n",
    "\n",
    "def same_tool(experiments, tool):\n",
    "    \"\"\"Selects only experiments comparing the same tool\"\"\"\n",
    "    exp1, exp2 = experiments\n",
    "    tool_exp1= exp1.split('_')[0]\n",
    "    tool_exp2= exp2.split('_')[0]\n",
    "    return tool_exp1.startswith(tool) and tool_exp2.startswith(tool)\n",
    "\n",
    "def same_number_threads(experiments):\n",
    "    \"\"\"Selects only experiments comparing the same number of processes and cores\"\"\"\n",
    "    exp1, exp2 = experiments\n",
    "    nodes_exp1, threads_exp1 = exp1.split('_')[-2:]\n",
    "    nodes_exp2, threads_exp2 = exp2.split('_')[-2:]\n",
    "    return (nodes_exp1 == nodes_exp2) and (threads_exp1 == threads_exp2) \n",
    "\n",
    "def same_thread_library(experiments):\n",
    "    \"\"\"Selects only experiments comparing the same parallelization library\"\"\"\n",
    "    exp1, exp2 = experiments\n",
    "    library_exp1 = exp1.split('_')[1]\n",
    "    library_exp2 = exp2.split('_')[1]\n",
    "    return library_exp1 == library_exp2\n",
    "\n",
    "def same_strategy(experiments):\n",
    "    \"\"\"Selects only experiments comparing the same strategy\"\"\"\n",
    "    exp1, exp2 = experiments\n",
    "    strategy_exp1 = exp1.split('_')[2]\n",
    "    strategy_exp2 = exp2.split('_')[2]\n",
    "    return strategy_exp1 == strategy_exp2\n",
    "\n",
    "def only_couvreur_strategy(experiments):\n",
    "    \"\"\"Selects only experiments comparing couvreur emptiness check algorithm\"\"\"\n",
    "    exp1, exp2 = experiments\n",
    "    strategy_exp1 = exp1.split('_')[2]\n",
    "    strategy_exp2 = exp2.split('_')[2]\n",
    "    return strategy_exp1.startswith('couv99') and strategy_exp2.startswith('couv99')\n",
    "\n",
    "def compare_threads_library(experiments):\n",
    "    \"\"\"Compares parallization libraries used in pmc-sog. \n",
    "    \n",
    "    It selects experiments where the tool is only pmc-sog and the strategy, number of threads, \n",
    "    number of processus are the same.\n",
    "    \"\"\"\n",
    "    return same_tool(experiments, 'pmc-sog') and \\\n",
    "            same_strategy(experiments) and \\\n",
    "            same_number_threads(experiments) and \\\n",
    "            not same_thread_library(experiments)\n",
    "\n",
    "def compare_couvreur_strategies(experiments):\n",
    "    \"\"\"Compares couvreurs strategies used in pmc-sog. \n",
    "    \n",
    "    It selects experiments where the tool is only pmc-sog, the strategy is couvreur, and \n",
    "    the parallelization library, number of threads, number of processus are the same.\n",
    "    \"\"\"\n",
    "    return only_couvreur_strategy(experiments) and \\\n",
    "            same_thread_library(experiments) and \\\n",
    "            same_number_threads(experiments)\n",
    "\n",
    "def same_distributed_number_threads(experiments):\n",
    "    \"\"\"Selects only experiments where the multiplication of theirs nodes with cores are the same.\"\"\"\n",
    "    exp1, exp2 = experiments\n",
    "    nodes_exp1, threads_exp1 = exp1.split('_')[-2:]\n",
    "    nodes_exp2, threads_exp2 = exp2.split('_')[-2:]\n",
    "    return (int(nodes_exp1) * int(threads_exp1)) == (int(nodes_exp2) * int(threads_exp2))\n",
    "\n",
    "def compare_tools(experiments):\n",
    "    \"\"\"Compares pmc-sog and pnml2lts-mc using the DFS algorithm. \n",
    "    \n",
    "    It selects experiments where the tools are not the same, the exploration algorithm is DFS and \n",
    "    the number of processus and cores are the same.\n",
    "    \"\"\"\n",
    "    return not (same_tool(experiments, 'pmc-sog') or same_tool(experiments,'pnml2lts-mc')) and \\\n",
    "            versus_dfs(experiments)\n",
    "\n",
    "def compare_multithreading(experiments):\n",
    "    \"\"\"Compares the sequential and multi-core version of pmc-sog. \n",
    "    \n",
    "    It selects experiments where the tools is pmc-sog, the parallelization library, the emptiness check \n",
    "    strategy are the same. Here the number of processus and cores are different.\n",
    "    \"\"\"\n",
    "    return same_tool(experiments, 'pmc-sog') and \\\n",
    "            same_thread_library(experiments) and \\\n",
    "            same_strategy(experiments) and \\\n",
    "            versus_sequential(experiments)\n",
    "\n",
    "def against_hybrid(experiments):\n",
    "    \"\"\"Selects only experiments comparing with hybrid mode\"\"\"\n",
    "    exp1, exp2 = experiments\n",
    "    library_exp1 = exp1.split('_')[1]\n",
    "    library_exp2 = exp2.split('_')[1]\n",
    "    return (library_exp1 == 'otf') or (library_exp2 == 'otf')\n",
    "\n",
    "\n",
    "def compare_distributed(experiments):\n",
    "    \"\"\"Compares the hybrid version of pmc-sog\"\"\"\n",
    "    return same_tool(experiments, 'pmc-sog') and \\\n",
    "        same_strategy(experiments) and \\\n",
    "        same_distributed_number_threads(experiments) and \\\n",
    "        against_hybrid(experiments)\n",
    "\n",
    "def compare_others(experiments):\n",
    "    return (not compare_threads_library(experiments)) and \\\n",
    "        (not compare_couvreur_strategies(experiments)) and \\\n",
    "        (not compare_tools(experiments)) and \\\n",
    "        (not compare_multithreading(experiments)) and \\\n",
    "        (not compare_distributed(experiments))\n",
    "\n",
    "# Plots to be created\n",
    "plots = {\n",
    "    'compare_thread_library': compare_threads_library,\n",
    "    'compare_couvreur_algorithm': compare_couvreur_strategies,\n",
    "    'compare_tools': compare_tools,\n",
    "    'compare_multicore': compare_multithreading,\n",
    "    'compare_distributed': compare_distributed,\n",
    "    'others' : compare_others\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Root folder\n",
    "PROJECT_FOLDER = os.path.abspath(os.pardir)\n",
    "\n",
    "# csv file with the output\n",
    "csv_file = os.path.join(PROJECT_FOLDER, \"results\", \"output.csv\")\n",
    "\n",
    "# formulas folder\n",
    "FORMULAS_FOLDER = os.path.join(PROJECT_FOLDER, \"formulas\")\n",
    "\n",
    "# Output folder\n",
    "OUTPUT_FOLDER = os.path.join(PROJECT_FOLDER,\"results\", \"figures\")\n",
    "create_folder(OUTPUT_FOLDER)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>formula</th>\n",
       "      <th>tool</th>\n",
       "      <th>time</th>\n",
       "      <th>property</th>\n",
       "      <th>explored_states</th>\n",
       "      <th>error</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>robot20</td>\n",
       "      <td>1</td>\n",
       "      <td>pmc-sog_otfC_couv99-default_1_16</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TIME LIMIT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>robot20</td>\n",
       "      <td>1</td>\n",
       "      <td>pmc-sog_otfC_couv99-shy_1_16</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TIME LIMIT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>robot20</td>\n",
       "      <td>1</td>\n",
       "      <td>pmc-sog_otfPR_couv99-default_1_16</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TIME LIMIT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>robot20</td>\n",
       "      <td>1</td>\n",
       "      <td>pmc-sog_otfPR_couv99-shy_1_16</td>\n",
       "      <td>2.737</td>\n",
       "      <td>F</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>robot20</td>\n",
       "      <td>1</td>\n",
       "      <td>pmc-sog_otfP_couv99-default_1_16</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TIME LIMIT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2395</th>\n",
       "      <td>robot20</td>\n",
       "      <td>200</td>\n",
       "      <td>pmc-sog_otf_couv99-default_4_4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TIME LIMIT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2396</th>\n",
       "      <td>robot20</td>\n",
       "      <td>200</td>\n",
       "      <td>pmc-sog_otf_couv99-shy_2_8</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TIME LIMIT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2397</th>\n",
       "      <td>robot20</td>\n",
       "      <td>200</td>\n",
       "      <td>pmc-sog_otf_couv99-shy_4_4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TIME LIMIT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2398</th>\n",
       "      <td>robot20</td>\n",
       "      <td>200</td>\n",
       "      <td>pnml2lts-mc_dfs_1_16</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.875346e+09</td>\n",
       "      <td>TABLE FULL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2399</th>\n",
       "      <td>robot20</td>\n",
       "      <td>200</td>\n",
       "      <td>pnml2lts-mc_ndfs_1_16</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TIME LIMIT</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2400 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        model  formula                               tool   time property  \\\n",
       "0     robot20        1   pmc-sog_otfC_couv99-default_1_16    NaN      NaN   \n",
       "1     robot20        1       pmc-sog_otfC_couv99-shy_1_16    NaN      NaN   \n",
       "2     robot20        1  pmc-sog_otfPR_couv99-default_1_16    NaN      NaN   \n",
       "3     robot20        1      pmc-sog_otfPR_couv99-shy_1_16  2.737        F   \n",
       "4     robot20        1   pmc-sog_otfP_couv99-default_1_16    NaN      NaN   \n",
       "...       ...      ...                                ...    ...      ...   \n",
       "2395  robot20      200     pmc-sog_otf_couv99-default_4_4    NaN      NaN   \n",
       "2396  robot20      200         pmc-sog_otf_couv99-shy_2_8    NaN      NaN   \n",
       "2397  robot20      200         pmc-sog_otf_couv99-shy_4_4    NaN      NaN   \n",
       "2398  robot20      200               pnml2lts-mc_dfs_1_16    NaN      NaN   \n",
       "2399  robot20      200              pnml2lts-mc_ndfs_1_16    NaN      NaN   \n",
       "\n",
       "      explored_states       error  \n",
       "0                 NaN  TIME LIMIT  \n",
       "1                 NaN  TIME LIMIT  \n",
       "2                 NaN  TIME LIMIT  \n",
       "3                 NaN          OK  \n",
       "4                 NaN  TIME LIMIT  \n",
       "...               ...         ...  \n",
       "2395              NaN  TIME LIMIT  \n",
       "2396              NaN  TIME LIMIT  \n",
       "2397              NaN  TIME LIMIT  \n",
       "2398     2.875346e+09  TABLE FULL  \n",
       "2399              NaN  TIME LIMIT  \n",
       "\n",
       "[2400 rows x 7 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# read data\n",
    "df = pd.read_csv(csv_file)\n",
    "\n",
    "# merge the information related to the experiment (# nodes, # threads, strategy) to the tool column\n",
    "df['tool'] = df[['tool', 'strategy', 'num_nodes', 'num_threads']].astype(str).apply('_'.join, axis=1)\n",
    "df = df.drop(columns=['strategy', 'num_nodes', 'num_threads'])\n",
    "\n",
    "# FIX: filtering philo20 experiments because there is a problem with the generated formulas\n",
    "df = df[df.model != \"philo20\"]\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>formula</th>\n",
       "      <th>tool</th>\n",
       "      <th>time</th>\n",
       "      <th>property</th>\n",
       "      <th>explored_states</th>\n",
       "      <th>error</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>robot20</td>\n",
       "      <td>1</td>\n",
       "      <td>pmc-sog_otfC_couv99-default_1_16</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TIME LIMIT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>robot20</td>\n",
       "      <td>1</td>\n",
       "      <td>pmc-sog_otfC_couv99-shy_1_16</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TIME LIMIT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>robot20</td>\n",
       "      <td>1</td>\n",
       "      <td>pmc-sog_otfPR_couv99-default_1_16</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TIME LIMIT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>robot20</td>\n",
       "      <td>1</td>\n",
       "      <td>pmc-sog_otfPR_couv99-shy_1_16</td>\n",
       "      <td>2.737</td>\n",
       "      <td>F</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>robot20</td>\n",
       "      <td>1</td>\n",
       "      <td>pmc-sog_otfP_couv99-default_1_16</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TIME LIMIT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1763</th>\n",
       "      <td>robot20</td>\n",
       "      <td>200</td>\n",
       "      <td>pmc-sog_otf_couv99-default_4_4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TIME LIMIT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1764</th>\n",
       "      <td>robot20</td>\n",
       "      <td>200</td>\n",
       "      <td>pmc-sog_otf_couv99-shy_2_8</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TIME LIMIT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1765</th>\n",
       "      <td>robot20</td>\n",
       "      <td>200</td>\n",
       "      <td>pmc-sog_otf_couv99-shy_4_4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TIME LIMIT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1766</th>\n",
       "      <td>robot20</td>\n",
       "      <td>200</td>\n",
       "      <td>pnml2lts-mc_dfs_1_16</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.875346e+09</td>\n",
       "      <td>TABLE FULL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1767</th>\n",
       "      <td>robot20</td>\n",
       "      <td>200</td>\n",
       "      <td>pnml2lts-mc_ndfs_1_16</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TIME LIMIT</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1768 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        model  formula                               tool   time property  \\\n",
       "0     robot20        1   pmc-sog_otfC_couv99-default_1_16    NaN      NaN   \n",
       "1     robot20        1       pmc-sog_otfC_couv99-shy_1_16    NaN      NaN   \n",
       "2     robot20        1  pmc-sog_otfPR_couv99-default_1_16    NaN      NaN   \n",
       "3     robot20        1      pmc-sog_otfPR_couv99-shy_1_16  2.737        F   \n",
       "4     robot20        1   pmc-sog_otfP_couv99-default_1_16    NaN      NaN   \n",
       "...       ...      ...                                ...    ...      ...   \n",
       "1763  robot20      200     pmc-sog_otf_couv99-default_4_4    NaN      NaN   \n",
       "1764  robot20      200         pmc-sog_otf_couv99-shy_2_8    NaN      NaN   \n",
       "1765  robot20      200         pmc-sog_otf_couv99-shy_4_4    NaN      NaN   \n",
       "1766  robot20      200               pnml2lts-mc_dfs_1_16    NaN      NaN   \n",
       "1767  robot20      200              pnml2lts-mc_ndfs_1_16    NaN      NaN   \n",
       "\n",
       "      explored_states       error  \n",
       "0                 NaN  TIME LIMIT  \n",
       "1                 NaN  TIME LIMIT  \n",
       "2                 NaN  TIME LIMIT  \n",
       "3                 NaN          OK  \n",
       "4                 NaN  TIME LIMIT  \n",
       "...               ...         ...  \n",
       "1763              NaN  TIME LIMIT  \n",
       "1764              NaN  TIME LIMIT  \n",
       "1765              NaN  TIME LIMIT  \n",
       "1766     2.875346e+09  TABLE FULL  \n",
       "1767              NaN  TIME LIMIT  \n",
       "\n",
       "[1768 rows x 7 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# filtering runtime errors\n",
    "df = df[(df.error != \"SEGMENTATION FAULT\") & \\\n",
    "        (df.error != \"ABORTED\") & \\\n",
    "        (df.error != \"TERMINATE\") & \\\n",
    "        (df.error != \"MDD\")]\n",
    "\n",
    "df = df.reset_index(drop=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>property</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model</th>\n",
       "      <th>formula</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"11\" valign=\"top\">robot20</th>\n",
       "      <th>1</th>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>T</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>T</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>196</th>\n",
       "      <td>T</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>197</th>\n",
       "      <td>T</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>198</th>\n",
       "      <td>T</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199</th>\n",
       "      <td>T</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>200</th>\n",
       "      <td>T</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>200 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                property\n",
       "model   formula         \n",
       "robot20 1              F\n",
       "        2              T\n",
       "        3              F\n",
       "        4              F\n",
       "        5              T\n",
       "...                  ...\n",
       "        196            T\n",
       "        197            T\n",
       "        198            T\n",
       "        199            T\n",
       "        200            T\n",
       "\n",
       "[200 rows x 1 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# ground truth for properties\n",
    "frames = []\n",
    "\n",
    "formula_results = glob.glob(os.path.join(FORMULAS_FOLDER, \"**/formula_results\"), recursive=True)\n",
    "for f in formula_results:\n",
    "    model, out_file = f.split('/')[-2:]\n",
    "    \n",
    "    tmp_df = pd.read_csv(f, sep=\";\", header=None, names=[\"formula\", \"property\"])\n",
    "    tmp_df[\"model\"] = model\n",
    "    frames.append(tmp_df)\n",
    "    \n",
    "p_df = pd.concat(frames)\n",
    "p_df = p_df.reindex(columns=[\"model\", \"formula\", \"property\"])\n",
    "p_df = p_df[p_df['model'].isin(df.model.unique())]\n",
    "p_df['property'] = p_df['property'].replace([True, False], [\"T\", \"F\"])\n",
    "p_df = p_df.set_index([\"model\", \"formula\"])\n",
    "p_df.sort_index(inplace=True)\n",
    "\n",
    "p_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th colspan=\"10\" halign=\"left\">time</th>\n",
       "      <th>...</th>\n",
       "      <th colspan=\"10\" halign=\"left\">error</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>tool</th>\n",
       "      <th>pmc-sog_otfC_couv99-default_1_16</th>\n",
       "      <th>pmc-sog_otfC_couv99-shy_1_16</th>\n",
       "      <th>pmc-sog_otfPR_couv99-default_1_16</th>\n",
       "      <th>pmc-sog_otfPR_couv99-shy_1_16</th>\n",
       "      <th>pmc-sog_otfP_couv99-default_1_16</th>\n",
       "      <th>pmc-sog_otfP_couv99-shy_1_16</th>\n",
       "      <th>pmc-sog_otf_couv99-default_2_8</th>\n",
       "      <th>pmc-sog_otf_couv99-default_4_4</th>\n",
       "      <th>pmc-sog_otf_couv99-shy_2_8</th>\n",
       "      <th>pmc-sog_otf_couv99-shy_4_4</th>\n",
       "      <th>...</th>\n",
       "      <th>pmc-sog_otfPR_couv99-default_1_16</th>\n",
       "      <th>pmc-sog_otfPR_couv99-shy_1_16</th>\n",
       "      <th>pmc-sog_otfP_couv99-default_1_16</th>\n",
       "      <th>pmc-sog_otfP_couv99-shy_1_16</th>\n",
       "      <th>pmc-sog_otf_couv99-default_2_8</th>\n",
       "      <th>pmc-sog_otf_couv99-default_4_4</th>\n",
       "      <th>pmc-sog_otf_couv99-shy_2_8</th>\n",
       "      <th>pmc-sog_otf_couv99-shy_4_4</th>\n",
       "      <th>pnml2lts-mc_dfs_1_16</th>\n",
       "      <th>pnml2lts-mc_ndfs_1_16</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model</th>\n",
       "      <th>formula</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"11\" valign=\"top\">robot20</th>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.737</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>TIME LIMIT</td>\n",
       "      <td>OK</td>\n",
       "      <td>TIME LIMIT</td>\n",
       "      <td>TIME LIMIT</td>\n",
       "      <td>TIME LIMIT</td>\n",
       "      <td>TIME LIMIT</td>\n",
       "      <td>TIME LIMIT</td>\n",
       "      <td>TIME LIMIT</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>OK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.714</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.915</td>\n",
       "      <td>0.883</td>\n",
       "      <td>0.777</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.416</td>\n",
       "      <td>1.052</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>OK</td>\n",
       "      <td>OK</td>\n",
       "      <td>OK</td>\n",
       "      <td>TIME LIMIT</td>\n",
       "      <td>OK</td>\n",
       "      <td>OK</td>\n",
       "      <td>TIME LIMIT</td>\n",
       "      <td>TIME LIMIT</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>TIME LIMIT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>271.853</td>\n",
       "      <td>NaN</td>\n",
       "      <td>266.638</td>\n",
       "      <td>NaN</td>\n",
       "      <td>324.036</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>297.056</td>\n",
       "      <td>1053.045</td>\n",
       "      <td>...</td>\n",
       "      <td>TIME LIMIT</td>\n",
       "      <td>OK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TIME LIMIT</td>\n",
       "      <td>OK</td>\n",
       "      <td>OK</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>OK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.022</td>\n",
       "      <td>0.023</td>\n",
       "      <td>0.015</td>\n",
       "      <td>0.015</td>\n",
       "      <td>0.033</td>\n",
       "      <td>0.026</td>\n",
       "      <td>0.112</td>\n",
       "      <td>0.126</td>\n",
       "      <td>0.109</td>\n",
       "      <td>0.124</td>\n",
       "      <td>...</td>\n",
       "      <td>OK</td>\n",
       "      <td>OK</td>\n",
       "      <td>OK</td>\n",
       "      <td>OK</td>\n",
       "      <td>OK</td>\n",
       "      <td>OK</td>\n",
       "      <td>OK</td>\n",
       "      <td>OK</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>OK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>NaN</td>\n",
       "      <td>7.658</td>\n",
       "      <td>81.240</td>\n",
       "      <td>7.984</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6.793</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.276</td>\n",
       "      <td>20.551</td>\n",
       "      <td>...</td>\n",
       "      <td>OK</td>\n",
       "      <td>OK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "      <td>OK</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>OK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>196</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.149</td>\n",
       "      <td>0.110</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>OK</td>\n",
       "      <td>OK</td>\n",
       "      <td>TIME LIMIT</td>\n",
       "      <td>TIME LIMIT</td>\n",
       "      <td>TIME LIMIT</td>\n",
       "      <td>TIME LIMIT</td>\n",
       "      <td>TIME LIMIT</td>\n",
       "      <td>TIME LIMIT</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>OK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>197</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.731</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TABLE FULL</td>\n",
       "      <td>UNKNOWN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>198</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>12.215</td>\n",
       "      <td>3.962</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>OK</td>\n",
       "      <td>OK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TIME LIMIT</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>UNKNOWN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199</th>\n",
       "      <td>263.468</td>\n",
       "      <td>NaN</td>\n",
       "      <td>120.204</td>\n",
       "      <td>1.042</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>245.885</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>OK</td>\n",
       "      <td>OK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OK</td>\n",
       "      <td>TIME LIMIT</td>\n",
       "      <td>TIME LIMIT</td>\n",
       "      <td>TIME LIMIT</td>\n",
       "      <td>TABLE FULL</td>\n",
       "      <td>TIME LIMIT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>200</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>194.795</td>\n",
       "      <td>0.388</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>OK</td>\n",
       "      <td>OK</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TIME LIMIT</td>\n",
       "      <td>TIME LIMIT</td>\n",
       "      <td>TIME LIMIT</td>\n",
       "      <td>TIME LIMIT</td>\n",
       "      <td>TABLE FULL</td>\n",
       "      <td>TIME LIMIT</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>200 rows × 48 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            time                               \\\n",
       "tool            pmc-sog_otfC_couv99-default_1_16 pmc-sog_otfC_couv99-shy_1_16   \n",
       "model   formula                                                                 \n",
       "robot20 1                                    NaN                          NaN   \n",
       "        2                                  0.714                          NaN   \n",
       "        3                                    NaN                      271.853   \n",
       "        4                                  0.022                        0.023   \n",
       "        5                                    NaN                        7.658   \n",
       "...                                          ...                          ...   \n",
       "        196                                  NaN                          NaN   \n",
       "        197                                  NaN                          NaN   \n",
       "        198                                  NaN                          NaN   \n",
       "        199                              263.468                          NaN   \n",
       "        200                                  NaN                          NaN   \n",
       "\n",
       "                                                   \\\n",
       "tool            pmc-sog_otfPR_couv99-default_1_16   \n",
       "model   formula                                     \n",
       "robot20 1                                     NaN   \n",
       "        2                                   0.915   \n",
       "        3                                     NaN   \n",
       "        4                                   0.015   \n",
       "        5                                  81.240   \n",
       "...                                           ...   \n",
       "        196                                 1.149   \n",
       "        197                                   NaN   \n",
       "        198                                12.215   \n",
       "        199                               120.204   \n",
       "        200                               194.795   \n",
       "\n",
       "                                               \\\n",
       "tool            pmc-sog_otfPR_couv99-shy_1_16   \n",
       "model   formula                                 \n",
       "robot20 1                               2.737   \n",
       "        2                               0.883   \n",
       "        3                             266.638   \n",
       "        4                               0.015   \n",
       "        5                               7.984   \n",
       "...                                       ...   \n",
       "        196                             0.110   \n",