{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "ZERO = 10e-5\n", "TIMEOUT = 10 * 60 # 10 minutes = 6000 seconds" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "import os\n", "import glob\n", "import re\n", "import pandas as pd\n", "import numpy as np\n", "import plotly.io as pio\n", "import plotly.express as px\n", "import plotly.graph_objs as go\n", "from itertools import combinations \n", "import plotly.figure_factory as ff\n", "from plotly.subplots import make_subplots\n", "\n", "# render figures in notebook\n", "pio.renderers.default = \"notebook_connected\"\n", "\n", "# templates figures\n", "px.defaults.template = \"simple_white\"\n", "pio.templates.default = \"simple_white\"\n", "\n", "# layout for all figures\n", "LAYOUT_FIGURES = dict(\n", " autosize=False,\n", " width = 500,\n", " height = 500,\n", " xaxis = dict(\n", " constrain=\"domain\",\n", " mirror=True,\n", " showexponent=\"all\",\n", " exponentformat=\"power\"\n", " ),\n", " yaxis = dict(\n", " scaleanchor = \"x\",\n", " scaleratio = 1,\n", " mirror=True,\n", " showexponent=\"all\",\n", " exponentformat=\"power\"\n", " ),\n", " title = dict(\n", " y = 0.9,\n", " x = 0.5,\n", " xanchor = 'center',\n", " yanchor = 'top'\n", " )\n", ")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Auxiliary Functions" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "def create_folder(path):\n", " \"\"\"Creates a folder if it does not exist\n", " \n", " Parameters\n", " ----------\n", " path : str\n", " Path of the new folder\n", " \n", " Examples\n", " --------\n", " \n", " >>> create_folder('./results')\n", " \"\"\"\n", " if not os.path.exists(path):\n", " os.makedirs(path)" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "def create_figure(df, model):\n", " \"\"\"Creates a scatter figure showing the time taken by each tool to verify each property of a model\n", " \n", " Parameters\n", " ----------\n", " df : pandas.Dataframe\n", " Dataframe containing the results of the experiments\n", " model : string\n", " model to be plotted\n", " \n", " Returns\n", " -------\n", " plotly.graph_objects.Figure\n", " Scatter figure\n", " \n", " Examples\n", " --------\n", " \n", " >>> import os\n", " >>> import pandas as pd\n", " >>> csv_file = os.path.join(\"results\", \"output.csv\")\n", " >>> df = pd.read_csv(csv_file)\n", " >>> fig = create_figure(df, 'philo10')\n", " \"\"\"\n", " model_df = df[df.model == model]\n", "\n", " figure = px.scatter(model_df, \n", " x=\"formula\", y=\"time\",\n", " title=model, \n", " color=\"tool\", \n", " symbol_sequence=['x'])\n", "\n", " figure.update_layout(yaxis_title=\"time (s)\", title=LAYOUT_FIGURES['title'])\n", " return figure" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "def get_axis_title(experiment, show_strategy=True):\n", " \"\"\"Get the axis title of a figure depending on the experiment being plotted\n", " \n", " Parameters\n", " ----------\n", " experiment : str\n", " String with the experiment information\n", " show_strategy : bool, optional\n", " Flag to show the information related to the strategy used by the tool\n", " \n", " Returns\n", " -------\n", " str\n", " axis title\n", " \n", " Examples\n", " --------\n", " \n", " >>> get_axis_title('pmc-sog_otfL_couv99-default_1_1', True)\n", " pmc-sog (Lace, strategy: couv99-default, # cores: 1)\n", " \"\"\"\n", " information = experiment.split('_')\n", " tool_name = information[0]\n", " \n", " info = []\n", " library_dic = {\n", " 'otfL': 'Lace',\n", " 'otfP': 'Pthreads',\n", " 'otfC': 'Cthreads',\n", " 'otfPR': 'otfPR',\n", " 'otf': 'Hybrid'\n", " }\n", " \n", " #if (len(information) == 5):\n", " # info.append(\"mode \" + library_dic[information[1]])\n", "\n", " #if (show_strategy):\n", " # info.append('strategy: {}'.format(information[-3]))\n", "\n", " nb_nodes = int(information[-2])\n", " if (nb_nodes):\n", " info.append('# nodes: {}'.format(nb_nodes))\n", "\n", " info.append('# cores: {}'.format(information[-1]))\n", "\n", " title = 'Verification time (seconds)
{} ({})'.format(tool_name, ', '.join(info))\n", " \n", " return title" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "def filter_errors(df_exp1, df_exp2):\n", " \"\"\"Returns dataframes of specific experiments without errors\"\"\"\n", " \n", " nan_1 = df_exp1[df_exp1.isna().any(axis=1)].index\n", " nan_2 = df_exp2[df_exp2.isna().any(axis=1)].index\n", "\n", " df_exp1 = df_exp1.drop(nan_2)\n", " df_exp2 = df_exp2.drop(nan_1)\n", "\n", " df_exp1 = df_exp1.dropna()\n", " df_exp2 = df_exp2.dropna()\n", " \n", " return df_exp1, df_exp2\n", "\n", "\n", "def get_info(info):\n", " \"\"\"Get some statistics from a table for a specific model and experiment\"\"\" \n", " time_limit = len(info[(info.error == \"TIME LIMIT\") | (info.error == \"TIMEOUT\")])\n", " error = len(info[(info.error != \"TIME LIMIT\") & (info.error != \"TIMEOUT\") & (info.error != \"OK\") & (info.error != 'MDD') & (info.error != 'TABLE FULL')])\n", " memory = len(info[(info.error == 'MDD') | (info.error == 'TABLE FULL')])\n", " ok = len(info[info.error == \"OK\"])\n", " \n", " if ((time_limit + error + ok + memory) != len(info)): raise Exception(\"Some information is missing in the table\")\n", " \n", " return {\n", " \"time limit\": time_limit,\n", " \"error\": error,\n", " \"memory\": memory,\n", " \"OK\": ok\n", " }\n", "\n", "\n", "def get_best_times(table_time, table_error, model, exp1, exp2):\n", " exp1 = pd.DataFrame({\"times_exp1\": table_time.loc[model][exp1], \"errors_exp1\": table_error.loc[model][exp1]})\n", " exp2 = pd.DataFrame({\"times_exp2\": table_time.loc[model][exp2], \"errors_exp2\": table_error.loc[model][exp2]})\n", "\n", " exp1, exp2 = filter_errors(exp1, exp2)\n", " df_ = pd.concat([exp1, exp2], axis=1, sort=False)\n", "\n", " df_ = df_[df_[\"times_exp1\"] != df_[\"times_exp2\"]]\n", " df_['best'] = np.where((df_[\"times_exp1\"] < df_[\"times_exp2\"]), \"exp1\", \"exp2\")\n", " count = df_.groupby([\"best\"]).size()\n", "\n", " return count.get(\"exp1\",0), count.get(\"exp2\",0)\n", "\n", "\n", "def get_table(df_time, df_errors, model, exp1, exp2):\n", " \"\"\"Creates a table with some statistics from a dataframe for a model and experiments\"\"\"\n", " rows=[[\"Experiment\", \"Time Limit\", \"Memory\", \"Unknown Error\", \"OK\", \"Faster\"]]\n", " \n", " df_exp1 = pd.DataFrame({\"error\": df_errors.loc[model][exp1]})\n", " df_exp2 = pd.DataFrame({\"error\": df_errors.loc[model][exp2]})\n", " df_exp1, df_exp2 = filter_errors(df_exp1, df_exp2)\n", " \n", " info1 = get_info(df_exp1)\n", " info2 = get_info(df_exp2)\n", " \n", " best1, best2 = get_best_times(df_time, df_errors, model, exp1, exp2)\n", " \n", " for (experiment, info, best) in [(exp1, info1, best1), (exp2, info2, best2)]:\n", " rows.append([experiment, info[\"time limit\"], info[\"memory\"], info[\"error\"], info[\"OK\"], best])\n", " \n", " return ff.create_table(rows)\n" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "scrolled": false }, "outputs": [], "source": [ "import webbrowser\n", "\n", "def get_filename(base_path, tool, model, model_instance, formula):\n", " \"\"\"Returns the absolute path of the experiment log\n", " \n", " Parameters\n", " ----------\n", " base_path : string\n", " Path of the folder where logs are saved\n", " tool : string\n", " Tool name\n", " model : string\n", " Model name\n", " model_instance : string\n", " Name of the model instance\n", " formula : string\n", " Identifier of the formula\n", " \n", " Returns\n", " -------\n", " string\n", " Absolute path of the log file\n", " \n", " \"\"\"\n", " information = tool.split('_')\n", " \n", " tool_name = information[0]\n", " tool_configuration = '_'.join(information[:-2])\n", " nb_nodes = information[-2]\n", " nb_cores = information[-1]\n", " \n", " experiment_folder = os.path.join(base_path, tool_name, tool_configuration, model, model_instance)\n", " filename = f'{tool_name}_{model_instance}-n{nb_nodes}-th{nb_cores}-f{formula}'\n", " absolute_path = os.path.join(experiment_folder, filename)\n", " \n", " return absolute_path\n", "\n", "def open_logs_callback(trace, points, selector):\n", " \"\"\"Callback that open the log files when clicking on a point of the figure\n", " \n", " Parameters\n", " ----------\n", " trace : plotly.graph_objects.Figure\n", " the figure to attach the callback\n", " points : plotly.callbacks.Points \n", " points of the figure selected\n", " selector: plotly.callbacks.InputDeviceState \n", " Device information \n", " \"\"\"\n", " inds = points.point_inds\n", " if (inds):\n", " index = inds[0]\n", "\n", " formula, error_x, error_y = trace['customdata'][index]\n", " model_instance = trace['meta']['model']\n", " model = ''.join(c for c in model_instance if not c.isdigit())\n", " tools = trace['meta']['tools']\n", " logs_folder = trace['meta']['folder']\n", "\n", " filename_x = get_filename(logs_folder, tools['x'], model, model_instance, formula)\n", " filename_y = get_filename(logs_folder, tools['y'], model, model_instance, formula)\n", "\n", " for f in [filename_x, filename_y]:\n", " webbrowser.open(f'file://{f}.err')\n", " webbrowser.open(f'file://{f}.out')\n", "\n", "OPEN_LOGS_CALLBACK_JS = \"\"\"\n", "function get_filename (base_path, tool, model_instance, formula) {\n", " const information = tool.split('_');\n", " const size = information.length;\n", "\n", " const tool_name = information[0];\n", " const tool_configuration = information.slice(0, size - 2).join('_');\n", " const nb_nodes = information[size - 2];\n", " const nb_cores = information[size - 1];\n", "\n", " const model = model_instance.replace(/[0-9]/g, '');\n", "\n", " const experiment_folder = `${base_path}/${tool_name}/${tool_configuration}/${model}/${model_instance}`;\n", " const filename = `${tool_name}_${model_instance}-n${nb_nodes}-th${nb_cores}-f${formula}`;\n", "\n", " return `${experiment_folder}/${filename}`;\n", "}\n", "\n", "const plots = document.getElementsByClassName(\"plotly-graph-div js-plotly-plot\");\n", "const myPlot = plots[0];\n", "\n", "myPlot.on('plotly_click', function(data){\n", " const points = data.points;\n", " if (points.length != 1) {return ;}\n", " \n", " const myPoint = points[0];\n", " const formula = myPoint.customdata[0];\n", " const meta = myPoint.data.meta;\n", " \n", " const href = window.location.href.split('/');\n", " const base_path = href.splice(0,href.length-4).join('/');\n", " \n", " const filename_x = get_filename(base_path, meta.tools.x, meta.model, formula);\n", " const filename_y = get_filename(base_path, meta.tools.y, meta.model, formula);\n", " \n", " console.log('x: ' + filename_x);\n", " window.open(`${filename_x}.err`);\n", " window.open(`${filename_x}.out`);\n", " \n", " console.log('y: ' + filename_y);\n", " window.open(`${filename_y}.err`);\n", " window.open(`${filename_y}.out`);\n", "});\n", "\"\"\"\n" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "def create_figure_explored_states(table_explored_states, model):\n", " \"\"\"Creates figure showing the number of explorated states during the verification \n", " \n", " Parameters\n", " ----------\n", " table_explored_states : pandas.Dataframe\n", " Dataframe with the explorated states of each experiment\n", " model : string\n", " Model to be analyzed\n", " \n", " Returns\n", " -------\n", " plotly.graph_objects.Figure\n", " Scatter figure\n", " \"\"\"\n", " colors={'T': 'green', 'F': 'red'}\n", " float_formatter = \"{:.2E}\".format\n", "\n", " table_model = table_explored_states[table_explored_states.property != 'U']\n", " table_model = table_model[table_model.model == model]\n", "\n", " table_stats = table_model.groupby(['property']).agg(['mean']) \n", "\n", " fig = go.Figure()\n", " max_x = 0\n", " for p in table_stats.index:\n", " data = table_model[table_model.property==p]\n", " stats = table_stats.loc[p]\n", " \n", " x_axis = np.arange(1, data['formula'].count()+1, 1)\n", " max_x = max(max_x, x_axis[-1]+1)\n", " mean = stats['explored_states','mean']\n", " \n", " figure = px.scatter(data, \n", " x=x_axis, \n", " y=\"explored_states\",\n", " title=model, \n", " color='property',\n", " color_discrete_map=colors,\n", " symbol_sequence=[\"circle\"])\n", "\n", " line = go.Scatter(x=[x_axis[0], x_axis[-1]], \n", " y=[mean, mean],\n", " mode='lines', showlegend=False, \n", " line=dict(color=colors[p], width=1.5))\n", "\n", " fig.add_trace(figure['data'][0])\n", " fig.add_trace(line)\n", " \n", " fig.add_annotation(x=1, \n", " y=mean,\n", " font=dict(color=colors[p]),\n", " text=f\"mean = {float_formatter(mean)}\")\n", "\n", " fig.update_layout(title_text=model, title=LAYOUT_FIGURES['title'], \n", " width = 500, height = 500, margin=dict(r=110))\n", " \n", " fig.update_xaxes(title=\"formula\", range=[0, max_x])\n", " fig.update_yaxes(title=\"# explored states\")\n", " \n", " fig.update_annotations(dict(\n", " showarrow=False,\n", " xanchor=\"left\",\n", " yanchor=\"middle\",\n", " xref='paper'))\n", " \n", " return fig" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "def create_log_figure(table, table_errors, model, tool_x, tool_y, show_strategy=True, callback=None):\n", " \"\"\"Creates a Scatter figure in logarithmic scale comparing the performance of two tools\n", " \n", " Parameters\n", " ----------\n", " table : pandas.Dataframe\n", " Dataframe with the times of each experiment\n", " table_errors : pandas.Dataframe\n", " Dataframe with the errors of each experiment\n", " model : string\n", " Model to be analyzed\n", " tool_x : string\n", " Tool to be compared and plotted on the x-axis\n", " tool_y : string\n", " Tool to be compared and plotted on the y-axis\n", " show_strategy : bool\n", " Flag to show the stretagy used by the tools\n", " callback : function\n", " Function to be called when clicking on a point\n", " \n", " Returns\n", " -------\n", " plotly.graph_objects.Figure\n", " Scatter figure\n", " \n", " Examples\n", " --------\n", " >>> import os\n", " >>> import pandas as pd\n", " >>> csv_file = os.path.join(\"results\", \"output.csv\")\n", " >>> df = pd.read_csv(csv_file)\n", " >>> table = df.set_index(['model', 'formula', 'tool'], drop=True).unstack('tool')\n", " >>> fig = create_log_figure(table['time'], table['error'], 'philo10', 'pmc-sog_otfL_couv99-default_1_8', 'pmc-sog_otfP_couv99-default_1_8')\n", " \"\"\"\n", " try:\n", " min_value = ZERO\n", " max_value = TIMEOUT\n", " \n", " min_value_log = np.log10(min_value)\n", " max_value_log = np.log10(max_value)\n", "\n", " table_model = table.loc[model]\n", " table_errors_model = table_error.loc[model]\n", " \n", " full_table_x = pd.concat([table_model[tool_x],table_model['property'], table_errors_model[tool_x]], axis=1)\n", " full_table_x.columns = ['time', 'property', 'error']\n", "\n", " full_table_y = pd.concat([table_model[tool_y],table_model['property'], table_errors_model[tool_y]], axis=1)\n", " full_table_y.columns = ['time', 'property', 'error']\n", " \n", " full_table_x, full_table_y = filter_errors(full_table_x, full_table_y)\n", "\n", " traces = [\n", " {\"property\": 'T', \"color\":\"green\", \"label\": \"True\"},\n", " {\"property\": 'F', \"color\":\"red\", \"label\": \"False\"},\n", " {\"property\": 'U', \"color\":\"black\", \"label\": \"Unknown\"}\n", " ]\n", "\n", " figures = []\n", " for t in traces:\n", " # filter by verification output\n", " table_x = full_table_x[full_table_x.property == t['property']]\n", " table_y = full_table_y[full_table_y.property == t['property']]\n", "\n", " # custom data\n", " custom_data = list(zip(table_x.index, table_x.error,table_y.error))\n", " \n", " # tools\n", " metainfo = {\n", " 'model': model, \n", " 'tools': {'x': tool_x, 'y': tool_y},\n", " 'folder': os.path.join(os.path.abspath(os.pardir), \"results\")\n", " }\n", "\n", " figures.append(go.Scatter(x=table_x.time,\n", " y=table_y.time,\n", " name=t['label'],\n", " mode='markers',\n", " marker_symbol='circle-open',\n", " marker_color=t['color'],\n", " meta = metainfo,\n", " customdata=custom_data,\n", " hovertemplate =\n", " 'Formula # %{customdata[0]}
' +\n", " '
Times:
' +\n", " 'x: %{x} s' +\n", " '
y: %{y} s
' +\n", " '
Errors:
' +\n", " 'x: %{customdata[1]}
' +\n", " 'y: %{customdata[2]}',\n", " )) \n", "\n", " # Line\n", " figures.append(go.Scatter(x=[min_value, max_value], \n", " y=[min_value, max_value],\n", " mode='lines', showlegend=False,\n", " line=dict(color='black', width=1)))\n", "\n", " # Create figure\n", " figure = go.FigureWidget(figures)\n", " figure.update_layout(LAYOUT_FIGURES,\n", " title_text=model,\n", " hoverlabel=dict(bgcolor=\"white\", align='auto'),\n", " legend_title_text='property',\n", " xaxis=dict(type='log', autorange=False, range=[min_value_log, max_value_log]),\n", " yaxis=dict(type='log', autorange=False, range=[min_value_log, max_value_log]),\n", " xaxis_title=get_axis_title(tool_x, show_strategy),\n", " yaxis_title=get_axis_title(tool_y, show_strategy))\n", "\n", " # Add event\n", " if callback is not None:\n", " for i in range(len(figure.data)):\n", " figure.data[i].on_click(callback)\n", " \n", " return figure\n", " except Exception as e:\n", " print(\"Error when ploting model: {} - tool_x: {} - tool_y: {}\".format(model, tool_x, tool_y))\n", " print(e)" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "# Experiment filters\n", "\n", "def versus_dfs(experiments):\n", " \"\"\"Selects only experiments using DFS strategy\"\"\"\n", " exp1, exp2 = experiments\n", " strategy_exp1= exp1.split('_')[1]\n", " strategy_exp2= exp2.split('_')[1]\n", " \n", " return strategy_exp1 == 'dfs' or strategy_exp2 == 'dfs'\n", "\n", "def versus_sequential(experiments):\n", " \"\"\"Selects only experiments run sequentially \"\"\"\n", " exp1, exp2 = experiments\n", " nodes_exp1, threads_exp1 = exp1.split('_')[-2:]\n", " nodes_exp2, threads_exp2 = exp2.split('_')[-2:]\n", "\n", " return (nodes_exp1 == '1' and nodes_exp2 == '1') and \\\n", " (threads_exp1 == '1' or threads_exp2 == '1')\n", "\n", "def same_tool(experiments, tool):\n", " \"\"\"Selects only experiments comparing the same tool\"\"\"\n", " exp1, exp2 = experiments\n", " tool_exp1= exp1.split('_')[0]\n", " tool_exp2= exp2.split('_')[0]\n", " return tool_exp1.startswith(tool) and tool_exp2.startswith(tool)\n", "\n", "def same_number_threads(experiments):\n", " \"\"\"Selects only experiments comparing the same number of processes and cores\"\"\"\n", " exp1, exp2 = experiments\n", " nodes_exp1, threads_exp1 = exp1.split('_')[-2:]\n", " nodes_exp2, threads_exp2 = exp2.split('_')[-2:]\n", " return (nodes_exp1 == nodes_exp2) and (threads_exp1 == threads_exp2) \n", "\n", "def same_thread_library(experiments):\n", " \"\"\"Selects only experiments comparing the same parallelization library\"\"\"\n", " exp1, exp2 = experiments\n", " library_exp1 = exp1.split('_')[1]\n", " library_exp2 = exp2.split('_')[1]\n", " return library_exp1 == library_exp2\n", "\n", "def same_strategy(experiments):\n", " \"\"\"Selects only experiments comparing the same strategy\"\"\"\n", " exp1, exp2 = experiments\n", " strategy_exp1 = exp1.split('_')[2]\n", " strategy_exp2 = exp2.split('_')[2]\n", " return strategy_exp1 == strategy_exp2\n", "\n", "def only_couvreur_strategy(experiments):\n", " \"\"\"Selects only experiments comparing couvreur emptiness check algorithm\"\"\"\n", " exp1, exp2 = experiments\n", " strategy_exp1 = exp1.split('_')[2]\n", " strategy_exp2 = exp2.split('_')[2]\n", " return strategy_exp1.startswith('couv99') and strategy_exp2.startswith('couv99')\n", "\n", "def compare_threads_library(experiments):\n", " \"\"\"Compares parallization libraries used in pmc-sog. \n", " \n", " It selects experiments where the tool is only pmc-sog and the strategy, number of threads, \n", " number of processus are the same.\n", " \"\"\"\n", " return same_tool(experiments, 'pmc-sog') and \\\n", " same_strategy(experiments) and \\\n", " same_number_threads(experiments) and \\\n", " not same_thread_library(experiments)\n", "\n", "def compare_couvreur_strategies(experiments):\n", " \"\"\"Compares couvreurs strategies used in pmc-sog. \n", " \n", " It selects experiments where the tool is only pmc-sog, the strategy is couvreur, and \n", " the parallelization library, number of threads, number of processus are the same.\n", " \"\"\"\n", " return only_couvreur_strategy(experiments) and \\\n", " same_thread_library(experiments) and \\\n", " same_number_threads(experiments)\n", "\n", "def same_distributed_number_threads(experiments):\n", " \"\"\"Selects only experiments where the multiplication of theirs nodes with cores are the same.\"\"\"\n", " exp1, exp2 = experiments\n", " nodes_exp1, threads_exp1 = exp1.split('_')[-2:]\n", " nodes_exp2, threads_exp2 = exp2.split('_')[-2:]\n", " return (int(nodes_exp1) * int(threads_exp1)) == (int(nodes_exp2) * int(threads_exp2))\n", "\n", "def compare_tools(experiments):\n", " \"\"\"Compares pmc-sog and pnml2lts-mc using the DFS algorithm. \n", " \n", " It selects experiments where the tools are not the same, the exploration algorithm is DFS and \n", " the number of processus and cores are the same.\n", " \"\"\"\n", " return not (same_tool(experiments, 'pmc-sog') or same_tool(experiments,'pnml2lts-mc')) and \\\n", " versus_dfs(experiments)\n", "\n", "def compare_multithreading(experiments):\n", " \"\"\"Compares the sequential and multi-core version of pmc-sog. \n", " \n", " It selects experiments where the tools is pmc-sog, the parallelization library, the emptiness check \n", " strategy are the same. Here the number of processus and cores are different.\n", " \"\"\"\n", " return same_tool(experiments, 'pmc-sog') and \\\n", " same_thread_library(experiments) and \\\n", " same_strategy(experiments) and \\\n", " versus_sequential(experiments)\n", "\n", "def against_hybrid(experiments):\n", " \"\"\"Selects only experiments comparing with hybrid mode\"\"\"\n", " exp1, exp2 = experiments\n", " library_exp1 = exp1.split('_')[1]\n", " library_exp2 = exp2.split('_')[1]\n", " return (library_exp1 == 'otf') or (library_exp2 == 'otf')\n", "\n", "\n", "def compare_distributed(experiments):\n", " \"\"\"Compares the hybrid version of pmc-sog\"\"\"\n", " return same_tool(experiments, 'pmc-sog') and \\\n", " same_strategy(experiments) and \\\n", " same_distributed_number_threads(experiments) and \\\n", " against_hybrid(experiments)\n", "\n", "def compare_others(experiments):\n", " return (not compare_threads_library(experiments)) and \\\n", " (not compare_couvreur_strategies(experiments)) and \\\n", " (not compare_tools(experiments)) and \\\n", " (not compare_multithreading(experiments)) and \\\n", " (not compare_distributed(experiments))\n", "\n", "# Plots to be created\n", "plots = {\n", " 'compare_thread_library': compare_threads_library,\n", " 'compare_couvreur_algorithm': compare_couvreur_strategies,\n", " 'compare_tools': compare_tools,\n", " 'compare_multicore': compare_multithreading,\n", " 'compare_distributed': compare_distributed,\n", " 'others' : compare_others\n", "}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Load Data" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [], "source": [ "# Root folder\n", "PROJECT_FOLDER = os.path.abspath(os.pardir)\n", "\n", "# csv file with the output\n", "csv_file = os.path.join(PROJECT_FOLDER, \"results\", \"output.csv\")\n", "\n", "# formulas folder\n", "FORMULAS_FOLDER = os.path.join(PROJECT_FOLDER, \"formulas\")\n", "\n", "# Output folder\n", "OUTPUT_FOLDER = os.path.join(PROJECT_FOLDER,\"results\", \"figures\")\n", "create_folder(OUTPUT_FOLDER)" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
modelformulatooltimepropertyexplored_stateserror
0philo101pmc-sog_otfC_couv99-default_1_166.894FNaNOK
1philo101pmc-sog_otfC_couv99-shy_1_167.070FNaNOK
2philo101pmc-sog_otfP_couv99-default_1_167.622FNaNOK
3philo101pmc-sog_otfP_couv99-shy_1_168.650FNaNOK
4philo101pmc-sog_otf_couv99-default_2_811.579FNaNOK
........................
27195tring5200pmc-sog_otf_couv99-default_2_160.829FNaNOK
27196tring5200pmc-sog_otf_couv99-shy_2_80.807FNaNOK
27197tring5200pmc-sog_otf_couv99-shy_2_160.813FNaNOK
27198tring5200pnml2lts-mc_dfs_1_160.160NaN1540.0OK
27199tring5200pnml2lts-mc_ndfs_1_160.270F9043.0OK
\n", "

24400 rows × 7 columns

\n", "
" ], "text/plain": [ " model formula tool time property \\\n", "0 philo10 1 pmc-sog_otfC_couv99-default_1_16 6.894 F \n", "1 philo10 1 pmc-sog_otfC_couv99-shy_1_16 7.070 F \n", "2 philo10 1 pmc-sog_otfP_couv99-default_1_16 7.622 F \n", "3 philo10 1 pmc-sog_otfP_couv99-shy_1_16 8.650 F \n", "4 philo10 1 pmc-sog_otf_couv99-default_2_8 11.579 F \n", "... ... ... ... ... ... \n", "27195 tring5 200 pmc-sog_otf_couv99-default_2_16 0.829 F \n", "27196 tring5 200 pmc-sog_otf_couv99-shy_2_8 0.807 F \n", "27197 tring5 200 pmc-sog_otf_couv99-shy_2_16 0.813 F \n", "27198 tring5 200 pnml2lts-mc_dfs_1_16 0.160 NaN \n", "27199 tring5 200 pnml2lts-mc_ndfs_1_16 0.270 F \n", "\n", " explored_states error \n", "0 NaN OK \n", "1 NaN OK \n", "2 NaN OK \n", "3 NaN OK \n", "4 NaN OK \n", "... ... ... \n", "27195 NaN OK \n", "27196 NaN OK \n", "27197 NaN OK \n", "27198 1540.0 OK \n", "27199 9043.0 OK \n", "\n", "[24400 rows x 7 columns]" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# read data\n", "df = pd.read_csv(csv_file)\n", "\n", "# merge the information related to the experiment (# nodes, # threads, strategy) to the tool column\n", "df['tool'] = df[['tool', 'strategy', 'num_nodes', 'num_threads']].astype(str).apply('_'.join, axis=1)\n", "df = df.drop(columns=['strategy', 'num_nodes', 'num_threads'])\n", "\n", "# FIX: filtering philo20 experiments because there is a problem with the generated formulas\n", "df = df[df.model != \"philo20\"]\n", "df" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
modelformulatooltimepropertyexplored_stateserror
0philo101pmc-sog_otfC_couv99-default_1_166.894FNaNOK
1philo101pmc-sog_otfC_couv99-shy_1_167.070FNaNOK
2philo101pmc-sog_otfP_couv99-default_1_167.622FNaNOK
3philo101pmc-sog_otfP_couv99-shy_1_168.650FNaNOK
4philo101pmc-sog_otf_couv99-default_2_811.579FNaNOK
........................
21645tring5200pmc-sog_otf_couv99-default_2_160.829FNaNOK
21646tring5200pmc-sog_otf_couv99-shy_2_80.807FNaNOK
21647tring5200pmc-sog_otf_couv99-shy_2_160.813FNaNOK
21648tring5200pnml2lts-mc_dfs_1_160.160NaN1540.0OK
21649tring5200pnml2lts-mc_ndfs_1_160.270F9043.0OK
\n", "

21650 rows × 7 columns

\n", "
" ], "text/plain": [ " model formula tool time property \\\n", "0 philo10 1 pmc-sog_otfC_couv99-default_1_16 6.894 F \n", "1 philo10 1 pmc-sog_otfC_couv99-shy_1_16 7.070 F \n", "2 philo10 1 pmc-sog_otfP_couv99-default_1_16 7.622 F \n", "3 philo10 1 pmc-sog_otfP_couv99-shy_1_16 8.650 F \n", "4 philo10 1 pmc-sog_otf_couv99-default_2_8 11.579 F \n", "... ... ... ... ... ... \n", "21645 tring5 200 pmc-sog_otf_couv99-default_2_16 0.829 F \n", "21646 tring5 200 pmc-sog_otf_couv99-shy_2_8 0.807 F \n", "21647 tring5 200 pmc-sog_otf_couv99-shy_2_16 0.813 F \n", "21648 tring5 200 pnml2lts-mc_dfs_1_16 0.160 NaN \n", "21649 tring5 200 pnml2lts-mc_ndfs_1_16 0.270 F \n", "\n", " explored_states error \n", "0 NaN OK \n", "1 NaN OK \n", "2 NaN OK \n", "3 NaN OK \n", "4 NaN OK \n", "... ... ... \n", "21645 NaN OK \n", "21646 NaN OK \n", "21647 NaN OK \n", "21648 1540.0 OK \n", "21649 9043.0 OK \n", "\n", "[21650 rows x 7 columns]" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# filtering runtime errors\n", "df = df[(df.error != \"SEGMENTATION FAULT\") & \\\n", " (df.error != \"ABORTED\") & \\\n", " (df.error != \"TERMINATE\") & \\\n", " (df.error != \"MDD\")]\n", "\n", "df = df.reset_index(drop=True)\n", "df" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
property
modelformula
philo101F
2F
3F
4F
5F
.........
tring5196F
197F
198F
199F
200F
\n", "

1800 rows × 1 columns

\n", "
" ], "text/plain": [ " property\n", "model formula \n", "philo10 1 F\n", " 2 F\n", " 3 F\n", " 4 F\n", " 5 F\n", "... ...\n", "tring5 196 F\n", " 197 F\n", " 198 F\n", " 199 F\n", " 200 F\n", "\n", "[1800 rows x 1 columns]" ] }, "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# ground truth for properties\n", "frames = []\n", "\n", "formula_results = glob.glob(os.path.join(FORMULAS_FOLDER, \"**/formula_results\"), recursive=True)\n", "for f in formula_results:\n", " model, out_file = f.split('/')[-2:]\n", " \n", " tmp_df = pd.read_csv(f, sep=\";\", header=None, names=[\"formula\", \"property\"])\n", " tmp_df[\"model\"] = model\n", " frames.append(tmp_df)\n", " \n", "p_df = pd.concat(frames)\n", "p_df = p_df.reindex(columns=[\"model\", \"formula\", \"property\"])\n", "p_df = p_df[p_df['model'].isin(df.model.unique())]\n", "p_df['property'] = p_df['property'].replace([True, False], [\"T\", \"F\"])\n", "p_df = p_df.set_index([\"model\", \"formula\"])\n", "p_df.sort_index(inplace=True)\n", "\n", "p_df" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
time...error
toolpmc-sog_otfC_couv99-default_1_1pmc-sog_otfC_couv99-default_1_16pmc-sog_otfC_couv99-shy_1_1pmc-sog_otfC_couv99-shy_1_16pmc-sog_otfP_couv99-default_1_1pmc-sog_otfP_couv99-default_1_16pmc-sog_otfP_couv99-shy_1_1pmc-sog_otfP_couv99-shy_1_16pmc-sog_otf_couv99-default_2_16pmc-sog_otf_couv99-default_2_8...pmc-sog_otf_couv99-default_4_16pmc-sog_otf_couv99-default_4_8pmc-sog_otf_couv99-shy_2_16pmc-sog_otf_couv99-shy_2_8pmc-sog_otf_couv99-shy_4_16pmc-sog_otf_couv99-shy_4_8pnml2lts-mc_dfs_1_1pnml2lts-mc_dfs_1_16pnml2lts-mc_ndfs_1_1pnml2lts-mc_ndfs_1_16
modelformula
philo101NaN6.894NaN7.070NaN7.622NaN8.65022.02511.579...OKOKOKOKOKOKNaNOKNaNOK
2NaN4.048NaN3.976NaN4.092NaN4.48013.09310.419...OKOKOKOKOKOKNaNOKNaNOK
3NaN3.291NaN3.623NaN3.369NaN3.91510.4786.654...OKOKOKOKOKOKNaNOKNaNOK
4NaN3.375NaN4.280NaN4.478NaN4.28518.1398.911...OKOKOKOKOKOKNaNOKNaNOK
5NaN5.874NaN5.811NaN7.777NaN7.03224.45311.391...OKOKOKOKOKOKNaNOKNaNOK
.....................................................................
tring5196NaN0.075NaN0.075NaN0.104NaN0.1080.8260.831...NaNNaNOKOKNaNNaNNaNOKNaNOK
197NaN0.109NaN0.109NaN0.131NaN0.1300.8190.833...NaNNaNOKOKNaNNaNNaNOKNaNOK
198NaN0.130NaN0.134NaN0.144NaN0.1520.8400.833...NaNNaNOKOKNaNNaNNaNOKNaNOK
199NaN0.105NaN0.106NaN0.114NaN0.1220.8440.836...NaNNaNOKOKNaNNaNNaNOKNaNOK
200NaN0.080NaN0.074NaN0.086NaN0.0930.8290.816...NaNNaNOKOKNaNNaNNaNOKNaNOK
\n", "

1800 rows × 80 columns

\n", "
" ], "text/plain": [ " time \\\n", "tool pmc-sog_otfC_couv99-default_1_1 \n", "model formula \n", "philo10 1 NaN \n", " 2 NaN \n", " 3 NaN \n", " 4 NaN \n", " 5 NaN \n", "... ... \n", "tring5 196 NaN \n", " 197 NaN \n", " 198 NaN \n", " 199 NaN \n", " 200 NaN \n", "\n", " \\\n", "tool pmc-sog_otfC_couv99-default_1_16 pmc-sog_otfC_couv99-shy_1_1 \n", "model formula \n", "philo10 1 6.894 NaN \n", " 2 4.048 NaN \n", " 3 3.291 NaN \n", " 4 3.375 NaN \n", " 5 5.874 NaN \n", "... ... ... \n", "tring5 196 0.075 NaN \n", " 197 0.109 NaN \n", " 198 0.130 NaN \n", " 199 0.105 NaN \n", " 200 0.080 NaN \n", "\n", " \\\n", "tool pmc-sog_otfC_couv99-shy_1_16 pmc-sog_otfP_couv99-default_1_1 \n", "model formula \n", "philo10 1 7.070 NaN \n", " 2 3.976 NaN \n", " 3 3.623 NaN \n", " 4 4.280 NaN \n", " 5 5.811 NaN \n", "... ... ... \n", "tring5 196 0.075 NaN \n", " 197 0.109 NaN \n", " 198 0.134 NaN \n", " 199 0.106 NaN \n", " 200 0.074 NaN \n", "\n", " \\\n", "tool pmc-sog_otfP_couv99-default_1_16 pmc-sog_otfP_couv99-shy_1_1 \n", "model formula \n", "philo10 1 7.622 NaN \n", " 2 4.092 NaN \n", " 3 3.369 NaN \n", " 4 4.478 NaN \n", " 5 7.777 NaN \n", "... ... ... \n", "tring5 196 0.104 NaN \n", " 197 0.131 NaN \n", " 198 0.144 NaN \n", " 199 0.114 NaN \n", " 200 0.086 NaN \n", "\n", " \\\n", "tool pmc-sog_otfP_couv99-shy_1_16 pmc-sog_otf_couv99-default_2_16 \n", "model formula \n", "philo10 1 8.650 22.025 \n", " 2 4.480 13.093 \n", " 3 3.915 10.478 \n", " 4 4.285 18.139 \n", " 5 7.032 24.453 \n", "... ... ... \n", "tring5 196 0.108 0.826 \n", " 197 0.130 0.819 \n", " 198 0.152 0.840 \n", " 199 0.122 0.844 \n", " 200 0.093 0.829 \n", "\n", " ... \\\n", "tool pmc-sog_otf_couv99-default_2_8 ... \n", "model formula ... \n", "philo10 1 11.579 ... \n", " 2 10.419 ... \n", " 3 6.654 ... \n", " 4 8.911 ... \n", " 5 11.391 ... \n", "... ... ... \n", "tring5 196 0.831 ... \n", " 197 0.833 ... \n", " 198 0.833 ... \n", " 199 0.836 ... \n", " 200 0.816 ... \n", "\n", " error \\\n", "tool pmc-sog_otf_couv99-default_4_16 \n", "model formula \n", "philo10 1 OK \n", " 2 OK \n", " 3 OK \n", " 4 OK \n", " 5 OK \n", "... ... \n", "tring5 196 NaN \n", " 197 NaN \n", " 198 NaN \n", " 199 NaN \n", " 200 NaN \n", "\n", " \\\n", "tool pmc-sog_otf_couv99-default_4_8 pmc-sog_otf_couv99-shy_2_16 \n", "model formula \n", "philo10 1 OK OK \n", " 2 OK OK \n", " 3 OK OK \n", " 4 OK OK \n", " 5 OK OK \n", "... ... ... \n", "tring5 196 NaN OK \n", " 197 NaN OK \n", " 198 NaN OK \n", " 199 NaN OK \n", " 200 NaN OK \n", "\n", " \\\n", "tool pmc-sog_otf_couv99-shy_2_8 pmc-sog_otf_couv99-shy_4_16 \n", "model formula \n", "philo10 1 OK OK \n", " 2 OK OK \n", " 3 OK OK \n", " 4 OK OK \n", " 5 OK OK \n", "... ... ... \n", "tring5 196 OK NaN \n", " 197 OK NaN \n", " 198 OK NaN \n", " 199 OK NaN \n", " 200 OK NaN \n", "\n", " \\\n", "tool pmc-sog_otf_couv99-shy_4_8 pnml2lts-mc_dfs_1_1 \n", "model formula \n", "philo10 1 OK NaN \n", " 2 OK NaN \n", " 3 OK NaN \n", " 4 OK NaN \n", " 5 OK NaN \n", "... ... ... \n", "tring5 196 NaN NaN \n", " 197 NaN NaN \n", " 198 NaN NaN \n", " 199 NaN NaN \n", " 200 NaN NaN \n", "\n", " \\\n", "tool pnml2lts-mc_dfs_1_16 pnml2lts-mc_ndfs_1_1 \n", "model formula \n", "philo10 1 OK NaN \n", " 2 OK NaN \n", " 3 OK NaN \n", " 4 OK NaN \n", " 5 OK NaN \n", "... ... ... \n", "tring5 196 OK NaN \n", " 197 OK NaN \n", " 198 OK NaN \n", " 199 OK NaN \n", " 200 OK NaN \n", "\n", " \n", "tool pnml2lts-mc_ndfs_1_16 \n", "model formula \n", "philo10 1 OK \n", " 2 OK \n", " 3 OK \n", " 4 OK \n", " 5 OK \n", "... ... \n", "tring5 196 OK \n", " 197 OK \n", " 198 OK \n", " 199 OK \n", " 200 OK \n", "\n", "[1800 rows x 80 columns]" ] }, "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# table with times, verification output and error for each experiment\n", "table = df.set_index(['model', 'formula', 'tool'], drop=True).unstack('tool')\n", "table" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Preprocessing of data" ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
pmc-sog_otfC_couv99-default_1_1pmc-sog_otfC_couv99-default_1_16pmc-sog_otfC_couv99-shy_1_1pmc-sog_otfC_couv99-shy_1_16pmc-sog_otfP_couv99-default_1_1pmc-sog_otfP_couv99-default_1_16pmc-sog_otfP_couv99-shy_1_1pmc-sog_otfP_couv99-shy_1_16pmc-sog_otf_couv99-default_2_16pmc-sog_otf_couv99-default_2_8...pmc-sog_otf_couv99-default_4_8pmc-sog_otf_couv99-shy_2_16pmc-sog_otf_couv99-shy_2_8pmc-sog_otf_couv99-shy_4_16pmc-sog_otf_couv99-shy_4_8pnml2lts-mc_dfs_1_1pnml2lts-mc_dfs_1_16pnml2lts-mc_ndfs_1_1pnml2lts-mc_ndfs_1_16property
modelformula
philo101600.06.894600.07.070600.07.622600.08.65022.02511.579...3.95022.80812.0034.4504.295600.00.73600.00.21F
2600.04.048600.03.976600.04.092600.04.48013.09310.419...2.41812.8669.9142.5072.280600.00.60600.00.14F
3600.03.291600.03.623600.03.369600.03.91510.4786.654...1.91911.4228.2562.1362.094600.01.24600.00.20F
4600.03.375600.04.280600.04.478600.04.28518.1398.911...2.67915.9079.4733.0112.799600.00.65600.00.27F
5600.05.874600.05.811600.07.777600.07.03224.45311.391...4.00124.32710.8544.4154.034600.00.67600.00.11F
.....................................................................
tring5196600.00.075600.00.075600.00.104600.00.1080.8260.831...600.0000.8360.828600.000600.000600.00.33600.00.38F
197600.00.109600.00.109600.00.131600.00.1300.8190.833...600.0000.8350.830600.000600.000600.00.32600.00.49F
198600.00.130600.00.134600.00.144600.00.1520.8400.833...600.0000.8470.842600.000600.000600.00.47600.00.41F
199600.00.105600.00.106600.00.114600.00.1220.8440.836...600.0000.8350.828600.000600.000600.00.19600.00.25F
200600.00.080600.00.074600.00.086600.00.0930.8290.816...600.0000.8130.807600.000600.000600.00.16600.00.27F
\n", "

1800 rows × 21 columns

\n", "
" ], "text/plain": [ " pmc-sog_otfC_couv99-default_1_1 \\\n", "model formula \n", "philo10 1 600.0 \n", " 2 600.0 \n", " 3 600.0 \n", " 4 600.0 \n", " 5 600.0 \n", "... ... \n", "tring5 196 600.0 \n", " 197 600.0 \n", " 198 600.0 \n", " 199 600.0 \n", " 200 600.0 \n", "\n", " pmc-sog_otfC_couv99-default_1_16 \\\n", "model formula \n", "philo10 1 6.894 \n", " 2 4.048 \n", " 3 3.291 \n", " 4 3.375 \n", " 5 5.874 \n", "... ... \n", "tring5 196 0.075 \n", " 197 0.109 \n", " 198 0.130 \n", " 199 0.105 \n", " 200 0.080 \n", "\n", " pmc-sog_otfC_couv99-shy_1_1 pmc-sog_otfC_couv99-shy_1_16 \\\n", "model formula \n", "philo10 1 600.0 7.070 \n", " 2 600.0 3.976 \n", " 3 600.0 3.623 \n", " 4 600.0 4.280 \n", " 5 600.0 5.811 \n", "... ... ... \n", "tring5 196 600.0 0.075 \n", " 197 600.0 0.109 \n", " 198 600.0 0.134 \n", " 199 600.0 0.106 \n", " 200 600.0 0.074 \n", "\n", " pmc-sog_otfP_couv99-default_1_1 \\\n", "model formula \n", "philo10 1 600.0 \n", " 2 600.0 \n", " 3 600.0 \n", " 4 600.0 \n", " 5 600.0 \n", "... ... \n", "tring5 196 600.0 \n", " 197 600.0 \n", " 198 600.0 \n", " 199 600.0 \n", " 200 600.0 \n", "\n", " pmc-sog_otfP_couv99-default_1_16 \\\n", "model formula \n", "philo10 1 7.622 \n", " 2 4.092 \n", " 3 3.369 \n", " 4 4.478 \n", " 5 7.777 \n", "... ... \n", "tring5 196 0.104 \n", " 197 0.131 \n", " 198 0.144 \n", " 199 0.114 \n", " 200 0.086 \n", "\n", " pmc-sog_otfP_couv99-shy_1_1 pmc-sog_otfP_couv99-shy_1_16 \\\n", "model formula \n", "philo10 1 600.0 8.650 \n", " 2 600.0 4.480 \n", " 3 600.0 3.915 \n", " 4 600.0 4.285 \n", " 5 600.0 7.032 \n", "... ... ... \n", "tring5 196 600.0 0.108 \n", " 197 600.0 0.130 \n", " 198 600.0 0.152 \n", " 199 600.0 0.122 \n", " 200 600.0 0.093 \n", "\n", " pmc-sog_otf_couv99-default_2_16 \\\n", "model formula \n", "philo10 1 22.025 \n", " 2 13.093 \n", " 3 10.478 \n", " 4 18.139 \n", " 5 24.453 \n", "... ... \n", "tring5 196 0.826 \n", " 197 0.819 \n", " 198 0.840 \n", " 199 0.844 \n", " 200 0.829 \n", "\n", " pmc-sog_otf_couv99-default_2_8 ... \\\n", "model formula ... \n", "philo10 1 11.579 ... \n", " 2 10.419 ... \n", " 3 6.654 ... \n", " 4 8.911 ... \n", " 5 11.391 ... \n", "... ... ... \n", "tring5 196 0.831 ... \n", " 197 0.833 ... \n", " 198 0.833 ... \n", " 199 0.836 ... \n", " 200 0.816 ... \n", "\n", " pmc-sog_otf_couv99-default_4_8 pmc-sog_otf_couv99-shy_2_16 \\\n", "model formula \n", "philo10 1 3.950 22.808 \n", " 2 2.418 12.866 \n", " 3 1.919 11.422 \n", " 4 2.679 15.907 \n", " 5 4.001 24.327 \n", "... ... ... \n", "tring5 196 600.000 0.836 \n", " 197 600.000 0.835 \n", " 198 600.000 0.847 \n", " 199 600.000 0.835 \n", " 200 600.000 0.813 \n", "\n", " pmc-sog_otf_couv99-shy_2_8 pmc-sog_otf_couv99-shy_4_16 \\\n", "model formula \n", "philo10 1 12.003 4.450 \n", " 2 9.914 2.507 \n", " 3 8.256 2.136 \n", " 4 9.473 3.011 \n", " 5 10.854 4.415 \n", "... ... ... \n", "tring5 196 0.828 600.000 \n", " 197 0.830 600.000 \n", " 198 0.842 600.000 \n", " 199 0.828 600.000 \n", " 200 0.807 600.000 \n", "\n", " pmc-sog_otf_couv99-shy_4_8 pnml2lts-mc_dfs_1_1 \\\n", "model formula \n", "philo10 1 4.295 600.0 \n", " 2 2.280 600.0 \n", " 3 2.094 600.0 \n", " 4 2.799 600.0 \n", " 5 4.034 600.0 \n", "... ... ... \n", "tring5 196 600.000 600.0 \n", " 197 600.000 600.0 \n", " 198 600.000 600.0 \n", " 199 600.000 600.0 \n", " 200 600.000 600.0 \n", "\n", " pnml2lts-mc_dfs_1_16 pnml2lts-mc_ndfs_1_1 \\\n", "model formula \n", "philo10 1 0.73 600.0 \n", " 2 0.60 600.0 \n", " 3 1.24 600.0 \n", " 4 0.65 600.0 \n", " 5 0.67 600.0 \n", "... ... ... \n", "tring5 196 0.33 600.0 \n", " 197 0.32 600.0 \n", " 198 0.47 600.0 \n", " 199 0.19 600.0 \n", " 200 0.16 600.0 \n", "\n", " pnml2lts-mc_ndfs_1_16 property \n", "model formula \n", "philo10 1 0.21 F \n", " 2 0.14 F \n", " 3 0.20 F \n", " 4 0.27 F \n", " 5 0.11 F \n", "... ... ... \n", "tring5 196 0.38 F \n", " 197 0.49 F \n", " 198 0.41 F \n", " 199 0.25 F \n", " 200 0.27 F \n", "\n", "[1800 rows x 21 columns]" ] }, "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# table with times for each experiment\n", "table_time = table['time'].copy()\n", "\n", "# replace non finished experiments with a dummy value, e.g. timeout\n", "table_time.fillna(TIMEOUT, inplace=True)\n", "\n", "# replace 0.00 time for 10^(-5), we cannot plot log(0)\n", "table_time.replace(0.0, ZERO, inplace=True)\n", "\n", "# add verification output to the table\n", "table_time = pd.concat([table_time, p_df], axis=1)\n", "\n", "table_time" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
pmc-sog_otfC_couv99-default_1_1pmc-sog_otfC_couv99-default_1_16pmc-sog_otfC_couv99-shy_1_1pmc-sog_otfC_couv99-shy_1_16pmc-sog_otfP_couv99-default_1_1pmc-sog_otfP_couv99-default_1_16pmc-sog_otfP_couv99-shy_1_1pmc-sog_otfP_couv99-shy_1_16pmc-sog_otf_couv99-default_2_16pmc-sog_otf_couv99-default_2_8...pmc-sog_otf_couv99-default_4_8pmc-sog_otf_couv99-shy_2_16pmc-sog_otf_couv99-shy_2_8pmc-sog_otf_couv99-shy_4_16pmc-sog_otf_couv99-shy_4_8pnml2lts-mc_dfs_1_1pnml2lts-mc_dfs_1_16pnml2lts-mc_ndfs_1_1pnml2lts-mc_ndfs_1_16property
modelformula
philo101UFUFUFUFFF...FFFFFUUUFF
2UFUFUFUFFF...FFFFFUUUFF
3UFUFUFUFFF...FFFFFUUUFF
4UFUFUFUFFF...FFFFFUUUFF
5UFUFUFUFFF...FFFFFUUUFF
\n", "

5 rows × 21 columns

\n", "
" ], "text/plain": [ " pmc-sog_otfC_couv99-default_1_1 \\\n", "model formula \n", "philo10 1 U \n", " 2 U \n", " 3 U \n", " 4 U \n", " 5 U \n", "\n", " pmc-sog_otfC_couv99-default_1_16 pmc-sog_otfC_couv99-shy_1_1 \\\n", "model formula \n", "philo10 1 F U \n", " 2 F U \n", " 3 F U \n", " 4 F U \n", " 5 F U \n", "\n", " pmc-sog_otfC_couv99-shy_1_16 pmc-sog_otfP_couv99-default_1_1 \\\n", "model formula \n", "philo10 1 F U \n", " 2 F U \n", " 3 F U \n", " 4 F U \n", " 5 F U \n", "\n", " pmc-sog_otfP_couv99-default_1_16 pmc-sog_otfP_couv99-shy_1_1 \\\n", "model formula \n", "philo10 1 F U \n", " 2 F U \n", " 3 F U \n", " 4 F U \n", " 5 F U \n", "\n", " pmc-sog_otfP_couv99-shy_1_16 pmc-sog_otf_couv99-default_2_16 \\\n", "model formula \n", "philo10 1 F F \n", " 2 F F \n", " 3 F F \n", " 4 F F \n", " 5 F F \n", "\n", " pmc-sog_otf_couv99-default_2_8 ... \\\n", "model formula ... \n", "philo10 1 F ... \n", " 2 F ... \n", " 3 F ... \n", " 4 F ... \n", " 5 F ... \n", "\n", " pmc-sog_otf_couv99-default_4_8 pmc-sog_otf_couv99-shy_2_16 \\\n", "model formula \n", "philo10 1 F F \n", " 2 F F \n", " 3 F F \n", " 4 F F \n", " 5 F F \n", "\n", " pmc-sog_otf_couv99-shy_2_8 pmc-sog_otf_couv99-shy_4_16 \\\n", "model formula \n", "philo10 1 F F \n", " 2 F F \n", " 3 F F \n", " 4 F F \n", " 5 F F \n", "\n", " pmc-sog_otf_couv99-shy_4_8 pnml2lts-mc_dfs_1_1 \\\n", "model formula \n", "philo10 1 F U \n", " 2 F U \n", " 3 F U \n", " 4 F U \n", " 5 F U \n", "\n", " pnml2lts-mc_dfs_1_16 pnml2lts-mc_ndfs_1_1 \\\n", "model formula \n", "philo10 1 U U \n", " 2 U U \n", " 3 U U \n", " 4 U U \n", " 5 U U \n", "\n", " pnml2lts-mc_ndfs_1_16 property \n", "model formula \n", "philo10 1 F F \n", " 2 F F \n", " 3 F F \n", " 4 F F \n", " 5 F F \n", "\n", "[5 rows x 21 columns]" ] }, "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# table with verification output for each experiment\n", "table_property = table['property'].copy()\n", "\n", "# replace non finished experiments with a dummy value\n", "table_property.fillna('U', inplace=True)\n", "\n", "# add ground truth to the table\n", "table_property = pd.concat([table_property, p_df], axis=1)\n", "\n", "table_property.head()" ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
toolpmc-sog_otfC_couv99-default_1_1pmc-sog_otfC_couv99-default_1_16pmc-sog_otfC_couv99-shy_1_1pmc-sog_otfC_couv99-shy_1_16pmc-sog_otfP_couv99-default_1_1pmc-sog_otfP_couv99-default_1_16pmc-sog_otfP_couv99-shy_1_1pmc-sog_otfP_couv99-shy_1_16pmc-sog_otf_couv99-default_2_16pmc-sog_otf_couv99-default_2_8pmc-sog_otf_couv99-default_4_16pmc-sog_otf_couv99-default_4_8pmc-sog_otf_couv99-shy_2_16pmc-sog_otf_couv99-shy_2_8pmc-sog_otf_couv99-shy_4_16pmc-sog_otf_couv99-shy_4_8pnml2lts-mc_dfs_1_1pnml2lts-mc_dfs_1_16pnml2lts-mc_ndfs_1_1pnml2lts-mc_ndfs_1_16
modelformula
philo101NaNOKNaNOKNaNOKNaNOKOKOKOKOKOKOKOKOKNaNOKNaNOK
2NaNOKNaNOKNaNOKNaNOKOKOKOKOKOKOKOKOKNaNOKNaNOK
3NaNOKNaNOKNaNOKNaNOKOKOKOKOKOKOKOKOKNaNOKNaNOK
4NaNOKNaNOKNaNOKNaNOKOKOKOKOKOKOKOKOKNaNOKNaNOK
5NaNOKNaNOKNaNOKNaNOKOKOKOKOKOKOKOKOKNaNOKNaNOK
\n", "
" ], "text/plain": [ "tool pmc-sog_otfC_couv99-default_1_1 \\\n", "model formula \n", "philo10 1 NaN \n", " 2 NaN \n", " 3 NaN \n", " 4 NaN \n", " 5 NaN \n", "\n", "tool pmc-sog_otfC_couv99-default_1_16 pmc-sog_otfC_couv99-shy_1_1 \\\n", "model formula \n", "philo10 1 OK NaN \n", " 2 OK NaN \n", " 3 OK NaN \n", " 4 OK NaN \n", " 5 OK NaN \n", "\n", "tool pmc-sog_otfC_couv99-shy_1_16 pmc-sog_otfP_couv99-default_1_1 \\\n", "model formula \n", "philo10 1 OK NaN \n", " 2 OK NaN \n", " 3 OK NaN \n", " 4 OK NaN \n", " 5 OK NaN \n", "\n", "tool pmc-sog_otfP_couv99-default_1_16 pmc-sog_otfP_couv99-shy_1_1 \\\n", "model formula \n", "philo10 1 OK NaN \n", " 2 OK NaN \n", " 3 OK NaN \n", " 4 OK NaN \n", " 5 OK NaN \n", "\n", "tool pmc-sog_otfP_couv99-shy_1_16 pmc-sog_otf_couv99-default_2_16 \\\n", "model formula \n", "philo10 1 OK OK \n", " 2 OK OK \n", " 3 OK OK \n", " 4 OK OK \n", " 5 OK OK \n", "\n", "tool pmc-sog_otf_couv99-default_2_8 \\\n", "model formula \n", "philo10 1 OK \n", " 2 OK \n", " 3 OK \n", " 4 OK \n", " 5 OK \n", "\n", "tool pmc-sog_otf_couv99-default_4_16 \\\n", "model formula \n", "philo10 1 OK \n", " 2 OK \n", " 3 OK \n", " 4 OK \n", " 5 OK \n", "\n", "tool pmc-sog_otf_couv99-default_4_8 pmc-sog_otf_couv99-shy_2_16 \\\n", "model formula \n", "philo10 1 OK OK \n", " 2 OK OK \n", " 3 OK OK \n", " 4 OK OK \n", " 5 OK OK \n", "\n", "tool pmc-sog_otf_couv99-shy_2_8 pmc-sog_otf_couv99-shy_4_16 \\\n", "model formula \n", "philo10 1 OK OK \n", " 2 OK OK \n", " 3 OK OK \n", " 4 OK OK \n", " 5 OK OK \n", "\n", "tool pmc-sog_otf_couv99-shy_4_8 pnml2lts-mc_dfs_1_1 \\\n", "model formula \n", "philo10 1 OK NaN \n", " 2 OK NaN \n", " 3 OK NaN \n", " 4 OK NaN \n", " 5 OK NaN \n", "\n", "tool pnml2lts-mc_dfs_1_16 pnml2lts-mc_ndfs_1_1 \\\n", "model formula \n", "philo10 1 OK NaN \n", " 2 OK NaN \n", " 3 OK NaN \n", " 4 OK NaN \n", " 5 OK NaN \n", "\n", "tool pnml2lts-mc_ndfs_1_16 \n", "model formula \n", "philo10 1 OK \n", " 2 OK \n", " 3 OK \n", " 4 OK \n", " 5 OK " ] }, "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# table with error for each experiment\n", "table_error = table['error'].copy()\n", "\n", "table_error.head()" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
modelformulaexplored_statesproperty
0philo101373978.0F
1philo10251192.0F
2philo1031877419.0F
3philo104177147.0F
4philo105293788.0F
\n", "
" ], "text/plain": [ " model formula explored_states property\n", "0 philo10 1 373978.0 F\n", "1 philo10 2 51192.0 F\n", "2 philo10 3 1877419.0 F\n", "3 philo10 4 177147.0 F\n", "4 philo10 5 293788.0 F" ] }, "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# table with explored states for each experiment using ltsmin\n", "table_explored_states = table.copy()\n", "table_explored_states = table_explored_states['explored_states']\n", "table_explored_states = table_explored_states[['pnml2lts-mc_dfs_1_16']]\n", "table_explored_states = table_explored_states.rename(columns={\"pnml2lts-mc_dfs_1_16\": \"explored_states\"})\n", "\n", "# add verification output to the table\n", "table_explored_states = pd.concat([table_explored_states, p_df], axis=1)\n", "\n", "# reshape\n", "table_explored_states = table_explored_states.reset_index()\n", "\n", "table_explored_states.head()" ] }, { "cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
meanminmax
modelproperty
philo10F4.372575e+0551192.06.910975e+06
T1.461107e+0548115.01.117801e+06
philo5F6.317120e+031108.04.433200e+04
T2.107000e+031000.06.974000e+03
robot20F2.018056e+098372365.03.152441e+09
\n", "
" ], "text/plain": [ " mean min max\n", "model property \n", "philo10 F 4.372575e+05 51192.0 6.910975e+06\n", " T 1.461107e+05 48115.0 1.117801e+06\n", "philo5 F 6.317120e+03 1108.0 4.433200e+04\n", " T 2.107000e+03 1000.0 6.974000e+03\n", "robot20 F 2.018056e+09 8372365.0 3.152441e+09" ] }, "execution_count": 20, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# calculate the stats of the number of explored states\n", "\n", "table_explored_states_stats = table_explored_states.groupby(['model', 'property']).agg(['mean', 'min', 'max'])\n", "table_explored_states_stats = table_explored_states_stats['explored_states']\n", "table_explored_states_stats.head()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Examples" ] }, { "cell_type": "code", "execution_count": 21, "metadata": {}, "outputs": [ { "data": { "text/html": [ " \n", " " ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "create_figure_explored_states(table_explored_states, 'robot20')" ] }, { "cell_type": "code", "execution_count": 22, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "create_figure(df, \"robot20\")" ] }, { "cell_type": "code", "execution_count": 23, "metadata": { "scrolled": false }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "815ef8455a4042deae34be23c417342e", "version_major": 2, "version_minor": 0 }, "text/plain": [ "FigureWidget({\n", " 'data': [{'customdata': [[2, 'TIME LIMIT', 'UNKNOWN'], [5, 'OK', 'UNKNOWN'],\n", " …" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "log_figure = create_log_figure(table_time, table_error, \"robot20\", \"pmc-sog_otf_couv99-shy_2_8\", \"pnml2lts-mc_dfs_1_16\", True, open_logs_callback)\n", "log_figure" ] }, { "cell_type": "code", "execution_count": 24, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "table = get_table(table_time, table_error, \"robot20\", \"pmc-sog_otf_couv99-default_2_8\", \"pnml2lts-mc_dfs_1_16\")\n", "table" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Generate Figures" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# models\n", "models = df.model.unique()\n", "\n", "# tools \n", "tools = df.tool.unique()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# create all the figures of explored states\n", "\n", "folder = os.path.join(OUTPUT_FOLDER, 'explored-states')\n", "create_folder(folder)\n", "\n", "for model in models:\n", " try:\n", " fig = create_figure_explored_states(table_explored_states, model)\n", " \n", " # save figures in html and pdf\n", " fig.write_html(os.path.join(folder, model + '.html'), include_plotlyjs='cdn')\n", " fig.write_image(os.path.join(folder, model + '.pdf'))\n", " except KeyError:\n", " print(\"Error: {} was not plotted\".format(model))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# create all the figures formula vs time\n", "\n", "folder = os.path.join(OUTPUT_FOLDER, 'time-plots')\n", "create_folder(folder)\n", "\n", "for model in models:\n", " try:\n", " fig = create_figure(df, model)\n", " \n", " # save figures in html and pdf\n", " fig.write_html(os.path.join(folder, model + '.html'), include_plotlyjs='cdn')\n", " fig.write_image(os.path.join(folder, model + '.pdf'))\n", " except KeyError:\n", " print(\"Error: {} was not plotted\".format(model))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# create all the log figures\n", "\n", "tools_pairs = [sorted(t) for t in (combinations(tools, 2))]\n", "\n", "for plot, filter_method in plots.items():\n", " axes = list(filter(filter_method, tools_pairs))\n", " \n", " for model in models:\n", " folder = os.path.join(OUTPUT_FOLDER, plot, model)\n", " create_folder(folder)\n", " \n", " for axe in axes:\n", " try:\n", " show_strategy = plot == 'compare_couvreur_algorithm'\n", " fig = create_log_figure(table_time, table_error, model, axe[0], axe[1], show_strategy)\n", " table = get_table(table_time, table_error, model, axe[0], axe[1])\n", " \n", " # save figures in html and pdf\n", " figure_name = os.path.join(folder, '{}-{}-VS-{}-log'.format(model, axe[0], axe[1]))\n", " \n", " with open(figure_name + '.html', 'w') as f:\n", " f.write(fig.to_html(full_html=False, include_plotlyjs='cdn', post_script=OPEN_LOGS_CALLBACK_JS))\n", " f.write(table.to_html(full_html=False, include_plotlyjs='cdn', post_script=OPEN_LOGS_CALLBACK_JS))\n", " \n", " fig.write_image(figure_name + '.pdf')\n", " except KeyError:\n", " print(\"Error: {} was not plotted\".format(model))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" } }, "nbformat": 4, "nbformat_minor": 4 }