{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "ZERO = 20e-5\n", "TIMEOUT = 10 * 60 # 10 minutes = 600 seconds" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "import os\n", "import glob\n", "import re\n", "import pandas as pd\n", "import numpy as np\n", "import plotly.io as pio\n", "import plotly.express as px\n", "import plotly.graph_objs as go\n", "from itertools import combinations \n", "import plotly.figure_factory as ff\n", "from plotly.subplots import make_subplots\n", "\n", "# render figures in notebook\n", "pio.renderers.default = \"notebook_connected\"\n", "\n", "# templates figures\n", "px.defaults.template = \"simple_white\"\n", "pio.templates.default = \"simple_white\"\n", "\n", "# layout for all figures\n", "LAYOUT_FIGURES = dict(\n", " autosize=False,\n", " width = 500,\n", " height = 500,\n", " xaxis = dict(\n", " constrain=\"domain\",\n", " mirror=True,\n", " showexponent=\"all\",\n", " exponentformat=\"power\"\n", " ),\n", " yaxis = dict(\n", " scaleanchor = \"x\",\n", " scaleratio = 1,\n", " mirror=True,\n", " showexponent=\"all\",\n", " exponentformat=\"power\"\n", " ),\n", " title = dict(\n", " y = 0.9,\n", " x = 0.5,\n", " xanchor = 'center',\n", " yanchor = 'top'\n", " )\n", ")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Auxiliary Functions" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "def create_folder(path):\n", " \"\"\"Creates a folder if it does not exist\n", " \n", " Parameters\n", " ----------\n", " path : str\n", " Path of the new folder\n", " \n", " Examples\n", " --------\n", " \n", " >>> create_folder('./results')\n", " \"\"\"\n", " if not os.path.exists(path):\n", " os.makedirs(path)" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "def create_figure(df, model):\n", " \"\"\"Creates a scatter figure showing the time taken by each tool to verify each property of a model\n", " \n", " Parameters\n", " ----------\n", " df : pandas.Dataframe\n", " Dataframe containing the results of the experiments\n", " model : string\n", " model to be plotted\n", " \n", " Returns\n", " -------\n", " plotly.graph_objects.Figure\n", " Scatter figure\n", " \n", " Examples\n", " --------\n", " \n", " >>> import os\n", " >>> import pandas as pd\n", " >>> csv_file = os.path.join(\"results\", \"output.csv\")\n", " >>> df = pd.read_csv(csv_file)\n", " >>> fig = create_figure(df, 'philo10')\n", " \"\"\"\n", " model_df = df[df.model == model]\n", "\n", " figure = px.scatter(model_df, \n", " x=\"formula\", y=\"time\",\n", " title=model, \n", " color=\"tool\", \n", " symbol_sequence=['x'])\n", "\n", " figure.update_layout(yaxis_title=\"time (s)\", title=LAYOUT_FIGURES['title'])\n", " return figure" ] }, { "cell_type": "code", "execution_count": 30, "metadata": {}, "outputs": [], "source": [ "def get_axis_title(experiment, show_strategy=True):\n", " \"\"\"Get the axis title of a figure depending on the experiment being plotted\n", " \n", " Parameters\n", " ----------\n", " experiment : str\n", " String with the experiment information\n", " show_strategy : bool, optional\n", " Flag to show the information related to the strategy used by the tool\n", " \n", " Returns\n", " -------\n", " str\n", " axis title\n", " \n", " Examples\n", " --------\n", " \n", " >>> get_axis_title('pmc-sog_otfL_couv99-default_1_1', True)\n", " pmc-sog (Lace, strategy: couv99-default, # cores: 1)\n", " \"\"\"\n", " information = experiment.split('_')\n", " tool_name = information[0]\n", " \n", " info = []\n", " library_dic = {\n", " 'otfL': 'Lace',\n", " 'otfP': 'Pthreads',\n", " 'otfC': 'Cthreads',\n", " 'otfPR': 'otfPR',\n", " 'otf': 'Hybrid'\n", " }\n", " \n", " if (len(information) == 5):\n", " info.append(library_dic[information[1]])\n", "\n", " if (show_strategy):\n", " info.append('strategy: {}'.format(information[-3]))\n", "\n", " nb_nodes = int(information[-2])\n", " if (nb_nodes > 1):\n", " info.append('# nodes: {}'.format(nb_nodes))\n", "\n", " info.append('# cores: {}'.format(information[-1]))\n", "\n", " title = '{} ({})'.format(tool_name, ', '.join(info))\n", " \n", " return title" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "def filter_errors(df_exp1, df_exp2):\n", " \"\"\"Returns dataframes of specific experiments without errors\"\"\"\n", " \n", " nan_1 = df_exp1[df_exp1.isna().any(axis=1)].index\n", " nan_2 = df_exp2[df_exp2.isna().any(axis=1)].index\n", "\n", " df_exp1 = df_exp1.drop(nan_2)\n", " df_exp2 = df_exp2.drop(nan_1)\n", "\n", " df_exp1 = df_exp1.dropna()\n", " df_exp2 = df_exp2.dropna()\n", " \n", " return df_exp1, df_exp2\n", "\n", "\n", "def get_info(info):\n", " \"\"\"Get some statistics from a table for a specific model and experiment\"\"\" \n", " time_limit = len(info[(info.error == \"TIME LIMIT\") | (info.error == \"TIMEOUT\")])\n", " error = len(info[(info.error != \"TIME LIMIT\") & (info.error != \"TIMEOUT\") & (info.error != \"OK\") & (info.error != 'MDD') & (info.error != 'TABLE FULL')])\n", " memory = len(info[(info.error == 'MDD') | (info.error == 'TABLE FULL')])\n", " ok = len(info[info.error == \"OK\"])\n", " \n", " if ((time_limit + error + ok + memory) != len(info)): raise Exception(\"Some information is missing in the table\")\n", " \n", " return {\n", " \"time limit\": time_limit,\n", " \"error\": error,\n", " \"memory\": memory,\n", " \"OK\": ok\n", " }\n", "\n", "\n", "def get_best_times(table_time, table_error, model, exp1, exp2):\n", " exp1 = pd.DataFrame({\"times_exp1\": table_time.loc[model][exp1], \"errors_exp1\": table_error.loc[model][exp1]})\n", " exp2 = pd.DataFrame({\"times_exp2\": table_time.loc[model][exp2], \"errors_exp2\": table_error.loc[model][exp2]})\n", "\n", " exp1, exp2 = filter_errors(exp1, exp2)\n", " df_ = pd.concat([exp1, exp2], axis=1, sort=False)\n", "\n", " df_ = df_[df_[\"times_exp1\"] != df_[\"times_exp2\"]]\n", " df_['best'] = np.where((df_[\"times_exp1\"] < df_[\"times_exp2\"]), \"exp1\", \"exp2\")\n", " count = df_.groupby([\"best\"]).size()\n", "\n", " return count.get(\"exp1\",0), count.get(\"exp2\",0)\n", "\n", "\n", "def get_table(df_time, df_errors, model, exp1, exp2):\n", " \"\"\"Creates a table with some statistics from a dataframe for a model and experiments\"\"\"\n", " rows=[[\"Experiment\", \"Time Limit\", \"Memory\", \"Unknown Error\", \"OK\", \"Faster\"]]\n", " \n", " df_exp1 = pd.DataFrame({\"error\": df_errors.loc[model][exp1]})\n", " df_exp2 = pd.DataFrame({\"error\": df_errors.loc[model][exp2]})\n", " df_exp1, df_exp2 = filter_errors(df_exp1, df_exp2)\n", " \n", " info1 = get_info(df_exp1)\n", " info2 = get_info(df_exp2)\n", " \n", " best1, best2 = get_best_times(df_time, df_errors, model, exp1, exp2)\n", " \n", " for (experiment, info, best) in [(exp1, info1, best1), (exp2, info2, best2)]:\n", " rows.append([experiment, info[\"time limit\"], info[\"memory\"], info[\"error\"], info[\"OK\"], best])\n", " \n", " return ff.create_table(rows)\n" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "scrolled": false }, "outputs": [], "source": [ "import webbrowser\n", "\n", "def get_filename(base_path, tool, model, model_instance, formula):\n", " \"\"\"Returns the absolute path of the experiment log\n", " \n", " Parameters\n", " ----------\n", " base_path : string\n", " Path of the folder where logs are saved\n", " tool : string\n", " Tool name\n", " model : string\n", " Model name\n", " model_instance : string\n", " Name of the model instance\n", " formula : string\n", " Identifier of the formula\n", " \n", " Returns\n", " -------\n", " string\n", " Absolute path of the log file\n", " \n", " \"\"\"\n", " information = tool.split('_')\n", " \n", " tool_name = information[0]\n", " tool_configuration = '_'.join(information[:-2])\n", " nb_nodes = information[-2]\n", " nb_cores = information[-1]\n", " \n", " experiment_folder = os.path.join(base_path, tool_name, tool_configuration, model, model_instance)\n", " filename = f'{tool_name}_{model_instance}-n{nb_nodes}-th{nb_cores}-f{formula}'\n", " absolute_path = os.path.join(experiment_folder, filename)\n", " \n", " return absolute_path\n", "\n", "def open_logs_callback(trace, points, selector):\n", " \"\"\"Callback that open the log files when clicking on a point of the figure\n", " \n", " Parameters\n", " ----------\n", " trace : plotly.graph_objects.Figure\n", " the figure to attach the callback\n", " points : plotly.callbacks.Points \n", " points of the figure selected\n", " selector: plotly.callbacks.InputDeviceState \n", " Device information \n", " \"\"\"\n", " inds = points.point_inds\n", " if (inds):\n", " index = inds[0]\n", "\n", " formula, error_x, error_y = trace['customdata'][index]\n", " model_instance = trace['meta']['model']\n", " model = ''.join(c for c in model_instance if not c.isdigit())\n", " tools = trace['meta']['tools']\n", " logs_folder = trace['meta']['folder']\n", "\n", " filename_x = get_filename(logs_folder, tools['x'], model, model_instance, formula)\n", " filename_y = get_filename(logs_folder, tools['y'], model, model_instance, formula)\n", "\n", " for f in [filename_x, filename_y]:\n", " webbrowser.open(f'file://{f}.err')\n", " webbrowser.open(f'file://{f}.out')\n", "\n", "OPEN_LOGS_CALLBACK_JS = \"\"\"\n", "function get_filename (base_path, tool, model_instance, formula) {\n", " const information = tool.split('_');\n", " const size = information.length;\n", "\n", " const tool_name = information[0];\n", " const tool_configuration = information.slice(0, size - 2).join('_');\n", " const nb_nodes = information[size - 2];\n", " const nb_cores = information[size - 1];\n", "\n", " const model = model_instance.replace(/[0-9]/g, '');\n", "\n", " const experiment_folder = `${base_path}/${tool_name}/${tool_configuration}/${model}/${model_instance}`;\n", " const filename = `${tool_name}_${model_instance}-n${nb_nodes}-th${nb_cores}-f${formula}`;\n", "\n", " return `${experiment_folder}/${filename}`;\n", "}\n", "\n", "const plots = document.getElementsByClassName(\"plotly-graph-div js-plotly-plot\");\n", "const myPlot = plots[0];\n", "\n", "myPlot.on('plotly_click', function(data){\n", " const points = data.points;\n", " if (points.length != 1) {return ;}\n", " \n", " const myPoint = points[0];\n", " const formula = myPoint.customdata[0];\n", " const meta = myPoint.data.meta;\n", " \n", " const href = window.location.href.split('/');\n", " const base_path = href.splice(0,href.length-4).join('/');\n", " \n", " const filename_x = get_filename(base_path, meta.tools.x, meta.model, formula);\n", " const filename_y = get_filename(base_path, meta.tools.y, meta.model, formula);\n", " \n", " console.log('x: ' + filename_x);\n", " window.open(`${filename_x}.err`);\n", " window.open(`${filename_x}.out`);\n", " \n", " console.log('y: ' + filename_y);\n", " window.open(`${filename_y}.err`);\n", " window.open(`${filename_y}.out`);\n", "});\n", "\"\"\"\n" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "def create_figure_explored_states(table_explored_states, model):\n", " \"\"\"Creates figure showing the number of explorated states during the verification \n", " \n", " Parameters\n", " ----------\n", " table_explored_states : pandas.Dataframe\n", " Dataframe with the explorated states of each experiment\n", " model : string\n", " Model to be analyzed\n", " \n", " Returns\n", " -------\n", " plotly.graph_objects.Figure\n", " Scatter figure\n", " \"\"\"\n", " colors={'T': 'green', 'F': 'red'}\n", " float_formatter = \"{:.2E}\".format\n", "\n", " table_model = table_explored_states[table_explored_states.property != 'U']\n", " table_model = table_model[table_model.model == model]\n", "\n", " table_stats = table_model.groupby(['property']).agg(['mean']) \n", "\n", " fig = go.Figure()\n", " max_x = 0\n", " for p in table_stats.index:\n", " data = table_model[table_model.property==p]\n", " stats = table_stats.loc[p]\n", " \n", " x_axis = np.arange(1, data['formula'].count()+1, 1)\n", " max_x = max(max_x, x_axis[-1]+1)\n", " mean = stats['explored_states','mean']\n", " \n", " figure = px.scatter(data, \n", " x=x_axis, \n", " y=\"explored_states\",\n", " title=model, \n", " color='property',\n", " color_discrete_map=colors,\n", " symbol_sequence=[\"circle\"])\n", "\n", " line = go.Scatter(x=[x_axis[0], x_axis[-1]], \n", " y=[mean, mean],\n", " mode='lines', showlegend=False, \n", " line=dict(color=colors[p], width=1.5))\n", "\n", " fig.add_trace(figure['data'][0])\n", " fig.add_trace(line)\n", " \n", " fig.add_annotation(x=1, \n", " y=mean,\n", " font=dict(color=colors[p]),\n", " text=f\"mean = {float_formatter(mean)}\")\n", "\n", " fig.update_layout(title_text=model, title=LAYOUT_FIGURES['title'], \n", " width = 500, height = 500, margin=dict(r=110))\n", " \n", " fig.update_xaxes(title=\"formula\", range=[0, max_x])\n", " fig.update_yaxes(title=\"# explored states\")\n", " \n", " fig.update_annotations(dict(\n", " showarrow=False,\n", " xanchor=\"left\",\n", " yanchor=\"middle\",\n", " xref='paper'))\n", " \n", " return fig" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "def create_log_figure(table, table_errors, model, tool_x, tool_y, show_strategy=True, callback=None):\n", " \"\"\"Creates a Scatter figure in logarithmic scale comparing the performance of two tools\n", " \n", " Parameters\n", " ----------\n", " table : pandas.Dataframe\n", " Dataframe with the times of each experiment\n", " table_errors : pandas.Dataframe\n", " Dataframe with the errors of each experiment\n", " model : string\n", " Model to be analyzed\n", " tool_x : string\n", " Tool to be compared and plotted on the x-axis\n", " tool_y : string\n", " Tool to be compared and plotted on the y-axis\n", " show_strategy : bool\n", " Flag to show the stretagy used by the tools\n", " callback : function\n", " Function to be called when clicking on a point\n", " \n", " Returns\n", " -------\n", " plotly.graph_objects.Figure\n", " Scatter figure\n", " \n", " Examples\n", " --------\n", " >>> import os\n", " >>> import pandas as pd\n", " >>> csv_file = os.path.join(\"results\", \"output.csv\")\n", " >>> df = pd.read_csv(csv_file)\n", " >>> table = df.set_index(['model', 'formula', 'tool'], drop=True).unstack('tool')\n", " >>> fig = create_log_figure(table['time'], table['error'], 'philo10', 'pmc-sog_otfL_couv99-default_1_8', 'pmc-sog_otfP_couv99-default_1_8')\n", " \"\"\"\n", " try:\n", " min_value = ZERO\n", " max_value = TIMEOUT\n", " \n", " min_value_log = np.log10(min_value)\n", " max_value_log = np.log10(max_value)\n", "\n", " table_model = table.loc[model]\n", " table_errors_model = table_error.loc[model]\n", " \n", " full_table_x = pd.concat([table_model[tool_x],table_model['property'], table_errors_model[tool_x]], axis=1)\n", " full_table_x.columns = ['time', 'property', 'error']\n", "\n", " full_table_y = pd.concat([table_model[tool_y],table_model['property'], table_errors_model[tool_y]], axis=1)\n", " full_table_y.columns = ['time', 'property', 'error']\n", " \n", " full_table_x, full_table_y = filter_errors(full_table_x, full_table_y)\n", "\n", " traces = [\n", " {\"property\": 'T', \"color\":\"green\"},\n", " {\"property\": 'F', \"color\":\"red\"},\n", " {\"property\": 'U', \"color\":\"black\"}\n", " ]\n", "\n", " figures = []\n", " for t in traces:\n", " # filter by verification output\n", " table_x = full_table_x[full_table_x.property == t['property']]\n", " table_y = full_table_y[full_table_y.property == t['property']]\n", "\n", " # custom data\n", " custom_data = list(zip(table_x.index, table_x.error,table_y.error))\n", " \n", " # tools\n", " metainfo = {\n", " 'model': model, \n", " 'tools': {'x': tool_x, 'y': tool_y},\n", " 'folder': os.path.join(os.path.abspath(os.pardir), \"results\")\n", " }\n", "\n", " figures.append(go.Scatter(x=table_x.time,\n", " y=table_y.time,\n", " name=t['property'],\n", " mode='markers',\n", " marker_symbol='circle-open',\n", " marker_color=t['color'],\n", " meta = metainfo,\n", " customdata=custom_data,\n", " hovertemplate =\n", " 'Formula # %{customdata[0]}
' +\n", " '
Times:
' +\n", " 'x: %{x} s' +\n", " '
y: %{y} s
' +\n", " '
Errors:
' +\n", " 'x: %{customdata[1]}
' +\n", " 'y: %{customdata[2]}',\n", " )) \n", "\n", " # Line\n", " figures.append(go.Scatter(x=[min_value, max_value], \n", " y=[min_value, max_value],\n", " mode='lines', showlegend=False,\n", " line=dict(color='black', width=1)))\n", "\n", " # Create figure\n", " figure = go.FigureWidget(figures)\n", " figure.update_layout(LAYOUT_FIGURES,\n", " title_text=model,\n", " hoverlabel=dict(bgcolor=\"white\", align='auto'),\n", " legend_title_text='property',\n", " xaxis=dict(type='log', autorange=False, range=[min_value_log, max_value_log]),\n", " yaxis=dict(type='log', autorange=False, range=[min_value_log, max_value_log]),\n", " xaxis_title=get_axis_title(tool_x, show_strategy),\n", " yaxis_title=get_axis_title(tool_y, show_strategy))\n", "\n", " # Add event\n", " if callback is not None:\n", " for i in range(len(figure.data)):\n", " figure.data[i].on_click(callback)\n", " \n", " return figure\n", " except Exception as e:\n", " print(\"Error when ploting model: {} - tool_x: {} - tool_y: {}\".format(model, tool_x, tool_y))\n", " print(e)" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "# Experiment filters\n", "\n", "def versus_dfs(experiments):\n", " \"\"\"Selects only experiments using DFS strategy\"\"\"\n", " exp1, exp2 = experiments\n", " strategy_exp1= exp1.split('_')[1]\n", " strategy_exp2= exp2.split('_')[1]\n", " \n", " return strategy_exp1 == 'dfs' or strategy_exp2 == 'dfs'\n", "\n", "def versus_sequential(experiments):\n", " \"\"\"Selects only experiments run sequentially \"\"\"\n", " exp1, exp2 = experiments\n", " nodes_exp1, threads_exp1 = exp1.split('_')[-2:]\n", " nodes_exp2, threads_exp2 = exp2.split('_')[-2:]\n", "\n", " return (nodes_exp1 == '1' and nodes_exp2 == '1') and \\\n", " (threads_exp1 == '1' or threads_exp2 == '1')\n", "\n", "def same_tool(experiments, tool):\n", " \"\"\"Selects only experiments comparing the same tool\"\"\"\n", " exp1, exp2 = experiments\n", " tool_exp1= exp1.split('_')[0]\n", " tool_exp2= exp2.split('_')[0]\n", " return tool_exp1.startswith(tool) and tool_exp2.startswith(tool)\n", "\n", "def same_number_threads(experiments):\n", " \"\"\"Selects only experiments comparing the same number of processes and cores\"\"\"\n", " exp1, exp2 = experiments\n", " nodes_exp1, threads_exp1 = exp1.split('_')[-2:]\n", " nodes_exp2, threads_exp2 = exp2.split('_')[-2:]\n", " return (nodes_exp1 == nodes_exp2) and (threads_exp1 == threads_exp2) \n", "\n", "def same_thread_library(experiments):\n", " \"\"\"Selects only experiments comparing the same parallelization library\"\"\"\n", " exp1, exp2 = experiments\n", " library_exp1 = exp1.split('_')[1]\n", " library_exp2 = exp2.split('_')[1]\n", " return library_exp1 == library_exp2\n", "\n", "def same_strategy(experiments):\n", " \"\"\"Selects only experiments comparing the same strategy\"\"\"\n", " exp1, exp2 = experiments\n", " strategy_exp1 = exp1.split('_')[2]\n", " strategy_exp2 = exp2.split('_')[2]\n", " return strategy_exp1 == strategy_exp2\n", "\n", "def only_couvreur_strategy(experiments):\n", " \"\"\"Selects only experiments comparing couvreur emptiness check algorithm\"\"\"\n", " exp1, exp2 = experiments\n", " strategy_exp1 = exp1.split('_')[2]\n", " strategy_exp2 = exp2.split('_')[2]\n", " return strategy_exp1.startswith('couv99') and strategy_exp2.startswith('couv99')\n", "\n", "def compare_threads_library(experiments):\n", " \"\"\"Compares parallization libraries used in pmc-sog. \n", " \n", " It selects experiments where the tool is only pmc-sog and the strategy, number of threads, \n", " number of processus are the same.\n", " \"\"\"\n", " return same_tool(experiments, 'pmc-sog') and \\\n", " same_strategy(experiments) and \\\n", " same_number_threads(experiments) and \\\n", " not same_thread_library(experiments)\n", "\n", "def compare_couvreur_strategies(experiments):\n", " \"\"\"Compares couvreurs strategies used in pmc-sog. \n", " \n", " It selects experiments where the tool is only pmc-sog, the strategy is couvreur, and \n", " the parallelization library, number of threads, number of processus are the same.\n", " \"\"\"\n", " return only_couvreur_strategy(experiments) and \\\n", " same_thread_library(experiments) and \\\n", " same_number_threads(experiments)\n", "\n", "def same_distributed_number_threads(experiments):\n", " \"\"\"Selects only experiments where the multiplication of theirs nodes with cores are the same.\"\"\"\n", " exp1, exp2 = experiments\n", " nodes_exp1, threads_exp1 = exp1.split('_')[-2:]\n", " nodes_exp2, threads_exp2 = exp2.split('_')[-2:]\n", " return (int(nodes_exp1) * int(threads_exp1)) == (int(nodes_exp2) * int(threads_exp2))\n", "\n", "def compare_tools(experiments):\n", " \"\"\"Compares pmc-sog and pnml2lts-mc using the DFS algorithm. \n", " \n", " It selects experiments where the tools are not the same, the exploration algorithm is DFS and \n", " the number of processus and cores are the same.\n", " \"\"\"\n", " return not (same_tool(experiments, 'pmc-sog') or same_tool(experiments,'pnml2lts-mc')) and \\\n", " versus_dfs(experiments)\n", "\n", "def compare_multithreading(experiments):\n", " \"\"\"Compares the sequential and multi-core version of pmc-sog. \n", " \n", " It selects experiments where the tools is pmc-sog, the parallelization library, the emptiness check \n", " strategy are the same. Here the number of processus and cores are different.\n", " \"\"\"\n", " return same_tool(experiments, 'pmc-sog') and \\\n", " same_thread_library(experiments) and \\\n", " same_strategy(experiments) and \\\n", " versus_sequential(experiments)\n", "\n", "def against_hybrid(experiments):\n", " \"\"\"Selects only experiments comparing with hybrid mode\"\"\"\n", " exp1, exp2 = experiments\n", " library_exp1 = exp1.split('_')[1]\n", " library_exp2 = exp2.split('_')[1]\n", " return (library_exp1 == 'otf') or (library_exp2 == 'otf')\n", "\n", "\n", "def compare_distributed(experiments):\n", " \"\"\"Compares the hybrid version of pmc-sog\"\"\"\n", " return same_tool(experiments, 'pmc-sog') and \\\n", " same_strategy(experiments) and \\\n", " same_distributed_number_threads(experiments) and \\\n", " against_hybrid(experiments)\n", "\n", "def compare_others(experiments):\n", " return (not compare_threads_library(experiments)) and \\\n", " (not compare_couvreur_strategies(experiments)) and \\\n", " (not compare_tools(experiments)) and \\\n", " (not compare_multithreading(experiments)) and \\\n", " (not compare_distributed(experiments))\n", "\n", "# Plots to be created\n", "plots = {\n", " 'compare_thread_library': compare_threads_library,\n", " 'compare_couvreur_algorithm': compare_couvreur_strategies,\n", " 'compare_tools': compare_tools,\n", " 'compare_multicore': compare_multithreading,\n", " 'compare_distributed': compare_distributed,\n", " 'others' : compare_others\n", "}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Load Data" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [], "source": [ "# Root folder\n", "PROJECT_FOLDER = os.path.abspath(os.pardir)\n", "\n", "# csv file with the output\n", "csv_file = os.path.join(PROJECT_FOLDER, \"results\", \"output.csv\")\n", "\n", "# formulas folder\n", "FORMULAS_FOLDER = os.path.join(PROJECT_FOLDER, \"formulas\")\n", "\n", "# Output folder\n", "OUTPUT_FOLDER = os.path.join(PROJECT_FOLDER,\"results\", \"figures\")\n", "create_folder(OUTPUT_FOLDER)" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
modelformulatooltimepropertyexplored_stateserror
0robot201pmc-sog_otfC_couv99-default_1_16NaNNaNNaNTIME LIMIT
1robot201pmc-sog_otfC_couv99-shy_1_16NaNNaNNaNTIME LIMIT
2robot201pmc-sog_otfPR_couv99-default_1_16NaNNaNNaNTIME LIMIT
3robot201pmc-sog_otfPR_couv99-shy_1_162.737FNaNOK
4robot201pmc-sog_otfP_couv99-default_1_16NaNNaNNaNTIME LIMIT
........................
2395robot20200pmc-sog_otf_couv99-default_4_4NaNNaNNaNTIME LIMIT
2396robot20200pmc-sog_otf_couv99-shy_2_8NaNNaNNaNTIME LIMIT
2397robot20200pmc-sog_otf_couv99-shy_4_4NaNNaNNaNTIME LIMIT
2398robot20200pnml2lts-mc_dfs_1_16NaNNaN2.875346e+09TABLE FULL
2399robot20200pnml2lts-mc_ndfs_1_16NaNNaNNaNTIME LIMIT
\n", "

2400 rows × 7 columns

\n", "
" ], "text/plain": [ " model formula tool time property \\\n", "0 robot20 1 pmc-sog_otfC_couv99-default_1_16 NaN NaN \n", "1 robot20 1 pmc-sog_otfC_couv99-shy_1_16 NaN NaN \n", "2 robot20 1 pmc-sog_otfPR_couv99-default_1_16 NaN NaN \n", "3 robot20 1 pmc-sog_otfPR_couv99-shy_1_16 2.737 F \n", "4 robot20 1 pmc-sog_otfP_couv99-default_1_16 NaN NaN \n", "... ... ... ... ... ... \n", "2395 robot20 200 pmc-sog_otf_couv99-default_4_4 NaN NaN \n", "2396 robot20 200 pmc-sog_otf_couv99-shy_2_8 NaN NaN \n", "2397 robot20 200 pmc-sog_otf_couv99-shy_4_4 NaN NaN \n", "2398 robot20 200 pnml2lts-mc_dfs_1_16 NaN NaN \n", "2399 robot20 200 pnml2lts-mc_ndfs_1_16 NaN NaN \n", "\n", " explored_states error \n", "0 NaN TIME LIMIT \n", "1 NaN TIME LIMIT \n", "2 NaN TIME LIMIT \n", "3 NaN OK \n", "4 NaN TIME LIMIT \n", "... ... ... \n", "2395 NaN TIME LIMIT \n", "2396 NaN TIME LIMIT \n", "2397 NaN TIME LIMIT \n", "2398 2.875346e+09 TABLE FULL \n", "2399 NaN TIME LIMIT \n", "\n", "[2400 rows x 7 columns]" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# read data\n", "df = pd.read_csv(csv_file)\n", "\n", "# merge the information related to the experiment (# nodes, # threads, strategy) to the tool column\n", "df['tool'] = df[['tool', 'strategy', 'num_nodes', 'num_threads']].astype(str).apply('_'.join, axis=1)\n", "df = df.drop(columns=['strategy', 'num_nodes', 'num_threads'])\n", "\n", "# FIX: filtering philo20 experiments because there is a problem with the generated formulas\n", "df = df[df.model != \"philo20\"]\n", "df" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
modelformulatooltimepropertyexplored_stateserror
0robot201pmc-sog_otfC_couv99-default_1_16NaNNaNNaNTIME LIMIT
1robot201pmc-sog_otfC_couv99-shy_1_16NaNNaNNaNTIME LIMIT
2robot201pmc-sog_otfPR_couv99-default_1_16NaNNaNNaNTIME LIMIT
3robot201pmc-sog_otfPR_couv99-shy_1_162.737FNaNOK
4robot201pmc-sog_otfP_couv99-default_1_16NaNNaNNaNTIME LIMIT
........................
1763robot20200pmc-sog_otf_couv99-default_4_4NaNNaNNaNTIME LIMIT
1764robot20200pmc-sog_otf_couv99-shy_2_8NaNNaNNaNTIME LIMIT
1765robot20200pmc-sog_otf_couv99-shy_4_4NaNNaNNaNTIME LIMIT
1766robot20200pnml2lts-mc_dfs_1_16NaNNaN2.875346e+09TABLE FULL
1767robot20200pnml2lts-mc_ndfs_1_16NaNNaNNaNTIME LIMIT
\n", "

1768 rows × 7 columns

\n", "
" ], "text/plain": [ " model formula tool time property \\\n", "0 robot20 1 pmc-sog_otfC_couv99-default_1_16 NaN NaN \n", "1 robot20 1 pmc-sog_otfC_couv99-shy_1_16 NaN NaN \n", "2 robot20 1 pmc-sog_otfPR_couv99-default_1_16 NaN NaN \n", "3 robot20 1 pmc-sog_otfPR_couv99-shy_1_16 2.737 F \n", "4 robot20 1 pmc-sog_otfP_couv99-default_1_16 NaN NaN \n", "... ... ... ... ... ... \n", "1763 robot20 200 pmc-sog_otf_couv99-default_4_4 NaN NaN \n", "1764 robot20 200 pmc-sog_otf_couv99-shy_2_8 NaN NaN \n", "1765 robot20 200 pmc-sog_otf_couv99-shy_4_4 NaN NaN \n", "1766 robot20 200 pnml2lts-mc_dfs_1_16 NaN NaN \n", "1767 robot20 200 pnml2lts-mc_ndfs_1_16 NaN NaN \n", "\n", " explored_states error \n", "0 NaN TIME LIMIT \n", "1 NaN TIME LIMIT \n", "2 NaN TIME LIMIT \n", "3 NaN OK \n", "4 NaN TIME LIMIT \n", "... ... ... \n", "1763 NaN TIME LIMIT \n", "1764 NaN TIME LIMIT \n", "1765 NaN TIME LIMIT \n", "1766 2.875346e+09 TABLE FULL \n", "1767 NaN TIME LIMIT \n", "\n", "[1768 rows x 7 columns]" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# filtering runtime errors\n", "df = df[(df.error != \"SEGMENTATION FAULT\") & \\\n", " (df.error != \"ABORTED\") & \\\n", " (df.error != \"TERMINATE\") & \\\n", " (df.error != \"MDD\")]\n", "\n", "df = df.reset_index(drop=True)\n", "df" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
property
modelformula
robot201F
2T
3F
4F
5T
......
196T
197T
198T
199T
200T
\n", "

200 rows × 1 columns

\n", "
" ], "text/plain": [ " property\n", "model formula \n", "robot20 1 F\n", " 2 T\n", " 3 F\n", " 4 F\n", " 5 T\n", "... ...\n", " 196 T\n", " 197 T\n", " 198 T\n", " 199 T\n", " 200 T\n", "\n", "[200 rows x 1 columns]" ] }, "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# ground truth for properties\n", "frames = []\n", "\n", "formula_results = glob.glob(os.path.join(FORMULAS_FOLDER, \"**/formula_results\"), recursive=True)\n", "for f in formula_results:\n", " model, out_file = f.split('/')[-2:]\n", " \n", " tmp_df = pd.read_csv(f, sep=\";\", header=None, names=[\"formula\", \"property\"])\n", " tmp_df[\"model\"] = model\n", " frames.append(tmp_df)\n", " \n", "p_df = pd.concat(frames)\n", "p_df = p_df.reindex(columns=[\"model\", \"formula\", \"property\"])\n", "p_df = p_df[p_df['model'].isin(df.model.unique())]\n", "p_df['property'] = p_df['property'].replace([True, False], [\"T\", \"F\"])\n", "p_df = p_df.set_index([\"model\", \"formula\"])\n", "p_df.sort_index(inplace=True)\n", "\n", "p_df" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
time...error
toolpmc-sog_otfC_couv99-default_1_16pmc-sog_otfC_couv99-shy_1_16pmc-sog_otfPR_couv99-default_1_16pmc-sog_otfPR_couv99-shy_1_16pmc-sog_otfP_couv99-default_1_16pmc-sog_otfP_couv99-shy_1_16pmc-sog_otf_couv99-default_2_8pmc-sog_otf_couv99-default_4_4pmc-sog_otf_couv99-shy_2_8pmc-sog_otf_couv99-shy_4_4...pmc-sog_otfPR_couv99-default_1_16pmc-sog_otfPR_couv99-shy_1_16pmc-sog_otfP_couv99-default_1_16pmc-sog_otfP_couv99-shy_1_16pmc-sog_otf_couv99-default_2_8pmc-sog_otf_couv99-default_4_4pmc-sog_otf_couv99-shy_2_8pmc-sog_otf_couv99-shy_4_4pnml2lts-mc_dfs_1_16pnml2lts-mc_ndfs_1_16
modelformula
robot201NaNNaNNaN2.737NaNNaNNaNNaNNaNNaN...TIME LIMITOKTIME LIMITTIME LIMITTIME LIMITTIME LIMITTIME LIMITTIME LIMITUNKNOWNOK
20.714NaN0.9150.8830.777NaN0.4161.052NaNNaN...OKOKOKTIME LIMITOKOKTIME LIMITTIME LIMITUNKNOWNTIME LIMIT
3NaN271.853NaN266.638NaN324.036NaNNaN297.0561053.045...TIME LIMITOKNaNOKNaNTIME LIMITOKOKUNKNOWNOK
40.0220.0230.0150.0150.0330.0260.1120.1260.1090.124...OKOKOKOKOKOKOKOKUNKNOWNOK
5NaN7.65881.2407.984NaN6.793NaNNaN9.27620.551...OKOKNaNOKNaNNaNOKOKUNKNOWNOK
..................................................................
196NaNNaN1.1490.110NaNNaNNaNNaNNaNNaN...OKOKTIME LIMITTIME LIMITTIME LIMITTIME LIMITTIME LIMITTIME LIMITUNKNOWNOK
197NaNNaNNaN3.731NaNNaNNaNNaNNaNNaN...NaNOKNaNNaNNaNNaNNaNNaNTABLE FULLUNKNOWN
198NaNNaN12.2153.962NaNNaNNaNNaNNaNNaN...OKOKNaNNaNNaNNaNNaNTIME LIMITUNKNOWNUNKNOWN
199263.468NaN120.2041.042NaNNaN245.885NaNNaNNaN...OKOKNaNNaNOKTIME LIMITTIME LIMITTIME LIMITTABLE FULLTIME LIMIT
200NaNNaN194.7950.388NaNNaNNaNNaNNaNNaN...OKOKNaNNaNTIME LIMITTIME LIMITTIME LIMITTIME LIMITTABLE FULLTIME LIMIT
\n", "

200 rows × 48 columns

\n", "
" ], "text/plain": [ " time \\\n", "tool pmc-sog_otfC_couv99-default_1_16 pmc-sog_otfC_couv99-shy_1_16 \n", "model formula \n", "robot20 1 NaN NaN \n", " 2 0.714 NaN \n", " 3 NaN 271.853 \n", " 4 0.022 0.023 \n", " 5 NaN 7.658 \n", "... ... ... \n", " 196 NaN NaN \n", " 197 NaN NaN \n", " 198 NaN NaN \n", " 199 263.468 NaN \n", " 200 NaN NaN \n", "\n", " \\\n", "tool pmc-sog_otfPR_couv99-default_1_16 \n", "model formula \n", "robot20 1 NaN \n", " 2 0.915 \n", " 3 NaN \n", " 4 0.015 \n", " 5 81.240 \n", "... ... \n", " 196 1.149 \n", " 197 NaN \n", " 198 12.215 \n", " 199 120.204 \n", " 200 194.795 \n", "\n", " \\\n", "tool pmc-sog_otfPR_couv99-shy_1_16 \n", "model formula \n", "robot20 1 2.737 \n", " 2 0.883 \n", " 3 266.638 \n", " 4 0.015 \n", " 5 7.984 \n", "... ... \n", " 196 0.110 \n", " 197 3.731 \n", " 198 3.962 \n", " 199 1.042 \n", " 200 0.388 \n", "\n", " \\\n", "tool pmc-sog_otfP_couv99-default_1_16 pmc-sog_otfP_couv99-shy_1_16 \n", "model formula \n", "robot20 1 NaN NaN \n", " 2 0.777 NaN \n", " 3 NaN 324.036 \n", " 4 0.033 0.026 \n", " 5 NaN 6.793 \n", "... ... ... \n", " 196 NaN NaN \n", " 197 NaN NaN \n", " 198 NaN NaN \n", " 199 NaN NaN \n", " 200 NaN NaN \n", "\n", " \\\n", "tool pmc-sog_otf_couv99-default_2_8 pmc-sog_otf_couv99-default_4_4 \n", "model formula \n", "robot20 1 NaN NaN \n", " 2 0.416 1.052 \n", " 3 NaN NaN \n", " 4 0.112 0.126 \n", " 5 NaN NaN \n", "... ... ... \n", " 196 NaN NaN \n", " 197 NaN NaN \n", " 198 NaN NaN \n", " 199 245.885 NaN \n", " 200 NaN NaN \n", "\n", " ... \\\n", "tool pmc-sog_otf_couv99-shy_2_8 pmc-sog_otf_couv99-shy_4_4 ... \n", "model formula ... \n", "robot20 1 NaN NaN ... \n", " 2 NaN NaN ... \n", " 3 297.056 1053.045 ... \n", " 4 0.109 0.124 ... \n", " 5 9.276 20.551 ... \n", "... ... ... ... \n", " 196 NaN NaN ... \n", " 197 NaN NaN ... \n", " 198 NaN NaN ... \n", " 199 NaN NaN ... \n", " 200 NaN NaN ... \n", "\n", " error \\\n", "tool pmc-sog_otfPR_couv99-default_1_16 \n", "model formula \n", "robot20 1 TIME LIMIT \n", " 2 OK \n", " 3 TIME LIMIT \n", " 4 OK \n", " 5 OK \n", "... ... \n", " 196 OK \n", " 197 NaN \n", " 198 OK \n", " 199 OK \n", " 200 OK \n", "\n", " \\\n", "tool pmc-sog_otfPR_couv99-shy_1_16 \n", "model formula \n", "robot20 1 OK \n", " 2 OK \n", " 3 OK \n", " 4 OK \n", " 5 OK \n", "... ... \n", " 196 OK \n", " 197 OK \n", " 198 OK \n", " 199 OK \n", " 200 OK \n", "\n", " \\\n", "tool pmc-sog_otfP_couv99-default_1_16 pmc-sog_otfP_couv99-shy_1_16 \n", "model formula \n", "robot20 1 TIME LIMIT TIME LIMIT \n", " 2 OK TIME LIMIT \n", " 3 NaN OK \n", " 4 OK OK \n", " 5 NaN OK \n", "... ... ... \n", " 196 TIME LIMIT TIME LIMIT \n", " 197 NaN NaN \n", " 198 NaN NaN \n", " 199 NaN NaN \n", " 200 NaN NaN \n", "\n", " \\\n", "tool pmc-sog_otf_couv99-default_2_8 pmc-sog_otf_couv99-default_4_4 \n", "model formula \n", "robot20 1 TIME LIMIT TIME LIMIT \n", " 2 OK OK \n", " 3 NaN TIME LIMIT \n", " 4 OK OK \n", " 5 NaN NaN \n", "... ... ... \n", " 196 TIME LIMIT TIME LIMIT \n", " 197 NaN NaN \n", " 198 NaN NaN \n", " 199 OK TIME LIMIT \n", " 200 TIME LIMIT TIME LIMIT \n", "\n", " \\\n", "tool pmc-sog_otf_couv99-shy_2_8 pmc-sog_otf_couv99-shy_4_4 \n", "model formula \n", "robot20 1 TIME LIMIT TIME LIMIT \n", " 2 TIME LIMIT TIME LIMIT \n", " 3 OK OK \n", " 4 OK OK \n", " 5 OK OK \n", "... ... ... \n", " 196 TIME LIMIT TIME LIMIT \n", " 197 NaN NaN \n", " 198 NaN TIME LIMIT \n", " 199 TIME LIMIT TIME LIMIT \n", " 200 TIME LIMIT TIME LIMIT \n", "\n", " \n", "tool pnml2lts-mc_dfs_1_16 pnml2lts-mc_ndfs_1_16 \n", "model formula \n", "robot20 1 UNKNOWN OK \n", " 2 UNKNOWN TIME LIMIT \n", " 3 UNKNOWN OK \n", " 4 UNKNOWN OK \n", " 5 UNKNOWN OK \n", "... ... ... \n", " 196 UNKNOWN OK \n", " 197 TABLE FULL UNKNOWN \n", " 198 UNKNOWN UNKNOWN \n", " 199 TABLE FULL TIME LIMIT \n", " 200 TABLE FULL TIME LIMIT \n", "\n", "[200 rows x 48 columns]" ] }, "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# table with times, verification output and error for each experiment\n", "table = df.set_index(['model', 'formula', 'tool'], drop=True).unstack('tool')\n", "table" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Preprocessing of data" ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
pmc-sog_otfC_couv99-default_1_16pmc-sog_otfC_couv99-shy_1_16pmc-sog_otfPR_couv99-default_1_16pmc-sog_otfPR_couv99-shy_1_16pmc-sog_otfP_couv99-default_1_16pmc-sog_otfP_couv99-shy_1_16pmc-sog_otf_couv99-default_2_8pmc-sog_otf_couv99-default_4_4pmc-sog_otf_couv99-shy_2_8pmc-sog_otf_couv99-shy_4_4pnml2lts-mc_dfs_1_16pnml2lts-mc_ndfs_1_16property
modelformula
robot201600.000600.000600.0002.737600.000600.000600.000600.000600.000600.000600.00.88F
20.714600.0000.9150.8830.777600.0000.4161.052600.000600.000600.0600.00T
3600.000271.853600.000266.638600.000324.036600.000600.000297.0561053.045600.00.24F
40.0220.0230.0150.0150.0330.0260.1120.1260.1090.124600.00.43F
5600.0007.65881.2407.984600.0006.793600.000600.0009.27620.551600.0789.07T
..........................................
196600.000600.0001.1490.110600.000600.000600.000600.000600.000600.000600.0763.14T
197600.000600.000600.0003.731600.000600.000600.000600.000600.000600.000600.0600.00T
198600.000600.00012.2153.962600.000600.000600.000600.000600.000600.000600.0600.00T
199263.468600.000120.2041.042600.000600.000245.885600.000600.000600.000600.0600.00T
200600.000600.000194.7950.388600.000600.000600.000600.000600.000600.000600.0600.00T
\n", "

200 rows × 13 columns

\n", "
" ], "text/plain": [ " pmc-sog_otfC_couv99-default_1_16 \\\n", "model formula \n", "robot20 1 600.000 \n", " 2 0.714 \n", " 3 600.000 \n", " 4 0.022 \n", " 5 600.000 \n", "... ... \n", " 196 600.000 \n", " 197 600.000 \n", " 198 600.000 \n", " 199 263.468 \n", " 200 600.000 \n", "\n", " pmc-sog_otfC_couv99-shy_1_16 \\\n", "model formula \n", "robot20 1 600.000 \n", " 2 600.000 \n", " 3 271.853 \n", " 4 0.023 \n", " 5 7.658 \n", "... ... \n", " 196 600.000 \n", " 197 600.000 \n", " 198 600.000 \n", " 199 600.000 \n", " 200 600.000 \n", "\n", " pmc-sog_otfPR_couv99-default_1_16 \\\n", "model formula \n", "robot20 1 600.000 \n", " 2 0.915 \n", " 3 600.000 \n", " 4 0.015 \n", " 5 81.240 \n", "... ... \n", " 196 1.149 \n", " 197 600.000 \n", " 198 12.215 \n", " 199 120.204 \n", " 200 194.795 \n", "\n", " pmc-sog_otfPR_couv99-shy_1_16 \\\n", "model formula \n", "robot20 1 2.737 \n", " 2 0.883 \n", " 3 266.638 \n", " 4 0.015 \n", " 5 7.984 \n", "... ... \n", " 196 0.110 \n", " 197 3.731 \n", " 198 3.962 \n", " 199 1.042 \n", " 200 0.388 \n", "\n", " pmc-sog_otfP_couv99-default_1_16 \\\n", "model formula \n", "robot20 1 600.000 \n", " 2 0.777 \n", " 3 600.000 \n", " 4 0.033 \n", " 5 600.000 \n", "... ... \n", " 196 600.000 \n", " 197 600.000 \n", " 198 600.000 \n", " 199 600.000 \n", " 200 600.000 \n", "\n", " pmc-sog_otfP_couv99-shy_1_16 pmc-sog_otf_couv99-default_2_8 \\\n", "model formula \n", "robot20 1 600.000 600.000 \n", " 2 600.000 0.416 \n", " 3 324.036 600.000 \n", " 4 0.026 0.112 \n", " 5 6.793 600.000 \n", "... ... ... \n", " 196 600.000 600.000 \n", " 197 600.000 600.000 \n", " 198 600.000 600.000 \n", " 199 600.000 245.885 \n", " 200 600.000 600.000 \n", "\n", " pmc-sog_otf_couv99-default_4_4 pmc-sog_otf_couv99-shy_2_8 \\\n", "model formula \n", "robot20 1 600.000 600.000 \n", " 2 1.052 600.000 \n", " 3 600.000 297.056 \n", " 4 0.126 0.109 \n", " 5 600.000 9.276 \n", "... ... ... \n", " 196 600.000 600.000 \n", " 197 600.000 600.000 \n", " 198 600.000 600.000 \n", " 199 600.000 600.000 \n", " 200 600.000 600.000 \n", "\n", " pmc-sog_otf_couv99-shy_4_4 pnml2lts-mc_dfs_1_16 \\\n", "model formula \n", "robot20 1 600.000 600.0 \n", " 2 600.000 600.0 \n", " 3 1053.045 600.0 \n", " 4 0.124 600.0 \n", " 5 20.551 600.0 \n", "... ... ... \n", " 196 600.000 600.0 \n", " 197 600.000 600.0 \n", " 198 600.000 600.0 \n", " 199 600.000 600.0 \n", " 200 600.000 600.0 \n", "\n", " pnml2lts-mc_ndfs_1_16 property \n", "model formula \n", "robot20 1 0.88 F \n", " 2 600.00 T \n", " 3 0.24 F \n", " 4 0.43 F \n", " 5 789.07 T \n", "... ... ... \n", " 196 763.14 T \n", " 197 600.00 T \n", " 198 600.00 T \n", " 199 600.00 T \n", " 200 600.00 T \n", "\n", "[200 rows x 13 columns]" ] }, "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# table with times for each experiment\n", "table_time = table['time'].copy()\n", "\n", "# replace non finished experiments with a dummy value, e.g. timeout\n", "table_time.fillna(TIMEOUT, inplace=True)\n", "\n", "# replace 0.00 time for 10^(-5), we cannot plot log(0)\n", "table_time.replace(0.0, ZERO, inplace=True)\n", "\n", "# add verification output to the table\n", "table_time = pd.concat([table_time, p_df], axis=1)\n", "\n", "table_time" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
pmc-sog_otfC_couv99-default_1_16pmc-sog_otfC_couv99-shy_1_16pmc-sog_otfPR_couv99-default_1_16pmc-sog_otfPR_couv99-shy_1_16pmc-sog_otfP_couv99-default_1_16pmc-sog_otfP_couv99-shy_1_16pmc-sog_otf_couv99-default_2_8pmc-sog_otf_couv99-default_4_4pmc-sog_otf_couv99-shy_2_8pmc-sog_otf_couv99-shy_4_4pnml2lts-mc_dfs_1_16pnml2lts-mc_ndfs_1_16property
modelformula
robot201UUUFUUUUUUUFF
2FUFFFUFFUUUUT
3UFUFUFUUFFUFF
4TTTTTTTTTTUFF
5UFFFUFUUFFUFT
\n", "
" ], "text/plain": [ " pmc-sog_otfC_couv99-default_1_16 pmc-sog_otfC_couv99-shy_1_16 \\\n", "model formula \n", "robot20 1 U U \n", " 2 F U \n", " 3 U F \n", " 4 T T \n", " 5 U F \n", "\n", " pmc-sog_otfPR_couv99-default_1_16 \\\n", "model formula \n", "robot20 1 U \n", " 2 F \n", " 3 U \n", " 4 T \n", " 5 F \n", "\n", " pmc-sog_otfPR_couv99-shy_1_16 \\\n", "model formula \n", "robot20 1 F \n", " 2 F \n", " 3 F \n", " 4 T \n", " 5 F \n", "\n", " pmc-sog_otfP_couv99-default_1_16 pmc-sog_otfP_couv99-shy_1_16 \\\n", "model formula \n", "robot20 1 U U \n", " 2 F U \n", " 3 U F \n", " 4 T T \n", " 5 U F \n", "\n", " pmc-sog_otf_couv99-default_2_8 pmc-sog_otf_couv99-default_4_4 \\\n", "model formula \n", "robot20 1 U U \n", " 2 F F \n", " 3 U U \n", " 4 T T \n", " 5 U U \n", "\n", " pmc-sog_otf_couv99-shy_2_8 pmc-sog_otf_couv99-shy_4_4 \\\n", "model formula \n", "robot20 1 U U \n", " 2 U U \n", " 3 F F \n", " 4 T T \n", " 5 F F \n", "\n", " pnml2lts-mc_dfs_1_16 pnml2lts-mc_ndfs_1_16 property \n", "model formula \n", "robot20 1 U F F \n", " 2 U U T \n", " 3 U F F \n", " 4 U F F \n", " 5 U F T " ] }, "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# table with verification output for each experiment\n", "table_property = table['property'].copy()\n", "\n", "# replace non finished experiments with a dummy value\n", "table_property.fillna('U', inplace=True)\n", "\n", "# add ground truth to the table\n", "table_property = pd.concat([table_property, p_df], axis=1)\n", "\n", "table_property.head()" ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
toolpmc-sog_otfC_couv99-default_1_16pmc-sog_otfC_couv99-shy_1_16pmc-sog_otfPR_couv99-default_1_16pmc-sog_otfPR_couv99-shy_1_16pmc-sog_otfP_couv99-default_1_16pmc-sog_otfP_couv99-shy_1_16pmc-sog_otf_couv99-default_2_8pmc-sog_otf_couv99-default_4_4pmc-sog_otf_couv99-shy_2_8pmc-sog_otf_couv99-shy_4_4pnml2lts-mc_dfs_1_16pnml2lts-mc_ndfs_1_16
modelformula
robot201TIME LIMITTIME LIMITTIME LIMITOKTIME LIMITTIME LIMITTIME LIMITTIME LIMITTIME LIMITTIME LIMITUNKNOWNOK
2OKTIME LIMITOKOKOKTIME LIMITOKOKTIME LIMITTIME LIMITUNKNOWNTIME LIMIT
3NaNOKTIME LIMITOKNaNOKNaNTIME LIMITOKOKUNKNOWNOK
4OKOKOKOKOKOKOKOKOKOKUNKNOWNOK
5NaNOKOKOKNaNOKNaNNaNOKOKUNKNOWNOK
\n", "
" ], "text/plain": [ "tool pmc-sog_otfC_couv99-default_1_16 pmc-sog_otfC_couv99-shy_1_16 \\\n", "model formula \n", "robot20 1 TIME LIMIT TIME LIMIT \n", " 2 OK TIME LIMIT \n", " 3 NaN OK \n", " 4 OK OK \n", " 5 NaN OK \n", "\n", "tool pmc-sog_otfPR_couv99-default_1_16 \\\n", "model formula \n", "robot20 1 TIME LIMIT \n", " 2 OK \n", " 3 TIME LIMIT \n", " 4 OK \n", " 5 OK \n", "\n", "tool pmc-sog_otfPR_couv99-shy_1_16 \\\n", "model formula \n", "robot20 1 OK \n", " 2 OK \n", " 3 OK \n", " 4 OK \n", " 5 OK \n", "\n", "tool pmc-sog_otfP_couv99-default_1_16 pmc-sog_otfP_couv99-shy_1_16 \\\n", "model formula \n", "robot20 1 TIME LIMIT TIME LIMIT \n", " 2 OK TIME LIMIT \n", " 3 NaN OK \n", " 4 OK OK \n", " 5 NaN OK \n", "\n", "tool pmc-sog_otf_couv99-default_2_8 pmc-sog_otf_couv99-default_4_4 \\\n", "model formula \n", "robot20 1 TIME LIMIT TIME LIMIT \n", " 2 OK OK \n", " 3 NaN TIME LIMIT \n", " 4 OK OK \n", " 5 NaN NaN \n", "\n", "tool pmc-sog_otf_couv99-shy_2_8 pmc-sog_otf_couv99-shy_4_4 \\\n", "model formula \n", "robot20 1 TIME LIMIT TIME LIMIT \n", " 2 TIME LIMIT TIME LIMIT \n", " 3 OK OK \n", " 4 OK OK \n", " 5 OK OK \n", "\n", "tool pnml2lts-mc_dfs_1_16 pnml2lts-mc_ndfs_1_16 \n", "model formula \n", "robot20 1 UNKNOWN OK \n", " 2 UNKNOWN TIME LIMIT \n", " 3 UNKNOWN OK \n", " 4 UNKNOWN OK \n", " 5 UNKNOWN OK " ] }, "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# table with error for each experiment\n", "table_error = table['error'].copy()\n", "\n", "table_error.head()" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
modelformulaexplored_statesproperty
0robot201NaNF
1robot202NaNT
2robot203NaNF
3robot204NaNF
4robot205NaNT
\n", "
" ], "text/plain": [ " model formula explored_states property\n", "0 robot20 1 NaN F\n", "1 robot20 2 NaN T\n", "2 robot20 3 NaN F\n", "3 robot20 4 NaN F\n", "4 robot20 5 NaN T" ] }, "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# table with explored states for each experiment using ltsmin\n", "table_explored_states = table.copy()\n", "table_explored_states = table_explored_states['explored_states']\n", "table_explored_states = table_explored_states[['pnml2lts-mc_dfs_1_16']]\n", "table_explored_states = table_explored_states.rename(columns={\"pnml2lts-mc_dfs_1_16\": \"explored_states\"})\n", "\n", "# add verification output to the table\n", "table_explored_states = pd.concat([table_explored_states, p_df], axis=1)\n", "\n", "# reshape\n", "table_explored_states = table_explored_states.reset_index()\n", "\n", "table_explored_states.head()" ] }, { "cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
meanminmax
modelproperty
robot20F2.490964e+098372365.04.292339e+09
T2.200687e+091096.04.294409e+09
\n", "
" ], "text/plain": [ " mean min max\n", "model property \n", "robot20 F 2.490964e+09 8372365.0 4.292339e+09\n", " T 2.200687e+09 1096.0 4.294409e+09" ] }, "execution_count": 20, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# calculate the stats of the number of explored states\n", "\n", "table_explored_states_stats = table_explored_states.groupby(['model', 'property']).agg(['mean', 'min', 'max'])\n", "table_explored_states_stats = table_explored_states_stats['explored_states']\n", "table_explored_states_stats.head()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Examples" ] }, { "cell_type": "code", "execution_count": 21, "metadata": {}, "outputs": [ { "data": { "text/html": [ " \n", " " ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "
\n", " \n", " \n", "
\n", " \n", "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "create_figure_explored_states(table_explored_states, 'robot20')" ] }, { "cell_type": "code", "execution_count": 22, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", " \n", " \n", "
\n", " \n", "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "create_figure(df, \"robot20\")" ] }, { "cell_type": "code", "execution_count": 23, "metadata": { "scrolled": false }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "1eb8c522834d491e993cb8b676a47519", "version_major": 2, "version_minor": 0 }, "text/plain": [ "FigureWidget({\n", " 'data': [{'customdata': [[2, 'OK', 'UNKNOWN'], [6, 'OK', 'UNKNOWN'], [15,\n", " …" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "log_figure = create_log_figure(table_time, table_error, \"robot20\", \"pmc-sog_otf_couv99-default_4_4\", \"pnml2lts-mc_dfs_1_16\", True, open_logs_callback)\n", "log_figure" ] }, { "cell_type": "code", "execution_count": 24, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", " \n", " \n", "
\n", " \n", "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "table = get_table(table_time, table_error, \"robot20\", \"pmc-sog_otf_couv99-default_4_4\", \"pnml2lts-mc_dfs_1_16\")\n", "table" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Generate Figures" ] }, { "cell_type": "code", "execution_count": 25, "metadata": {}, "outputs": [], "source": [ "# models\n", "models = df.model.unique()\n", "\n", "# tools \n", "tools = df.tool.unique()" ] }, { "cell_type": "code", "execution_count": 26, "metadata": {}, "outputs": [], "source": [ "# create all the figures of explored states\n", "\n", "folder = os.path.join(OUTPUT_FOLDER, 'explored-states')\n", "create_folder(folder)\n", "\n", "for model in models:\n", " try:\n", " fig = create_figure_explored_states(table_explored_states, model)\n", " \n", " # save figures in html and pdf\n", " fig.write_html(os.path.join(folder, model + '.html'), include_plotlyjs='cdn')\n", " fig.write_image(os.path.join(folder, model + '.pdf'))\n", " except KeyError:\n", " print(\"Error: {} was not plotted\".format(model))" ] }, { "cell_type": "code", "execution_count": 31, "metadata": {}, "outputs": [], "source": [ "# create all the figures formula vs time\n", "\n", "folder = os.path.join(OUTPUT_FOLDER, 'time-plots')\n", "create_folder(folder)\n", "\n", "for model in models:\n", " try:\n", " fig = create_figure(df, model)\n", " \n", " # save figures in html and pdf\n", " fig.write_html(os.path.join(folder, model + '.html'), include_plotlyjs='cdn')\n", " fig.write_image(os.path.join(folder, model + '.pdf'))\n", " except KeyError:\n", " print(\"Error: {} was not plotted\".format(model))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# create all the log figures\n", "\n", "tools_pairs = [sorted(t) for t in (combinations(tools, 2))]\n", "\n", "for plot, filter_method in plots.items():\n", " axes = list(filter(filter_method, tools_pairs))\n", " \n", " for model in models:\n", " folder = os.path.join(OUTPUT_FOLDER, plot, model)\n", " create_folder(folder)\n", " \n", " for axe in axes:\n", " try:\n", " show_strategy = plot == 'compare_couvreur_algorithm'\n", " fig = create_log_figure(table_time, table_error, model, axe[0], axe[1], show_strategy)\n", " table = get_table(table_time, table_error, model, axe[0], axe[1])\n", " \n", " # save figures in html and pdf\n", " figure_name = os.path.join(folder, '{}-{}-VS-{}-log'.format(model, axe[0], axe[1]))\n", " \n", " with open(figure_name + '.html', 'w') as f:\n", " f.write(fig.to_html(full_html=False, include_plotlyjs='cdn', post_script=OPEN_LOGS_CALLBACK_JS))\n", " f.write(table.to_html(full_html=False, include_plotlyjs='cdn', post_script=OPEN_LOGS_CALLBACK_JS))\n", " \n", " fig.write_image(figure_name + '.pdf')\n", " except KeyError:\n", " print(\"Error: {} was not plotted\".format(model))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3.6.9 64-bit ('pmc-sog': virtualenv)", "language": "python", "name": "python36964bitpmcsogvirtualenvc93e492ad45248cd9c4c98eb05b48c1f" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.9" } }, "nbformat": 4, "nbformat_minor": 4 }