Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
PMC-SOG
experiments
hybrid
Commits
58754391
Commit
58754391
authored
Feb 14, 2022
by
Jaime Arias
Browse files
add oar script generator
parent
84fa1863
Changes
1
Show whitespace changes
Inline
Side-by-side
scripts/sbatch_generator.py
View file @
58754391
#!/usr/bin/env python3
import
os
import
stat
import
sys
from
itertools
import
product
from
functools
import
reduce
...
...
@@ -20,6 +21,18 @@ module load gcc/8.3.0/openmpi/3.1.4
"""
oar_header
=
"""
\
#!/bin/bash
#
#OAR --name {experiment_name}_{model_instance}_n{nodes}-th{threads}
#OAR --resource /nodes={nodes}/cpu=1/core={threads},walltime={timeout}
#OAR --stderr {error_file}
#OAR --stdout {output_file}
# Experiments
"""
def
create_folder
(
path
):
"""Creates a folder if it does not exist
...
...
@@ -245,6 +258,74 @@ def generate_experiment_name(tool_dict):
return
f
"
{
tool_name
}
_
{
tool_parameters
}
"
def
generate_oar
(
tool_dict
,
nodes
,
threads
,
model_dict
,
formulas_ids
,
timeout
,
paths
):
"""Generates a slurm batch of a experiment to be executed on the cluster
Parameters
---------
tool_dict : dict
Dictionary with an instance of a tool
nodes : int
Number of nodes
threads : int
Number of threads
model_dict : dict
Dictionary with a model information
formulas_ids : list of int
List of ids of the formulas to be verified
timeout : int
Timeout of the experiment
paths : dict
Dictionary with the paths of the project
"""
tool_name
=
tool_dict
[
"name"
]
model_name
=
model_dict
[
'name'
]
model_instances
=
model_dict
[
'instances'
]
experiment_name
=
generate_experiment_name
(
tool_dict
)
# folder where oar scripts will be saved
oar_folder
=
os
.
path
.
join
(
paths
[
'oar'
],
'experiments'
,
tool_name
,
experiment_name
,
model_name
)
create_folder
(
oar_folder
)
# print srun command for each model_instance
for
model_instance
in
model_instances
:
oar_name
=
f
"
{
model_instance
}
-n
{
nodes
}
-th
{
threads
}
.oar"
oar_file
=
os
.
path
.
join
(
oar_folder
,
oar_name
)
# folder where the outputs will be saved
output_folder
=
os
.
path
.
join
(
paths
[
'results'
],
tool_name
,
experiment_name
,
model_name
,
model_instance
)
create_folder
(
output_folder
)
error_file
=
f
"
{
output_folder
}
/
{
experiment_name
}
.err"
output_file
=
f
"
{
output_folder
}
/
{
experiment_name
}
.out"
header
=
oar_header
.
format
(
experiment_name
=
experiment_name
,
model_instance
=
model_instance
,
nodes
=
nodes
,
threads
=
threads
,
timeout
=
"2:00:00"
,
error_file
=
error_file
,
output_file
=
output_file
)
with
open
(
oar_file
,
'w'
)
as
f
:
f
.
write
(
header
)
for
formula
in
formulas_ids
:
command
=
tool_command
(
tool_dict
,
threads
,
model_name
,
model_instance
,
formula
,
paths
)
f
.
write
(
command
)
f
.
write
(
"
\n\n
"
)
# give oar script the exec right
st
=
os
.
stat
(
oar_file
)
os
.
chmod
(
oar_file
,
st
.
st_mode
|
stat
.
S_IEXEC
)
def
generate_sbatch
(
tool_dict
,
nodes
,
threads
,
model_dict
,
formulas_ids
,
timeout
,
paths
):
"""Generates a slurm batch of a experiment to be executed on the cluster
...
...
@@ -322,6 +403,8 @@ def create_default_paths():
'results'
:
os
.
path
.
join
(
base_folder
,
"results"
),
# Folder where the slurm batches will be saved
'slurm'
:
os
.
path
.
join
(
base_folder
,
"slurm"
),
# Folder where the oar batches will be saved
'oar'
:
os
.
path
.
join
(
base_folder
,
"oar"
),
# Folder where the tool are saved
'tools'
:
os
.
path
.
join
(
base_folder
,
"tools"
)
}
...
...
@@ -348,8 +431,8 @@ def explode_tool(tool):
return
result
def
generate_multiple_
s
batchs
(
tools
,
models
,
formulas
,
nodes_list
,
threads_list
,
timeout
,
paths
):
def
generate_multiple_batchs
(
tools
,
models
,
formulas
,
nodes_list
,
threads_list
,
timeout
,
paths
,
launcher
):
"""Generates the slurm batch for several experiments
Parameters
...
...
@@ -368,6 +451,8 @@ def generate_multiple_sbatchs(tools, models, formulas, nodes_list,
Time in minutes of each experiment
paths : dict
Dictionary with the paths of the project
cluster : string
Name of the cluster
"""
for
tool_dict
in
tools
:
tools_dict
=
explode_tool
(
tool_dict
)
...
...
@@ -376,22 +461,32 @@ def generate_multiple_sbatchs(tools, models, formulas, nodes_list,
for
model
in
models
:
for
nodes
in
nodes_list
:
for
threads
in
threads_list
:
if
launcher
==
"slurm"
:
generate_sbatch
(
tool
,
nodes
,
threads
,
model
,
formulas
,
timeout
,
paths
)
elif
launcher
==
"oar"
:
generate_oar
(
tool
,
nodes
,
threads
,
model
,
formulas
,
timeout
,
paths
)
else
:
print
(
f
"
{
launcher
}
is not supported"
)
sys
.
exit
(
0
)
if
__name__
==
'__main__'
:
# Default paths
paths
=
create_default_paths
()
# slurm or oar
launcher
=
"oar"
# Timeout: 20 minutes
timeout
=
20
# Number of nodes
nodes
=
[
4
]
nodes
=
[
1
]
# Number of threads
threads
=
[
4
]
threads
=
[
16
]
# Formulas to be verified
nb_formulas
=
200
...
...
@@ -431,5 +526,5 @@ if __name__ == '__main__':
# }
}]
generate_multiple_
s
batchs
(
tools
,
models
,
formulas
,
nodes
,
threads
,
timeout
,
paths
)
generate_multiple_batchs
(
tools
,
models
,
formulas
,
nodes
,
threads
,
timeout
,
paths
,
launcher
)
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment