Skip to content

Base

get_file_suffix(dir, suffix)

Returns a list of all files in dir which end in suffix.

This is the same function that is in utils.load but cannot do a relative import due to slurm job submission stuff.

Parameters:

Name Type Description Default
dir str

Directory of interest.

 required
suffix str

Usually the file type of interest e.g. .nml or .txt.

 required

Returns:

Type Description
List[str]

List of all files with the correct suffix.
Source code in isca_tools/run/base.py 
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
def get_file_suffix(dir: str, suffix: str) -> List[str]:
    """
    Returns a list of all files in `dir` which end in `suffix`.

    This is the same function that is in `utils.load` but cannot do a relative import due to slurm job submission
    stuff.

    Args:
        dir: Directory of interest.
        suffix: Usually the file type of interest e.g. `.nml` or `.txt`.

    Returns:
        List of all files with the correct `suffix`.
    """
    file_name = []
    for file in os.listdir(dir):
        if file.endswith(suffix):
            file_name += [file]
    return file_name

get_unique_dir_name(base_dir)

Return a unique directory name by appending a number if needed. E.g., 'results', 'results_1', 'results_2', ...

Parameters:

Name Type Description Default
base_dir str

Path to directory

 required

Returns:

Name Type Description
base_dir str

Unique directory name
Source code in isca_tools/run/base.py 
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
def get_unique_dir_name(base_dir: str) -> str:
    """
    Return a unique directory name by appending a number if needed.
    E.g., 'results', 'results_1', 'results_2', ...

    Args:
        base_dir: Path to directory

    Returns:
        base_dir: Unique directory name
    """
    if not os.path.exists(base_dir):
        return base_dir

    i = 1
    while True:
        new_dir = f"{base_dir}_{i}"
        if not os.path.exists(new_dir):
            return new_dir
        i += 1

pbs_job_name(path, max_length=15)

Convert a file path to a PBS-safe job name.

Uses the filename without its final extension, replaces unsupported characters with underscores, and ensures the name starts with a letter.

Parameters:

Name Type Description Default
path str

File path used to generate the job name.

 required
max_length int

Maximum number of characters in the job name. Defaults to 15. Must be positive.

 15

Returns:

Type Description
str

A non-empty job name containing only ASCII letters, digits,
str

underscores, and hyphens, starting with a letter and truncated to
str

max_length characters. Uniqueness is not guaranteed.

Raises:

Type Description
ValueError

If max_length is less than 1.
Source code in isca_tools/run/base.py 
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
def pbs_job_name(path: str, max_length: int = 15) -> str:
    """Convert a file path to a PBS-safe job name.

    Uses the filename without its final extension, replaces unsupported
    characters with underscores, and ensures the name starts with a letter.

    Args:
        path: File path used to generate the job name.
        max_length: Maximum number of characters in the job name. Defaults
            to 15. Must be positive.

    Returns:
        A non-empty job name containing only ASCII letters, digits,
        underscores, and hyphens, starting with a letter and truncated to
        max_length characters. Uniqueness is not guaranteed.

    Raises:
        ValueError: If max_length is less than 1.
    """
    path = path.replace('/','_')      # otherwise PBS will think it's a directory and only consider final part of the path
    if max_length < 1:
        raise ValueError("max_length must be at least 1.")

    name = re.sub(r"[^A-Za-z0-9_-]+", "_", Path(path).stem).strip("_-")

    if not name:
        name = "job"
    elif not re.match(r"[A-Za-z]", name):
        name = f"j_{name}"

    return name[:max_length]

run_experiment(namelist_file, diag_table_file, slurm=False, scheduler='slurm')

This splits the total simulation up into jobs based on info specified in namelist_file and then runs run_job for each job. If slurm==True, then each job will be submitted to a Slurm queue. Alternatively, scheduler='pbs' submits jobs to Imperial's PBS scheduler.

Parameters:

Name Type Description Default
namelist_file str

File path to namelist nml file for the experiment. This specifies the physical parameters used for the simulation. Also contains experiment_details section which contains the following:

  • name: string. Name of experiment e.g. data saved in folder $GFDL_DATA/{name}
  • input_dir: string. Directory containing any input files e.g. namelist.nml or co2.nc.
  • n_months_total: int. Total duration of simulation in months.
  • n_months_job: int. Approximate duration of each job of the simulation in months. E.g. if n_months_total=12 and n_months_job=6, the experiment would be split up into 2 jobs each of length 6 months.
  • n_nodes: int. Number of nodes to run the job on.
  • n_cores: int. Number of cores for each node to run the job on.
  • memory_per_node: string. Optional memory request per node for PBS, e.g. '16gb'.
  • resolution: string. Horizontal resolution of experiment (options are T21, T42 or T85).
  • partition: string. Slurm queue to submit the job to. Ignored for PBS.
  • overwrite_data: bool. If this is True and data already exists in $GFDL_DATA/{name}, then it will be overwritten. If it is False and the data exists, an error will occur.
  • compile: bool. If True, it will recompile the codebase before running the experiment.
  • max_walltime: string. Maximum job walltime; PBS on Imperial uses HH:MM:SS format.
  • delete_restart_files: bool. If True, only the restart file for the final month will be kept. Otherwise, a restart file will be generated for every month.
 required
diag_table_file str

File path to the diagnostic table file for the experiment. This specifies the outputs of the experiment.

 required
slurm bool

If True, will split each job to a Slurm queue. Otherwise, it will just loop over each job locally.

 False
scheduler str

Scheduler to use when submitting jobs. Set to 'pbs' for Imperial HPC; Slurm remains the default scheduler for backward compatibility.

 'slurm'
Source code in isca_tools/run/base.py 
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
def run_experiment(namelist_file: str, diag_table_file: str, slurm: bool = False,
                   scheduler: str = 'slurm'):
    """
    This splits the total simulation up into jobs based on info specified in `namelist_file` and
    then runs `run_job` for each job. If `slurm==True`, then each job will be submitted to a *Slurm*
    queue. Alternatively, `scheduler='pbs'` submits jobs to Imperial's PBS scheduler.

    Args:
        namelist_file: File path to namelist `nml` file for the experiment.
            This specifies the physical parameters used for the simulation.
            Also contains `experiment_details` section which contains the following:

            - `name`: *string*. Name of experiment e.g. data saved in folder `$GFDL_DATA/{name}`
            - `input_dir`: *string*. Directory containing any input files e.g. `namelist.nml` or `co2.nc`.
            - `n_months_total`: *int*. Total duration of simulation in months.
            - `n_months_job`: *int*. Approximate duration of each job of the simulation in months.
            E.g. if `n_months_total=12` and `n_months_job=6`, the experiment would be split up into 2 jobs each
            of length 6 months.
            - `n_nodes`: *int*. Number of nodes to run the job on.
            - `n_cores`: *int*. Number of cores for each node to run the job on.
            - `memory_per_node`: *string*. Optional memory request per node for PBS, e.g. `'16gb'`.
            - `resolution`: *string*. Horizontal resolution of experiment (options are `T21`, `T42` or `T85`).
            - `partition`: *string*. *Slurm* queue to submit the job to. Ignored for PBS.
            - `overwrite_data`: *bool*. If this is `True` and data already exists in `$GFDL_DATA/{name}`,
                then it will be overwritten. If it is `False` and the data exists, an error will occur.
            - `compile`: *bool*. If `True`, it will recompile the codebase before running the experiment.
            - `max_walltime`: *string*. Maximum job walltime; PBS on Imperial uses `HH:MM:SS` format.
            - `delete_restart_files`: *bool*. If `True`, only the restart file for the final month will be kept.
                Otherwise, a restart file will be generated for every month.
        diag_table_file: File path to the diagnostic table file for the experiment.
            This specifies the outputs of the experiment.
        slurm: If `True`, will split each job to a *Slurm* queue. Otherwise, it will just loop over each
            job locally.
        scheduler: Scheduler to use when submitting jobs. Set to `'pbs'` for Imperial HPC; Slurm remains
            the default scheduler for backward compatibility.
    """
    if scheduler not in ('slurm', 'pbs'):
        raise ValueError(f"Unsupported scheduler {scheduler!r}; expected 'slurm' or 'pbs'.")

    exp_details = f90nml.read(namelist_file)['experiment_details']

    if exp_details['n_months_total'] % exp_details['n_months_job'] == 0:
        # If you give n_months_job which is exact multiple of n_months_total, do this many jobs
        n_jobs = int(np.ceil(exp_details['n_months_total'] / exp_details['n_months_job']))
    else:
        # Split simulation equally between jobs so no more than 1.5 multiplied by exp_details['n_months_job'] months
        # would be in each job.
        n_jobs = int(np.ceil(exp_details['n_months_total'] / (1.5 * exp_details['n_months_job'])))
    # month_jobs[i] are the months to simulate in job i.
    month_jobs = np.array_split(np.arange(1, exp_details['n_months_total'] + 1), n_jobs)

    run_dir = os.path.dirname(os.path.realpath(__file__))
    if scheduler == 'pbs':
        submit_script = os.path.join(run_dir, 'imperial', 'run_pbs.sh')
        exp_details.setdefault('nodelist', '-')
    elif 'nodelist' not in exp_details:
        # if not given, allow all available nodes
        submit_script = os.path.join(run_dir, 'run_slurm_no_namelist.sh')
        exp_details['nodelist'] = ''    # just set to empty string
    else:
        print(f"Submitting to node: {exp_details['nodelist']}")
        submit_script = os.path.join(run_dir, 'run_slurm.sh')
    # Run this base.py script as main if using a batch scheduler
    # Because doing this, cannot have any relative imports in this file
    run_job_script = os.path.realpath(__file__)

    # Get directory name to save debugging and timing info
    # If directory already exists, will add a number e.g. console_output_1 until finds one that exists.
    dir_output = os.path.join(os.environ['GFDL_DATA'], exp_details['name'], 'console_output')
    dir_output = get_unique_dir_name(dir_output)
    os.makedirs(dir_output, exist_ok=False)

    prev_job_id = ''
    # Iterate over all jobs
    for month_job in month_jobs:
        if slurm or scheduler == 'pbs':
            # Different script for first vs subsequent scripts, as they are dependent on previous job.
            job_submit_script = submit_script if prev_job_id == '' else submit_script.replace('.sh', '_depend.sh')
            partition = '-' if scheduler == 'pbs' else exp_details['partition']
            dependency_arg = prev_job_id or ('-' if scheduler == 'pbs' else '')
            job_name = pbs_job_name(exp_details['name'])
            memory_request = ''
            if scheduler == 'pbs' and exp_details.get('memory_per_node'):
                memory_request = f":mem={exp_details['memory_per_node']}"
            cmd = (
                f"bash {job_submit_script} "
                f"{job_name} {month_job[0]} {len(month_job)} "
                f"{partition} {exp_details['n_nodes']} {exp_details['n_cores']} "
                f"{namelist_file} {diag_table_file} {exp_details['max_walltime']} {run_job_script} {dir_output} "
                f"{exp_details['nodelist']} {dependency_arg} {memory_request}"
            )
            output = subprocess.check_output(cmd, shell=True).decode("utf-8").strip()   # get job just submitted info
            print(f"{output}{prev_job_id if prev_job_id=='' else ' (dependency: job '+prev_job_id+')'}")
            prev_job_id = output.split()[-1]  # Save this job id (last word) for the next submission
        else:
            run_job(namelist_file, diag_table_file, month_job[0], len(month_job))

run_job(namelist_file, diag_table_file, month_start, month_duration)

Runs simulation for month_duration months starting with the month indicated by month_start. Will throw an error if no data found for months prior to month_start.

Output will be saved in $GFDL_DATA/{exp_name} with a different folder for each month.

Extended from original script. Args: namelist_file: File path to namelist nml file for the experiment. This specifies the physical parameters used for the simulation. Also contains experiment_details section which contains the following:

    - `name`: *string*. Name of experiment e.g. data saved in folder `$GFDL_DATA/{name}`
    - `input_dir`: *string*. Directory containing any input files e.g. `namelist.nml` or `co2.nc`.
    - `n_months_total`: *int*. Total duration of simulation in months.
    - `n_months_job`: *int*. Approximate duration of each job of the simulation in months.
    E.g. if `n_months_total=12` and `n_months_job=6`, the experiment would be split up into 2 jobs each
    of length 6 months.
    - `n_nodes`: *int*. Number of nodes to run job on (*Slurm* info).
    - `n_cores`: *int*. Number of cores for each node to run job on (*Slurm* info).
    - `resolution`: *string*. Horizontal resolution of experiment (options are `T21`, `T42` or `T85`).
    - `partition`: *string*. *Slurm* queue to submit job to.
    - `overwrite_data`: *bool*. If this is `True` and data already exists in `$GFDL_DATA/{name}`,
        then it will be overwritten. If it is `False` and the data exists, an error will occur.
    - `compile`: *bool*. If `True`, it will recompile the codebase before running the experiment.
    - `max_walltime`: *string*. Maximum time that job can run on *Slurm*. E.g. 1 hour would be '01:00:00'.
    - `delete_restart_files`: *bool*. If `True`, only the restart file for the final month will be kept.
        Otherwise, a restart file will be generated for every month.
diag_table_file: File path to the diagnostic table file for the experiment.
    This specifies the outputs of the experiment.
month_start: Index of month at which this job starts the simulation (starting with 1).
month_duration: How many months to run simulation for in this job.
Source code in isca_tools/run/base.py 
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
def run_job(namelist_file: str, diag_table_file: str, month_start: int, month_duration: int):
    """
    Runs simulation for `month_duration` months starting with the month indicated by `month_start`.
    Will throw an error if no data found for months prior to `month_start`.

    Output will be saved in `$GFDL_DATA/{exp_name}` with a different folder for each month.

    Extended from [original script](https://github.com/ExeClim/Isca/blob/master/exp/run_isca/isca).
    Args:
        namelist_file: File path to namelist `nml` file for the experiment.
            This specifies the physical parameters used for the simulation.
            Also contains `experiment_details` section which contains the following:

            - `name`: *string*. Name of experiment e.g. data saved in folder `$GFDL_DATA/{name}`
            - `input_dir`: *string*. Directory containing any input files e.g. `namelist.nml` or `co2.nc`.
            - `n_months_total`: *int*. Total duration of simulation in months.
            - `n_months_job`: *int*. Approximate duration of each job of the simulation in months.
            E.g. if `n_months_total=12` and `n_months_job=6`, the experiment would be split up into 2 jobs each
            of length 6 months.
            - `n_nodes`: *int*. Number of nodes to run job on (*Slurm* info).
            - `n_cores`: *int*. Number of cores for each node to run job on (*Slurm* info).
            - `resolution`: *string*. Horizontal resolution of experiment (options are `T21`, `T42` or `T85`).
            - `partition`: *string*. *Slurm* queue to submit job to.
            - `overwrite_data`: *bool*. If this is `True` and data already exists in `$GFDL_DATA/{name}`,
                then it will be overwritten. If it is `False` and the data exists, an error will occur.
            - `compile`: *bool*. If `True`, it will recompile the codebase before running the experiment.
            - `max_walltime`: *string*. Maximum time that job can run on *Slurm*. E.g. 1 hour would be '01:00:00'.
            - `delete_restart_files`: *bool*. If `True`, only the restart file for the final month will be kept.
                Otherwise, a restart file will be generated for every month.
        diag_table_file: File path to the diagnostic table file for the experiment.
            This specifies the outputs of the experiment.
        month_start: Index of month at which this job starts the simulation (starting with 1).
        month_duration: How many months to run simulation for in this job.
    """
    namelist = f90nml.read(namelist_file)
    diag_table = DiagTable.from_file(diag_table_file)
    exp_details = namelist['experiment_details']

    if 'column_nml' in namelist:
        # Different code base if using column
        cb = ColumnCodeBase.from_directory(GFDL_BASE)
        if 'idealized_moist_phys_nml' in namelist and 'do_socrates_radiation' in namelist['idealized_moist_phys_nml']:
            if namelist['idealized_moist_phys_nml']['do_socrates_radiation']:
                # Different codebase if using socrates
                cb = SocColumnCodeBase.from_directory(GFDL_BASE)
    else:
        cb = IscaCodeBase.from_directory(GFDL_BASE)
        if 'idealized_moist_phys_nml' in namelist and 'do_socrates_radiation' in namelist['idealized_moist_phys_nml']:
            if namelist['idealized_moist_phys_nml']['do_socrates_radiation']:
                # Different codebase if using socrates
                cb = SocratesCodeBase.from_directory(GFDL_BASE)

    if exp_details['compile']:
        cb.compile()

    exp = Experiment(exp_details['name'], codebase=cb)
    # When passing to experiment, will have additional namelist called 'experiment_details' but does not
    # seem to be an issue.
    exp.namelist = namelist
    exp.diag_table = diag_table
    exp.inputfiles = [namelist_file, diag_table_file]

    # Get any additional input files e.g. co2 concentration or land - all of which should have a .nc suffix.
    nc_files = get_file_suffix(exp_details['input_dir'], '.nc')
    if len(nc_files) > 0:
        nc_files = [os.path.join(exp_details['input_dir'], val) for val in nc_files]
        exp.inputfiles += nc_files

    exp.set_resolution(exp_details['resolution'])  # set resolution
    if month_start == 1:
        # If first month, then there is no restart file to use
        use_restart = False
    else:
        use_restart = True
    exp.run(month_start, use_restart=use_restart, num_cores=exp_details['n_cores'],
            overwrite_data=exp_details['overwrite_data'])
    for i in range(month_start + 1, month_start + month_duration):
        # For all months but first, use latest restart file to set up simulation.
        exp.run(i, num_cores=exp_details['n_cores'], overwrite_data=exp_details['overwrite_data'])
        if exp_details['delete_restart_files']:
            # delete restart file for month previous to current month as no longer needed and quite large file
            restart_file = os.path.join(os.environ['GFDL_DATA'], exp_details['name'], 'restarts',
                                        'res%04d.tar.gz' % (i - 1))
            if os.path.exists(restart_file):
                os.remove(restart_file)
            if i == month_start + 1 and use_restart:
                # If not the first job, delete the last restart file of the last job
                restart_file = os.path.join(os.environ['GFDL_DATA'], exp_details['name'], 'restarts',
                                            'res%04d.tar.gz' % (i - 2))
                if os.path.exists(restart_file):
                    os.remove(restart_file)