<core_client_version>7.16.6</core_client_version>
<![CDATA[
<message>
process exited with code 1 (0x1, -255)</message>
<stderr_txt>
thub/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2205] DR_HOOK_HASHBITS=16
[EC_DRHOOK:pyroxeen:1:1:920111:920111] [20221129:144222:1669729342:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2213] DR_HOOK_NCALLSTACK=0
[EC_DRHOOK:pyroxeen:1:1:920111:920111] [20221129:144222:1669729342:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2221] DR_HOOK_HARAKIRI_TIMEOUT=500
[EC_DRHOOK:pyroxeen:1:1:920111:920111] [20221129:144222:1669729342:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2228] DR_HOOK_TRAPFPE=1
[EC_DRHOOK:pyroxeen:1:1:920111:920111] [20221129:144222:1669729342:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2235] DR_HOOK_TRAPFPE_INVALID=1
[EC_DRHOOK:pyroxeen:1:1:920111:920111] [20221129:144222:1669729342:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2242] DR_HOOK_TRAPFPE_DIVBYZERO=1
[EC_DRHOOK:pyroxeen:1:1:920111:920111] [20221129:144222:1669729342:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2249] DR_HOOK_TRAPFPE_OVERFLOW=1
[EC_DRHOOK:pyroxeen:1:1:920111:920111] [20221129:144222:1669729342:0.000] [ignore_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1227] DR_HOOK_IGNORE_SIGNALS=<undef>
[EC_DRHOOK:pyroxeen:1:1:920111:920111] [20221129:144222:1669729342:0.000] [restore_default_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1178] DR_HOOK_RESTORE_DEFAULT_SIGNALS=<undef>
[EC_DRHOOK:pyroxeen:1:1:920111:920111] [20221129:144222:1669729342:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1937] New signal handler 'signal_drhook' for signal#6 (SIGABRT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:pyroxeen:1:1:920111:920111] [20221129:144222:1669729342:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1938] New signal handler 'signal_drhook' for signal#7 (SIGBUS) at 0x81f820 (old at (nil))
[EC_DRHOOK:pyroxeen:1:1:920111:920111] [20221129:144222:1669729342:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1939] New signal handler 'signal_drhook' for signal#11 (SIGSEGV) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:pyroxeen:1:1:920111:920111] [20221129:144222:1669729342:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1944] New signal handler 'signal_drhook' for signal#16 (SIGSTKFLT) at 0x81f820 (old at (nil))
[EC_DRHOOK:pyroxeen:1:1:920111:920111] [20221129:144222:1669729342:0.000] [trapfpe_treatment@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1149] DR_HOOK enables SIGFPE-related floating point trapping since DRHOOK_TRAPFPE=1
[EC_DRHOOK:pyroxeen:1:1:920111:920111] [20221129:144222:1669729342:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1948] New signal handler 'signal_drhook' for signal#8 (SIGFPE) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:pyroxeen:1:1:920111:920111] [20221129:144222:1669729342:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1949] New signal handler 'signal_drhook' for signal#4 (SIGILL) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:pyroxeen:1:1:920111:920111] [20221129:144222:1669729342:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1951] New signal handler 'signal_drhook' for signal#5 (SIGTRAP) at 0x81f820 (old at (nil))
[EC_DRHOOK:pyroxeen:1:1:920111:920111] [20221129:144222:1669729342:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1952] New signal handler 'signal_drhook' for signal#2 (SIGINT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:pyroxeen:1:1:920111:920111] [20221129:144222:1669729342:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1960] New signal handler 'signal_drhook' for signal#3 (SIGQUIT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:pyroxeen:1:1:920111:920111] [20221129:144222:1669729342:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1961] New signal handler 'signal_drhook' for signal#15 (SIGTERM) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:pyroxeen:1:1:920111:920111] [20221129:144222:1669729342:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1966] New signal handler 'signal_drhook' for signal#24 (SIGXCPU) at 0x81f820 (old at (nil))
[EC_DRHOOK:pyroxeen:1:1:920111:920111] [20221129:144222:1669729342:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1968] New signal handler 'signal_drhook' for signal#25 (SIGXFSZ) at 0x81f820 (old at (nil))
[EC_DRHOOK:pyroxeen:1:1:920111:920111] [20221129:144222:1669729342:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1973] New signal handler 'signal_drhook' for signal#31 (SIGSYS) at 0x81f820 (old at (nil))
[EC_DRHOOK:pyroxeen:1:1:920111:920111] [20221129:144222:1669729342:0.000] [catch_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1111] DR_HOOK_CATCH_SIGNALS=<undef>
14:43:13 STEP 1368 H=1368:00 +CPU= 45.016
14:43:39 STEP 1369 H=1369:00 +CPU= 25.207
14:44:07 STEP 1370 H=1370:00 +CPU= 27.446
14:44:47 STEP 1371 H=1371:00 +CPU= 38.797
14:45:14 STEP 1372 H=1372:00 +CPU= 26.643
14:45:41 STEP 1373 H=1373:00 +CPU= 26.793
14:46:21 STEP 1374 H=1374:00 +CPU= 39.358
14:46:47 STEP 1375 H=1375:00 +CPU= 25.674
14:47:16 STEP 1376 H=1376:00 +CPU= 27.279
14:47:56 STEP 1377 H=1377:00 +CPU= 39.064
14:48:23 STEP 1378 H=1378:00 +CPU= 26.625
14:48:51 STEP 1379 H=1379:00 +CPU= 26.539
14:49:39 STEP 1380 H=1380:00 +CPU= 46.145
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMGGhpi1+001380
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMSHhpi1+001380
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMUAhpi1+001380
14:50:05 STEP 1381 H=1381:00 +CPU= 25.438
14:50:35 STEP 1382 H=1382:00 +CPU= 27.805
14:51:14 STEP 1383 H=1383:00 +CPU= 38.600
14:51:41 STEP 1384 H=1384:00 +CPU= 26.217
14:52:08 STEP 1385 H=1385:00 +CPU= 26.822
14:52:48 STEP 1386 H=1386:00 +CPU= 39.567
14:53:14 STEP 1387 H=1387:00 +CPU= 25.361
14:53:42 STEP 1388 H=1388:00 +CPU= 27.025
14:54:22 STEP 1389 H=1389:00 +CPU= 39.004
14:54:51 STEP 1390 H=1390:00 +CPU= 27.301
14:55:18 STEP 1391 H=1391:00 +CPU= 27.151
14:56:06 STEP 1392 H=1392:00 +CPU= 46.582
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMGGhpi1+001392
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMSHhpi1+001392
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMUAhpi1+001392
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMGGhpi1+001368
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMSHhpi1+001368
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMUAhpi1+001368
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMGGhpi1+001380
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMSHhpi1+001380
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMUAhpi1+001380
Zipping up the intermediate file: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_0120_2021050100_123_945_12163209_0_r1264279961_1.zip
Uploading the intermediate file: upload_file_1.zip
14:56:33 STEP 1393 H=1393:00 +CPU= 25.989
Uploading trickle at timestep: 5011200
14:57:00 STEP 1394 H=1394:00 +CPU= 26.218
14:57:42 STEP 1395 H=1395:00 +CPU= 39.837
14:58:09 STEP 1396 H=1396:00 +CPU= 26.331
14:58:34 STEP 1397 H=1397:00 +CPU= 24.945
14:59:17 STEP 1398 H=1398:00 +CPU= 42.584
14:59:43 STEP 1399 H=1399:00 +CPU= 24.713
15:00:10 STEP 1400 H=1400:00 +CPU= 26.773
15:00:51 STEP 1401 H=1401:00 +CPU= 39.493
15:01:17 STEP 1402 H=1402:00 +CPU= 25.472
15:01:43 STEP 1403 H=1403:00 +CPU= 25.981
15:02:29 STEP 1404 H=1404:00 +CPU= 43.653
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMGGhpi1+001404
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMSHhpi1+001404
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMUAhpi1+001404
15:02:55 STEP 1405 H=1405:00 +CPU= 25.179
15:03:24 STEP 1406 H=1406:00 +CPU= 27.898
15:04:08 STEP 1407 H=1407:00 +CPU= 40.820
15:04:35 STEP 1408 H=1408:00 +CPU= 26.749
15:05:03 STEP 1409 H=1409:00 +CPU= 26.853
15:05:47 STEP 1410 H=1410:00 +CPU= 41.741
15:06:14 STEP 1411 H=1411:00 +CPU= 26.436
15:06:44 STEP 1412 H=1412:00 +CPU= 27.834
15:07:27 STEP 1413 H=1413:00 +CPU= 42.068
15:07:55 STEP 1414 H=1414:00 +CPU= 26.705
15:08:22 STEP 1415 H=1415:00 +CPU= 26.594
15:09:09 STEP 1416 H=1416:00 +CPU= 45.202
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMGGhpi1+001416
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMSHhpi1+001416
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMUAhpi1+001416
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMGGhpi1+001392
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMSHhpi1+001392
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMUAhpi1+001392
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMGGhpi1+001404
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMSHhpi1+001404
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMUAhpi1+001404
Zipping up the intermediate file: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_0120_2021050100_123_945_12163209_0_r1264279961_2.zip
Uploading the intermediate file: upload_file_2.zip
Uploading trickle at timestep: 5097600
15:09:34 STEP 1417 H=1417:00 +CPU= 25.211
15:10:04 STEP 1418 H=1418:00 +CPU= 28.037
15:10:43 STEP 1419 H=1419:00 +CPU= 37.861
15:11:09 STEP 1420 H=1420:00 +CPU= 25.826
15:11:36 STEP 1421 H=1421:00 +CPU= 26.582
15:12:16 STEP 1422 H=1422:00 +CPU= 39.374
15:12:43 STEP 1423 H=1423:00 +CPU= 25.976
15:13:10 STEP 1424 H=1424:00 +CPU= 26.376
15:13:51 STEP 1425 H=1425:00 +CPU= 40.009
15:14:18 STEP 1426 H=1426:00 +CPU= 26.228
15:14:44 STEP 1427 H=1427:00 +CPU= 25.820
15:15:33 STEP 1428 H=1428:00 +CPU= 45.559
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMGGhpi1+001428
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMSHhpi1+001428
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMUAhpi1+001428
15:15:58 STEP 1429 H=1429:00 +CPU= 25.362
15:16:24 STEP 1430 H=1430:00 +CPU= 25.376
15:17:06 STEP 1431 H=1431:00 +CPU= 40.577
15:17:33 STEP 1432 H=1432:00 +CPU= 25.812
15:17:58 STEP 1433 H=1433:00 +CPU= 25.104
15:18:40 STEP 1434 H=1434:00 +CPU= 40.586
15:19:06 STEP 1435 H=1435:00 +CPU= 25.414
15:19:33 STEP 1436 H=1436:00 +CPU= 26.590
15:20:15 STEP 1437 H=1437:00 +CPU= 40.400
15:20:43 STEP 1438 H=1438:00 +CPU= 27.344
15:21:13 STEP 1439 H=1439:00 +CPU= 27.926
15:22:02 STEP 1440 H=1440:00 +CPU= 47.776
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMGGhpi1+001440
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMSHhpi1+001440
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMUAhpi1+001440
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMGGhpi1+001416
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMSHhpi1+001416
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMUAhpi1+001416
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMGGhpi1+001428
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMSHhpi1+001428
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMUAhpi1+001428
Zipping up the intermediate file: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_0120_2021050100_123_945_12163209_0_r1264279961_3.zip
Uploading the intermediate file: upload_file_3.zip
Uploading trickle at timestep: 5184000
15:22:28 STEP 1441 H=1441:00 +CPU= 25.055
15:22:58 STEP 1442 H=1442:00 +CPU= 28.711
15:23:37 STEP 1443 H=1443:00 +CPU= 38.492
15:24:03 STEP 1444 H=1444:00 +CPU= 25.871
15:24:32 STEP 1445 H=1445:00 +CPU= 28.050
15:25:12 STEP 1446 H=1446:00 +CPU= 38.930
15:25:38 STEP 1447 H=1447:00 +CPU= 25.418
15:26:06 STEP 1448 H=1448:00 +CPU= 27.082
15:26:46 STEP 1449 H=1449:00 +CPU= 38.750
15:27:13 STEP 1450 H=1450:00 +CPU= 26.440
15:27:38 STEP 1451 H=1451:00 +CPU= 24.601
15:28:24 STEP 1452 H=1452:00 +CPU= 44.336
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMGGhpi1+001452
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMSHhpi1+001452
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMUAhpi1+001452
15:28:51 STEP 1453 H=1453:00 +CPU= 25.854
15:29:18 STEP 1454 H=1454:00 +CPU= 26.659
15:29:58 STEP 1455 H=1455:00 +CPU= 39.708
15:30:24 STEP 1456 H=1456:00 +CPU= 25.738
15:30:52 STEP 1457 H=1457:00 +CPU= 26.988
15:31:33 STEP 1458 H=1458:00 +CPU= 39.903
15:31:59 STEP 1459 H=1459:00 +CPU= 24.767
15:32:29 STEP 1460 H=1460:00 +CPU= 28.316
15:33:08 STEP 1461 H=1461:00 +CPU= 38.552
15:33:34 STEP 1462 H=1462:00 +CPU= 25.753
15:34:04 STEP 1463 H=1463:00 +CPU= 27.554
15:34:50 STEP 1464 H=1464:00 +CPU= 45.207
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMGGhpi1+001464
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMSHhpi1+001464
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMUAhpi1+001464
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMGGhpi1+001440
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMSHhpi1+001440
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMUAhpi1+001440
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMGGhpi1+001452
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMSHhpi1+001452
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMUAhpi1+001452
Zipping up the intermediate file: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_0120_2021050100_123_945_12163209_0_r1264279961_4.zip
Uploading the intermediate file: upload_file_4.zip
15:35:18 STEP 1465 H=1465:00 +CPU= 26.891
Uploading trickle at timestep: 5270400
15:35:45 STEP 1466 H=1466:00 +CPU= 26.266
15:36:24 STEP 1467 H=1467:00 +CPU= 39.080
15:36:52 STEP 1468 H=1468:00 +CPU= 27.064
15:37:18 STEP 1469 H=1469:00 +CPU= 25.191
15:38:00 STEP 1470 H=1470:00 +CPU= 40.825
15:38:26 STEP 1471 H=1471:00 +CPU= 25.283
15:38:52 STEP 1472 H=1472:00 +CPU= 26.425
15:39:34 STEP 1473 H=1473:00 +CPU= 40.220
15:40:00 STEP 1474 H=1474:00 +CPU= 26.051
15:40:26 STEP 1475 H=1475:00 +CPU= 25.158
15:41:13 STEP 1476 H=1476:00 +CPU= 44.886
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMGGhpi1+001476
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMSHhpi1+001476
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMUAhpi1+001476
15:41:42 STEP 1477 H=1477:00 +CPU= 27.263
15:42:08 STEP 1478 H=1478:00 +CPU= 25.791
15:42:47 STEP 1479 H=1479:00 +CPU= 38.281
Quit request received from BOINC client, ending the child process
(argv0) ../../projects/climateprediction.net/oifs_43r3_ps_1.01_x86_64-pc-linux-gnu
(argv1) start_date: 2021050100
(argv2) exptid: hpi1
(argv3) unique_member_id: 0120
(argv4) batchid: 945
(argv5) wuid: 12163209
(argv6) fclen: 123
(argv7) app_name: oifs_43r3_ps
(argv8) nthreads: 1
Working directory is: /var/lib/boinc-client/slots/3
Project directory is: /var/lib/boinc-client/projects/climateprediction.net/
app name: oifs_43r3_ps
version: 1.01
Location of temp folder: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209
..mkdir for temp folder for results failed
Copying: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_app_1.01_x86_64-pc-linux-gnu.zip to: /var/lib/boinc-client/slots/3/oifs_43r3_ps_app_1.01_x86_64-pc-linux-gnu.zip
Unzipping the app zip file: /var/lib/boinc-client/slots/3/oifs_43r3_ps_app_1.01_x86_64-pc-linux-gnu.zip
Copying the namelist files from: ../../projects/climateprediction.net/jf_67cb0c72c97fe1c38e734f399c610d16 to: /var/lib/boinc-client/slots/3/oifs_43r3_ps_0120_2021050100_123_945_12163209.zip
Unzipping the namelist zip file: /var/lib/boinc-client/slots/3/oifs_43r3_ps_0120_2021050100_123_945_12163209.zip
ic_ancil_file: ic_ancil_12163209
ifsdata_file: ifsdata_12163209
climate_data_file: clim_data_12163209
horiz_resolution: 159
vert_resolution: 60
grid_type: l_2
upload_interval: 24
utstep: 3600
Copying IC ancils from: ../../projects/climateprediction.net/jf_f6b249c1bbdd8b0b2d9b7cfe1d7f36ce to: /var/lib/boinc-client/slots/3/ic_ancil_12163209.zip
Unzipping the IC ancils zip file: /var/lib/boinc-client/slots/3/ic_ancil_12163209.zip
..mkdir for ifsdata folder failed
Copying the ifsdata_file from: ../../projects/climateprediction.net/jf_9a5ab440cf540a120bd06629cab8bab4 to: /var/lib/boinc-client/slots/3/ifsdata/ifsdata_12163209.zip
Unzipping the ifsdata_zip file: /var/lib/boinc-client/slots/3/ifsdata/ifsdata_12163209.zip
..mkdir for the climate data folder failed
Copying the climate data file from: ../../projects/climateprediction.net/jf_f32c668cf7829426ffd94699b48b9a2e to: /var/lib/boinc-client/slots/3/159l_2/clim_data_12163209.zip
Unzipping the climate data zip file: /var/lib/boinc-client/slots/3/159l_2/clim_data_12163209.zip
total_length_of_simulation: 10627200
result_base_name: oifs_43r3_ps_0120_2021050100_123_945_12163209_0_r1264279961
The child process has been launched with process id: 928700
Executing the command: /var/lib/boinc-client/slots/3/./master.exe
[EC_DRHOOK:hostname:myproc:omptid:pid:unixtid] [YYYYMMDD:HHMMSS:epoch:walltime] [function@file:lineno] -- Max OpenMP threads = 1
[EC_DRHOOK:pyroxeen:1:1:928700:928700] [20221129:154522:1669733122:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2091] fp = 0x3129fe0
[EC_DRHOOK:pyroxeen:1:1:928700:928700] [20221129:154522:1669733122:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2098] DR_HOOK_ALLOW_COREDUMP=-1
[EC_DRHOOK:pyroxeen:1:1:928700:928700] [20221129:154522:1669733122:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2104] Hardlimit for core file is now 0 (0x0)
[EC_DRHOOK:pyroxeen:1:1:928700:928700] [20221129:154522:1669733122:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2122] DR_HOOK_PROFILE_PROC=-1
[EC_DRHOOK:pyroxeen:1:1:928700:928700] [20221129:154522:1669733122:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2128] DR_HOOK_PROFILE_LIMIT=-10.000
[EC_DRHOOK:pyroxeen:1:1:928700:928700] [20221129:154522:1669733122:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2194] DR_HOOK_RANDOM_MEMSTAT=0 (RAND_MAX=2147483647)
[EC_DRHOOK:pyroxeen:1:1:928700:928700] [20221129:154522:1669733122:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2205] DR_HOOK_HASHBITS=16
[EC_DRHOOK:pyroxeen:1:1:928700:928700] [20221129:154522:1669733122:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2213] DR_HOOK_NCALLSTACK=0
[EC_DRHOOK:pyroxeen:1:1:928700:928700] [20221129:154522:1669733122:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2221] DR_HOOK_HARAKIRI_TIMEOUT=500
[EC_DRHOOK:pyroxeen:1:1:928700:928700] [20221129:154522:1669733122:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2228] DR_HOOK_TRAPFPE=1
[EC_DRHOOK:pyroxeen:1:1:928700:928700] [20221129:154522:1669733122:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2235] DR_HOOK_TRAPFPE_INVALID=1
[EC_DRHOOK:pyroxeen:1:1:928700:928700] [20221129:154522:1669733122:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2242] DR_HOOK_TRAPFPE_DIVBYZERO=1
[EC_DRHOOK:pyroxeen:1:1:928700:928700] [20221129:154522:1669733122:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2249] DR_HOOK_TRAPFPE_OVERFLOW=1
[EC_DRHOOK:pyroxeen:1:1:928700:928700] [20221129:154522:1669733122:0.000] [ignore_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1227] DR_HOOK_IGNORE_SIGNALS=<undef>
[EC_DRHOOK:pyroxeen:1:1:928700:928700] [20221129:154522:1669733122:0.000] [restore_default_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1178] DR_HOOK_RESTORE_DEFAULT_SIGNALS=<undef>
[EC_DRHOOK:pyroxeen:1:1:928700:928700] [20221129:154522:1669733122:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1937] New signal handler 'signal_drhook' for signal#6 (SIGABRT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:pyroxeen:1:1:928700:928700] [20221129:154522:1669733122:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1938] New signal handler 'signal_drhook' for signal#7 (SIGBUS) at 0x81f820 (old at (nil))
[EC_DRHOOK:pyroxeen:1:1:928700:928700] [20221129:154522:1669733122:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1939] New signal handler 'signal_drhook' for signal#11 (SIGSEGV) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:pyroxeen:1:1:928700:928700] [20221129:154522:1669733122:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1944] New signal handler 'signal_drhook' for signal#16 (SIGSTKFLT) at 0x81f820 (old at (nil))
[EC_DRHOOK:pyroxeen:1:1:928700:928700] [20221129:154522:1669733122:0.000] [trapfpe_treatment@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1149] DR_HOOK enables SIGFPE-related floating point trapping since DRHOOK_TRAPFPE=1
[EC_DRHOOK:pyroxeen:1:1:928700:928700] [20221129:154522:1669733122:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1948] New signal handler 'signal_drhook' for signal#8 (SIGFPE) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:pyroxeen:1:1:928700:928700] [20221129:154522:1669733122:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1949] New signal handler 'signal_drhook' for signal#4 (SIGILL) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:pyroxeen:1:1:928700:928700] [20221129:154522:1669733122:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1951] New signal handler 'signal_drhook' for signal#5 (SIGTRAP) at 0x81f820 (old at (nil))
[EC_DRHOOK:pyroxeen:1:1:928700:928700] [20221129:154522:1669733122:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1952] New signal handler 'signal_drhook' for signal#2 (SIGINT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:pyroxeen:1:1:928700:928700] [20221129:154522:1669733122:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1960] New signal handler 'signal_drhook' for signal#3 (SIGQUIT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:pyroxeen:1:1:928700:928700] [20221129:154522:1669733122:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1961] New signal handler 'signal_drhook' for signal#15 (SIGTERM) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:pyroxeen:1:1:928700:928700] [20221129:154522:1669733122:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1966] New signal handler 'signal_drhook' for signal#24 (SIGXCPU) at 0x81f820 (old at (nil))
[EC_DRHOOK:pyroxeen:1:1:928700:928700] [20221129:154522:1669733122:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1968] New signal handler 'signal_drhook' for signal#25 (SIGXFSZ) at 0x81f820 (old at (nil))
[EC_DRHOOK:pyroxeen:1:1:928700:928700] [20221129:154522:1669733122:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1973] New signal handler 'signal_drhook' for signal#31 (SIGSYS) at 0x81f820 (old at (nil))
[EC_DRHOOK:pyroxeen:1:1:928700:928700] [20221129:154522:1669733122:0.000] [catch_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1111] DR_HOOK_CATCH_SIGNALS=<undef>
15:46:15 STEP 1464 H=1464:00 +CPU= 46.185
15:46:41 STEP 1465 H=1465:00 +CPU= 24.736
15:47:10 STEP 1466 H=1466:00 +CPU= 26.446
15:47:49 STEP 1467 H=1467:00 +CPU= 39.180
15:48:18 STEP 1468 H=1468:00 +CPU= 27.406
15:48:46 STEP 1469 H=1469:00 +CPU= 27.575
15:49:28 STEP 1470 H=1470:00 +CPU= 41.544
15:49:55 STEP 1471 H=1471:00 +CPU= 25.837
15:50:23 STEP 1472 H=1472:00 +CPU= 27.509
15:51:04 STEP 1473 H=1473:00 +CPU= 39.881
15:51:32 STEP 1474 H=1474:00 +CPU= 27.421
15:51:59 STEP 1475 H=1475:00 +CPU= 25.738
15:52:46 STEP 1476 H=1476:00 +CPU= 44.455
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMGGhpi1+001476
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMSHhpi1+001476
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMUAhpi1+001476
15:53:12 STEP 1477 H=1477:00 +CPU= 25.854
15:53:41 STEP 1478 H=1478:00 +CPU= 27.585
15:54:22 STEP 1479 H=1479:00 +CPU= 39.696
15:54:50 STEP 1480 H=1480:00 +CPU= 27.826
15:55:18 STEP 1481 H=1481:00 +CPU= 26.751
15:55:58 STEP 1482 H=1482:00 +CPU= 39.306
15:56:24 STEP 1483 H=1483:00 +CPU= 25.304
15:56:52 STEP 1484 H=1484:00 +CPU= 27.913
15:57:31 STEP 1485 H=1485:00 +CPU= 37.781
15:57:58 STEP 1486 H=1486:00 +CPU= 26.316
15:58:28 STEP 1487 H=1487:00 +CPU= 28.801
15:59:15 STEP 1488 H=1488:00 +CPU= 46.371
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMGGhpi1+001488
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMSHhpi1+001488
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMUAhpi1+001488
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMGGhpi1+001464
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMSHhpi1+001464
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMUAhpi1+001464
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMGGhpi1+001476
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMSHhpi1+001476
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMUAhpi1+001476
Zipping up the intermediate file: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_0120_2021050100_123_945_12163209_0_r1264279961_1.zip
Uploading the intermediate file: upload_file_1.zip
Uploading trickle at timestep: 5356800
15:59:41 STEP 1489 H=1489:00 +CPU= 25.479
16:00:09 STEP 1490 H=1490:00 +CPU= 27.878
16:00:49 STEP 1491 H=1491:00 +CPU= 38.748
16:01:16 STEP 1492 H=1492:00 +CPU= 26.856
16:01:43 STEP 1493 H=1493:00 +CPU= 26.761
16:02:26 STEP 1494 H=1494:00 +CPU= 41.624
16:02:54 STEP 1495 H=1495:00 +CPU= 26.734
16:03:23 STEP 1496 H=1496:00 +CPU= 27.379
16:04:02 STEP 1497 H=1497:00 +CPU= 37.877
16:04:29 STEP 1498 H=1498:00 +CPU= 26.134
16:04:57 STEP 1499 H=1499:00 +CPU= 27.411
16:05:43 STEP 1500 H=1500:00 +CPU= 43.332
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMGGhpi1+001500
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMSHhpi1+001500
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMUAhpi1+001500
16:06:09 STEP 1501 H=1501:00 +CPU= 25.788
16:06:38 STEP 1502 H=1502:00 +CPU= 26.787
16:07:16 STEP 1503 H=1503:00 +CPU= 37.503
16:07:43 STEP 1504 H=1504:00 +CPU= 26.563
16:08:12 STEP 1505 H=1505:00 +CPU= 27.537
16:08:52 STEP 1506 H=1506:00 +CPU= 39.190
16:09:18 STEP 1507 H=1507:00 +CPU= 25.975
16:09:49 STEP 1508 H=1508:00 +CPU= 28.362
16:10:28 STEP 1509 H=1509:00 +CPU= 38.316
16:10:55 STEP 1510 H=1510:00 +CPU= 26.322
16:11:26 STEP 1511 H=1511:00 +CPU= 28.972
16:12:14 STEP 1512 H=1512:00 +CPU= 45.669
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMGGhpi1+001512
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMSHhpi1+001512
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMUAhpi1+001512
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMGGhpi1+001488
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMSHhpi1+001488
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMUAhpi1+001488
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMGGhpi1+001500
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMSHhpi1+001500
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMUAhpi1+001500
Zipping up the intermediate file: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_0120_2021050100_123_945_12163209_0_r1264279961_2.zip
Uploading the intermediate file: upload_file_2.zip
16:12:42 STEP 1513 H=1513:00 +CPU= 26.481
Uploading trickle at timestep: 5443200
16:13:11 STEP 1514 H=1514:00 +CPU= 27.369
16:13:50 STEP 1515 H=1515:00 +CPU= 38.761
16:14:18 STEP 1516 H=1516:00 +CPU= 27.468
16:14:46 STEP 1517 H=1517:00 +CPU= 26.932
16:15:26 STEP 1518 H=1518:00 +CPU= 38.855
16:15:53 STEP 1519 H=1519:00 +CPU= 27.147
16:16:23 STEP 1520 H=1520:00 +CPU= 27.833
16:17:02 STEP 1521 H=1521:00 +CPU= 37.840
16:17:29 STEP 1522 H=1522:00 +CPU= 26.758
16:17:58 STEP 1523 H=1523:00 +CPU= 27.272
16:18:42 STEP 1524 H=1524:00 +CPU= 42.953
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMGGhpi1+001524
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMSHhpi1+001524
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMUAhpi1+001524
16:19:11 STEP 1525 H=1525:00 +CPU= 27.076
16:19:39 STEP 1526 H=1526:00 +CPU= 27.639
16:20:18 STEP 1527 H=1527:00 +CPU= 38.811
16:20:46 STEP 1528 H=1528:00 +CPU= 26.571
16:21:13 STEP 1529 H=1529:00 +CPU= 27.081
16:21:53 STEP 1530 H=1530:00 +CPU= 38.827
16:22:19 STEP 1531 H=1531:00 +CPU= 25.754
16:22:48 STEP 1532 H=1532:00 +CPU= 28.482
16:23:31 STEP 1533 H=1533:00 +CPU= 41.162
16:23:58 STEP 1534 H=1534:00 +CPU= 26.713
16:24:26 STEP 1535 H=1535:00 +CPU= 27.433
16:25:17 STEP 1536 H=1536:00 +CPU= 48.337
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMGGhpi1+001536
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMSHhpi1+001536
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMUAhpi1+001536
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMGGhpi1+001512
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMSHhpi1+001512
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMUAhpi1+001512
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMGGhpi1+001524
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMSHhpi1+001524
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMUAhpi1+001524
Zipping up the intermediate file: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_0120_2021050100_123_945_12163209_0_r1264279961_3.zip
Uploading the intermediate file: upload_file_3.zip
16:25:44 STEP 1537 H=1537:00 +CPU= 26.390
Uploading trickle at timestep: 5529600
16:26:13 STEP 1538 H=1538:00 +CPU= 26.780
16:26:55 STEP 1539 H=1539:00 +CPU= 41.285
16:27:22 STEP 1540 H=1540:00 +CPU= 26.678
16:27:52 STEP 1541 H=1541:00 +CPU= 26.823
16:28:33 STEP 1542 H=1542:00 +CPU= 40.601
16:28:59 STEP 1543 H=1543:00 +CPU= 25.898
16:29:28 STEP 1544 H=1544:00 +CPU= 27.729
16:30:09 STEP 1545 H=1545:00 +CPU= 40.216
16:30:36 STEP 1546 H=1546:00 +CPU= 26.081
16:31:03 STEP 1547 H=1547:00 +CPU= 26.954
16:31:49 STEP 1548 H=1548:00 +CPU= 44.490
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMGGhpi1+001548
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMSHhpi1+001548
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMUAhpi1+001548
16:32:15 STEP 1549 H=1549:00 +CPU= 25.438
16:32:43 STEP 1550 H=1550:00 +CPU= 26.647
16:33:24 STEP 1551 H=1551:00 +CPU= 39.477
16:33:53 STEP 1552 H=1552:00 +CPU= 27.545
16:34:21 STEP 1553 H=1553:00 +CPU= 26.915
16:35:01 STEP 1554 H=1554:00 +CPU= 39.447
16:35:27 STEP 1555 H=1555:00 +CPU= 25.675
16:35:56 STEP 1556 H=1556:00 +CPU= 27.656
16:36:35 STEP 1557 H=1557:00 +CPU= 38.729
16:37:03 STEP 1558 H=1558:00 +CPU= 27.086
16:37:33 STEP 1559 H=1559:00 +CPU= 28.472
16:38:20 STEP 1560 H=1560:00 +CPU= 45.378
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMGGhpi1+001560
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMSHhpi1+001560
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMUAhpi1+001560
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMGGhpi1+001536
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMSHhpi1+001536
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMUAhpi1+001536
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMGGhpi1+001548
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMSHhpi1+001548
Adding to the zip: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209/ICMUAhpi1+001548
Zipping up the intermediate file: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_0120_2021050100_123_945_12163209_0_r1264279961_4.zip
Uploading the intermediate file: upload_file_4.zip
Uploading trickle at timestep: 5616000
16:38:49 STEP 1561 H=1561:00 +CPU= 26.958
16:39:17 STEP 1562 H=1562:00 +CPU= 27.542
16:39:56 STEP 1563 H=1563:00 +CPU= 38.040
16:40:23 STEP 1564 H=1564:00 +CPU= 27.024
16:40:51 STEP 1565 H=1565:00 +CPU= 26.810
16:41:32 STEP 1566 H=1566:00 +CPU= 39.527
16:41:59 STEP 1567 H=1567:00 +CPU= 26.133
16:42:27 STEP 1568 H=1568:00 +CPU= 27.140
16:43:06 STEP 1569 H=1569:00 +CPU= 38.188
16:43:34 STEP 1570 H=1570:00 +CPU= 27.113
16:44:00 STEP 1571 H=1571:00 +CPU= 25.357
16:44:49 STEP 1572 H=1572:00 +CPU= 46.411
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMGGhpi1+001572
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMSHhpi1+001572
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMUAhpi1+001572
16:45:17 STEP 1573 H=1573:00 +CPU= 26.608
16:45:43 STEP 1574 H=1574:00 +CPU= 25.625
Quit request received from BOINC client, ending the child process
(argv0) ../../projects/climateprediction.net/oifs_43r3_ps_1.01_x86_64-pc-linux-gnu
(argv1) start_date: 2021050100
(argv2) exptid: hpi1
(argv3) unique_member_id: 0120
(argv4) batchid: 945
(argv5) wuid: 12163209
(argv6) fclen: 123
(argv7) app_name: oifs_43r3_ps
(argv8) nthreads: 1
Working directory is: /var/lib/boinc-client/slots/3
Project directory is: /var/lib/boinc-client/projects/climateprediction.net/
app name: oifs_43r3_ps
version: 1.01
Location of temp folder: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209
..mkdir for temp folder for results failed
Copying: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_app_1.01_x86_64-pc-linux-gnu.zip to: /var/lib/boinc-client/slots/3/oifs_43r3_ps_app_1.01_x86_64-pc-linux-gnu.zip
Unzipping the app zip file: /var/lib/boinc-client/slots/3/oifs_43r3_ps_app_1.01_x86_64-pc-linux-gnu.zip
Copying the namelist files from: ../../projects/climateprediction.net/jf_67cb0c72c97fe1c38e734f399c610d16 to: /var/lib/boinc-client/slots/3/oifs_43r3_ps_0120_2021050100_123_945_12163209.zip
Unzipping the namelist zip file: /var/lib/boinc-client/slots/3/oifs_43r3_ps_0120_2021050100_123_945_12163209.zip
ic_ancil_file: ic_ancil_12163209
ifsdata_file: ifsdata_12163209
climate_data_file: clim_data_12163209
horiz_resolution: 159
vert_resolution: 60
grid_type: l_2
upload_interval: 24
utstep: 3600
Copying IC ancils from: ../../projects/climateprediction.net/jf_f6b249c1bbdd8b0b2d9b7cfe1d7f36ce to: /var/lib/boinc-client/slots/3/ic_ancil_12163209.zip
Unzipping the IC ancils zip file: /var/lib/boinc-client/slots/3/ic_ancil_12163209.zip
..mkdir for ifsdata folder failed
Copying the ifsdata_file from: ../../projects/climateprediction.net/jf_9a5ab440cf540a120bd06629cab8bab4 to: /var/lib/boinc-client/slots/3/ifsdata/ifsdata_12163209.zip
Unzipping the ifsdata_zip file: /var/lib/boinc-client/slots/3/ifsdata/ifsdata_12163209.zip
..mkdir for the climate data folder failed
Copying the climate data file from: ../../projects/climateprediction.net/jf_f32c668cf7829426ffd94699b48b9a2e to: /var/lib/boinc-client/slots/3/159l_2/clim_data_12163209.zip
Unzipping the climate data zip file: /var/lib/boinc-client/slots/3/159l_2/clim_data_12163209.zip
total_length_of_simulation: 10627200
result_base_name: oifs_43r3_ps_0120_2021050100_123_945_12163209_0_r1264279961
The child process has been launched with process id: 937890
Executing the command: /var/lib/boinc-client/slots/3/./master.exe
[EC_DRHOOK:hostname:myproc:omptid:pid:unixtid] [YYYYMMDD:HHMMSS:epoch:walltime] [function@file:lineno] -- Max OpenMP threads = 1
[EC_DRHOOK:pyroxeen:1:1:937890:937890] [20221129:164707:1669736827:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2091] fp = 0x3129fe0
[EC_DRHOOK:pyroxeen:1:1:937890:937890] [20221129:164707:1669736827:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2098] DR_HOOK_ALLOW_COREDUMP=-1
[EC_DRHOOK:pyroxeen:1:1:937890:937890] [20221129:164707:1669736827:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2104] Hardlimit for core file is now 0 (0x0)
[EC_DRHOOK:pyroxeen:1:1:937890:937890] [20221129:164707:1669736827:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2122] DR_HOOK_PROFILE_PROC=-1
[EC_DRHOOK:pyroxeen:1:1:937890:937890] [20221129:164707:1669736827:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2128] DR_HOOK_PROFILE_LIMIT=-10.000
[EC_DRHOOK:pyroxeen:1:1:937890:937890] [20221129:164707:1669736827:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2194] DR_HOOK_RANDOM_MEMSTAT=0 (RAND_MAX=2147483647)
[EC_DRHOOK:pyroxeen:1:1:937890:937890] [20221129:164707:1669736827:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2205] DR_HOOK_HASHBITS=16
[EC_DRHOOK:pyroxeen:1:1:937890:937890] [20221129:164707:1669736827:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2213] DR_HOOK_NCALLSTACK=0
[EC_DRHOOK:pyroxeen:1:1:937890:937890] [20221129:164707:1669736827:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2221] DR_HOOK_HARAKIRI_TIMEOUT=500
[EC_DRHOOK:pyroxeen:1:1:937890:937890] [20221129:164707:1669736827:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2228] DR_HOOK_TRAPFPE=1
[EC_DRHOOK:pyroxeen:1:1:937890:937890] [20221129:164707:1669736827:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2235] DR_HOOK_TRAPFPE_INVALID=1
[EC_DRHOOK:pyroxeen:1:1:937890:937890] [20221129:164707:1669736827:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2242] DR_HOOK_TRAPFPE_DIVBYZERO=1
[EC_DRHOOK:pyroxeen:1:1:937890:937890] [20221129:164707:1669736827:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2249] DR_HOOK_TRAPFPE_OVERFLOW=1
[EC_DRHOOK:pyroxeen:1:1:937890:937890] [20221129:164707:1669736827:0.000] [ignore_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1227] DR_HOOK_IGNORE_SIGNALS=<undef>
[EC_DRHOOK:pyroxeen:1:1:937890:937890] [20221129:164707:1669736827:0.000] [restore_default_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1178] DR_HOOK_RESTORE_DEFAULT_SIGNALS=<undef>
[EC_DRHOOK:pyroxeen:1:1:937890:937890] [20221129:164707:1669736827:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1937] New signal handler 'signal_drhook' for signal#6 (SIGABRT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:pyroxeen:1:1:937890:937890] [20221129:164707:1669736827:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1938] New signal handler 'signal_drhook' for signal#7 (SIGBUS) at 0x81f820 (old at (nil))
[EC_DRHOOK:pyroxeen:1:1:937890:937890] [20221129:164707:1669736827:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1939] New signal handler 'signal_drhook' for signal#11 (SIGSEGV) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:pyroxeen:1:1:937890:937890] [20221129:164707:1669736827:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1944] New signal handler 'signal_drhook' for signal#16 (SIGSTKFLT) at 0x81f820 (old at (nil))
[EC_DRHOOK:pyroxeen:1:1:937890:937890] [20221129:164707:1669736827:0.000] [trapfpe_treatment@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1149] DR_HOOK enables SIGFPE-related floating point trapping since DRHOOK_TRAPFPE=1
[EC_DRHOOK:pyroxeen:1:1:937890:937890] [20221129:164707:1669736827:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1948] New signal handler 'signal_drhook' for signal#8 (SIGFPE) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:pyroxeen:1:1:937890:937890] [20221129:164707:1669736827:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1949] New signal handler 'signal_drhook' for signal#4 (SIGILL) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:pyroxeen:1:1:937890:937890] [20221129:164707:1669736827:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1951] New signal handler 'signal_drhook' for signal#5 (SIGTRAP) at 0x81f820 (old at (nil))
[EC_DRHOOK:pyroxeen:1:1:937890:937890] [20221129:164707:1669736827:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1952] New signal handler 'signal_drhook' for signal#2 (SIGINT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:pyroxeen:1:1:937890:937890] [20221129:164707:1669736827:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1960] New signal handler 'signal_drhook' for signal#3 (SIGQUIT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:pyroxeen:1:1:937890:937890] [20221129:164707:1669736827:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1961] New signal handler 'signal_drhook' for signal#15 (SIGTERM) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:pyroxeen:1:1:937890:937890] [20221129:164707:1669736827:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1966] New signal handler 'signal_drhook' for signal#24 (SIGXCPU) at 0x81f820 (old at (nil))
[EC_DRHOOK:pyroxeen:1:1:937890:937890] [20221129:164707:1669736827:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1968] New signal handler 'signal_drhook' for signal#25 (SIGXFSZ) at 0x81f820 (old at (nil))
[EC_DRHOOK:pyroxeen:1:1:937890:937890] [20221129:164707:1669736827:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1973] New signal handler 'signal_drhook' for signal#31 (SIGSYS) at 0x81f820 (old at (nil))
[EC_DRHOOK:pyroxeen:1:1:937890:937890] [20221129:164707:1669736827:0.000] [catch_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1111] DR_HOOK_CATCH_SIGNALS=<undef>
16:48:00 STEP 1560 H=1560:00 +CPU= 45.594
16:48:27 STEP 1561 H=1561:00 +CPU= 26.644
16:48:55 STEP 1562 H=1562:00 +CPU= 26.936
16:49:35 STEP 1563 H=1563:00 +CPU= 39.013
16:50:04 STEP 1564 H=1564:00 +CPU= 27.332
16:50:31 STEP 1565 H=1565:00 +CPU= 26.736
16:51:14 STEP 1566 H=1566:00 +CPU= 40.872
16:51:43 STEP 1567 H=1567:00 +CPU= 28.178
16:52:12 STEP 1568 H=1568:00 +CPU= 28.183
16:52:51 STEP 1569 H=1569:00 +CPU= 38.427
16:53:19 STEP 1570 H=1570:00 +CPU= 27.046
16:53:46 STEP 1571 H=1571:00 +CPU= 25.241
16:54:29 STEP 1572 H=1572:00 +CPU= 42.518
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMGGhpi1+001572
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMSHhpi1+001572
Moving to projects directory: /var/lib/boinc-client/slots/3/ICMUAhpi1+001572
16:54:58 STEP 1573 H=1573:00 +CPU= 27.834
16:55:24 STEP 1574 H=1574:00 +CPU= 25.631
16:56:04 STEP 1575 H=1575:00 +CPU= 39.203
16:56:32 STEP 1576 H=1576:00 +CPU= 27.100
16:56:58 STEP 1577 H=1577:00 +CPU= 24.758
16:57:39 STEP 1578 H=1578:00 +CPU= 40.641
16:58:05 STEP 1579 H=1579:00 +CPU= 25.468
16:58:31 STEP 1580 H=1580:00 +CPU= 25.331
16:59:12 STEP 1581 H=1581:00 +CPU= 40.553
16:59:39 STEP 1582 H=1582:00 +CPU= 26.954
17:00:05 STEP 1583 H=1583:00 +CPU= 25.284
ECCODE(argv0) ../../projects/climateprediction.net/oifs_43r3_ps_1.01_x86_64-pc-linux-gnu
(argv1) start_date: 2021050100
(argv2) exptid: hpi1
(argv3) unique_member_id: 0120
(argv4) batchid: 945
(argv5) wuid: 12163209
(argv6) fclen: 123
(argv7) app_name: oifs_43r3_ps
(argv8) nthreads: 1
Working directory is: /var/lib/boinc-client/slots/3
Project directory is: /var/lib/boinc-client/projects/climateprediction.net/
app name: oifs_43r3_ps
version: 1.01
Location of temp folder: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_12163209
..mkdir for temp folder for results failed
Copying: /var/lib/boinc-client/projects/climateprediction.net/oifs_43r3_ps_app_1.01_x86_64-pc-linux-gnu.zip to: /var/lib/boinc-client/slots/3/oifs_43r3_ps_app_1.01_x86_64-pc-linux-gnu.zip
Unzipping the app zip file: /var/lib/boinc-client/slots/3/oifs_43r3_ps_app_1.01_x86_64-pc-linux-gnu.zip
Copying the namelist files from: ../../projects/climateprediction.net/jf_67cb0c72c97fe1c38e734f399c610d16 to: /var/lib/boinc-client/slots/3/oifs_43r3_ps_0120_2021050100_123_945_12163209.zip
Unzipping the namelist zip file: /var/lib/boinc-client/slots/3/oifs_43r3_ps_0120_2021050100_123_945_12163209.zip
ic_ancil_file: ic_ancil_12163209
ifsdata_file: ifsdata_12163209
climate_data_file: clim_data_12163209
horiz_resolution: 159
vert_resolution: 60
grid_type: l_2
upload_interval: 24
utstep: 3600
Copying IC ancils from: ../../projects/climateprediction.net/jf_f6b249c1bbdd8b0b2d9b7cfe1d7f36ce to: /var/lib/boinc-client/slots/3/ic_ancil_12163209.zip
Unzipping the IC ancils zip file: /var/lib/boinc-client/slots/3/ic_ancil_12163209.zip
..mkdir for ifsdata folder failed
Copying the ifsdata_file from: ../../projects/climateprediction.net/jf_9a5ab440cf540a120bd06629cab8bab4 to: /var/lib/boinc-client/slots/3/ifsdata/ifsdata_12163209.zip
Unzipping the ifsdata_zip file: /var/lib/boinc-client/slots/3/ifsdata/ifsdata_12163209.zip
..mkdir for the climate data folder failed
Copying the climate data file from: ../../projects/climateprediction.net/jf_f32c668cf7829426ffd94699b48b9a2e to: /var/lib/boinc-client/slots/3/159l_2/clim_data_12163209.zip
Unzipping the climate data zip file: /var/lib/boinc-client/slots/3/159l_2/clim_data_12163209.zip
total_length_of_simulation: 10627200
result_base_name: oifs_43r3_ps_0120_2021050100_123_945_12163209_0_r1264279961
The child process has been launched with process id: 959920
Executing the command: /var/lib/boinc-client/slots/3/./master.exe
[EC_DRHOOK:hostname:myproc:omptid:pid:unixtid] [YYYYMMDD:HHMMSS:epoch:walltime] [function@file:lineno] -- Max OpenMP threads = 1
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192724:1669746444:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2091] fp = 0x3129fe0
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192724:1669746444:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2098] DR_HOOK_ALLOW_COREDUMP=-1
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192724:1669746444:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2104] Hardlimit for core file is now 0 (0x0)
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192724:1669746444:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2122] DR_HOOK_PROFILE_PROC=-1
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192724:1669746444:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2128] DR_HOOK_PROFILE_LIMIT=-10.000
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192724:1669746444:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2194] DR_HOOK_RANDOM_MEMSTAT=0 (RAND_MAX=2147483647)
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192724:1669746444:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2205] DR_HOOK_HASHBITS=16
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192724:1669746444:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2213] DR_HOOK_NCALLSTACK=0
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192724:1669746444:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2221] DR_HOOK_HARAKIRI_TIMEOUT=500
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192724:1669746444:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2228] DR_HOOK_TRAPFPE=1
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192724:1669746444:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2235] DR_HOOK_TRAPFPE_INVALID=1
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192724:1669746444:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2242] DR_HOOK_TRAPFPE_DIVBYZERO=1
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192724:1669746444:0.000] [process_options@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:2249] DR_HOOK_TRAPFPE_OVERFLOW=1
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192724:1669746444:0.000] [ignore_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1227] DR_HOOK_IGNORE_SIGNALS=<undef>
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192724:1669746444:0.000] [restore_default_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1178] DR_HOOK_RESTORE_DEFAULT_SIGNALS=<undef>
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192724:1669746444:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1937] New signal handler 'signal_drhook' for signal#6 (SIGABRT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192724:1669746444:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1938] New signal handler 'signal_drhook' for signal#7 (SIGBUS) at 0x81f820 (old at (nil))
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192724:1669746444:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1939] New signal handler 'signal_drhook' for signal#11 (SIGSEGV) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192724:1669746444:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1944] New signal handler 'signal_drhook' for signal#16 (SIGSTKFLT) at 0x81f820 (old at (nil))
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192724:1669746444:0.000] [trapfpe_treatment@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1149] DR_HOOK enables SIGFPE-related floating point trapping since DRHOOK_TRAPFPE=1
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192724:1669746444:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1948] New signal handler 'signal_drhook' for signal#8 (SIGFPE) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192724:1669746444:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1949] New signal handler 'signal_drhook' for signal#4 (SIGILL) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192724:1669746444:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1951] New signal handler 'signal_drhook' for signal#5 (SIGTRAP) at 0x81f820 (old at (nil))
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192724:1669746444:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1952] New signal handler 'signal_drhook' for signal#2 (SIGINT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192724:1669746444:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1960] New signal handler 'signal_drhook' for signal#3 (SIGQUIT) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192724:1669746444:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1961] New signal handler 'signal_drhook' for signal#15 (SIGTERM) at 0x81f820 (old at 0x1ce8bb0)
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192724:1669746444:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1966] New signal handler 'signal_drhook' for signal#24 (SIGXCPU) at 0x81f820 (old at (nil))
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192724:1669746444:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1968] New signal handler 'signal_drhook' for signal#25 (SIGXFSZ) at 0x81f820 (old at (nil))
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192724:1669746444:0.000] [signal_drhook_init@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1973] New signal handler 'signal_drhook' for signal#31 (SIGSYS) at 0x81f820 (old at (nil))
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192724:1669746444:0.000] [catch_signals@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1111] DR_HOOK_CATCH_SIGNALS=<undef>
ABORT! 1 !! *** WAVE MODEL HAS ABORTED
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192815:1669746495:51.127] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3843] 1721 MB (maxheap), 3520 MB (maxrss), 0 MB (maxstack)
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192815:1669746495:51.127] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : MASTER
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192815:1669746495:51.127] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CNT0
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192815:1669746495:51.127] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CNT1
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192815:1669746495:51.127] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CNT2
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192815:1669746495:51.127] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CNT3
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192815:1669746495:51.127] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : CNT4
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192815:1669746495:51.127] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : WVCOUPLE
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192815:1669746495:51.127] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : WAVEMDL
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192815:1669746495:51.127] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : INITMDL
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192815:1669746495:51.127] [c_drhook_print_@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:3897] : USERIN
SDL_TRACEBACK: Calling INTEL_TRBK, THRD = 1
Process 1 thread 1 calling linux_trbk from intel_trbk()
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:240] Backtrace(s) for program '/var/lib/boinc-client/slots/3/./master.exe' : sigcontextptr=0x7ffcaeb4a3a0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:277] Backtrace (size = 19) with addr2line-cmd
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:302] /usr/bin/addr2line -fs -e '/var/lib/boinc-client/slots/3/./master.exe' 0x839273 0x839879 0x8c2957 0x81d89e 0x15f8aca 0x15d51c8 0x166b07f 0x15d7ff0 0x8711e0 0x7e0165 0x55e4ad 0x5471c9 0x546412 0x545e28 0x40708d 0x40220f 0x4021a2 0x1dca390 0x40206e
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [0]: [0x839273] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [1]: [0x839879] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [2]: [0x8c2957] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [3]: [0x81d89e] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [4]: [0x15f8aca] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [5]: [0x15d51c8] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [6]: [0x166b07f] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [7]: [0x15d7ff0] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [8]: [0x8711e0] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [9]: [0x7e0165] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [10]: [0x55e4ad] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [11]: [0x5471c9] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [12]: [0x546412] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [13]: [0x545e28] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [14]: [0x40708d] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [15]: [0x40220f] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [16]: [0x4021a2] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [17]: [0x1dca390] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:319] [18]: [0x40206e] : ??() at ??:0
[LinuxTraceBack@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/utilities/linuxtrbk.c:370] End of backtrace(s)
SDL_TRACEBACK: Done INTEL_TRBK, THRD = 1
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192815:1669746495:51.165] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1538] Received signal#6 (SIGABRT) :: 1721MB (heap), 3520MB (maxrss), 0MB (maxstack), 0 (paging), nsigs = 1
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192815:1669746495:51.165] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1542] Also activating Harakiri-alarm (SIGALRM=14) to expire after 500s elapsed to prevent hangs, nsigs = 1
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192815:1669746495:51.165] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1544] Harakiri signal handler 'signal_harakiri' for signal#14 (SIGALRM) installed at 0x81f3c0 (old at (nil))
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192815:1669746495:51.165] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1617] Signal#6 was caused by unrecognized si_code [memaddr=0x7f000ea5b0], nsigs = 1
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192815:1669746495:51.165] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1686] Starting DrHook backtrace for signal#6, nsigs = 1
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192816:1669746496:52.167] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1734] DrHook backtrace done for signal#6, nsigs = 1
[EC_DRHOOK:pyroxeen:1:1:959920:959920] [20221129:192816:1669746496:52.167] [signal_drhook@/home/glenn/github/jamie_oifs43r3.git/src/ifsaux/support/drhook.c:1785] Calling previous signal handler at 0x1ce8bb0 for signal#6, nsigs = 1
forrtl: error (76): Abort trap signal
Image PC Routine Line Source
master.exe 0000000001CE916B Unknown Unknown Unknown
master.exe 000000000081FFF1 Unknown Unknown Unknown
master.exe 0000000001DC9090 Unknown Unknown Unknown
master.exe 0000000001DC903B Unknown Unknown Unknown
master.exe 00000000008C2880 sdl_mod_mp_sdl_sr 100 sdl_mod.F90
master.exe 000000000081D8AF abor1_ 41 abor1.F90
master.exe 00000000015F8ACA mpabort_ 50 mpabort.F
master.exe 00000000015D51C8 abort1_ 44 abort1.F
master.exe 000000000166B07F userin_ 553 userin.F
master.exe 00000000015D7FF0 initmdl_ 452 initmdl.F
master.exe 00000000008711E0 wavemdl_ 393 wavemdl.F
master.exe 00000000007E0165 wvcouple_ 260 wvcouple.F90
master.exe 000000000055E4AD cnt4_ 1283 cnt4.F90
master.exe 00000000005471C9 cnt3_ 267 cnt3.F90
master.exe 0000000000546412 cnt2_ 88 cnt2.F90
master.exe 0000000000545E28 cnt1_ 92 cnt1.F90
master.exe 000000000040708D cnt0_ 146 cnt0.F90
master.exe 000000000040220F MAIN__ 96 master.F90
master.exe 00000000004021A2 Unknown Unknown Unknown
master.exe 0000000001DCA390 Unknown Unknown Unknown
master.exe 000000000040206E Unknown Unknown Unknown
The child process terminated with status: 0
..Failed, model did not complete successfully
</stderr_txt>
]]>
