Added missing Fig. 9, Table III and Table IV

.gitignore

@@ -1,3 +1,8 @@
 **/.ipynb_checkpoints/
 task_slowdown/?_state_changes.json.gz
 **/*.gz
+
+figure_9/*.parquet/
+figure_9/?_task_count/
+figure_9/?_machine_count/
+table_iii/*.parquet/

figure_9/9a-total.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+i=$1
+
+cut ${i}_task_count/part-00000 -d, -f2-3 | \
+	sort | uniq -c | sed 's/^\s*//' | tr ' ' ',' | \
+	awk 'BEGIN {print "count,term,task_count"} {print $0}' > ${i}_term_task_count.csv

figure_9/9c-total.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+i=$1
+
+cut ${i}_machine_count/part-00000 -d, -f2-3 | \
+	sort | uniq -c | sed 's/^\s*//' | tr ' ' ',' | \
+	awk 'BEGIN {print "count,term,task_count"} {print $0}' > ${i}_term_machine_count.csv

figure_9/a_priority_exectime/part-00000

@@ -0,0 +1,296 @@
+7,115,1-2,5905
+7,105,60-1d,27729
+7,107,>=1d,194
+6,118,2-4,116
+-1,119,<1,3050
+6,0,60-1d,100
+7,116,10-60,21314
+6,119,1-2,33520
+5,115,4-10,1564
+7,114,1-2,12
+7,101,4-10,85
+7,101,2-4,72
+6,103,10-60,2656
+7,103,60-1d,25666
+-1,107,<1,52
+5,25,4-10,500
+6,118,>=1d,4
+5,119,60-1d,2532
+5,0,2-4,120
+5,107,2-4,968
+7,200,1-2,3671488
+7,400,60-1d,4
+7,200,>=1d,45919
+5,119,1-2,544
+7,107,4-10,44537
+6,0,4-10,648
+7,25,1-2,1132
+6,119,4-10,41776
+6,116,60-1d,93
+7,118,4-10,2876
+5,103,<1,133
+7,118,<1,4648
+5,0,<1,48
+7,114,<1,6
+7,114,60-1d,4
+7,115,<1,22628
+5,117,2-4,4
+7,119,60-1d,21704
+-1,4,<1,1
+7,25,60-1d,1465
+5,115,<1,11652
+4,200,4-10,4
+5,101,60-1d,5
+7,102,2-4,4
+6,115,4-10,129718
+6,115,10-60,183788
+-1,115,<1,2396
+5,118,60-1d,780
+6,25,4-10,21552
+6,119,>=1d,972
+5,107,60-1d,233
+7,114,10-60,4
+7,107,1-2,4812
+5,0,>=1d,8
+7,117,60-1d,600
+7,101,1-2,12
+6,25,60-1d,2940
+7,118,1-2,1296
+-1,101,1-2,22
+-1,103,<1,2775
+5,116,<1,24
+8,103,<1,134
+6,117,60-1d,4105
+7,103,<1,33437
+6,105,1-2,7419
+5,119,4-10,868
+7,360,<1,164
+6,115,60-1d,37715
+6,107,<1,13476
+8,105,<1,1325
+6,105,10-60,6383
+5,117,4-10,88
+6,105,<1,1677
+5,0,4-10,16
+5,360,60-1d,35680
+6,200,2-4,306904
+7,107,60-1d,1621
+7,119,<1,4094
+6,25,1-2,66116
+7,200,2-4,2068536
+6,200,1-2,79220
+6,25,<1,18240
+5,200,>=1d,2500
+6,0,<1,20500
+7,115,10-60,83925
+5,119,10-60,1500
+6,117,10-60,23168
+5,25,60-1d,64
+6,25,>=1d,92
+7,116,1-2,9050
+6,116,2-4,26
+7,118,10-60,9484
+6,360,<1,792
+5,105,<1,20
+6,101,>=1d,236
+-1,500,<1,27
+6,0,2-4,544
+5,115,10-60,6961
+-1,170,<1,3
+7,103,1-2,17985
+6,118,<1,448
+-1,0,<1,2052
+7,360,10-60,732
+7,360,2-4,4
+5,117,60-1d,54
+7,0,60-1d,7868
+7,0,>=1d,6660
+7,103,10-60,79764
+-1,105,<1,608
+6,118,1-2,92
+7,103,>=1d,2244
+7,200,10-60,1494988
+7,360,60-1d,824
+7,0,<1,39476
+-1,117,<1,35
+5,119,2-4,1348
+5,103,60-1d,13215
+6,115,>=1d,680
+-1,450,<1,4384
+5,103,4-10,14
+6,107,2-4,178
+6,200,<1,2960924
+6,116,10-60,64
+7,102,60-1d,32
+6,101,1-2,4
+6,107,10-60,5623
+6,118,60-1d,1540
+5,107,<1,20
+7,100,10-60,344
+7,25,4-10,1488
+6,115,1-2,67760
+6,103,1-2,4246
+5,117,<1,48
+7,116,>=1d,1760
+6,101,2-4,12
+-1,118,<1,260
+7,115,60-1d,25381
+-1,360,<1,5105
+6,105,4-10,6274
+4,115,60-1d,32
+7,0,2-4,18544
+7,116,<1,9058
+6,119,60-1d,14112
+7,118,>=1d,644
+7,119,4-10,35584
+7,100,60-1d,596
+5,103,1-2,52
+5,200,60-1d,11228
+7,117,2-4,788
+7,118,2-4,988
+5,117,10-60,40
+7,107,2-4,5403
+-1,40,<1,1
+7,360,1-2,4
+6,105,2-4,6090
+6,200,>=1d,5876
+6,115,<1,89844
+5,115,1-2,8165
+5,200,4-10,3120
+7,117,10-60,1144
+7,117,4-10,2138
+7,25,2-4,880
+6,115,2-4,114016
+5,118,10-60,236
+7,0,4-10,33840
+-1,205,<1,144
+5,119,<1,4392
+5,0,60-1d,904
+5,105,2-4,1338
+7,119,>=1d,4456
+6,105,60-1d,1469
+6,360,1-2,12
+7,25,10-60,1076
+4,200,>=1d,4
+7,100,<1,20
+5,117,1-2,4
+5,25,<1,408
+-1,24,<1,12
+7,25,<1,424
+7,200,4-10,1326784
+8,107,<1,16
+4,200,10-60,4
+5,105,60-1d,1314
+6,200,4-10,288520
+5,115,2-4,6002
+5,116,10-60,33
+6,0,>=1d,9
+4,25,<1,4
+6,117,4-10,19502
+6,103,>=1d,717
+6,119,2-4,28636
+6,116,1-2,20
+-1,25,<1,210
+5,25,>=1d,16
+7,101,<1,95
+7,105,1-2,136570
+5,200,<1,2196
+6,119,10-60,100308
+6,117,1-2,120
+5,116,60-1d,40
+-1,101,>=1d,3000
+7,103,4-10,53940
+8,115,<1,390
+5,25,2-4,388
+6,116,4-10,4
+7,105,10-60,109266
+5,103,10-60,9086
+7,116,2-4,8246
+-1,101,<1,79
+5,25,1-2,172
+7,101,60-1d,150
+6,119,<1,103092
+5,105,4-10,2149
+5,200,1-2,856
+6,117,<1,72
+5,360,10-60,40
+7,117,<1,4952
+6,103,2-4,2047
+5,105,10-60,3533
+6,118,4-10,56
+-1,210,<1,9
+7,115,4-10,35526
+6,116,<1,14920
+5,115,60-1d,7953
+6,117,2-4,3186
+7,360,>=1d,2316
+5,360,>=1d,28
+4,115,10-60,4
+6,360,>=1d,336
+6,200,10-60,359064
+4,200,60-1d,8
+5,107,4-10,12
+7,101,10-60,217
+5,103,2-4,47
+7,107,<1,7342
+6,107,1-2,80
+5,102,60-1d,12
+5,0,10-60,8808
+6,0,10-60,1116
+6,101,<1,7
+7,200,60-1d,432812
+5,200,10-60,52120
+7,119,10-60,134568
+8,119,<1,4
+6,103,4-10,2754
+5,25,10-60,284
+7,200,<1,4498622
+6,107,60-1d,134
+7,116,4-10,18092
+6,0,1-2,500
+7,103,2-4,20072
+5,119,>=1d,96
+7,119,2-4,24672
+7,105,>=1d,772
+7,105,2-4,118183
+5,105,>=1d,102
+6,360,60-1d,84
+-1,215,<1,9
+-1,100,<1,6
+7,115,>=1d,3982
+-1,214,<1,30
+5,107,1-2,3488
+7,0,1-2,14816
+7,100,1-2,4
+7,105,4-10,110820
+5,107,10-60,44
+6,118,10-60,80
+5,105,1-2,1652
+5,116,>=1d,14
+7,25,>=1d,156
+7,116,60-1d,5176
+7,117,1-2,8532
+5,118,>=1d,148
+6,103,<1,14168
+7,107,10-60,57949
+5,115,>=1d,235
+6,25,10-60,11620
+7,119,1-2,33748
+6,103,60-1d,1001
+-1,116,<1,397
+6,102,60-1d,16
+6,107,4-10,3167
+-1,200,<1,31280
+5,0,1-2,104
+6,200,60-1d,81620
+7,115,2-4,22775
+6,360,10-60,724
+5,103,>=1d,870
+7,105,<1,37580
+7,0,10-60,40128
+-1,400,<1,12
+5,200,2-4,1704
+6,25,2-4,18240
+7,100,4-10,16
+7,500,60-1d,8
+7,118,60-1d,16478

figure_9/b_priority_exectime/part-00000

@@ -0,0 +1,269 @@
+7,115,1-2,14942
+7,105,60-1d,41666
+-1,115,>=1d,702
+6,118,2-4,18960
+7,107,>=1d,285
+-1,119,<1,1340
+6,0,60-1d,164
+5,107,>=1d,12
+4,115,<1,4
+7,116,10-60,4514
+6,119,1-2,58280
+5,115,4-10,4146
+7,101,2-4,216
+7,101,4-10,116
+7,103,60-1d,258640
+6,103,10-60,38693
+-1,199,<1,348
+-1,107,<1,84
+5,25,4-10,140
+6,118,>=1d,240
+5,119,60-1d,1368
+5,0,2-4,48
+7,200,1-2,8392
+7,200,>=1d,4320
+5,119,1-2,444
+7,107,4-10,2048
+6,0,4-10,628
+7,25,1-2,121392
+6,119,4-10,24920
+6,116,60-1d,853
+7,118,4-10,6348
+5,103,<1,350
+7,118,<1,5982
+5,0,<1,392
+7,114,<1,2
+7,100,>=1d,72
+7,115,<1,5995
+7,119,60-1d,6748
+7,210,>=1d,16
+7,25,60-1d,516
+5,115,<1,6082
+5,101,60-1d,20
+6,115,10-60,178916
+6,115,4-10,148422
+-1,115,<1,2594
+6,25,4-10,181268
+7,50,<1,4000
+6,119,>=1d,4644
+5,107,60-1d,8
+7,107,1-2,1492
+5,0,>=1d,84
+7,101,1-2,340
+7,118,1-2,10376
+6,25,60-1d,2016
+5,100,60-1d,32
+-1,103,<1,9058
+8,103,<1,237
+6,117,60-1d,254
+7,103,<1,280803
+6,105,1-2,25741
+5,119,4-10,836
+6,115,60-1d,59879
+6,107,<1,17848
+6,105,10-60,324
+6,105,<1,23452
+5,0,4-10,44
+5,360,60-1d,35680
+6,200,2-4,81296
+6,360,2-4,4
+7,107,60-1d,2521
+7,119,<1,27304
+6,25,1-2,60612
+6,200,1-2,225200
+7,200,2-4,20736
+6,25,<1,56812
+5,200,>=1d,184
+7,80,<1,4
+6,0,<1,21896
+7,115,10-60,35121
+5,119,10-60,6656
+5,101,10-60,56
+6,117,10-60,195
+5,100,1-2,40
+6,105,>=1d,12
+6,25,>=1d,56
+5,25,60-1d,32
+7,116,1-2,3622
+6,116,2-4,8
+7,118,10-60,8132
+6,360,<1,56
+6,0,2-4,704
+5,115,10-60,11995
+-1,170,<1,3
+7,103,1-2,162097
+6,118,<1,416
+-1,0,<1,2002
+7,360,2-4,12
+7,360,10-60,4
+7,0,60-1d,4504
+7,0,>=1d,6596
+7,103,10-60,127547
+-1,105,<1,399
+6,118,1-2,20
+7,103,>=1d,44263
+7,200,10-60,46668
+7,360,60-1d,544
+6,107,>=1d,12
+7,0,<1,187742
+-1,117,<1,3
+5,119,2-4,172
+5,103,60-1d,13358
+6,115,>=1d,972
+-1,450,<1,3492
+5,103,4-10,1493
+6,107,2-4,4
+6,80,60-1d,4
+6,200,<1,372572
+6,116,10-60,308
+6,118,60-1d,500
+7,100,10-60,308
+7,25,4-10,11136
+6,103,1-2,24924
+6,115,1-2,221942
+6,100,4-10,4
+7,116,>=1d,752
+-1,118,<1,275
+7,115,60-1d,42488
+-1,360,<1,2301
+6,105,4-10,740
+4,115,60-1d,4
+7,0,2-4,124396
+7,116,<1,5385
+6,119,60-1d,22056
+7,118,>=1d,1712
+7,119,4-10,35828
+7,100,60-1d,1572
+5,103,1-2,56
+5,200,60-1d,612
+7,118,2-4,15520
+7,107,2-4,2936
+6,105,2-4,1616
+6,200,>=1d,1664
+6,115,<1,727677
+5,115,1-2,5785
+5,200,4-10,1000
+7,25,2-4,31336
+7,117,10-60,84
+7,117,4-10,4
+6,115,2-4,197272
+5,118,10-60,8
+7,0,4-10,112184
+-1,205,<1,486
+5,0,60-1d,1188
+5,119,<1,2468
+7,119,>=1d,1516
+6,105,60-1d,2558
+6,360,1-2,8
+7,100,2-4,4
+7,25,10-60,3016
+7,100,<1,14
+5,25,<1,2916
+-1,80,<1,1
+7,25,<1,147772
+7,200,4-10,27644
+5,105,60-1d,245
+6,200,4-10,121452
+5,115,2-4,1366
+5,116,10-60,85
+6,103,>=1d,35379
+6,117,4-10,268
+6,119,2-4,50864
+-1,25,<1,249
+5,25,>=1d,64
+7,101,<1,28
+7,105,1-2,389608
+5,200,<1,2084
+6,119,10-60,60348
+6,117,1-2,76
+5,116,60-1d,72
+7,103,4-10,102534
+8,115,<1,157
+5,100,10-60,228
+5,25,2-4,820
+6,116,4-10,50
+7,105,10-60,81291
+5,103,10-60,2986
+7,116,2-4,8981
+-1,101,<1,51
+5,25,1-2,15132
+7,101,60-1d,241
+6,119,<1,6376
+5,200,1-2,544
+7,215,>=1d,24
+5,360,10-60,44
+6,117,<1,4
+6,103,2-4,44885
+5,105,10-60,168
+6,118,4-10,8172
+-1,210,<1,27
+7,115,4-10,34946
+6,116,<1,124
+5,115,60-1d,5631
+7,360,>=1d,836
+6,117,2-4,156
+5,360,>=1d,120
+4,115,10-60,4
+6,360,>=1d,360
+6,200,10-60,72564
+5,103,2-4,5092
+7,101,10-60,284
+7,107,<1,496
+6,107,1-2,36
+5,100,<1,1532
+5,0,10-60,13196
+7,205,>=1d,304
+6,0,10-60,684
+7,200,60-1d,127964
+7,101,>=1d,4
+5,200,10-60,13132
+6,116,>=1d,84
+7,119,10-60,22024
+8,119,<1,8
+6,103,4-10,51358
+5,25,10-60,1164
+7,200,<1,327432
+6,107,60-1d,298
+7,116,4-10,10563
+6,0,1-2,464
+7,103,2-4,81510
+5,119,>=1d,48
+7,119,2-4,11104
+7,105,>=1d,1590
+7,105,2-4,61082
+5,105,>=1d,93
+6,360,60-1d,508
+-1,215,<1,27
+-1,100,<1,6
+7,115,>=1d,17023
+-1,214,<1,30
+7,0,1-2,93416
+7,100,1-2,16
+7,105,4-10,50860
+7,99,<1,2
+6,118,10-60,8144
+5,107,10-60,4
+5,116,>=1d,8
+7,25,>=1d,104
+6,100,10-60,4
+7,116,60-1d,6125
+6,103,<1,61775
+5,115,>=1d,2548
+7,107,10-60,908
+6,25,10-60,35880
+7,119,1-2,7468
+6,103,60-1d,8683
+-1,116,<1,147
+6,107,4-10,32
+-1,200,<1,26240
+5,0,1-2,76
+6,200,60-1d,67452
+7,115,2-4,34623
+5,103,>=1d,4533
+6,360,10-60,4
+7,105,<1,693575
+7,0,10-60,19196
+6,25,2-4,98020
+5,200,2-4,568
+7,100,4-10,16
+7,118,60-1d,5320

figure_9/c_priority_exectime/part-00000

@@ -0,0 +1,328 @@
+6,101,10-60,8
+7,115,1-2,38141
+7,105,60-1d,102018
+5,360,2-4,4
+6,118,2-4,27552
+7,107,>=1d,60
+-1,119,<1,7501
+6,0,60-1d,1140
+4,115,<1,20
+7,116,10-60,94753
+6,119,1-2,109588
+5,115,4-10,20887
+7,114,1-2,8880
+7,101,4-10,1205
+7,101,2-4,464
+7,103,60-1d,107032
+6,103,10-60,23212
+-1,107,<1,107
+5,25,4-10,204
+5,119,60-1d,6876
+5,0,2-4,64
+5,107,2-4,8
+6,117,>=1d,20
+7,200,1-2,117488
+7,200,>=1d,46011
+5,119,1-2,11432
+7,107,4-10,141234
+6,0,4-10,3896
+7,25,1-2,184
+6,119,4-10,72828
+6,116,60-1d,4659
+4,118,60-1d,4
+7,118,4-10,16959
+5,103,<1,351
+7,118,<1,9573
+5,0,<1,280
+7,114,<1,2932
+7,114,60-1d,2245
+7,100,>=1d,1200
+5,116,4-10,3204
+7,115,<1,65565
+5,117,2-4,11913
+7,119,60-1d,57168
+7,25,60-1d,6820
+6,114,60-1d,4
+5,115,<1,7651
+4,200,4-10,88
+5,101,60-1d,4
+-1,105,1-2,72
+6,115,10-60,367531
+6,115,4-10,457984
+-1,115,<1,5348
+5,118,60-1d,260
+6,25,4-10,119704
+7,114,10-60,4520
+6,119,>=1d,5500
+5,107,60-1d,152
+7,107,1-2,108616
+5,0,>=1d,56
+7,117,60-1d,640
+7,101,1-2,1616
+7,118,1-2,6628
+6,25,60-1d,21120
+5,100,60-1d,4
+-1,103,<1,2239
+5,116,<1,20
+8,103,<1,1020
+6,117,60-1d,128922
+7,103,<1,515277
+4,119,<1,32
+6,105,1-2,25318
+5,119,4-10,1936
+4,103,<1,32
+7,360,<1,166
+6,115,60-1d,70409
+6,107,<1,56329
+8,105,<1,24
+6,105,10-60,14552
+5,117,4-10,25390
+6,105,<1,22102
+5,0,4-10,96
+5,360,60-1d,35708
+6,200,2-4,781884
+4,103,4-10,32
+6,360,2-4,2072
+7,107,60-1d,5503
+7,119,<1,9863
+6,25,1-2,112029
+6,200,1-2,2408312
+7,200,2-4,610380
+6,25,<1,20940
+5,200,>=1d,780
+6,0,<1,220552
+7,115,10-60,113899
+5,119,10-60,4928
+6,117,10-60,702244
+6,25,>=1d,5360
+6,105,>=1d,1129
+5,25,60-1d,584
+7,116,1-2,66111
+6,116,2-4,1824
+7,118,10-60,24508
+6,360,<1,13504
+5,105,<1,56
+6,0,2-4,3876
+5,115,10-60,10092
+-1,170,<1,3
+7,201,60-1d,72
+5,116,2-4,94
+5,100,4-10,28
+7,103,1-2,420152
+4,117,10-60,1869
+6,118,<1,3024
+7,10,>=1d,4
+-1,0,<1,2108
+5,117,60-1d,38409
+7,360,10-60,200
+7,360,2-4,20
+7,0,60-1d,8288
+7,0,>=1d,6520
+7,360,4-10,44
+7,103,10-60,402023
+-1,105,<1,1287
+6,118,1-2,2392
+4,117,2-4,808
+7,103,>=1d,4198
+7,200,10-60,329220
+7,360,60-1d,908
+7,0,<1,31160
+-1,117,<1,521
+5,119,2-4,5240
+5,103,60-1d,6396
+6,115,>=1d,1122
+5,103,4-10,3334
+-1,450,<1,1625
+6,107,2-4,6694
+6,200,<1,2353908
+6,116,10-60,8949
+7,3,<1,2
+6,107,10-60,14488
+4,117,4-10,1033
+6,118,60-1d,520
+5,107,<1,36
+7,100,10-60,10992
+7,25,4-10,1812
+6,115,1-2,250381
+6,103,1-2,12600
+5,117,<1,93200
+7,116,>=1d,64
+6,101,2-4,4
+-1,118,<1,64
+7,115,60-1d,32024
+-1,360,<1,4608
+6,105,4-10,8592
+7,0,2-4,36076
+7,116,<1,46125
+6,119,60-1d,22292
+7,118,>=1d,244
+7,119,4-10,91976
+7,100,60-1d,12332
+5,103,1-2,237
+5,200,60-1d,123408
+7,117,2-4,353
+7,118,2-4,20605
+5,117,10-60,73865
+4,119,10-60,4
+7,107,2-4,142022
+4,117,60-1d,334
+6,105,2-4,14358
+6,200,>=1d,23436
+6,115,<1,287093
+5,100,>=1d,32
+7,117,>=1d,40
+5,200,4-10,10116
+5,115,1-2,8968
+7,117,10-60,23586
+7,117,4-10,18285
+7,25,2-4,344
+6,115,2-4,321609
+5,118,10-60,64
+7,0,4-10,67208
+-1,205,<1,48
+5,119,<1,5088
+5,0,60-1d,300
+5,105,2-4,215
+7,119,>=1d,12904
+6,105,60-1d,10790
+6,360,1-2,948
+7,25,10-60,3264
+5,116,1-2,48
+4,200,<1,24
+5,117,1-2,18457
+7,100,<1,160
+5,118,4-10,12
+5,25,<1,432
+7,25,<1,228
+7,200,4-10,507648
+8,107,<1,12
+4,200,10-60,1824
+5,105,60-1d,5290
+6,200,4-10,2510684
+5,115,2-4,14544
+5,116,10-60,9684
+6,0,>=1d,16
+6,117,4-10,1089845
+6,103,>=1d,1601
+8,115,10-60,18
+6,119,2-4,60324
+6,116,1-2,4153
+-1,25,<1,2470
+7,101,<1,10346
+5,25,>=1d,96
+4,117,1-2,1000
+7,105,1-2,262337
+5,200,<1,12676
+4,115,2-4,12
+6,119,10-60,89964
+6,117,1-2,142224
+5,116,60-1d,8227
+7,103,4-10,472350
+8,115,<1,5520
+5,100,10-60,64
+5,118,<1,8
+6,116,4-10,3972
+5,25,2-4,200
+7,114,4-10,4604
+5,103,10-60,37488
+7,105,10-60,160081
+7,116,2-4,83850
+-1,101,<1,33
+4,117,<1,2925
+5,25,1-2,448
+7,101,60-1d,192
+8,115,2-4,42
+6,119,<1,537088
+5,105,4-10,1196
+5,200,1-2,3636
+6,117,<1,2339426
+5,360,10-60,40
+7,117,<1,379
+6,103,2-4,10315
+5,105,10-60,24262
+6,118,4-10,10772
+-1,210,<1,3
+7,115,4-10,32471
+6,116,<1,3536
+6,117,2-4,677400
+5,115,60-1d,3643
+7,360,>=1d,3044
+5,360,>=1d,68
+4,115,10-60,36
+6,360,>=1d,864
+5,118,2-4,8
+6,200,10-60,2855680
+4,200,60-1d,252
+5,107,4-10,28
+5,103,2-4,1972
+7,101,10-60,307
+7,107,<1,62446
+6,107,1-2,4585
+5,118,1-2,8
+5,0,10-60,4436
+6,0,10-60,1452
+6,101,<1,120
+7,200,60-1d,110032
+7,101,>=1d,24
+5,200,10-60,11700
+7,119,10-60,225016
+6,103,4-10,26551
+5,25,10-60,556
+7,200,<1,2186014
+6,107,60-1d,240
+7,114,2-4,6121
+6,0,1-2,270164
+7,116,4-10,80946
+6,101,4-10,4
+4,0,10-60,4
+7,103,2-4,451203
+5,119,>=1d,1012
+7,119,2-4,61508
+7,105,>=1d,1576
+7,105,2-4,187532
+5,105,>=1d,506
+6,360,60-1d,632
+-1,215,<1,3
+8,118,<1,36
+7,201,>=1d,8
+-1,100,<1,794
+4,103,2-4,36
+7,115,>=1d,10266
+4,103,1-2,24
+-1,214,<1,48
+5,107,1-2,48
+4,115,4-10,20
+7,201,10-60,40
+7,0,1-2,66588
+7,105,4-10,160737
+6,118,10-60,776
+5,107,10-60,1120
+5,105,1-2,162
+7,25,>=1d,3310
+4,115,1-2,20
+7,116,60-1d,45379
+7,117,1-2,112
+5,118,>=1d,24
+6,103,<1,53366
+7,107,10-60,56208
+5,115,>=1d,1688
+6,25,10-60,88889
+7,119,1-2,34645
+6,103,60-1d,7231
+-1,116,<1,67
+6,107,4-10,21277
+-1,200,<1,37198
+5,0,1-2,196
+4,200,2-4,8
+6,200,60-1d,454072
+7,115,2-4,27703
+5,103,>=1d,1461
+6,360,10-60,1656
+6,360,4-10,10392
+4,119,2-4,4
+7,105,<1,269254
+7,0,10-60,33872
+6,25,2-4,27430
+5,200,2-4,9036
+7,100,4-10,72
+7,118,60-1d,5714

figure_9/d_priority_exectime/part-00000

@@ -0,0 +1,313 @@
+6,101,10-60,528
+7,115,1-2,49131
+7,195,4-10,4
+7,105,60-1d,88156
+6,118,2-4,15116
+7,107,>=1d,580
+-1,119,<1,6122
+6,0,60-1d,84
+5,107,>=1d,12
+4,115,<1,4
+7,116,10-60,122151
+6,119,1-2,32420
+5,115,4-10,3017
+7,101,2-4,1024
+7,101,4-10,228
+7,103,60-1d,100588
+6,103,10-60,18353
+-1,199,<1,126
+-1,107,<1,91
+5,25,4-10,968
+6,118,>=1d,72
+5,119,60-1d,864
+5,0,2-4,32
+6,117,>=1d,17
+7,200,1-2,43048
+7,200,>=1d,52173
+5,119,1-2,600
+7,107,4-10,11015
+6,0,4-10,1320
+7,25,1-2,7928
+6,119,4-10,31056
+6,116,60-1d,1519
+7,118,4-10,6540
+5,103,<1,2728
+5,0,<1,524
+7,118,<1,207
+7,114,<1,16
+7,114,60-1d,8
+5,116,4-10,2396
+7,100,>=1d,708
+7,115,<1,38925
+5,117,2-4,20
+7,119,60-1d,59212
+7,25,60-1d,11405
+7,150,<1,2
+5,115,<1,5172
+5,101,60-1d,80
+6,115,10-60,271756
+6,115,4-10,185810
+-1,115,<1,4117
+5,118,60-1d,121
+6,25,4-10,39776
+5,114,>=1d,24
+6,119,>=1d,5384
+5,107,60-1d,256
+7,114,10-60,8
+7,110,<1,4
+7,107,1-2,11325
+5,0,>=1d,64
+7,197,>=1d,176
+7,117,60-1d,325
+7,101,1-2,417
+6,25,60-1d,31296
+7,118,1-2,1472
+-1,103,<1,2598
+7,199,4-10,4
+8,103,<1,211
+6,117,60-1d,14726
+7,103,<1,560948
+6,105,1-2,13861
+5,119,4-10,5732
+-1,250,<1,1
+7,360,<1,148
+6,115,60-1d,76253
+5,117,>=1d,8
+6,107,<1,17940
+8,105,<1,30
+6,105,10-60,11504
+5,117,4-10,1597
+6,105,<1,37189
+5,0,4-10,32
+5,360,60-1d,35700
+6,200,2-4,246984
+7,107,60-1d,5592
+7,119,<1,10010
+6,25,1-2,200208
+6,200,1-2,344988
+7,200,2-4,34148
+6,25,<1,33832
+5,200,>=1d,1364
+6,0,<1,30004
+7,115,10-60,29062
+5,119,10-60,1564
+5,101,10-60,136
+6,117,10-60,46100
+-1,197,<1,12
+6,25,>=1d,3272
+6,105,>=1d,404
+5,25,60-1d,220
+6,116,2-4,22653
+7,116,1-2,76025
+7,118,10-60,34832
+6,360,<1,28
+7,196,10-60,16
+6,0,2-4,940
+5,115,10-60,5418
+5,100,4-10,32
+7,201,60-1d,24
+5,116,2-4,51
+7,103,1-2,236246
+6,118,<1,1708
+-1,0,<1,2048
+5,117,60-1d,1463
+7,360,10-60,760
+7,0,60-1d,996
+7,0,>=1d,6572
+7,360,4-10,4
+7,103,10-60,305852
+-1,105,<1,1737
+7,197,10-60,136
+6,118,1-2,11156
+7,103,>=1d,8067
+7,200,10-60,1185368
+7,360,60-1d,460
+6,107,>=1d,60
+7,0,<1,4748
+-1,117,<1,7
+5,119,2-4,404
+5,103,60-1d,5205
+6,115,>=1d,1485
+-1,450,<1,1649
+5,103,4-10,224
+6,107,2-4,204
+6,200,<1,859852
+6,116,10-60,8596
+6,101,1-2,592
+6,107,10-60,264
+6,118,60-1d,820
+7,100,10-60,9848
+7,25,4-10,1220
+6,115,1-2,193055
+6,103,1-2,28339
+6,101,2-4,1380
+7,116,>=1d,973
+-1,118,<1,892
+7,115,60-1d,38189
+7,199,60-1d,204
+-1,360,<1,4411
+6,105,4-10,3734
+4,115,60-1d,4
+7,116,<1,27966
+7,0,2-4,1184
+6,119,60-1d,27700
+7,118,>=1d,10293
+7,100,60-1d,7432
+7,119,4-10,2884
+5,103,1-2,1440
+5,200,60-1d,10092
+7,117,2-4,20
+7,118,2-4,3087
+5,117,10-60,4085
+7,107,2-4,20157
+6,105,2-4,6346
+6,200,>=1d,23240
+6,115,<1,388328
+5,100,>=1d,8
+7,117,>=1d,8
+5,115,1-2,22354
+5,200,4-10,8248
+7,25,2-4,2052
+7,117,10-60,321
+7,117,4-10,8
+6,115,2-4,222493
+5,118,10-60,80
+6,101,60-1d,656
+7,0,4-10,3040
+-1,205,<1,30
+5,119,<1,1560
+5,0,60-1d,36
+5,105,2-4,80
+7,119,>=1d,17836
+6,105,60-1d,7958
+7,25,10-60,1344
+4,200,>=1d,4
+4,200,<1,8
+5,117,1-2,156
+7,100,<1,2
+5,25,<1,8504
+7,25,<1,44560
+7,200,4-10,158704
+8,107,<1,8
+4,200,10-60,52
+5,105,60-1d,2319
+6,200,4-10,223708
+5,115,2-4,15917
+7,195,60-1d,4
+5,116,10-60,3667
+-1,5,<1,3
+6,0,>=1d,20
+6,117,4-10,49976
+6,103,>=1d,120
+6,119,2-4,18784
+6,116,1-2,29274
+7,20,<1,2
+-1,25,<1,2067
+7,199,>=1d,176
+5,25,>=1d,96
+7,101,<1,10
+7,105,1-2,92346
+5,200,<1,13044
+6,119,10-60,50408
+6,117,1-2,26675
+5,116,60-1d,1821
+7,103,4-10,320786
+8,115,<1,327
+5,100,10-60,4
+5,118,<1,152
+6,116,4-10,6096
+5,25,2-4,2436
+7,197,60-1d,200
+7,105,10-60,128279
+5,103,10-60,19713
+7,116,2-4,83184
+-1,101,<1,81
+5,25,1-2,11680
+7,101,60-1d,1078
+6,119,<1,76144
+5,105,4-10,417
+5,200,1-2,4436
+6,117,<1,7659
+5,360,10-60,36
+7,117,<1,38
+6,103,2-4,11986
+5,105,10-60,9971
+6,118,4-10,1132
+-1,210,<1,3
+7,115,4-10,10881
+6,116,<1,11154
+6,117,2-4,42434
+7,360,>=1d,2892
+5,115,60-1d,3245
+5,360,>=1d,124
+6,360,>=1d,484
+5,118,2-4,48
+6,200,10-60,358248
+4,200,60-1d,16
+7,101,10-60,1266
+5,103,2-4,944
+7,107,<1,2452
+6,107,1-2,285
+5,118,1-2,4
+5,0,10-60,4156
+6,0,10-60,188
+7,200,60-1d,273064
+7,101,>=1d,88
+5,200,10-60,4832
+6,116,>=1d,85
+7,199,10-60,132
+7,119,10-60,40860
+6,103,4-10,7716
+5,25,10-60,440
+7,200,<1,851724
+6,107,60-1d,72
+7,116,4-10,99820
+6,0,1-2,1020
+6,101,4-10,456
+7,103,2-4,258714
+5,119,>=1d,328
+6,110,4-10,4
+7,105,>=1d,3094
+7,119,2-4,916
+7,105,2-4,83416
+5,105,>=1d,215
+6,360,60-1d,464
+-1,215,<1,3
+8,118,<1,232
+-1,100,<1,320
+7,115,>=1d,16417
+-1,214,<1,30
+4,115,4-10,4
+7,0,1-2,3784
+7,100,1-2,4
+7,105,4-10,88247
+6,118,10-60,2200
+5,107,10-60,144
+5,116,>=1d,37
+7,25,>=1d,3128
+7,116,60-1d,46077
+5,118,>=1d,44
+7,117,1-2,4
+6,103,<1,152937
+7,107,10-60,10883
+5,115,>=1d,2271
+6,25,10-60,13540
+7,119,1-2,288
+6,103,60-1d,7937
+-1,116,<1,247
+6,107,4-10,193
+7,197,4-10,4
+-1,200,<1,37500
+5,0,1-2,144
+6,200,60-1d,120240
+7,115,2-4,19153
+5,103,>=1d,495
+6,360,10-60,8
+7,190,10-60,16
+7,195,10-60,116
+7,105,<1,154731
+7,0,10-60,2024
+6,25,2-4,111492
+5,200,2-4,3544
+7,100,4-10,48
+7,118,60-1d,23462

figure_9/e_priority_exectime/part-00000

@@ -0,0 +1,292 @@
+5,101,4-10,116
+6,101,10-60,608
+7,115,1-2,37504
+7,105,60-1d,6208
+7,107,>=1d,396
+6,118,2-4,12
+6,0,60-1d,952
+-1,119,<1,161
+5,107,>=1d,12
+4,115,<1,64
+7,116,10-60,1532
+6,119,1-2,27848
+5,115,4-10,1924
+7,101,4-10,240
+7,101,2-4,72
+6,103,10-60,39252
+7,103,60-1d,68680
+-1,107,<1,138
+5,25,4-10,10136
+5,119,60-1d,32
+5,0,2-4,8
+5,107,2-4,4
+7,200,1-2,15320
+7,200,>=1d,19600
+5,119,1-2,36
+7,107,4-10,3008
+6,0,4-10,176
+7,25,1-2,45148
+6,119,4-10,3756
+7,118,4-10,268
+5,103,<1,1048
+5,0,<1,48
+-1,220,<1,6
+7,100,>=1d,1868
+7,115,<1,38373
+7,119,60-1d,1012
+7,210,>=1d,8
+7,25,60-1d,28060
+5,115,<1,16224
+5,101,60-1d,36
+6,115,4-10,94252
+6,115,10-60,87777
+-1,115,<1,653
+6,25,4-10,67072
+6,119,>=1d,88
+5,107,60-1d,8
+7,110,<1,2
+7,107,1-2,2732
+5,0,>=1d,52
+7,117,60-1d,636
+7,101,1-2,84
+6,25,60-1d,115580
+5,100,60-1d,72
+-1,103,<1,1648
+8,103,<1,540
+6,117,60-1d,112
+7,103,<1,115466
+6,105,1-2,6008
+5,119,4-10,4
+4,103,<1,8
+6,115,60-1d,14566
+5,117,>=1d,44
+6,107,<1,18253
+8,105,<1,4
+6,105,10-60,8348
+5,117,4-10,80
+6,105,<1,35376
+5,0,4-10,28
+5,360,60-1d,35696
+6,200,2-4,21292
+7,220,>=1d,40
+7,107,60-1d,5064
+7,119,<1,128
+6,25,1-2,73000
+6,200,1-2,34652
+7,200,2-4,14588
+7,220,10-60,4
+6,25,<1,41848
+5,200,>=1d,312
+6,0,<1,18284
+7,115,10-60,6308
+5,119,10-60,200
+5,101,10-60,116
+6,117,10-60,160
+5,100,1-2,68
+6,105,>=1d,1141
+6,25,>=1d,288
+5,25,60-1d,132
+7,116,1-2,651
+6,116,2-4,20
+7,118,10-60,3744
+5,105,<1,200
+6,360,<1,12
+6,0,2-4,80
+5,115,10-60,1656
+-1,170,<1,3
+7,201,60-1d,40
+5,100,4-10,80
+5,116,2-4,4
+7,103,1-2,73656
+6,118,<1,4
+-1,0,<1,1984
+7,360,10-60,8
+7,0,60-1d,3556
+7,0,>=1d,6484
+7,103,10-60,236708
+-1,105,<1,465
+6,118,1-2,12
+7,103,>=1d,3729
+7,200,10-60,1056660
+7,360,60-1d,56
+6,107,>=1d,124
+7,0,<1,44502
+-1,117,<1,2160
+5,119,2-4,4
+5,103,60-1d,3340
+6,115,>=1d,468
+-1,450,<1,1649
+5,103,4-10,616
+6,107,2-4,139
+6,200,<1,722880
+6,116,10-60,4
+6,101,1-2,1440
+6,107,10-60,245
+5,107,<1,520
+7,100,10-60,11172
+7,25,4-10,700
+6,115,1-2,79444
+6,103,1-2,7896
+6,101,2-4,456
+-1,118,<1,9
+7,115,60-1d,3140
+-1,360,<1,2139
+6,105,4-10,1708
+7,0,2-4,54316
+7,116,<1,28
+6,119,60-1d,2596
+7,100,60-1d,10996
+7,119,4-10,1580
+5,103,1-2,236
+5,200,60-1d,3220
+4,115,>=1d,8
+4,25,1-2,212
+7,107,2-4,1720
+6,105,2-4,248
+6,200,>=1d,1292
+6,115,<1,86313
+5,100,>=1d,32
+7,117,>=1d,6368
+5,115,1-2,2656
+5,200,4-10,540
+7,25,2-4,4852
+6,115,2-4,170892
+5,118,10-60,4
+-1,201,<1,6
+6,101,60-1d,92
+7,0,4-10,71420
+-1,205,<1,6
+5,119,<1,220
+5,0,60-1d,68
+5,105,2-4,4
+6,105,60-1d,9483
+7,119,>=1d,40
+7,100,2-4,188
+6,360,1-2,16
+7,25,10-60,44560
+5,116,1-2,4
+7,100,<1,40
+5,25,<1,14292
+7,25,<1,6222
+7,200,4-10,16272
+5,105,60-1d,4
+6,200,4-10,46408
+5,115,2-4,1388
+5,116,10-60,1245
+6,0,>=1d,4
+4,25,<1,264
+6,103,>=1d,1244
+6,117,4-10,64
+6,119,2-4,2212
+6,116,1-2,40
+-1,25,<1,1825
+7,101,<1,1936
+5,25,>=1d,12
+7,105,1-2,10692
+5,200,<1,448
+4,115,2-4,4
+6,119,10-60,7704
+6,117,1-2,220
+5,116,60-1d,5
+7,103,4-10,148156
+8,115,<1,1798
+5,100,10-60,152
+5,25,2-4,13992
+6,116,4-10,4
+7,105,10-60,8756
+5,103,10-60,62608
+7,116,2-4,816
+5,25,1-2,13640
+7,101,60-1d,756
+6,119,<1,11072
+5,105,4-10,12
+5,200,1-2,872
+7,215,>=1d,8
+6,117,<1,20
+5,360,10-60,36
+7,117,<1,8
+6,103,2-4,15484
+5,105,10-60,132
+-1,210,<1,6
+6,100,>=1d,16
+7,115,4-10,6040
+6,116,<1,3060
+5,115,60-1d,452
+6,117,2-4,324
+7,360,>=1d,620
+5,360,>=1d,36
+4,115,10-60,12
+6,360,>=1d,292
+6,200,10-60,183016
+5,107,4-10,68
+7,101,10-60,10016
+5,103,2-4,572
+7,107,<1,4432
+6,107,1-2,144
+5,118,1-2,8
+4,25,10-60,4
+5,100,<1,20
+5,0,10-60,4004
+6,0,10-60,1368
+7,205,>=1d,152
+6,101,<1,3548
+7,200,60-1d,57028
+5,200,10-60,2020
+7,119,10-60,6332
+6,103,4-10,69844
+4,25,2-4,128
+5,25,10-60,1944
+7,200,<1,107250
+6,107,60-1d,222
+7,116,4-10,525
+6,0,1-2,140
+6,101,4-10,640
+7,103,2-4,60528
+5,119,>=1d,28
+7,119,2-4,300
+7,105,>=1d,568
+7,105,2-4,2916
+5,105,>=1d,8
+6,360,60-1d,36
+-1,215,<1,6
+7,201,>=1d,8
+-1,100,<1,516
+4,103,2-4,4
+7,115,>=1d,1916
+5,107,1-2,68
+7,220,60-1d,8
+4,115,4-10,4
+7,0,1-2,54164
+7,100,1-2,40
+7,105,4-10,6868
+5,107,10-60,8076
+5,105,1-2,20
+7,25,>=1d,816
+4,115,1-2,12
+7,116,60-1d,341
+7,117,1-2,16
+6,103,<1,39188
+7,107,10-60,24804
+5,115,>=1d,272
+6,25,10-60,84412
+7,119,1-2,108
+6,103,60-1d,5052
+-1,116,<1,38
+6,107,4-10,203
+-1,200,<1,13470
+5,0,1-2,32
+6,200,60-1d,7528
+4,25,4-10,192
+7,115,2-4,20184
+5,103,>=1d,812
+6,360,10-60,4
+5,101,1-2,12
+7,105,<1,43090
+7,0,10-60,35696
+6,25,2-4,32696
+5,200,2-4,544
+7,100,4-10,164
+6,100,60-1d,4
+5,101,<1,4
+5,100,2-4,4
+7,118,60-1d,1640

figure_9/f_priority_exectime/part-00000

@@ -0,0 +1,283 @@
+6,101,10-60,8
+7,115,1-2,3152
+7,105,60-1d,5785
+6,118,2-4,268
+7,107,>=1d,104
+-1,119,<1,2086
+4,115,<1,8
+7,116,10-60,44344
+6,119,1-2,8444
+5,115,4-10,28100
+7,101,4-10,296
+7,101,2-4,68
+7,114,1-2,8
+7,103,60-1d,88558
+6,103,10-60,9460
+-1,107,<1,24
+5,25,4-10,4
+5,119,60-1d,368
+5,0,2-4,12
+6,117,>=1d,65
+7,200,1-2,82204
+7,200,>=1d,20964
+5,119,1-2,696
+7,107,4-10,2588
+6,0,4-10,64
+7,25,1-2,120
+6,119,4-10,13540
+6,116,60-1d,669
+7,118,4-10,6612
+5,103,<1,68
+7,118,<1,1266
+5,0,<1,100
+7,114,<1,16
+5,116,4-10,56
+7,115,<1,2096
+5,117,2-4,8213
+7,119,60-1d,7612
+7,25,60-1d,1044
+5,115,<1,26600
+6,115,4-10,330736
+6,115,10-60,246196
+-1,115,<1,1495
+5,118,60-1d,105
+6,25,4-10,2780
+6,119,>=1d,224
+5,107,60-1d,148
+7,114,10-60,4
+5,0,>=1d,100
+7,107,1-2,1024
+7,117,60-1d,1500
+6,25,60-1d,1700
+7,118,1-2,4564
+-1,103,<1,861
+5,116,<1,7486
+8,103,<1,10
+6,117,60-1d,1285
+7,103,<1,46801
+6,105,1-2,652
+5,119,4-10,440
+7,360,<1,2
+6,115,60-1d,28533
+5,117,>=1d,160
+6,107,<1,17896
+8,105,<1,4
+6,105,10-60,820
+5,117,4-10,917
+6,105,<1,2440
+5,0,4-10,44
+5,360,60-1d,35676
+6,200,2-4,112712
+6,360,2-4,176
+7,107,60-1d,3441
+7,119,<1,14901
+6,25,1-2,39708
+6,200,1-2,57736
+7,200,2-4,219884
+6,25,<1,768
+5,200,>=1d,304
+6,0,<1,18308
+7,115,10-60,10056
+5,119,10-60,700
+6,117,10-60,5237
+6,105,>=1d,30
+5,25,60-1d,60
+6,25,>=1d,180
+7,116,1-2,6871
+6,116,2-4,298
+7,118,10-60,7854
+6,360,<1,5412
+5,105,<1,32
+6,0,2-4,160
+5,115,10-60,7725
+5,116,2-4,788
+7,103,1-2,60380
+6,118,<1,404
+-1,0,<1,1954
+5,117,60-1d,204
+7,360,10-60,4
+7,0,60-1d,20364
+7,0,>=1d,6416
+7,103,10-60,117340
+-1,105,<1,286
+7,103,>=1d,1287
+7,200,10-60,171576
+7,360,60-1d,116
+7,0,<1,40658
+-1,117,<1,4122
+5,119,2-4,600
+5,103,60-1d,3953
+6,115,>=1d,1124
+-1,450,<1,1653
+5,103,4-10,200
+6,107,2-4,76
+6,200,<1,478460
+6,116,10-60,1124
+6,101,1-2,12
+6,107,10-60,84
+6,118,60-1d,56
+7,25,4-10,76
+6,115,1-2,350408
+5,117,<1,36482
+6,103,1-2,3460
+6,101,2-4,4
+7,116,>=1d,187
+-1,118,<1,197
+7,115,60-1d,10373
+-1,360,<1,3130
+6,105,4-10,2628
+7,0,2-4,11504
+7,116,<1,6660
+6,119,60-1d,5000
+7,118,>=1d,376
+7,119,4-10,15772
+7,100,60-1d,32
+5,114,1-2,4
+5,103,1-2,32
+5,200,60-1d,70020
+7,117,2-4,1164
+7,118,2-4,19084
+-1,105,60-1d,11
+5,117,10-60,445
+7,107,2-4,1344
+6,105,2-4,284
+6,200,>=1d,1988
+6,115,<1,340328
+7,117,>=1d,7384
+5,115,1-2,47132
+5,200,4-10,252
+7,117,4-10,232
+7,117,10-60,1066
+7,25,2-4,72
+6,115,2-4,310045
+5,118,10-60,152
+-1,201,<1,15
+6,101,60-1d,16
+7,0,4-10,30308
+5,0,60-1d,5744
+5,119,<1,604
+5,105,2-4,8
+7,119,>=1d,3976
+6,105,60-1d,516
+6,360,1-2,12224
+7,100,2-4,4
+5,116,1-2,4804
+7,25,10-60,236
+8,116,10-60,13
+6,114,<1,16
+4,200,<1,4
+5,117,1-2,15579
+5,118,4-10,4
+5,25,<1,20
+7,25,<1,50
+7,200,4-10,234376
+4,200,10-60,4
+5,105,60-1d,1076
+6,200,4-10,100736
+5,115,2-4,16064
+5,116,10-60,202
+6,117,4-10,5313
+6,103,>=1d,484
+6,119,2-4,2744
+6,116,1-2,1670
+7,20,<1,2
+-1,25,<1,227
+5,25,>=1d,20
+7,105,1-2,8656
+5,200,<1,3516
+4,115,2-4,4
+6,119,10-60,14220
+5,116,60-1d,69
+6,117,1-2,8439
+-1,101,>=1d,21
+7,103,4-10,23478
+8,115,<1,32
+5,118,<1,10
+6,116,4-10,388
+5,25,2-4,16
+7,114,4-10,4
+7,105,10-60,20157
+5,103,10-60,2044
+7,116,2-4,7327
+-1,101,<1,21
+5,25,1-2,16
+7,101,60-1d,216
+6,119,<1,14984
+5,105,4-10,8
+5,200,1-2,1180
+6,117,<1,28803
+5,360,10-60,36
+7,117,<1,1413
+6,103,2-4,2629
+5,105,10-60,356
+6,118,4-10,556
+6,116,<1,14580
+7,115,4-10,2949
+7,360,>=1d,807
+6,117,2-4,3614
+5,115,60-1d,1551
+5,360,>=1d,100
+4,115,10-60,8
+6,360,>=1d,328
+5,118,2-4,76
+6,200,10-60,61684
+4,200,60-1d,16
+5,103,2-4,616
+7,101,10-60,308
+7,107,<1,2000
+6,107,1-2,112
+5,0,10-60,14072
+6,0,10-60,2172
+6,101,<1,432
+7,200,60-1d,101960
+5,200,10-60,2100
+7,119,10-60,3348
+6,103,4-10,4920
+5,25,10-60,56
+7,200,<1,294660
+6,107,60-1d,12
+6,0,1-2,124
+7,116,4-10,7948
+7,103,2-4,36523
+5,119,>=1d,468
+7,119,2-4,2064
+7,105,>=1d,684
+7,105,2-4,9272
+5,105,>=1d,1356
+6,360,60-1d,8
+8,118,<1,4
+-1,100,<1,1
+7,115,>=1d,2641
+4,115,4-10,4
+7,0,1-2,11000
+7,105,4-10,13388
+5,107,10-60,96
+6,118,10-60,48
+5,105,1-2,24
+5,116,>=1d,8
+7,25,>=1d,332
+4,115,1-2,4
+7,116,60-1d,3410
+7,117,1-2,944
+5,118,>=1d,32
+6,103,<1,7716
+7,107,10-60,4496
+5,115,>=1d,2206
+6,25,10-60,9352
+7,119,1-2,4979
+6,103,60-1d,12265
+-1,116,<1,337
+6,107,4-10,144
+-1,200,<1,21597
+5,0,1-2,40
+6,200,60-1d,22876
+5,103,>=1d,2741
+7,115,2-4,2743
+6,360,10-60,16
+5,114,<1,4
+6,360,4-10,52
+7,105,<1,10291
+7,0,10-60,54656
+5,200,2-4,1112
+6,25,2-4,2992
+7,118,60-1d,4717

figure_9/figure9a-join.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+# coding: utf-8
+
+import json
+import pandas
+import findspark
+import pyspark
+import pyspark.sql
+import sys
+import gzip
+import os
+from pyspark import AccumulatorParam
+from pyspark.sql.functions import lit
+from pyspark.sql import Window
+from pyspark.sql.types import ByteType
+from pyspark.sql.functions import col
+
+if len(sys.argv) is not 4:
+    print(sys.argv[0] + " {cluster} {tmpdir} {maxram}")
+    sys.exit()
+
+cluster=sys.argv[1]
+findspark.init()
+
+DIR = os.path.dirname(__file__)
+NAME = cluster + "_figure9a.csv"
+
+spark = pyspark.sql.SparkSession.builder \
+  .appName("figure_9a_join") \
+  .config("spark.driver.maxResultSize", "128g") \
+  .config("spark.local.dir", sys.argv[2]) \
+  .config("spark.driver.memory", sys.argv[3]) \
+  .getOrCreate()
+sc = spark.sparkContext
+
+dfepath="/home/claudio/google_2019/collection_events/" + cluster + "/" + cluster + "_collection_events*.json.gz"
+#dfepath="/home/claudio/google_2019/collection_events/" + cluster + "/" + cluster + "_test.json"
+
+df = spark.read.json(dfepath)
+df = df.withColumnRenamed("collection_id", "jobid") \
+    .select("jobid", "collection_type", "time", "type")
+
+try:
+    df["collection_type"] = df["collection_type"].cast(ByteType())
+except:
+    df = df.withColumn("collection_type", lit(None).cast(ByteType()))
+
+df = df.filter(((col("collection_type").isNull()) | (col("collection_type") ==
+        "0")) & (col("time").isNotNull()))
+
+jpath = DIR + "/figure-9a-task-count-" + cluster + ".parquet"
+dfj = spark.read.parquet(jpath)
+
+df = df.join(dfj, 'jobid', 'left')
+
+MICROS = 1000000
+
+def for_each_job(data):
+    jobid = data[0]
+    ts = data[1]
+    global non
+
+    ts = sorted(ts, key=lambda x: x["time"])
+
+    l = len(ts)
+    last_term = -1
+    task_count = 0
+
+    for i,t in enumerate(ts):
+        if t["type"] >= 4 and t["type"] <= 8:
+            last_term = t["type"]
+        task_count = t["task_count"]
+
+    return str(jobid) + "," + str(last_term) + "," + str(task_count)
+
+def cleanup(x):
+    return {
+        "time": int(x.time),
+        "type": 0 if x.type is None else int(x.type),
+        "id": x.jobid,
+        "task_count": -1 if x.task_count is None else int(x.task_count)
+    }
+
+df2 = df.rdd \
+    .map(cleanup) \
+    .groupBy(lambda x: x["id"]) \
+    .repartition(100) \
+    .map(for_each_job) \
+    .coalesce(1) \
+    .saveAsTextFile(cluster + "_task_count")
+
+# vim: set ts=4 sw=4 et tw=80:

figure_9/figure9a.py

@@ -15,6 +15,7 @@ from pyspark.sql.functions import lit
 from pyspark.sql import Window
 from pyspark.sql.types import *
 from decimal import *
+import os
 
 if len(sys.argv) is not 5:
     print(sys.argv[0] + " {cluster} {tmpdir} {maxram} {basedir}")
@@ -22,6 +23,12 @@ if len(sys.argv) is not 5:
 
 cluster=sys.argv[1]
 
+if os.path.exists("/home/claudio/google_2019/thesis_queries/figure_9/figure-9a-task-count-" + cluster + "-working"):
+    print("already done")
+    sys.exit()
+
+os.system("touch /home/claudio/google_2019/thesis_queries/figure_9/figure-9a-task-count-" + cluster + "-working")
+
 spark = pyspark.sql.SparkSession.builder \
   .appName("figure_9a") \
   .config("spark.driver.maxResultSize", "128g") \
@@ -44,9 +51,10 @@ df = df.rdd.filter(lambda x: x.collection_id is not None and x.instance_index is
         (x.collection_type == 0 or x.collection_type is None)) \
         .map(lambda x: (x.collection_id, x.instance_index)) \
         .groupBy(lambda x: x[0]) \
-        .map(lambda x: [x[0], len(x[1])]) \
+        .map(lambda x: [x[0], len(set(x[1]))]) \
         .toDF(["jobid", "task_count"])
 
 df.write.parquet("/home/claudio/google_2019/thesis_queries/figure_9/figure-9a-task-count-" + cluster + ".parquet")
 
+os.system("rm /home/claudio/google_2019/thesis_queries/figure_9/figure-9a-task-count-" + cluster + "-working")
 # vim: set ts=4 sw=4 et tw=120:

figure_9/figure9c-join.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+# coding: utf-8
+
+import json
+import pandas
+import findspark
+import pyspark
+import pyspark.sql
+import sys
+import gzip
+import os
+from pyspark import AccumulatorParam
+from pyspark.sql.functions import lit
+from pyspark.sql import Window
+from pyspark.sql.types import ByteType
+from pyspark.sql.functions import col
+
+if len(sys.argv) is not 4:
+    print(sys.argv[0] + " {cluster} {tmpdir} {maxram}")
+    sys.exit()
+
+cluster=sys.argv[1]
+findspark.init()
+
+DIR = os.path.dirname(__file__)
+NAME = cluster + "_figure9a.csv"
+
+spark = pyspark.sql.SparkSession.builder \
+  .appName("figure_9a_join") \
+  .config("spark.driver.maxResultSize", "128g") \
+  .config("spark.local.dir", sys.argv[2]) \
+  .config("spark.driver.memory", sys.argv[3]) \
+  .getOrCreate()
+sc = spark.sparkContext
+
+dfepath="/home/claudio/google_2019/collection_events/" + cluster + "/" + cluster + "_collection_events*.json.gz"
+#dfepath="/home/claudio/google_2019/collection_events/" + cluster + "/" + cluster + "_test.json"
+
+df = spark.read.json(dfepath)
+df = df.withColumnRenamed("collection_id", "jobid") \
+    .select("jobid", "collection_type", "time", "type")
+
+try:
+    df["collection_type"] = df["collection_type"].cast(ByteType())
+except:
+    df = df.withColumn("collection_type", lit(None).cast(ByteType()))
+
+df = df.filter(((col("collection_type").isNull()) | (col("collection_type") ==
+        "0")) & (col("time").isNotNull()))
+
+jpath = DIR + "/figure-9c-machine-count-" + cluster + ".parquet"
+dfj = spark.read.parquet(jpath)
+
+df = df.join(dfj, 'jobid', 'left')
+
+MICROS = 1000000
+
+def for_each_job(data):
+    jobid = data[0]
+    ts = data[1]
+    global non
+
+    ts = sorted(ts, key=lambda x: x["time"])
+
+    l = len(ts)
+    last_term = -1
+    machine_count = 0
+
+    for i,t in enumerate(ts):
+        if t["type"] >= 4 and t["type"] <= 8:
+            last_term = t["type"]
+        machine_count = t["machine_count"]
+
+    return str(jobid) + "," + str(last_term) + "," + str(machine_count)
+
+def cleanup(x):
+    return {
+        "time": int(x.time),
+        "type": 0 if x.type is None else int(x.type),
+        "id": x.jobid,
+        "machine_count": -1 if x.machine_count is None else int(x.machine_count)
+    }
+
+df2 = df.rdd \
+    .map(cleanup) \
+    .groupBy(lambda x: x["id"]) \
+    .repartition(100) \
+    .map(for_each_job) \
+    .coalesce(1) \
+    .saveAsTextFile(cluster + "_machine_count")
+
+# vim: set ts=4 sw=4 et tw=80:

figure_9/figure9c.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+# coding: utf-8
+
+import os
+import json
+import pandas as pd
+import findspark
+findspark.init()
+import pyspark
+import pyspark.sql
+import sys
+import gzip
+from pyspark import AccumulatorParam
+from pyspark.sql.functions import lit
+from pyspark.sql import Window
+from pyspark.sql.types import *
+from decimal import *
+import os
+
+if len(sys.argv) is not 5:
+    print(sys.argv[0] + " {cluster} {tmpdir} {maxram} {basedir}")
+    sys.exit()
+
+cluster=sys.argv[1]
+
+DIR = os.path.dirname(__file__)
+NAME = "figure-9c-machine-count-" + cluster
+
+if os.path.exists(DIR + "/" + NAME + "-working"):
+    print("already done")
+    sys.exit()
+
+os.system("touch " + DIR + "/" + NAME + "-working")
+
+spark = pyspark.sql.SparkSession.builder \
+  .appName(NAME) \
+  .config("spark.driver.maxResultSize", "128g") \
+  .config("spark.local.dir", sys.argv[2]) \
+  .config("spark.driver.memory", sys.argv[3]) \
+  .getOrCreate()
+sc = spark.sparkContext
+
+# READING INSTANCE EVENTS DATA
+dfepath = sys.argv[4] + "/" + cluster + "/" + cluster + "_instance_events*.json.gz"
+#dfepath = sys.argv[4] + "/" + cluster + "/" + cluster + "_test.json"
+df = spark.read.json(dfepath)
+
+try:
+    df["collection_type"] = df["collection_type"].cast(ByteType())
+except:
+    df = df.withColumn("collection_type", lit(None).cast(ByteType()))
+
+df = df.rdd.filter(lambda x: x.collection_id is not None and x.machine_id is not None and
+        int(x.machine_id) is not -1 and int(x.machine_id) is not 0 and
+        (x.collection_type == 0 or x.collection_type is None)) \
+        .map(lambda x: (x.collection_id, int(x.machine_id))) \
+        .groupBy(lambda x: x[0]) \
+        .map(lambda x: [x[0], len(set(x[1]))]) \
+        .toDF(["jobid", "machine_count"])
+
+df.write.parquet(DIR + "/" + NAME + ".parquet")
+
+os.system("rm " + DIR + "/" + NAME + "-working")
+# vim: set ts=4 sw=4 et tw=120:

figure_9/g_priority_exectime/part-00000

@@ -0,0 +1,235 @@
+5,101,4-10,4
+6,101,10-60,124
+7,115,1-2,68
+7,105,60-1d,2740
+6,118,2-4,2900
+7,107,>=1d,28
+-1,119,<1,749
+5,107,>=1d,4
+7,116,10-60,514
+6,119,1-2,114484
+7,101,2-4,1228
+7,101,4-10,884
+5,115,4-10,666
+7,103,60-1d,33466
+6,103,10-60,24932
+-1,107,<1,27
+6,118,>=1d,20
+5,25,4-10,12
+5,119,60-1d,2268
+7,200,1-2,2784
+7,200,>=1d,6140
+5,119,1-2,12
+6,0,4-10,9020
+7,107,4-10,408
+7,25,1-2,68
+6,119,4-10,10816
+7,118,4-10,8
+5,103,<1,140
+6,450,>=1d,8
+5,0,<1,4
+7,100,>=1d,24
+7,115,<1,256
+7,119,60-1d,46288
+7,25,60-1d,72
+5,115,<1,1588
+6,115,10-60,53602
+6,115,4-10,41141
+-1,115,<1,1035
+6,25,4-10,4108
+6,119,>=1d,5248
+5,107,60-1d,8
+7,107,1-2,32
+5,0,>=1d,4
+7,101,1-2,240
+6,25,60-1d,100
+5,100,60-1d,16
+-1,103,<1,2655
+8,103,<1,8
+6,117,60-1d,128
+7,103,<1,29586
+6,105,1-2,2924
+6,115,60-1d,41863
+6,107,<1,17828
+6,105,10-60,2820
+6,105,<1,8836
+5,360,60-1d,35716
+6,200,2-4,127100
+7,107,60-1d,2640
+7,119,<1,128
+6,25,1-2,2440
+7,200,2-4,16888
+6,200,1-2,153048
+6,25,<1,8268
+5,200,>=1d,124
+6,0,<1,17660
+7,115,10-60,1204
+5,119,10-60,36
+5,101,10-60,20
+7,450,<1,48
+6,117,10-60,260
+5,100,1-2,48
+6,25,>=1d,40
+5,25,60-1d,8
+7,116,1-2,120
+6,116,2-4,5
+5,105,<1,68
+6,101,>=1d,407
+6,0,2-4,648
+5,115,10-60,1772
+-1,170,<1,3
+7,103,1-2,32344
+6,118,<1,4
+-1,0,<1,1951
+7,360,10-60,652
+7,0,60-1d,3116
+7,0,>=1d,6548
+7,103,10-60,52017
+-1,105,<1,60
+-1,50,<1,3
+6,118,1-2,3872
+7,103,>=1d,1194
+7,200,10-60,467980
+7,360,60-1d,320
+7,0,<1,10464
+-1,117,<1,1896
+5,119,2-4,4
+5,103,60-1d,4316
+6,115,>=1d,1396
+-1,450,<1,3516
+5,103,4-10,252
+6,107,2-4,56
+6,200,<1,818728
+6,101,1-2,160
+6,107,10-60,4
+6,118,60-1d,508
+7,100,10-60,2600
+7,25,4-10,1064
+6,115,1-2,24359
+6,103,1-2,4660
+6,101,2-4,264
+-1,118,<1,33
+7,115,60-1d,996
+-1,360,<1,2733
+6,105,4-10,312
+7,0,2-4,11036
+7,116,<1,32
+6,119,60-1d,18592
+7,118,>=1d,40
+7,119,4-10,5040
+7,100,60-1d,7780
+5,103,1-2,156
+5,200,60-1d,112
+7,107,2-4,684
+6,105,2-4,84
+6,200,>=1d,1780
+6,115,<1,56909
+7,117,>=1d,6368
+5,200,4-10,808
+5,115,1-2,536
+7,25,2-4,180
+6,115,2-4,34956
+6,101,60-1d,68
+7,0,4-10,3240
+5,0,60-1d,4
+7,119,>=1d,2732
+6,105,60-1d,8140
+6,360,1-2,24
+7,25,10-60,556
+7,25,<1,16
+7,200,4-10,3180
+5,105,60-1d,12
+6,200,4-10,201212
+5,115,2-4,1336
+5,116,10-60,5
+6,103,>=1d,9916
+6,117,4-10,296
+6,119,2-4,13032
+6,116,1-2,16
+-1,25,<1,171
+7,101,<1,22
+7,105,1-2,16360
+5,200,<1,296
+4,115,2-4,4
+6,119,10-60,20456
+6,117,1-2,104
+-1,101,>=1d,3240
+7,103,4-10,39384
+5,100,10-60,20
+6,116,4-10,20
+5,25,2-4,28
+5,103,10-60,41556
+7,105,10-60,6248
+7,116,2-4,144
+-1,101,<1,84
+7,101,60-1d,32
+6,119,<1,3992
+5,105,4-10,4
+5,200,1-2,1816
+5,360,10-60,40
+6,117,<1,4
+6,103,2-4,7044
+5,105,10-60,56
+6,118,4-10,2548
+7,115,4-10,572
+6,116,<1,124
+5,115,60-1d,1236
+6,117,2-4,180
+7,360,>=1d,900
+5,360,>=1d,112
+4,115,10-60,12
+6,360,>=1d,416
+6,200,10-60,211092
+5,103,2-4,1184
+7,101,10-60,204
+6,107,1-2,12
+5,100,<1,4
+5,0,10-60,3492
+6,0,10-60,3848
+6,101,<1,208
+7,200,60-1d,81696
+5,200,10-60,1580
+7,119,10-60,608672
+6,103,4-10,10372
+7,200,<1,106944
+-1,101,60-1d,198
+7,116,4-10,262
+6,0,1-2,140
+6,101,4-10,208
+7,103,2-4,30136
+7,119,2-4,2060
+7,105,>=1d,40
+7,105,2-4,2284
+5,105,>=1d,36
+6,360,60-1d,296
+-1,100,<1,27
+7,115,>=1d,1525
+4,103,10-60,4
+7,0,1-2,10296
+7,105,4-10,4020
+6,450,4-10,4
+6,118,10-60,896
+5,107,10-60,4
+5,105,1-2,20
+7,25,>=1d,128
+7,116,60-1d,164
+6,103,<1,6812
+5,115,>=1d,1012
+7,107,10-60,132
+6,25,10-60,5880
+7,119,1-2,2152
+6,103,60-1d,67220
+-1,116,<1,4
+6,107,4-10,20
+-1,200,<1,9875
+6,200,60-1d,49016
+5,103,>=1d,3091
+7,115,2-4,424
+6,360,10-60,8
+7,105,<1,19916
+7,0,10-60,6196
+6,25,2-4,13280
+5,200,2-4,1432
+7,100,4-10,40
+5,100,2-4,16
+7,118,60-1d,8

figure_9/h_priority_exectime/part-00000

@@ -0,0 +1,241 @@
+7,115,1-2,284
+7,105,60-1d,4264
+7,107,>=1d,5884
+6,118,2-4,16
+-1,119,<1,1172
+6,0,60-1d,20
+5,107,>=1d,20
+7,116,10-60,878
+6,119,1-2,3904
+5,115,4-10,308
+7,103,60-1d,31044
+6,103,10-60,2220
+-1,107,<1,531
+6,118,>=1d,184
+5,25,4-10,4
+5,119,60-1d,336
+5,0,2-4,8
+7,200,1-2,17132
+7,200,>=1d,24656
+5,119,1-2,416
+6,0,4-10,196
+7,107,4-10,1188
+7,25,1-2,16
+6,119,4-10,9676
+6,116,60-1d,76
+7,118,4-10,132
+5,103,<1,224
+7,118,<1,3954
+5,0,<1,124
+-1,220,<1,3
+7,100,>=1d,24
+5,116,4-10,24
+7,115,<1,3396
+7,119,60-1d,99996
+7,25,60-1d,172
+5,115,<1,206
+6,115,4-10,12893
+6,115,10-60,34008
+-1,115,<1,285
+5,118,60-1d,16
+6,25,4-10,2236
+6,119,>=1d,364
+7,107,1-2,20
+7,117,60-1d,4
+7,118,1-2,2276
+6,25,60-1d,300
+-1,103,<1,609
+-1,19,<1,10
+8,103,<1,16
+6,117,60-1d,224
+7,103,<1,28288
+5,119,4-10,4336
+6,105,1-2,52
+7,360,<1,2
+6,115,60-1d,9732
+6,107,<1,17812
+8,105,<1,2
+6,105,10-60,624
+6,105,<1,60
+5,360,60-1d,35680
+6,200,2-4,410788
+6,360,2-4,13164
+7,220,>=1d,12
+7,107,60-1d,18848
+7,119,<1,950
+6,25,1-2,384
+6,200,1-2,33100
+7,200,2-4,30556
+6,25,<1,480
+5,200,>=1d,228
+6,0,<1,23708
+7,115,10-60,32175
+5,119,10-60,7840
+6,117,10-60,1860
+6,25,>=1d,220
+5,25,60-1d,164
+6,116,2-4,19661
+7,116,1-2,1781
+7,118,10-60,1524
+6,360,<1,388
+6,101,>=1d,12
+6,0,2-4,420
+5,115,10-60,432
+5,116,2-4,780
+7,103,1-2,26220
+-1,0,<1,1972
+7,360,10-60,416
+7,0,60-1d,708
+7,0,>=1d,6460
+7,360,4-10,52
+7,103,10-60,83770
+-1,105,<1,99
+7,103,>=1d,1736
+7,200,10-60,464888
+7,360,60-1d,488
+7,0,<1,11392
+-1,117,<1,1902
+5,119,2-4,1916
+5,103,60-1d,644
+6,115,>=1d,1212
+-1,450,<1,1646
+5,103,4-10,16
+6,107,2-4,4
+6,200,<1,452704
+6,116,10-60,1088
+6,107,10-60,16
+6,118,60-1d,12
+7,25,4-10,24
+6,103,1-2,1056
+6,115,1-2,8692
+7,116,>=1d,16
+-1,118,<1,99
+7,115,60-1d,8769
+-1,360,<1,2786
+7,0,2-4,3152
+7,116,<1,1890
+6,119,60-1d,4460
+7,118,>=1d,396
+7,119,4-10,29524
+5,103,1-2,156
+5,200,60-1d,11368
+7,118,2-4,360
+7,107,2-4,76
+6,200,>=1d,10736
+6,115,<1,10848
+7,117,>=1d,6368
+5,200,4-10,1756
+5,115,1-2,116
+7,25,2-4,20
+7,117,10-60,4
+6,115,2-4,12249
+5,118,10-60,12
+-1,201,<1,3
+7,0,4-10,5240
+5,0,60-1d,108
+5,119,<1,416
+7,119,>=1d,372
+6,105,60-1d,116
+7,100,2-4,44
+6,360,1-2,12
+7,25,10-60,92
+5,116,1-2,80
+7,100,<1,12
+5,25,<1,212
+7,25,<1,118
+7,200,4-10,51200
+4,200,10-60,4
+5,105,60-1d,424
+6,200,4-10,180096
+5,115,2-4,1240
+5,116,10-60,60
+6,0,>=1d,120
+6,103,>=1d,744
+6,117,4-10,1100
+6,230,4-10,4
+6,119,2-4,21156
+6,116,1-2,8823
+-1,25,<1,420
+5,25,>=1d,32
+7,105,1-2,5136
+5,200,<1,5860
+6,119,10-60,14740
+6,230,60-1d,20
+6,117,1-2,1556
+5,116,60-1d,32
+7,103,4-10,20908
+5,100,10-60,20
+6,116,4-10,7035
+5,25,2-4,40
+7,105,10-60,4672
+5,103,10-60,492
+7,116,2-4,791
+-1,101,<1,30
+5,25,1-2,268
+6,119,<1,2896
+5,200,1-2,232
+6,117,<1,1184
+5,360,10-60,40
+6,103,2-4,1500
+5,105,10-60,648
+7,115,4-10,447
+6,116,<1,1007
+6,117,2-4,1052
+7,360,>=1d,684
+5,115,60-1d,696
+5,360,>=1d,136
+6,360,>=1d,476
+6,200,10-60,285716
+5,103,2-4,96
+7,107,<1,102
+6,107,1-2,32
+5,0,10-60,2364
+6,230,2-4,4
+6,0,10-60,2968
+7,200,60-1d,461636
+5,200,10-60,20736
+6,230,10-60,4
+7,119,10-60,46548
+6,103,4-10,2136
+5,25,10-60,64
+7,200,<1,9561724
+6,0,1-2,7988
+7,116,4-10,1568
+7,103,2-4,16196
+5,119,>=1d,8
+7,119,2-4,12708
+7,105,>=1d,12
+7,105,2-4,6756
+5,105,>=1d,4
+6,360,60-1d,716
+-1,100,<1,6
+7,115,>=1d,1287
+7,0,1-2,1860
+7,100,1-2,16
+7,105,4-10,3132
+5,116,>=1d,4
+7,25,>=1d,444
+7,116,60-1d,210
+5,118,>=1d,32
+6,103,<1,5072
+7,107,10-60,200
+5,115,>=1d,166
+6,25,10-60,740
+7,119,1-2,80
+6,103,60-1d,2565
+-1,116,<1,37
+6,107,4-10,28
+-1,200,<1,20773
+5,0,1-2,400
+6,200,60-1d,160780
+6,360,10-60,7384
+7,115,2-4,656
+5,103,>=1d,320
+6,360,4-10,1112
+7,230,<1,72
+7,105,<1,910
+7,0,10-60,2300
+6,25,2-4,18872
+5,200,2-4,152
+7,100,4-10,12
+7,118,60-1d,800

queries.txt

@@ -0,0 +1,44 @@
+The Google 2019 Borg cluster traces analysis were conducted by using Apache
+Spark and its Python 3 API (pyspark).  Spark was used to execute a series of
+queries to perform various sums and aggregations over the entire dataset
+provided by Google.
+
+In general, each query follows a general Map-Reduce template, where traces are
+first read, parsed, filtered by performing selections, projections and computing
+new derived fields. Then, the trace records are often grouped by one of their
+fields, clustering related data toghether before a reduce or fold operation is
+applied to each grouping.
+
+Most input data is in JSONL format and adheres to a schema Google profided in
+the form of a protobuffer specification located here:
+
+https://github.com/google/cluster-data/blob/master/clusterdata_trace_format_v3.proto
+
+On of the main quirks in the traces is that fields that have a "zero" value
+(i.e. a value like 0 or the empty string) are often omitted in the JSON object
+records. When reading the traces in Apache Spark is therefore necessary to check
+for this possibility and populate those zero fields when omitted.
+
+Most queries use only two or three fields in each trace records, while the
+original records often are made of a couple of dozen fields. In order to save
+memory during the query, a projection is often applied to the data by the means
+of a .map() operation over the entire trace set, performed using Spark's RDD
+API.
+
+Another operation that is often necessary to perform prior to the Map-Reduce core of
+each query is a record filtering process, which is often motivated by the
+presence of incomplete data (i.e. records which contain fields whose values is
+unknown). This filtering is performed using the .filter() operation of Spark's
+RDD API.
+
+The core of each query is often a groupBy followed by a map() operation on the
+aggregated data. The groupby groups the set of all records into several subsets
+of records each having something in common. Then, each of this small clusters is
+reduced with a .map() operation to a single record. The motivation behind this
+computation is often to analyze a time series of several different traces of
+programs. This is implemented by groupBy()-ing records by program id, and then
+map()-ing each program trace set by sorting by time the traces and computing the
+desired property in the form of a record.
+
+Sometimes intermediate results are saved in Spark's parquet format in order to
+compute and save intermediate results beforehand.

table_iii/bigtable.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+# coding: utf-8
+
+import json
+import pandas
+import findspark
+findspark.init()
+import pyspark
+import pyspark.sql
+import sys
+import gzip
+import os
+
+from pyspark import AccumulatorParam
+from pyspark.sql.functions import lit
+from pyspark.sql import Window
+from pyspark.sql.types import ByteType
+
+DIR = os.path.dirname(__file__)
+
+if len(sys.argv) != 4:
+    print(sys.argv[0] + " {cluster} {tmpdir} {maxram}")
+    sys.exit()
+
+cluster=sys.argv[1]
+
+if os.path.exists(DIR + "/bigtable-" + cluster + "-working") or os.path.exists(DIR + "/bigtable-" + cluster + ".parquet"):
+    print("already launched")
+    sys.exit()
+
+os.system("touch " + DIR + "/bigtable-" + cluster + "-working")
+
+spark = pyspark.sql.SparkSession.builder \
+  .appName("bigtable") \
+  .config("spark.driver.maxResultSize", "32g") \
+  .config("spark.local.dir", sys.argv[2]) \
+  .config("spark.driver.memory", sys.argv[3]) \
+  .getOrCreate()
+sc = spark.sparkContext
+
+df = spark.read.json("/home/claudio/google_2019/instance_events/" + cluster + "/" + cluster + "_instance_events*.json.gz")
+#df = spark.read.json("/home/claudio/google_2019/instance_events/" + cluster + "/" + cluster + "_test.json")
+
+try:
+    df["collection_type"] = df["collection_type"].cast(ByteType())
+except:
+    df = df.withColumn("collection_type", lit(None).cast(ByteType()))
+
+def for_each_task(data):
+    rowids = data[0].split("-")
+    jobid = rowids[0]
+    taskid = rowids[1]
+
+    ts = data[1]
+    ts = sorted(ts, key=lambda x: x["time"])
+    
+    last_term = -1
+    counts = [0] * 11
+    
+    for i,t in enumerate(ts):
+        if t["type"] >= 4 and t["type"] <= 8:
+            last_term = t["type"]
+        counts[t["type"]] += 1
+            
+    return (jobid, taskid, last_term, *counts)
+    
+def cleanup(x):
+    return {
+        "time": int(x.time),
+        "type": 0 if x.type is None else int(x.type),
+        "id": x.collection_id + "-" + x.instance_index,
+        "priority": -1 if x.priority is None else int(x.priority)
+    }
+
+
+df = df.rdd \
+    .filter(lambda x: x.collection_type is None or x.collection_type == 0) \
+    .filter(lambda x: x.time is not None and x.instance_index is not None and x.collection_id is not None) \
+    .map(cleanup) \
+    .groupBy(lambda x: x["id"]) \
+    .repartition(1000) \
+    .map(for_each_task) \
+    .toDF(["jobid", "taskid", "task_term", *["count_" + str(x) for x in range(0,11)]])
+
+df.write.parquet(DIR + "/bigtable-" + cluster + ".parquet")
+os.system("rm " + DIR + "/bigtable-" + cluster + "-working")
+

table_iii/table-iii-a.csv/part-00000-672a3964-a5e1-48b4-ace7-f2dde6a257b4-c000.csv

@@ -0,0 +1,7 @@
+task_term,%95,mean,avg_count_0,avg_count_1,avg_count_2,avg_count_3,avg_count_4,avg_count_5,avg_count_6,avg_count_7,avg_count_8,avg_count_9,avg_count_10
+7,58.0,27.395924919004106,5.835266629759437,0.7362657541797388,5.528681906150175,5.317656696475938,2.3495793687316646,0.21385905619823325,0.0034117154159850603,3.395995570544976,0.08957552916559747,0.8945408979149864,3.0310917944673723
+6,9.0,12.405369995608641,2.1847132402154554,0.8799252043981606,2.1926863641910397,2.1075212224047664,0.01932128901162818,0.0037793286646258285,2.1534317263499836,0.008150029740766142,0.008989201952316337,0.7198900340111737,2.1269623546687253
+5,108.0,50.03955645233451,10.198896275881816,0.9415567565632885,11.821643640723707,10.278011417527157,0.28777805615705365,11.061863580235867,0.002098283791764348,0.46765555933356,0.05314384652552837,0.3560170189158718,4.570892016678895
+8,7.0,8.847144735439395,1.7224956469208628,2.0535606410575316,0.7519899790341494,0.6955243239401585,0.08334813972495647,0.0018211861696457127,0.3841903272804804,1.3299100955900642,1.007933264631676,0.21771614370491454,0.5986549873849544
+4,2924.0,428.55068857072354,104.07974609774922,0.4735045965967858,102.01192332530044,98.2043818614248,73.69359521596917,0.7685530826862396,1.785086130405792E-4,28.76616421742349,0.8455007954133787,4.683940000105005,15.02320086944195
+-1,84.0,14.818523496614894,0.9975457984866587,0.7739496216646754,0.7931800079649542,0.7857277976901633,0.0,0.0,0.0,0.0,0.0,1.394578853046595,10.073541417761849

table_iii/table-iii-b.csv/part-00000-70af9f3d-0753-4a64-8755-4846f6634a84-c000.csv

@@ -0,0 +1,7 @@
+task_term,%95,mean,avg_count_0,avg_count_1,avg_count_2,avg_count_3,avg_count_4,avg_count_5,avg_count_6,avg_count_7,avg_count_8,avg_count_9,avg_count_10
+7,60.0,40.9010406775462,8.96533411212207,1.1236242609569156,8.817185962634415,8.21370173898341,3.351495825915401,0.2763050332235977,0.0036563227006368043,5.5410792131219875,0.033457306614610785,1.0084090015199039,3.566791899753253
+6,20.0,17.27759558548148,2.999636153710804,1.5549015403294333,2.9978729175206382,3.0121754750188137,0.02044427162564865,0.020628020320906527,2.942578848692604,0.011640483355261005,0.016277768205339063,1.025285737295677,2.6761543694063548
+5,260.0,86.77241946490753,19.370860965779514,1.665039341604111,20.905317678349576,19.52377119459658,0.5180609751118421,19.65679772757762,5.599398064708044E-4,0.6753923953175034,0.08852298377924372,0.778502977596575,3.5895932853884873
+8,14.0,25.69045523138833,4.756602112676056,4.94595699195171,2.7637072434607646,2.7792064889336014,0.2572308853118712,0.007419517102615694,1.9283513581488934,3.5154363682092553,2.0151534205231387,0.5090543259557344,2.212336519114688
+4,1578.0,345.70555920501823,84.1745817071757,0.36389221634014707,84.05783324961037,82.74729832425261,64.81651789739428,0.24021407318675375,0.0,17.961538956940633,1.028401406545848,1.7111171219908035,8.604164251581077
+-1,32.0,13.018129881628738,1.8312972351930517,1.1088478038599783,1.696303401460931,1.5939749765471007,0.0,0.0,0.0,0.0,0.0,1.1423670036154356,5.645339460952241

table_iii/table-iii-c.csv/part-00000-d78ac82e-37cf-4237-9d33-984a2ee00ee7-c000.csv

@@ -0,0 +1,7 @@
+task_term,%95,mean,avg_count_0,avg_count_1,avg_count_2,avg_count_3,avg_count_4,avg_count_5,avg_count_6,avg_count_7,avg_count_8,avg_count_9,avg_count_10
+7,32.0,24.230886526375954,5.060811887115921,1.573948615647822,4.942099494144457,4.633419469054492,1.5332365927048457,0.11608206045741715,0.003993994145486182,3.799111307139541,0.013669845891360154,0.5994831134666571,1.955030146607957
+6,18.0,15.242627654582822,2.5080490741507164,0.8746289625250901,2.507759818412733,2.5083757392544026,0.017929086508531398,0.012701162996762964,2.470653874806301,0.006019712671565393,0.006413712748438859,1.6979836625222418,2.6321128479860385
+5,156.0,187.03089386917128,37.82751435280233,0.4492334927406169,52.12291162115978,38.09865323383696,0.7728231070897311,48.44577283847411,2.0353779673727304,0.7560146539960415,0.1336869798261682,3.4452411393727793,2.9436644825000307
+8,28.0,22.385445850996728,4.159124580112361,4.263784782831913,2.2310592169920644,2.239757675326009,0.41136520323295145,0.007568978921273818,1.4122013113696037,2.751353175010635,1.9986651606941164,0.764950933654086,2.1456148328517153
+4,1748.0,404.10866897787474,99.20235081022125,0.6657593876597071,98.1256427235899,93.00203042224992,73.71552664381427,1.8128164926768464,1.6555001558117793E-4,22.90802235899034,0.5461981925833593,1.5693849330009348,12.560771463072609
+-1,96.0,21.315165633982065,1.8056665534218799,0.39915551744778605,1.6912731410329003,1.617318902711943,0.0,0.0,0.0,0.0,0.0,11.252972675650753,4.548778843716802

table_iii/table-iii-d.csv/part-00000-c8e9ade3-b218-4604-8b09-cef37116bbbc-c000.csv

@@ -0,0 +1,7 @@
+task_term,%95,mean,avg_count_0,avg_count_1,avg_count_2,avg_count_3,avg_count_4,avg_count_5,avg_count_6,avg_count_7,avg_count_8,avg_count_9,avg_count_10
+7,32.0,29.9538731682486,6.43207232989496,1.5230748800354594,6.178519054595344,5.715246820278079,1.96013363220935,0.1505207364496276,0.00238531977905695,4.682411234693338,0.016155970205843924,0.6766512877024795,2.6167019024050626
+6,18.0,23.105615155196222,3.882226655388177,0.7853315736358375,3.8814333232103873,3.8784288047713456,0.058650893015740224,0.01905116361354794,3.7890504581110585,0.009785355344056015,0.01869912861489448,2.684171827817283,4.098785971673896
+5,269.0,228.0049751947969,48.29500852789435,0.76380209623428,62.52257238094564,48.935145540371266,0.4963163742648672,58.96820970835309,0.809520102476274,2.040396033998344,0.3247535403146475,1.8731219170423423,2.9761289729018197
+8,20.0,17.065721040189125,3.330640353582074,6.579627916538185,0.12204748689485044,0.12741288930003083,0.014759995888580532,0.0035769349367869257,0.07928872443211019,4.636283276801316,1.9997944290266214,0.020803782505910164,0.15148525028266008
+4,1478.0,323.36613014614846,77.59419388155071,0.7205634398700903,77.45358278322871,74.52967454130295,62.000510181861166,0.7002681565636141,3.7330380085486573E-4,14.057513672251707,0.627592128267186,1.8045505733324207,13.877307484119035
+-1,103.0,27.86740294224059,1.5263777572007629,0.32681682436700127,1.1950660133805107,1.083569934937114,0.0,0.0,0.0,0.0,0.0,20.897217810428458,2.8383546019267443

table_iii/table-iii-e.csv/part-00000-c1bb7468-967e-4c18-803d-3fb0017f48e1-c000.csv

@@ -0,0 +1,7 @@
+task_term,%95,mean,avg_count_0,avg_count_1,avg_count_2,avg_count_3,avg_count_4,avg_count_5,avg_count_6,avg_count_7,avg_count_8,avg_count_9,avg_count_10
+7,258.0,55.877475071701696,13.063833371251633,1.1173576581695857,12.635772911133188,12.518487243606957,1.2879172287931027,0.056909481014429456,1.853925477924379E-4,12.159880185300567,0.05499691815902356,0.8632289725849989,2.1189057091404133
+6,14.0,11.97680562448181,2.0308401767744173,1.8078136128055937,2.030681585756855,2.0673681759413753,0.013879349234882188,0.008434976564836575,1.9986772902241174,0.008240876917293001,0.02664068237046031,0.3272714370177521,1.6569574608742277
+5,138.0,450.52693734836595,109.92164977879264,0.7346581989439133,112.89458755530184,110.18156843156844,0.45770301127443985,111.47104681033252,0.0,0.45570500927643787,0.18799058084772372,0.9267696589125161,3.295258313115456
+8,14.0,11.899908172635445,1.8243036424854606,4.883073155800428,0.07131925313743495,0.14937251300887663,0.0,0.0,0.03397612488521579,3.1310070400979493,1.7921640648913377,0.001224364860728497,0.013468013468013467
+4,310.0,84.64518885660571,18.720340845276752,0.8784253403976529,18.672697381806937,19.399654922642448,11.780753798699449,0.10611942606473455,8.952478615784034E-5,5.7909596243214425,0.6549551969138364,0.8367800376004102,7.804412758095889
+-1,34.0,7.349165461025442,1.3465088494906061,0.3400815635192289,0.7647814974820162,0.7079290024348072,0.0,0.0,0.0,0.0,0.0,0.4800250034026808,3.709839544696103

table_iii/table-iii-f.csv/part-00000-aa82bdc7-bb87-438d-8f13-3f58cffe5e86-c000.csv

@@ -0,0 +1,7 @@
+task_term,%95,mean,avg_count_0,avg_count_1,avg_count_2,avg_count_3,avg_count_4,avg_count_5,avg_count_6,avg_count_7,avg_count_8,avg_count_9,avg_count_10
+7,162.0,45.039556726309925,9.904786908733803,1.1339116462680816,9.580871276442581,9.527044091989895,0.3840652955842721,0.09842973652060591,0.0011776835385799425,9.804287186832369,0.037782929127022656,0.9119289931796342,3.6552709780930823
+6,20.0,19.899709298545638,3.136003779746111,1.363153013172663,3.132176993327313,3.1688171863715584,0.019381065809467326,0.003510430786726587,3.0078389040021283,0.09793363782817881,0.023707014910783277,1.3421624804362036,4.605024792154503
+5,220.0,164.04307315635555,38.08070549296094,0.8814581396930471,41.14745755027091,38.52359955566639,0.27935208904808323,39.25740744938904,2.267008229239872E-5,1.5497948357552538,0.2039967355081499,0.7303053660084786,3.388973271972977
+8,36.0,25.002218658112803,4.205987703822507,7.239160652232023,2.2070836674685914,1.4351242983159582,0.01181502272119754,9.0884790163058E-4,0.149585672280139,7.283533814488105,2.000427693130179,0.12675755145682974,0.3418337342956429
+4,510.0,302.26234698981665,71.70614966448653,0.1528874920958502,71.67736440254065,69.58731844049944,23.973621374708607,0.19239408439272535,9.437790801929084E-5,45.97999660239531,0.3747888294308068,2.028705040724067,16.589026680634596
+-1,24.0,7.784904898437184,1.5369704801518393,0.2055728304325001,1.2624157008440011,1.1139764971933934,0.0,0.0,0.0,0.0,0.0,0.5838953277066591,3.0820740621087914

table_iii/table-iii-g.csv/part-00000-da936423-e2e5-4295-8f16-1a46d2ace627-c000.csv

@@ -0,0 +1,7 @@
+task_term,%95,mean,avg_count_0,avg_count_1,avg_count_2,avg_count_3,avg_count_4,avg_count_5,avg_count_6,avg_count_7,avg_count_8,avg_count_9,avg_count_10
+7,641.0,130.05414273603333,32.198520675875095,0.7087585088856327,31.295298970192587,29.2657999265031,6.909204375453352,0.13507292467994678,3.287874849980418E-5,25.27576921314129,0.1311059582165447,0.7666908088528912,3.3678884954843866
+6,18.0,105.24041797047958,14.036486764374327,1.6319637413475843,14.19607983670794,14.492640248151833,0.015228464125191524,0.001655183202748528,14.153774713161157,0.004879189146047221,0.1582999659867196,4.1007516899574465,42.448658174318595
+5,40.0,40.12155252498581,8.696211692791879,1.8829566264107134,8.984963081189314,8.715229664186882,0.01611128987600368,8.592727844170227,0.0,0.3388833596491885,0.011310294399052415,0.7730361726305842,2.1101224996819647
+8,4602.25,576.3841201716738,86.26609442060087,3.995708154506438,85.08583690987125,87.78969957081546,1.9313304721030042,0.3605150214592275,48.09442060085837,35.59656652360515,3.5343347639484977,20.30472103004292,203.42489270386267
+4,2015.0,555.57474325323,137.80949771922224,0.005262353151041003,137.61768303560052,134.25940980097346,77.42905379577483,0.3031268316301827,0.0,58.299329782625314,0.6538187100226803,1.6500726281185494,7.547488596111211
+-1,30.0,9.503553056585716,1.5523627228137338,0.2196434980260797,1.465797344179926,1.173968178011724,0.0,0.0,0.0,0.0,0.0,0.6631175977987798,4.428663715755473

table_iii/table-iii-h.csv/part-00000-839e5540-25e3-4fe5-8029-29e51b46c9a4-c000.csv

@@ -0,0 +1,7 @@
+task_term,%95,mean,avg_count_0,avg_count_1,avg_count_2,avg_count_3,avg_count_4,avg_count_5,avg_count_6,avg_count_7,avg_count_8,avg_count_9,avg_count_10
+7,388.0,74.42554228920945,17.895524233498676,0.5817619993509507,17.243615488850107,17.2667966667992,0.6333384085077026,0.1696663924684502,2.3133795476427908E-4,17.17262432139971,0.06279913753825522,1.088900096658901,2.310284206182731
+6,22.0,23.978294402298292,3.6882077719058923,1.081887468785444,3.6829913707382578,3.7429685027798287,0.023699507085570847,0.014129364260089101,3.6325294190151904,0.011111315224644062,0.028481837780308786,1.6026920369253232,6.4695958077977425
+5,487.0,170.15370084864648,41.17235440365571,0.552648027001125,41.72223842660111,41.79343583760434,0.6004833534730614,37.59994166423601,0.0,2.8666472214120033,0.3438059919163298,1.5301054210592109,1.9720405016875704
+8,386.4,94.66666666666667,20.88,0.3333333333333333,20.826666666666668,25.36,1.4933333333333334,2.4,0.5733333333333334,14.04,3.48,0.4266666666666667,4.8533333333333335
+4,206.0,75.65806422555511,16.271750063805193,0.04424327042892102,16.219039469140807,16.8735606299449,6.7325436502574725,0.8371541383296551,0.0,7.164722484949482,0.4217448092600099,0.510081220255521,10.583224489183143
+-1,18.0,8.123506043876587,1.659467322235171,0.4401439301785517,1.4493905084342065,1.3649208469083083,0.0,0.0,0.0,0.0,0.0,0.5973187475012122,2.612264688619138

table_iii/table-iv-evts-a.csv/part-00000-19a54282-47cc-4106-a8fd-cf4a2832eff4-c000.csv

@@ -0,0 +1,5 @@
+7,2.6865411576392657,0.812926270023483,2.553259402830044,2.4814356052649793,1.0454192149907342,0.07386656162820535,0.461386667518275,1.1887195858954966,0.04461011602641108,0.6498649728387728,1.5212777851493517
+6,1.0981809684231005,0.6130836760164798,1.0987790998703284,1.0937889142310522,0.0046962306439975905,0.0013414780719794978,1.0726227447148902,0.02439614982399709,9.523577904594333E-4,0.566999709922109,0.6397623828541761
+5,3.2263807904568313,0.8149870827388062,3.121328830735432,2.968946761301456,0.694941716787136,0.6835561684129362,0.08595725931720571,1.8495867802311339,0.009730425479182963,0.48260478892287484,0.8415826679218458
+8,1.0,2.0086799418718826,0.0,0.0,0.0,0.0,0.0,1.0340815364675386,0.974598405404344,0.0916499744707592,0.0
+-1,84.51669763776025,0.6128597270976729,82.89136811273919,78.43768798331588,23.263951270275747,3.454473694810383,23.047597090019774,34.565607717784104,0.7077094276169171,9.177240424461292,57.698175220439836

table_iii/table-iv-evts-b.csv/part-00000-cfa16bf7-1d59-4758-a4e9-3e5117fff443-c000.csv

@@ -0,0 +1,6 @@
+7,5.293960967421654,1.3693264100673077,5.224224392552472,4.946995947424621,1.6964485460623222,0.15783270537625355,0.6137475711552597,3.008677502126142,0.012091563523474123,0.7718873806519629,2.999386103718944
+6,2.3887342955525983,0.43569440172917784,2.387499604783982,2.396668796042666,0.022379645943933938,0.008476485365510145,2.3493043120223276,0.012729369005291584,0.0064840577044129105,1.5899558364064372,1.3143858989799115
+5,9.082228823342376,0.9085572987732548,8.780879413066843,8.65149141074595,2.0033546676189595,1.9937652197355031,0.26658435954652787,4.944145420703646,0.03452602270416128,2.3009932553357197,3.4360679603461737
+8,1.9967805062811692,4.956789344107064,0.0,0.0,0.0,0.0,0.0,2.9599457104980744,1.9968751972728995,0.15283126065273656,0.0
+4,1.0,0.6666666666666666,1.0,1.0,1.0,0.0,0.0,0.0,0.0,0.6666666666666666,4.666666666666667
+-1,89.08430995718696,0.3986483257378175,88.05026088945665,81.26871318698608,34.681161241336575,0.7112415245158765,13.379532541509533,38.79418756440348,0.7804834173968188,6.769798682584763,20.037000284059943