More results

figure_7/b_priority/part-00000

@@ -0,0 +1,92 @@
+-1,103,39194
+7,103,15425580
+-1,0,123828
+6,101,25076
+5,215,338
+5,116,4
+5,107,2850
+5,205,356
+-1,100,2
+-1,450,22935
+4,25,1107
+7,210,1483
+5,115,46
+-1,200,1855
+5,105,111
+7,205,32316
+6,114,1999
+6,115,123
+4,205,3150
+5,25,10
+-1,107,126
+4,360,137
+5,450,594
+-1,115,5
+5,210,3
+6,450,174
+7,200,179507
+6,118,394
+6,119,108
+5,200,7308
+4,215,343
+-1,105,53
+4,200,15551
+6,116,215967
+7,116,2380357
+7,117,1849
+8,101,24
+4,101,77
+4,100,1
+7,119,52914
+4,450,2477
+-1,118,22
+-1,119,3
+8,200,3
+4,210,383
+7,450,2209
+4,0,90698
+-1,116,507
+6,107,1
+7,118,126145
+8,103,15257
+8,0,16520
+7,360,32794
+4,103,38678
+7,199,14141
+8,105,1
+5,360,207
+7,170,14
+7,115,4389
+6,105,11509
+5,0,159067
+-1,360,1
+5,103,407
+6,200,91396
+7,114,293319
+6,0,28112341
+5,199,80
+5,100,11
+4,116,34
+4,118,5
+-1,25,996
+6,25,19
+8,119,3
+7,105,2371672
+6,360,1203
+7,215,1188
+4,119,2024
+4,105,486
+7,0,33678129
+7,107,498509
+7,25,102875
+-1,199,214
+7,214,400
+6,103,24246
+5,118,4
+5,119,51
+4,199,39
+-1,80,1
+4,107,3
+7,101,12543
+7,100,87217
+4,115,81

figure_7/c_priority/part-00000

@@ -0,0 +1,93 @@
+7,170,30
+5,107,5
+6,118,98
+8,116,1
+5,100,7
+5,118,2
+-1,105,40596
+7,101,22432
+7,210,156
+7,0,101735739
+7,114,1003335
+6,200,40086
+7,360,38231
+6,105,11301
+-1,107,15
+7,107,11030791
+-1,117,1
+-1,360,22
+-1,200,1632
+4,215,2
+4,119,212
+6,107,24
+4,450,2371
+4,25,1714
+6,114,223
+6,360,1764
+-1,119,234
+7,105,38100701
+7,214,2560
+7,118,2367286
+4,200,14951
+4,103,76943
+8,105,55
+6,450,472
+-1,25,686
+6,117,259
+8,119,2
+-1,450,21762
+4,0,106433
+5,115,31
+-1,103,26335
+7,116,3873078
+7,10,3
+4,101,85
+-1,0,92708
+6,101,25060
+8,103,12545
+6,119,1523
+7,100,813
+7,205,2480
+-1,115,25
+8,200,22
+7,3,3
+5,205,8
+4,210,4
+4,114,4
+8,0,49832
+6,103,3434
+8,101,40
+7,450,6590
+8,115,5537
+5,119,543
+5,25,55
+5,0,307897
+7,115,118599
+6,25,903
+4,107,196
+6,115,249
+5,103,4065
+7,200,1042932
+-1,116,313
+4,105,1976
+5,200,11779
+4,118,86
+-1,100,2
+7,215,158
+6,0,106010494
+7,119,66216
+-1,118,1108
+5,105,785
+4,116,11
+5,450,10
+6,116,536
+8,118,139
+4,360,189
+5,116,1
+7,103,45555190
+5,360,184
+7,117,18
+4,205,72
+7,25,661
+4,100,5
+4,115,122

figure_7/d_priority/part-00000

@@ -0,0 +1,95 @@
+7,117,1
+8,107,3
+6,116,2205
+5,360,156
+4,199,22
+4,450,1904
+5,103,293
+7,114,1271146
+-1,118,669
+-1,101,4208
+4,119,1131
+7,107,5992354
+7,199,1990
+-1,360,17
+5,115,301
+8,115,5
+4,105,4466
+6,115,166
+5,200,17411
+7,119,91027
+5,105,2111
+7,25,537
+4,101,79
+8,0,43904
+5,101,3
+7,200,1161014
+-1,116,5022
+5,205,2
+7,360,38540
+-1,107,11905
+4,200,17918
+7,101,25997
+6,25,785
+4,107,70
+-1,115,2
+7,205,1117
+5,25,11
+5,118,5
+4,116,767
+4,103,59801
+4,115,410
+-1,105,2667
+7,103,30975747
+-1,200,4111
+7,214,1160
+8,119,1
+6,119,413
+5,199,36
+7,115,16547
+4,25,1494
+4,118,86
+4,210,3
+-1,100,1
+4,205,41
+-1,199,8
+8,101,2
+6,450,169
+-1,25,1991
+5,116,3
+4,0,72361
+-1,0,65833
+7,116,1739860
+5,0,120110
+6,0,22195803
+7,0,61178515
+5,450,44
+4,100,2
+6,117,6
+6,360,630
+-1,119,85
+8,116,1
+8,103,2688
+6,103,18727
+7,450,3779
+6,200,83851
+7,5,1
+7,118,1263171
+7,210,113
+6,107,21
+4,215,1
+5,119,815
+4,114,15
+8,118,759
+7,215,115
+8,105,1254
+7,100,386
+4,360,156
+6,118,684
+8,200,28
+6,101,25098
+-1,103,44797
+-1,450,21757
+6,105,10130
+7,105,16306729
+7,197,9

figure_7/e_priority/part-00000

@@ -0,0 +1,79 @@
+4,360,93
+7,201,52
+7,450,2022
+-1,210,104
+8,360,3
+6,119,32
+5,119,4
+4,101,6559
+7,114,26682
+7,115,127866
+5,100,1
+5,450,1
+4,100,7
+6,101,1000
+7,205,48095
+7,25,8227
+5,200,1176
+6,115,25
+4,0,69470
+6,360,3670
+6,103,6156
+7,210,2546
+5,103,146
+7,118,231
+7,119,6196
+8,0,909
+7,360,6003
+7,116,22406
+-1,117,2070
+4,107,11
+-1,105,4
+4,450,344
+7,117,1916638
+5,205,3
+-1,119,2
+7,215,2546
+-1,107,45
+4,105,13
+5,0,53842
+6,200,52049
+-1,205,101
+8,115,3
+7,103,13362894
+-1,0,61026
+8,103,2249
+5,105,8
+4,25,1179
+7,100,2793
+7,170,25
+4,115,120
+7,101,6274601
+5,107,59
+6,0,18718913
+6,450,82
+-1,450,12502
+7,107,17521
+-1,200,121987
+4,200,7808
+-1,215,48
+5,215,56
+4,210,41
+8,200,103
+6,105,1679
+4,119,749
+4,205,41
+4,117,7895
+7,105,447535
+5,115,4
+6,25,560
+5,25,28
+-1,25,484
+7,220,360
+5,117,620
+7,200,2473444
+4,116,8
+4,215,41
+7,0,24827204
+5,360,108
+4,103,28492

figure_7/f_priority/part-00000

@@ -0,0 +1,81 @@
+7,360,41986
+7,105,5536865
+5,107,3
+5,0,80835
+7,200,2330301
+5,200,5512
+4,107,18
+-1,0,41145
+-1,200,37261
+5,201,1
+7,201,1475
+4,105,1997
+7,0,15402800
+4,360,623
+6,450,257
+7,114,626313
+6,360,3433
+7,115,7238
+6,105,12390
+5,115,62
+8,0,2574
+-1,360,5
+-1,105,2697
+8,200,1
+5,116,1
+6,107,135
+5,117,653
+4,201,1234
+4,115,196
+4,0,135770
+7,118,1341512
+-1,117,6637
+-1,107,214
+4,114,303
+4,200,34474
+7,119,16518
+6,117,2
+-1,116,2456
+-1,25,984
+5,25,15
+5,119,72
+7,116,3282274
+-1,119,25
+5,450,3
+7,25,4093
+6,200,91296
+6,119,9624
+7,117,2322133
+7,103,13242196
+6,0,12923456
+-1,118,3272
+7,450,3551
+6,101,24996
+8,118,24
+-1,100,1
+-1,450,21360
+4,119,4455
+5,103,601
+6,118,28
+-1,103,7754
+8,117,1
+4,118,115
+8,103,1
+6,103,7905
+7,101,37003
+8,116,367
+4,450,720
+4,117,886
+4,103,28808
+5,360,190
+-1,115,4
+8,101,4
+4,116,1089
+4,25,1199
+6,115,285
+7,107,221447
+8,114,34438
+4,101,27
+5,105,274
+6,114,46
+6,25,41

figure_7/figure7a.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+# coding: utf-8
+
+import json
+import pandas
+import findspark
+findspark.init()
+import pyspark
+import pyspark.sql
+import sys
+import gzip
+import os
+
+from pyspark import AccumulatorParam
+from pyspark.sql.functions import lit
+from pyspark.sql import Window
+from pyspark.sql.types import ByteType
+
+if len(sys.argv) is not 4:
+    print(sys.argv[0] + " {cluster} {tmpdir} {maxram}")
+    sys.exit()
+
+cluster=sys.argv[1]
+
+if os.path.exists("/home/claudio/google_2019/thesis_queries/figure_7/" + cluster + "_priority"):
+    print("already computed")
+    sys.exit()
+
+spark = pyspark.sql.SparkSession.builder \
+  .appName("task_slowdown") \
+  .config("spark.driver.maxResultSize", "128g") \
+  .config("spark.local.dir", sys.argv[2]) \
+  .config("spark.driver.memory", sys.argv[3]) \
+  .getOrCreate()
+sc = spark.sparkContext
+
+df = spark.read.json("/home/claudio/google_2019/instance_events/" + cluster + "/" + cluster + "_instance_events*.json.gz")
+#df = spark.read.json("/home/claudio/google_2019/instance_events/" + cluster + "/" + cluster + "_test.json")
+
+try:
+    df["collection_type"] = df["collection_type"].cast(ByteType())
+except:
+    df = df.withColumn("collection_type", lit(None).cast(ByteType()))
+
+def for_each_task(data):
+    ts = data[1]
+    priority = -1
+    term = -1
+    min_time = -1
+    max_time = -1
+    for t in ts:
+        if (min_time == -1 or t["time"] <= min_time) and t["priority"] != -1:
+            priority = t["priority"]
+            min_time = t["time"]
+        if (max_time == -1 or t["time"] >= max_time) and t["type"] >= 4 and t["type"] <= 8:
+            term = t["type"]
+            max_time = t["time"]
+
+    return (term, priority)
+
+
+def cleanup(x):
+    return {
+        "time": int(x.time),
+        "type": 0 if x.type is None else int(x.type),
+        "id": x.collection_id + "-" + x.instance_index,
+        "priority": -1 if x.priority is None else int(x.priority)
+    }
+
+
+def sum_rows(xs):
+    csum = 0
+    for x in xs:
+        csum += x[3]
+    return csum
+
+
+df2 = df.rdd \
+    .filter(lambda x: x.collection_type is None or x.collection_type == 0) \
+    .filter(lambda x: x.time is not None and x.instance_index is not None and x.collection_id is not None) \
+    .map(cleanup) \
+    .groupBy(lambda x: x["id"]) \
+    .map(for_each_task) \
+    .groupBy(lambda x: x) \
+    .mapValues(lambda x: len(x)) \
+    .map(lambda x: str(x[0][0]) + "," + str(x[0][1]) + "," + str(x[1])) \
+    .coalesce(1) \
+    .saveAsTextFile(cluster + "_priority")
+
+# vim: set ts=4 sw=4 et tw=80:

figure_7/figure7c.py

@@ -107,7 +107,7 @@ result = df.rdd \
     .groupBy(lambda x: x[0]) \
     .flatMap(mark_next) \
     .groupBy(lambda x: x["mid"]) \
-    .partitionBy(10000, lambda x: random.randint(0, 10000-1)) \
+    .partitionBy(1000, lambda x: random.randint(0, 1000-1)) \
     .map(for_each_joined) \
     .fold({}, fold_resobjs)
 

figure_7/g_priority/part-00000

@@ -0,0 +1,73 @@
+4,118,504
+-1,450,15926
+-1,119,1
+6,360,1713
+-1,105,4
+4,0,67498
+7,117,1825000
+4,119,1244
+7,119,140556
+8,200,9
+-1,25,992
+6,450,643
+8,119,1
+4,450,1362
+-1,117,1842
+-1,200,1854
+4,105,32
+7,118,976
+-1,360,55
+4,200,5246
+5,0,889534
+5,360,143
+4,115,26
+6,103,2055
+7,115,1434
+5,103,32042
+7,360,103764
+6,0,5492855
+7,114,120
+-1,0,18820
+6,115,140
+6,25,16
+6,101,99978
+4,360,139
+5,100,3
+7,25,4439
+5,450,4
+7,0,9872724
+7,170,256
+7,50,112
+6,200,51527
+6,118,1425
+6,119,122
+7,105,132163
+7,200,132895
+5,119,29
+5,118,5
+7,107,23067
+4,101,8
+4,100,2
+5,117,12432
+4,170,4
+4,25,1405
+8,101,32
+-1,103,2099
+7,101,3528
+5,200,1114
+7,100,257
+-1,100,90
+4,103,756
+5,25,4
+5,115,120
+7,103,5971608
+7,450,5556
+-1,115,2
+5,107,1
+-1,101,90
+6,105,260
+8,105,1
+-1,107,20
+8,0,423
+4,117,254
+4,50,2

figure_7/h_priority/part-00000

@@ -0,0 +1,69 @@
+-1,360,4
+4,107,98
+6,360,1862
+6,25,79
+5,360,160
+5,0,62972
+7,201,202
+8,103,4
+6,118,506
+-1,105,2320
+7,25,6966
+-1,117,3423
+6,105,10000
+7,103,4974446
+8,360,1
+7,200,2030444
+-1,103,488
+7,220,360
+-1,200,28755
+4,103,3965
+6,119,3970
+7,105,688095
+5,450,6
+4,200,22340
+7,360,39551
+7,115,405
+8,105,13
+5,119,124
+6,116,134
+8,117,7
+6,0,6626413
+4,115,26
+4,201,160
+7,117,2009637
+4,450,1235
+-1,0,62880
+8,200,30
+4,117,106
+5,107,12
+6,117,1
+7,107,140588
+5,201,1
+4,25,514
+4,360,494
+6,103,24133
+7,116,12809
+7,450,2282
+4,0,35456
+6,115,15
+6,450,91
+5,117,242
+8,107,4
+5,25,17
+4,105,55
+6,200,57852
+5,200,1050
+5,103,7248
+7,119,2727180
+-1,450,13383
+8,0,16
+-1,107,5841
+5,105,161
+-1,119,453
+7,0,17096053
+4,119,2160
+6,101,25000
+7,118,49926
+-1,19,10
+5,115,4