more results for 7c

2021-04-19 12:27:09 +00:00 · 2021-04-19 12:27:09 +00:00 · ed8a9b6006
commit ed8a9b6006
parent 3cb4deb8a3
2 changed files with 650 additions and 10 deletions
--- a/figure_7/f_figure7c.csv
+++ b/figure_7/f_figure7c.csv
@ -0,0 +1,625 @@
+term,n_exec,count
+6,1,35245280
+6,3,15198454
+6,4,5903899
+6,18,25111
+6,21,16003
+6,23,13437
+6,17,26546
+6,10,100248
+6,7,470015
+6,2,15600345
+6,5,2555041
+6,6,992293
+6,8,260317
+6,9,155453
+6,11,85147
+6,12,69196
+6,13,56955
+6,14,46034
+6,15,38905
+6,16,32620
+6,19,20696
+6,20,17279
+6,22,15868
+6,24,13132
+6,28,9021
+6,29,8540
+6,32,7072
+6,35,6284
+6,38,5117
+6,40,4652
+6,43,3633
+6,44,3261
+6,46,2671
+6,48,2147
+6,50,1716
+6,51,1659
+6,57,802
+6,67,264
+6,70,251
+6,73,178
+6,77,144
+6,82,112
+6,84,92
+6,90,87
+6,92,83
+6,95,74
+6,97,46
+6,99,55
+6,101,63
+6,103,74
+6,105,55
+6,106,53
+6,108,52
+6,110,36
+6,112,35
+6,114,22
+6,115,47
+6,25,11013
+6,26,10163
+6,27,9512
+6,31,7607
+6,33,6886
+6,34,6377
+6,39,4899
+6,42,3786
+6,30,7871
+6,37,5875
+6,41,4289
+6,45,2992
+6,47,2354
+6,49,1889
+6,56,894
+6,60,551
+6,62,474
+6,64,378
+6,36,5917
+6,52,1349
+6,53,1322
+6,54,1068
+6,55,935
+6,61,484
+6,63,363
+6,72,171
+6,74,170
+6,79,123
+6,81,129
+6,83,89
+6,59,633
+6,66,330
+6,69,239
+6,71,190
+6,76,146
+6,58,667
+6,78,109
+6,85,77
+6,88,84
+6,91,74
+6,93,55
+6,65,297
+6,75,155
+6,89,60
+6,96,82
+6,98,60
+6,100,62
+6,104,44
+6,107,44
+6,87,76
+6,68,216
+6,80,78
+6,94,48
+6,109,46
+6,111,44
+6,113,41
+6,116,24
+6,118,26
+6,120,32
+6,122,16
+6,86,87
+6,102,53
+6,117,28
+6,123,12
+6,125,15
+6,127,6
+6,129,4
+6,119,19
+6,121,24
+6,126,15
+6,132,4
+6,136,4
+6,139,2
+6,142,2
+6,143,2
+6,124,15
+6,128,6
+6,130,6
+6,131,4
+6,133,3
+6,135,4
+4,2,12230534
+4,3,15012854
+4,5,1995991
+4,19,7999
+4,14,19463
+4,9,70428
+4,1,41374539
+4,4,4712657
+4,6,657358
+4,7,270223
+4,8,122163
+4,10,43346
+4,11,37038
+4,12,30093
+4,13,25297
+4,15,15885
+4,16,12425
+4,79,16
+4,80,20
+4,30,3092
+4,111,10
+4,63,64
+4,25,4224
+4,26,4052
+4,31,2873
+4,37,1906
+4,42,1117
+4,29,3313
+4,33,2578
+4,18,10267
+4,20,6482
+4,22,6081
+4,28,3563
+4,34,2471
+4,36,2139
+4,38,1755
+4,39,1622
+4,43,972
+4,48,499
+4,50,333
+4,17,10301
+4,21,6252
+4,24,5120
+4,23,5371
+4,32,2790
+4,27,3703
+4,40,1470
+4,35,2209
+4,41,1222
+4,44,760
+4,45,732
+4,46,686
+4,47,551
+4,49,359
+4,51,301
+4,52,244
+4,73,16
+4,76,29
+4,53,213
+4,57,110
+4,54,160
+4,55,179
+4,56,148
+4,64,47
+4,60,117
+4,62,31
+4,78,10
+4,100,4
+4,58,118
+4,89,4
+4,97,4
+4,59,112
+4,61,81
+4,66,34
+4,67,41
+4,84,6
+4,99,5
+4,104,6
+4,114,2
+4,115,4
+4,65,39
+4,96,6
+4,70,20
+4,75,24
+4,68,38
+4,71,16
+4,85,10
+4,69,17
+4,101,5
+4,126,6
+4,87,12
+4,92,10
+4,98,6
+4,77,6
+4,95,8
+4,109,4
+4,110,2
+4,120,5
+4,130,2
+4,131,5
+4,134,2
+4,135,2
+4,137,2
+4,138,2
+4,140,2
+4,141,2
+4,143,4
+4,93,10
+4,82,17
+4,81,6
+4,88,2
+4,74,16
+4,83,22
+4,86,10
+4,119,5
+4,72,16
+4,112,4
+4,90,2
+4,122,2
+4,123,6
+4,124,4
+4,105,2
+4,91,12
+4,103,2
+4,118,2
+4,128,2
+4,129,2
+4,132,2
+4,107,2
+4,94,2
+4,117,2
+4,106,2
+7,2,193731409
+7,3,221061272
+7,4,80804298
+7,5,34556187
+7,1,467883864
+7,6,12611903
+7,7,5437592
+7,8,2795883
+7,9,1665577
+7,10,1052880
+7,11,882586
+7,12,717089
+7,13,611134
+7,14,489138
+7,15,391883
+7,16,301795
+7,17,232464
+7,19,153558
+7,20,118651
+7,23,84141
+7,22,103748
+7,21,102660
+7,18,204175
+7,24,83007
+7,25,66490
+7,26,60392
+7,27,55926
+7,28,50839
+7,29,47655
+7,30,45661
+7,31,42856
+7,32,40618
+7,33,38372
+7,34,36430
+7,35,33928
+7,36,32118
+7,37,30064
+7,38,27429
+7,39,25306
+7,40,22893
+7,41,20964
+7,42,19031
+7,43,17024
+7,44,15677
+7,45,14051
+7,46,12272
+7,47,10861
+7,48,9710
+7,49,8694
+7,50,7848
+7,51,6949
+7,52,6053
+7,53,5183
+7,54,4567
+7,55,4174
+7,56,3742
+7,57,3186
+7,58,2947
+7,59,2663
+7,60,2333
+7,61,2152
+7,62,1834
+7,63,1670
+7,64,1490
+7,65,1421
+7,66,1293
+7,67,1166
+7,68,1024
+7,69,920
+7,70,910
+7,71,851
+7,72,790
+7,73,708
+7,74,764
+7,75,641
+7,76,629
+7,77,533
+7,78,554
+7,79,517
+7,80,492
+7,81,458
+7,82,464
+7,83,441
+7,84,408
+7,85,426
+7,86,380
+7,87,355
+7,88,373
+7,89,397
+7,90,364
+7,91,368
+7,92,320
+7,93,317
+7,94,343
+7,95,325
+7,96,298
+7,97,298
+7,98,296
+7,99,294
+7,100,294
+7,101,294
+7,102,314
+7,103,251
+7,104,261
+7,105,236
+7,107,224
+7,108,210
+7,109,180
+7,110,161
+7,111,149
+7,112,169
+7,113,146
+7,114,167
+7,115,126
+7,106,229
+7,116,126
+7,117,103
+7,118,102
+7,119,125
+7,120,121
+7,121,89
+7,122,57
+7,123,60
+7,124,50
+7,125,59
+7,126,37
+7,127,33
+7,128,24
+7,129,23
+7,130,12
+7,131,12
+7,132,14
+7,133,11
+7,134,6
+7,135,8
+7,136,3
+7,137,2
+7,138,1
+7,139,1
+7,140,1
+7,141,1
+7,142,1
+7,143,5
+5,2,2195039
+5,3,2443598
+5,4,993197
+5,5,473494
+5,22,7929
+5,18,17758
+5,8,89614
+5,1,4804686
+5,6,215463
+5,7,125858
+5,9,82769
+5,10,75842
+5,11,67418
+5,12,58772
+5,13,51365
+5,14,44099
+5,15,36525
+5,16,30367
+5,17,23681
+5,19,15301
+5,31,1987
+5,42,713
+5,49,288
+5,56,130
+5,94,8
+5,109,11
+5,23,6317
+5,24,5292
+5,32,1881
+5,30,2202
+5,38,1018
+5,27,3063
+5,28,2664
+5,20,10712
+5,26,3429
+5,34,1616
+5,40,900
+5,45,476
+5,21,9379
+5,37,1213
+5,25,4199
+5,35,1350
+5,36,1289
+5,39,958
+5,29,2564
+5,33,1648
+5,41,779
+5,43,645
+5,46,464
+5,47,398
+5,62,70
+5,73,21
+5,44,575
+5,54,188
+5,60,84
+5,76,17
+5,48,332
+5,52,209
+5,55,163
+5,59,70
+5,71,31
+5,72,10
+5,75,20
+5,90,16
+5,51,234
+5,57,104
+5,53,177
+5,67,30
+5,68,32
+5,99,6
+5,101,7
+5,104,4
+5,50,238
+5,83,14
+5,95,16
+5,64,48
+5,85,12
+5,91,6
+5,113,2
+5,84,19
+5,89,9
+5,61,87
+5,69,29
+5,111,2
+5,115,6
+5,66,33
+5,86,6
+5,93,8
+5,70,30
+5,87,16
+5,58,73
+5,118,4
+5,124,2
+5,128,1
+5,65,32
+5,78,13
+5,100,11
+5,96,6
+5,97,8
+5,120,2
+5,122,2
+5,77,8
+5,92,10
+5,63,51
+5,105,6
+5,110,6
+5,116,4
+5,98,6
+5,79,14
+5,106,10
+5,131,2
+5,134,2
+5,107,4
+5,74,14
+5,102,4
+5,81,6
+5,88,2
+5,82,5
+5,125,2
+5,129,2
+5,112,6
+5,108,5
+5,103,8
+5,80,7
+5,119,2
+8,2,1228513
+8,1,2653400
+8,3,1166871
+8,7,35895
+8,10,4297
+8,4,468165
+8,5,205458
+8,6,78937
+8,8,18685
+8,9,9392
+8,11,4182
+8,12,2923
+8,23,152
+8,14,1174
+8,22,140
+8,41,40
+8,13,1770
+8,15,978
+8,16,577
+8,17,314
+8,21,82
+8,27,38
+8,33,30
+8,52,16
+8,57,2
+8,30,44
+8,54,10
+8,19,264
+8,48,12
+8,32,32
+8,18,577
+8,20,142
+8,26,63
+8,37,36
+8,35,66
+8,24,178
+8,38,34
+8,28,92
+8,39,44
+8,34,70
+8,45,26
+8,31,50
+8,80,4
+8,25,80
+8,29,44
+8,70,2
+8,43,44
+8,50,18
+8,47,20
+8,36,34
+8,51,6
+8,60,4
+8,40,24
+8,42,30
+8,46,24
+8,44,14
+8,56,6
+8,53,2
+8,58,8
+8,86,4
+8,55,2
+8,49,12
+8,74,2
+8,85,2
+-1,2,4225
+-1,3,2902
+-1,1,18381
+-1,5,963
+-1,6,495
+-1,7,430
+-1,4,1397
+-1,8,129
+-1,9,55
+-1,12,39
+-1,14,9
+-1,17,5
+-1,11,32
+-1,13,20
+-1,10,68
+-1,15,5
+-1,25,1
+-1,26,1
+-1,23,1
+-1,16,4
+-1,19,2
+-1,33,2
--- a/figure_8/figure8-cd-only.py
+++ b/figure_8/figure8-cd-only.py
@ -16,12 +16,25 @@ from pyspark.sql import Window
 from pyspark.sql.types import *
 from decimal import *

+CHECKDIR = "/home/claudio/google_2019/thesis_queries/figure_8/" 
+
 if len(sys.argv) is not 4:
    print(sys.argv[0] + " {cluster} {tmpdir} {maxram}")
    sys.exit()

 cluster=sys.argv[1]

+if os.path.exists(CHECKDIR + cluster + "_figure8cd.json"):
+    print("already computed")
+    sys.exit()
+
+if os.path.exists(CHECKDIR + cluster + "_figure8cd_working"):
+    print("already in execution")
+    sys.exit()
+
+os.system("touch " + CHECKDIR + cluster + "_figure8cd_working")
+
+
 spark = pyspark.sql.SparkSession.builder \
  .appName("task_slowdown") \
  .config("spark.driver.maxResultSize", "128g") \
@ -30,12 +43,15 @@ spark = pyspark.sql.SparkSession.builder \
  .getOrCreate()
 sc = spark.sparkContext

+def tabid(x):
+    return Decimal(x.collection_id) + Decimal(x.instance_index) / Decimal(2**64)
+
 #
 # READING INSTANCE EVENTS DATA
 #
 dfepath = "/home/claudio/google_2019/instance_events/" + cluster + "/" + cluster + "_instance_events*.json.gz"
+#dfepath = "/home/claudio/google_2019/instance_events/" + cluster + "/" + cluster + "_test.json"
 df = spark.read.json(dfepath)
-df = df.limit(10000)

 # READING MACHINE EVENTS DATA, sort them and save them as broadcast variable
 print("Starting to read machine events...")
@ -56,7 +72,7 @@ print("Converting to broadcast variable...")
 dfm = sc.broadcast([tuple(r) for r in dfm.to_numpy()])
 print("Done with machine events.")

-df = dfe.rdd \
+df = df.rdd \
    .filter(lambda x: x.time is not None and x.type is not None and x.machine_id is not None and
            x.instance_index is not None and x.collection_id is not None and x.resource_request is not None and
            x.resource_request.cpus is not None and x.resource_request.memory is not None) \
@ -68,12 +84,12 @@ df = dfe.rdd \
 def get_machine_time_resources(machine_id, time):
    def aux(i, j):
        if i == j:
-            return dfm.value[i] if dfm.value[i][1] == machine_id else None
+            return i if dfm.value[i][1] == machine_id else None
        elif i + 1 == j:
            if dfm.value[i][1] == machine_id:
-                return dfm.value[i]
+                return i
            elif dfm.value[j][1] == machine_id:
-                return dfm.value[j]
+                return j
            else:
                return None

@ -170,20 +186,19 @@ def fold_resobjs(ro1, ro2):
                    ro1[k][kk] = [sum(x) for x in zip(ro1[k][kk], ro2[k][kk])]
        return ro1
 
-# TODO: partition by id and in the for-each-row 
-# function implement lookup to dfm.value to understand its memory capacity
-
 import random

 result = df.rdd \
    .groupBy(lambda x: x.mid) \
-    .partitionBy(10000, lambda x: random.randint(0, 10000-1)) \
+    .partitionBy(1000, lambda x: random.randint(0, 1000-1)) \
    .map(for_each_joined) \
    .fold(None, fold_resobjs)

 d = os.path.dirname(os.path.realpath(__file__))

-with open(d + "/" + cluster + "_figure8.json", "w") as f:
+with open(d + "/" + cluster + "_figure8cd.json", "w") as f:
    json.dump(result, f)

+os.system("rm " + CHECKDIR + cluster + "_figure8cd_working")
+
 # vim: set ts=4 sw=4 et tw=120: