Results for 7c

This commit is contained in:
Claudio Maggioni 2021-04-18 19:58:27 +00:00
parent baa7ce4c7c
commit a23dbe28e5
2 changed files with 507 additions and 7 deletions

475
figure_7/a_figure7c.csv Normal file
View File

@ -0,0 +1,475 @@
term,n_exec,count
7,1,391323386
7,2,104682127
7,3,61873236
7,4,17411163
7,5,5172753
7,6,1486512
7,7,474469
7,8,250632
7,9,135549
7,10,103986
7,11,82490
7,12,64904
7,13,62738
7,14,53403
7,15,51435
7,16,49406
7,17,47969
7,19,45116
7,20,83878
7,21,56441
7,22,40996
7,23,39294
7,24,36682
7,25,35147
7,26,33427
7,27,31685
7,28,29824
7,29,28196
7,30,27186
7,31,25940
7,32,24411
7,33,22205
7,34,19263
7,35,17215
7,36,15877
7,37,14576
7,38,13677
7,18,46872
7,39,12635
7,40,11790
7,41,10665
7,42,9662
7,43,8892
7,44,8156
7,45,7622
7,47,6283
7,46,6918
7,48,5845
7,49,5252
7,50,4921
7,51,4631
7,52,4200
7,53,3826
7,54,3513
7,55,3128
7,56,2903
7,57,2648
7,58,2454
7,59,2280
7,60,2073
7,61,1844
7,62,1781
7,63,1580
7,64,1495
7,65,1320
7,66,1260
7,67,1166
7,68,1058
7,69,940
7,70,820
7,71,744
7,72,710
7,73,565
7,74,517
7,75,440
7,76,376
7,77,351
7,78,287
7,79,201
7,80,197
7,81,171
7,82,145
7,83,126
7,84,125
7,85,99
7,86,88
7,87,76
7,88,65
7,89,62
7,90,52
7,91,52
7,92,35
7,93,31
7,94,22
7,95,27
7,96,23
7,97,11
7,98,9
7,99,10
7,100,10
7,101,9
7,102,2
6,1,80534713
6,2,16553975
6,37,820
6,29,983
6,3,9294919
6,38,729
6,9,11609
6,4,2325273
6,12,3392
6,15,1477
6,5,646748
6,6,167352
6,7,46680
6,11,6538
6,25,1093
6,28,962
6,32,896
6,35,963
6,16,1404
6,23,1165
6,8,28636
6,33,1634
6,34,1137
6,39,758
6,10,6764
6,24,1112
6,14,1742
6,18,1241
6,20,10888
6,36,803
6,40,678
6,41,703
6,17,1220
6,27,1012
6,19,1184
6,21,3945
6,30,913
6,26,1045
6,31,945
6,44,495
6,45,492
6,46,479
6,42,552
6,50,354
6,52,341
6,54,255
6,47,453
6,57,225
6,13,3010
6,22,1201
6,48,385
6,43,584
6,62,160
6,64,130
6,66,117
6,58,181
6,59,173
6,56,193
6,61,175
6,49,403
6,53,323
6,55,249
6,60,182
6,63,128
6,69,97
6,71,82
6,72,60
6,74,67
6,68,87
6,51,317
6,70,84
6,65,139
6,67,126
6,77,35
6,83,18
6,86,15
6,90,5
6,73,61
6,78,32
6,79,21
6,81,20
6,82,17
6,85,7
6,88,7
6,76,36
6,87,12
6,89,9
6,92,4
6,80,20
6,75,44
6,84,16
6,93,5
6,96,4
6,97,5
6,98,4
6,99,3
6,101,3
6,91,6
6,95,4
6,94,2
6,100,2
8,1,32144983
8,2,6151070
8,3,3334677
8,4,784436
8,5,203923
8,6,47719
8,7,11081
8,15,207
8,22,159
8,9,2393
8,11,1014
8,30,79
8,8,7509
8,10,1387
8,12,554
8,13,620
8,18,173
8,19,162
8,21,1203
8,23,103
8,24,99
8,25,131
8,36,63
8,37,51
8,42,42
8,17,162
8,27,75
8,26,112
8,28,79
8,31,97
8,34,190
8,39,48
8,35,104
8,20,4128
8,16,186
8,53,19
8,61,15
8,29,100
8,32,71
8,46,24
8,52,24
8,14,303
8,40,45
8,41,32
8,43,46
8,44,27
8,33,84
8,45,21
8,58,6
8,59,5
8,48,21
8,57,11
8,89,4
8,38,62
8,54,7
8,51,13
8,63,4
8,73,2
8,74,2
8,50,14
8,60,9
8,68,4
8,65,5
8,49,16
8,75,2
8,62,4
8,66,6
8,47,12
8,70,4
8,55,1
8,56,2
8,81,1
8,72,1
8,71,1
4,1,55548512
4,2,12554434
4,3,6644659
4,21,4408
4,30,1004
4,35,753
4,36,685
4,25,1347
4,14,2482
4,10,7847
4,4,1907654
4,5,570836
4,6,159595
4,8,23612
4,9,10337
4,11,5563
4,18,1929
4,19,1915
4,23,1743
4,13,3677
4,15,2377
4,28,1142
4,7,51196
4,17,2067
4,29,1093
4,31,933
4,12,3948
4,26,1274
4,38,556
4,37,637
4,20,5323
4,22,1791
4,24,1439
4,32,872
4,34,869
4,27,1183
4,39,535
4,42,391
4,46,289
4,41,443
4,55,117
4,16,2280
4,33,800
4,50,227
4,44,345
4,47,270
4,48,309
4,49,241
4,40,462
4,43,398
4,45,314
4,52,175
4,54,136
4,62,65
4,53,142
4,63,79
4,51,184
4,56,114
4,60,82
4,64,60
4,66,53
4,69,36
4,70,28
4,61,79
4,57,107
4,59,107
4,67,51
4,58,87
4,71,24
4,65,52
4,68,32
4,76,117
4,75,29
4,89,3
4,73,27
4,74,20
4,77,10
4,78,18
4,79,11
4,80,10
4,81,4
4,82,8
4,72,27
4,91,1
4,92,3
4,102,1
4,87,1
4,85,6
4,84,2
4,94,1
4,83,3
4,90,2
4,86,1
4,93,1
5,1,13139241
5,18,1071
5,29,732
5,2,4461294
5,16,1132
5,20,1173
5,22,1027
5,39,352
5,24,894
5,23,974
5,19,1020
5,7,49050
5,6,116558
5,5,362914
5,9,11812
5,3,2723445
5,10,7022
5,26,851
5,4,978358
5,12,2382
5,8,21628
5,27,790
5,13,1925
5,21,1027
5,34,562
5,37,413
5,15,1203
5,33,702
5,11,4845
5,25,872
5,31,652
5,17,1020
5,28,717
5,36,433
5,49,124
5,50,137
5,52,103
5,54,99
5,40,318
5,56,77
5,30,680
5,32,741
5,45,201
5,14,1303
5,35,470
5,41,272
5,44,224
5,38,382
5,43,216
5,57,59
5,68,24
5,46,186
5,53,96
5,42,272
5,47,161
5,51,116
5,55,84
5,60,52
5,71,18
5,67,34
5,61,50
5,64,33
5,66,29
5,70,21
5,48,318
5,58,71
5,69,30
5,62,47
5,65,31
5,59,61
5,72,17
5,63,43
5,73,17
5,78,5
5,76,51
5,74,9
5,77,10
5,83,5
5,93,1
5,75,20
5,85,2
5,89,2
5,92,1
5,79,7
5,80,2
5,84,4
5,86,2
5,88,1
5,81,4
5,90,2
5,82,3
5,87,4
5,94,2
5,96,1
5,95,1
-1,1,15529
-1,2,1267
-1,5,41
-1,3,483
-1,4,149
-1,6,12
-1,9,1
-1,7,1
1 term n_exec count
2 7 1 391323386
3 7 2 104682127
4 7 3 61873236
5 7 4 17411163
6 7 5 5172753
7 7 6 1486512
8 7 7 474469
9 7 8 250632
10 7 9 135549
11 7 10 103986
12 7 11 82490
13 7 12 64904
14 7 13 62738
15 7 14 53403
16 7 15 51435
17 7 16 49406
18 7 17 47969
19 7 19 45116
20 7 20 83878
21 7 21 56441
22 7 22 40996
23 7 23 39294
24 7 24 36682
25 7 25 35147
26 7 26 33427
27 7 27 31685
28 7 28 29824
29 7 29 28196
30 7 30 27186
31 7 31 25940
32 7 32 24411
33 7 33 22205
34 7 34 19263
35 7 35 17215
36 7 36 15877
37 7 37 14576
38 7 38 13677
39 7 18 46872
40 7 39 12635
41 7 40 11790
42 7 41 10665
43 7 42 9662
44 7 43 8892
45 7 44 8156
46 7 45 7622
47 7 47 6283
48 7 46 6918
49 7 48 5845
50 7 49 5252
51 7 50 4921
52 7 51 4631
53 7 52 4200
54 7 53 3826
55 7 54 3513
56 7 55 3128
57 7 56 2903
58 7 57 2648
59 7 58 2454
60 7 59 2280
61 7 60 2073
62 7 61 1844
63 7 62 1781
64 7 63 1580
65 7 64 1495
66 7 65 1320
67 7 66 1260
68 7 67 1166
69 7 68 1058
70 7 69 940
71 7 70 820
72 7 71 744
73 7 72 710
74 7 73 565
75 7 74 517
76 7 75 440
77 7 76 376
78 7 77 351
79 7 78 287
80 7 79 201
81 7 80 197
82 7 81 171
83 7 82 145
84 7 83 126
85 7 84 125
86 7 85 99
87 7 86 88
88 7 87 76
89 7 88 65
90 7 89 62
91 7 90 52
92 7 91 52
93 7 92 35
94 7 93 31
95 7 94 22
96 7 95 27
97 7 96 23
98 7 97 11
99 7 98 9
100 7 99 10
101 7 100 10
102 7 101 9
103 7 102 2
104 6 1 80534713
105 6 2 16553975
106 6 37 820
107 6 29 983
108 6 3 9294919
109 6 38 729
110 6 9 11609
111 6 4 2325273
112 6 12 3392
113 6 15 1477
114 6 5 646748
115 6 6 167352
116 6 7 46680
117 6 11 6538
118 6 25 1093
119 6 28 962
120 6 32 896
121 6 35 963
122 6 16 1404
123 6 23 1165
124 6 8 28636
125 6 33 1634
126 6 34 1137
127 6 39 758
128 6 10 6764
129 6 24 1112
130 6 14 1742
131 6 18 1241
132 6 20 10888
133 6 36 803
134 6 40 678
135 6 41 703
136 6 17 1220
137 6 27 1012
138 6 19 1184
139 6 21 3945
140 6 30 913
141 6 26 1045
142 6 31 945
143 6 44 495
144 6 45 492
145 6 46 479
146 6 42 552
147 6 50 354
148 6 52 341
149 6 54 255
150 6 47 453
151 6 57 225
152 6 13 3010
153 6 22 1201
154 6 48 385
155 6 43 584
156 6 62 160
157 6 64 130
158 6 66 117
159 6 58 181
160 6 59 173
161 6 56 193
162 6 61 175
163 6 49 403
164 6 53 323
165 6 55 249
166 6 60 182
167 6 63 128
168 6 69 97
169 6 71 82
170 6 72 60
171 6 74 67
172 6 68 87
173 6 51 317
174 6 70 84
175 6 65 139
176 6 67 126
177 6 77 35
178 6 83 18
179 6 86 15
180 6 90 5
181 6 73 61
182 6 78 32
183 6 79 21
184 6 81 20
185 6 82 17
186 6 85 7
187 6 88 7
188 6 76 36
189 6 87 12
190 6 89 9
191 6 92 4
192 6 80 20
193 6 75 44
194 6 84 16
195 6 93 5
196 6 96 4
197 6 97 5
198 6 98 4
199 6 99 3
200 6 101 3
201 6 91 6
202 6 95 4
203 6 94 2
204 6 100 2
205 8 1 32144983
206 8 2 6151070
207 8 3 3334677
208 8 4 784436
209 8 5 203923
210 8 6 47719
211 8 7 11081
212 8 15 207
213 8 22 159
214 8 9 2393
215 8 11 1014
216 8 30 79
217 8 8 7509
218 8 10 1387
219 8 12 554
220 8 13 620
221 8 18 173
222 8 19 162
223 8 21 1203
224 8 23 103
225 8 24 99
226 8 25 131
227 8 36 63
228 8 37 51
229 8 42 42
230 8 17 162
231 8 27 75
232 8 26 112
233 8 28 79
234 8 31 97
235 8 34 190
236 8 39 48
237 8 35 104
238 8 20 4128
239 8 16 186
240 8 53 19
241 8 61 15
242 8 29 100
243 8 32 71
244 8 46 24
245 8 52 24
246 8 14 303
247 8 40 45
248 8 41 32
249 8 43 46
250 8 44 27
251 8 33 84
252 8 45 21
253 8 58 6
254 8 59 5
255 8 48 21
256 8 57 11
257 8 89 4
258 8 38 62
259 8 54 7
260 8 51 13
261 8 63 4
262 8 73 2
263 8 74 2
264 8 50 14
265 8 60 9
266 8 68 4
267 8 65 5
268 8 49 16
269 8 75 2
270 8 62 4
271 8 66 6
272 8 47 12
273 8 70 4
274 8 55 1
275 8 56 2
276 8 81 1
277 8 72 1
278 8 71 1
279 4 1 55548512
280 4 2 12554434
281 4 3 6644659
282 4 21 4408
283 4 30 1004
284 4 35 753
285 4 36 685
286 4 25 1347
287 4 14 2482
288 4 10 7847
289 4 4 1907654
290 4 5 570836
291 4 6 159595
292 4 8 23612
293 4 9 10337
294 4 11 5563
295 4 18 1929
296 4 19 1915
297 4 23 1743
298 4 13 3677
299 4 15 2377
300 4 28 1142
301 4 7 51196
302 4 17 2067
303 4 29 1093
304 4 31 933
305 4 12 3948
306 4 26 1274
307 4 38 556
308 4 37 637
309 4 20 5323
310 4 22 1791
311 4 24 1439
312 4 32 872
313 4 34 869
314 4 27 1183
315 4 39 535
316 4 42 391
317 4 46 289
318 4 41 443
319 4 55 117
320 4 16 2280
321 4 33 800
322 4 50 227
323 4 44 345
324 4 47 270
325 4 48 309
326 4 49 241
327 4 40 462
328 4 43 398
329 4 45 314
330 4 52 175
331 4 54 136
332 4 62 65
333 4 53 142
334 4 63 79
335 4 51 184
336 4 56 114
337 4 60 82
338 4 64 60
339 4 66 53
340 4 69 36
341 4 70 28
342 4 61 79
343 4 57 107
344 4 59 107
345 4 67 51
346 4 58 87
347 4 71 24
348 4 65 52
349 4 68 32
350 4 76 117
351 4 75 29
352 4 89 3
353 4 73 27
354 4 74 20
355 4 77 10
356 4 78 18
357 4 79 11
358 4 80 10
359 4 81 4
360 4 82 8
361 4 72 27
362 4 91 1
363 4 92 3
364 4 102 1
365 4 87 1
366 4 85 6
367 4 84 2
368 4 94 1
369 4 83 3
370 4 90 2
371 4 86 1
372 4 93 1
373 5 1 13139241
374 5 18 1071
375 5 29 732
376 5 2 4461294
377 5 16 1132
378 5 20 1173
379 5 22 1027
380 5 39 352
381 5 24 894
382 5 23 974
383 5 19 1020
384 5 7 49050
385 5 6 116558
386 5 5 362914
387 5 9 11812
388 5 3 2723445
389 5 10 7022
390 5 26 851
391 5 4 978358
392 5 12 2382
393 5 8 21628
394 5 27 790
395 5 13 1925
396 5 21 1027
397 5 34 562
398 5 37 413
399 5 15 1203
400 5 33 702
401 5 11 4845
402 5 25 872
403 5 31 652
404 5 17 1020
405 5 28 717
406 5 36 433
407 5 49 124
408 5 50 137
409 5 52 103
410 5 54 99
411 5 40 318
412 5 56 77
413 5 30 680
414 5 32 741
415 5 45 201
416 5 14 1303
417 5 35 470
418 5 41 272
419 5 44 224
420 5 38 382
421 5 43 216
422 5 57 59
423 5 68 24
424 5 46 186
425 5 53 96
426 5 42 272
427 5 47 161
428 5 51 116
429 5 55 84
430 5 60 52
431 5 71 18
432 5 67 34
433 5 61 50
434 5 64 33
435 5 66 29
436 5 70 21
437 5 48 318
438 5 58 71
439 5 69 30
440 5 62 47
441 5 65 31
442 5 59 61
443 5 72 17
444 5 63 43
445 5 73 17
446 5 78 5
447 5 76 51
448 5 74 9
449 5 77 10
450 5 83 5
451 5 93 1
452 5 75 20
453 5 85 2
454 5 89 2
455 5 92 1
456 5 79 7
457 5 80 2
458 5 84 4
459 5 86 2
460 5 88 1
461 5 81 4
462 5 90 2
463 5 82 3
464 5 87 4
465 5 94 2
466 5 96 1
467 5 95 1
468 -1 1 15529
469 -1 2 1267
470 -1 5 41
471 -1 3 483
472 -1 4 149
473 -1 6 12
474 -1 9 1
475 -1 7 1

View File

@ -17,12 +17,24 @@ from pyspark.sql.types import *
from decimal import *
import random
CHECKDIR = "/home/claudio/google_2019/thesis_queries/figure_7/"
if len(sys.argv) is not 4:
print(sys.argv[0] + " {cluster} {tmpdir} {maxram}")
sys.exit()
cluster=sys.argv[1]
if os.path.exists(CHECKDIR + cluster + "_figure7c.csv"):
print("already computed")
sys.exit()
if os.path.exists(CHECKDIR + cluster + "_figure7c_working"):
print("already in execution")
sys.exit()
os.system("touch " + CHECKDIR + cluster + "_figure7c_working")
spark = pyspark.sql.SparkSession.builder \
.appName("task_slowdown") \
.config("spark.driver.maxResultSize", "128g") \
@ -33,7 +45,7 @@ sc = spark.sparkContext
# READING INSTANCE EVENTS DATA
dfepath = "/home/claudio/google_2019/instance_events/" + cluster + "/" + cluster + "_instance_events*.json.gz"
#dfepath = "/home/claudio/google_2019/instance_events/" + cluster + "/" + cluster + "_test.json"
#dfepath = "/home/claudio/google_2019/instance_events/" + cluster + "/" + cluster + "_instance_events00000000000?.json.gz"
df = spark.read.json(dfepath)
def tabid(x):
@ -58,9 +70,11 @@ def tally_event(bucket, term, nexec):
def for_each_joined(x):
machine_id = x[0]
if x[0] is None:
return {}
ts = x[1]
ts = sorted(ts, key=lambda x: x["time"])
ts = sorted(ts, key=lambda x: x["time"] or -1)
in_execution = set()
chum = {}
@ -88,24 +102,34 @@ def fold_resobjs(ro1, ro2):
return ro1
def mark_next(data):
ts = data[1]
ts = sorted(ts, key=lambda z: z[1])
ts = list(data[1])
ts = sorted(ts, key=lambda z: z[1] or -1)
last_term = -1
for i in range(0, len(ts)):
t = ts[i]
ts[i] = {"id": t[0], "time": t[1], "type": t[2], "mid": t[3], "end": (i == len(ts) -1 or t[3] != ts[i+1][3])}
if ts[i]["type"] >= 4 or ts[i]["type"] <= 8:
if ts[i]["type"] >= 4 and ts[i]["type"] <= 8:
last_term = ts[i]["type"]
for t in ts:
t["term"] = last_term
return ts
def to_csv(result):
out = "term,n_exec,count\n"
for key in result.keys():
for key2 in result[key].keys():
out += str(key) + "," + str(key2) + "," + str(result[key][key2]) + "\n"
return out
result = df.rdd \
.filter(lambda x: x.time is not None and x.type is not None and
x.instance_index is not None and x.collection_id is not None) \
.map(lambda x: [tabid(x), int(x.time), int(x.type), x.machine_id]) \
.groupBy(lambda x: x[0]) \
.flatMap(mark_next) \
.filter(lambda x: x["mid"] is not None) \
.groupBy(lambda x: x["mid"]) \
.partitionBy(1000, lambda x: random.randint(0, 1000-1)) \
.map(for_each_joined) \
@ -113,7 +137,8 @@ result = df.rdd \
d = os.path.dirname(os.path.realpath(__file__))
with open(d + "/" + cluster + "_figure7c.json", "w") as f:
json.dump(result, f)
with open(d + "/" + cluster + "_figure7c.csv", "w") as f:
f.write(to_csv(result))
os.system("rm " + CHECKDIR + cluster + "_figure7c_working")
# vim: set ts=4 sw=4 et tw=120: