more results for 7c

This commit is contained in:
Claudio Maggioni 2021-04-19 12:27:09 +00:00
parent e0c92f6dbd
commit 9183651649
2 changed files with 650 additions and 10 deletions

625
figure_7/f_figure7c.csv Normal file
View File

@ -0,0 +1,625 @@
term,n_exec,count
6,1,35245280
6,3,15198454
6,4,5903899
6,18,25111
6,21,16003
6,23,13437
6,17,26546
6,10,100248
6,7,470015
6,2,15600345
6,5,2555041
6,6,992293
6,8,260317
6,9,155453
6,11,85147
6,12,69196
6,13,56955
6,14,46034
6,15,38905
6,16,32620
6,19,20696
6,20,17279
6,22,15868
6,24,13132
6,28,9021
6,29,8540
6,32,7072
6,35,6284
6,38,5117
6,40,4652
6,43,3633
6,44,3261
6,46,2671
6,48,2147
6,50,1716
6,51,1659
6,57,802
6,67,264
6,70,251
6,73,178
6,77,144
6,82,112
6,84,92
6,90,87
6,92,83
6,95,74
6,97,46
6,99,55
6,101,63
6,103,74
6,105,55
6,106,53
6,108,52
6,110,36
6,112,35
6,114,22
6,115,47
6,25,11013
6,26,10163
6,27,9512
6,31,7607
6,33,6886
6,34,6377
6,39,4899
6,42,3786
6,30,7871
6,37,5875
6,41,4289
6,45,2992
6,47,2354
6,49,1889
6,56,894
6,60,551
6,62,474
6,64,378
6,36,5917
6,52,1349
6,53,1322
6,54,1068
6,55,935
6,61,484
6,63,363
6,72,171
6,74,170
6,79,123
6,81,129
6,83,89
6,59,633
6,66,330
6,69,239
6,71,190
6,76,146
6,58,667
6,78,109
6,85,77
6,88,84
6,91,74
6,93,55
6,65,297
6,75,155
6,89,60
6,96,82
6,98,60
6,100,62
6,104,44
6,107,44
6,87,76
6,68,216
6,80,78
6,94,48
6,109,46
6,111,44
6,113,41
6,116,24
6,118,26
6,120,32
6,122,16
6,86,87
6,102,53
6,117,28
6,123,12
6,125,15
6,127,6
6,129,4
6,119,19
6,121,24
6,126,15
6,132,4
6,136,4
6,139,2
6,142,2
6,143,2
6,124,15
6,128,6
6,130,6
6,131,4
6,133,3
6,135,4
4,2,12230534
4,3,15012854
4,5,1995991
4,19,7999
4,14,19463
4,9,70428
4,1,41374539
4,4,4712657
4,6,657358
4,7,270223
4,8,122163
4,10,43346
4,11,37038
4,12,30093
4,13,25297
4,15,15885
4,16,12425
4,79,16
4,80,20
4,30,3092
4,111,10
4,63,64
4,25,4224
4,26,4052
4,31,2873
4,37,1906
4,42,1117
4,29,3313
4,33,2578
4,18,10267
4,20,6482
4,22,6081
4,28,3563
4,34,2471
4,36,2139
4,38,1755
4,39,1622
4,43,972
4,48,499
4,50,333
4,17,10301
4,21,6252
4,24,5120
4,23,5371
4,32,2790
4,27,3703
4,40,1470
4,35,2209
4,41,1222
4,44,760
4,45,732
4,46,686
4,47,551
4,49,359
4,51,301
4,52,244
4,73,16
4,76,29
4,53,213
4,57,110
4,54,160
4,55,179
4,56,148
4,64,47
4,60,117
4,62,31
4,78,10
4,100,4
4,58,118
4,89,4
4,97,4
4,59,112
4,61,81
4,66,34
4,67,41
4,84,6
4,99,5
4,104,6
4,114,2
4,115,4
4,65,39
4,96,6
4,70,20
4,75,24
4,68,38
4,71,16
4,85,10
4,69,17
4,101,5
4,126,6
4,87,12
4,92,10
4,98,6
4,77,6
4,95,8
4,109,4
4,110,2
4,120,5
4,130,2
4,131,5
4,134,2
4,135,2
4,137,2
4,138,2
4,140,2
4,141,2
4,143,4
4,93,10
4,82,17
4,81,6
4,88,2
4,74,16
4,83,22
4,86,10
4,119,5
4,72,16
4,112,4
4,90,2
4,122,2
4,123,6
4,124,4
4,105,2
4,91,12
4,103,2
4,118,2
4,128,2
4,129,2
4,132,2
4,107,2
4,94,2
4,117,2
4,106,2
7,2,193731409
7,3,221061272
7,4,80804298
7,5,34556187
7,1,467883864
7,6,12611903
7,7,5437592
7,8,2795883
7,9,1665577
7,10,1052880
7,11,882586
7,12,717089
7,13,611134
7,14,489138
7,15,391883
7,16,301795
7,17,232464
7,19,153558
7,20,118651
7,23,84141
7,22,103748
7,21,102660
7,18,204175
7,24,83007
7,25,66490
7,26,60392
7,27,55926
7,28,50839
7,29,47655
7,30,45661
7,31,42856
7,32,40618
7,33,38372
7,34,36430
7,35,33928
7,36,32118
7,37,30064
7,38,27429
7,39,25306
7,40,22893
7,41,20964
7,42,19031
7,43,17024
7,44,15677
7,45,14051
7,46,12272
7,47,10861
7,48,9710
7,49,8694
7,50,7848
7,51,6949
7,52,6053
7,53,5183
7,54,4567
7,55,4174
7,56,3742
7,57,3186
7,58,2947
7,59,2663
7,60,2333
7,61,2152
7,62,1834
7,63,1670
7,64,1490
7,65,1421
7,66,1293
7,67,1166
7,68,1024
7,69,920
7,70,910
7,71,851
7,72,790
7,73,708
7,74,764
7,75,641
7,76,629
7,77,533
7,78,554
7,79,517
7,80,492
7,81,458
7,82,464
7,83,441
7,84,408
7,85,426
7,86,380
7,87,355
7,88,373
7,89,397
7,90,364
7,91,368
7,92,320
7,93,317
7,94,343
7,95,325
7,96,298
7,97,298
7,98,296
7,99,294
7,100,294
7,101,294
7,102,314
7,103,251
7,104,261
7,105,236
7,107,224
7,108,210
7,109,180
7,110,161
7,111,149
7,112,169
7,113,146
7,114,167
7,115,126
7,106,229
7,116,126
7,117,103
7,118,102
7,119,125
7,120,121
7,121,89
7,122,57
7,123,60
7,124,50
7,125,59
7,126,37
7,127,33
7,128,24
7,129,23
7,130,12
7,131,12
7,132,14
7,133,11
7,134,6
7,135,8
7,136,3
7,137,2
7,138,1
7,139,1
7,140,1
7,141,1
7,142,1
7,143,5
5,2,2195039
5,3,2443598
5,4,993197
5,5,473494
5,22,7929
5,18,17758
5,8,89614
5,1,4804686
5,6,215463
5,7,125858
5,9,82769
5,10,75842
5,11,67418
5,12,58772
5,13,51365
5,14,44099
5,15,36525
5,16,30367
5,17,23681
5,19,15301
5,31,1987
5,42,713
5,49,288
5,56,130
5,94,8
5,109,11
5,23,6317
5,24,5292
5,32,1881
5,30,2202
5,38,1018
5,27,3063
5,28,2664
5,20,10712
5,26,3429
5,34,1616
5,40,900
5,45,476
5,21,9379
5,37,1213
5,25,4199
5,35,1350
5,36,1289
5,39,958
5,29,2564
5,33,1648
5,41,779
5,43,645
5,46,464
5,47,398
5,62,70
5,73,21
5,44,575
5,54,188
5,60,84
5,76,17
5,48,332
5,52,209
5,55,163
5,59,70
5,71,31
5,72,10
5,75,20
5,90,16
5,51,234
5,57,104
5,53,177
5,67,30
5,68,32
5,99,6
5,101,7
5,104,4
5,50,238
5,83,14
5,95,16
5,64,48
5,85,12
5,91,6
5,113,2
5,84,19
5,89,9
5,61,87
5,69,29
5,111,2
5,115,6
5,66,33
5,86,6
5,93,8
5,70,30
5,87,16
5,58,73
5,118,4
5,124,2
5,128,1
5,65,32
5,78,13
5,100,11
5,96,6
5,97,8
5,120,2
5,122,2
5,77,8
5,92,10
5,63,51
5,105,6
5,110,6
5,116,4
5,98,6
5,79,14
5,106,10
5,131,2
5,134,2
5,107,4
5,74,14
5,102,4
5,81,6
5,88,2
5,82,5
5,125,2
5,129,2
5,112,6
5,108,5
5,103,8
5,80,7
5,119,2
8,2,1228513
8,1,2653400
8,3,1166871
8,7,35895
8,10,4297
8,4,468165
8,5,205458
8,6,78937
8,8,18685
8,9,9392
8,11,4182
8,12,2923
8,23,152
8,14,1174
8,22,140
8,41,40
8,13,1770
8,15,978
8,16,577
8,17,314
8,21,82
8,27,38
8,33,30
8,52,16
8,57,2
8,30,44
8,54,10
8,19,264
8,48,12
8,32,32
8,18,577
8,20,142
8,26,63
8,37,36
8,35,66
8,24,178
8,38,34
8,28,92
8,39,44
8,34,70
8,45,26
8,31,50
8,80,4
8,25,80
8,29,44
8,70,2
8,43,44
8,50,18
8,47,20
8,36,34
8,51,6
8,60,4
8,40,24
8,42,30
8,46,24
8,44,14
8,56,6
8,53,2
8,58,8
8,86,4
8,55,2
8,49,12
8,74,2
8,85,2
-1,2,4225
-1,3,2902
-1,1,18381
-1,5,963
-1,6,495
-1,7,430
-1,4,1397
-1,8,129
-1,9,55
-1,12,39
-1,14,9
-1,17,5
-1,11,32
-1,13,20
-1,10,68
-1,15,5
-1,25,1
-1,26,1
-1,23,1
-1,16,4
-1,19,2
-1,33,2
1 term n_exec count
2 6 1 35245280
3 6 3 15198454
4 6 4 5903899
5 6 18 25111
6 6 21 16003
7 6 23 13437
8 6 17 26546
9 6 10 100248
10 6 7 470015
11 6 2 15600345
12 6 5 2555041
13 6 6 992293
14 6 8 260317
15 6 9 155453
16 6 11 85147
17 6 12 69196
18 6 13 56955
19 6 14 46034
20 6 15 38905
21 6 16 32620
22 6 19 20696
23 6 20 17279
24 6 22 15868
25 6 24 13132
26 6 28 9021
27 6 29 8540
28 6 32 7072
29 6 35 6284
30 6 38 5117
31 6 40 4652
32 6 43 3633
33 6 44 3261
34 6 46 2671
35 6 48 2147
36 6 50 1716
37 6 51 1659
38 6 57 802
39 6 67 264
40 6 70 251
41 6 73 178
42 6 77 144
43 6 82 112
44 6 84 92
45 6 90 87
46 6 92 83
47 6 95 74
48 6 97 46
49 6 99 55
50 6 101 63
51 6 103 74
52 6 105 55
53 6 106 53
54 6 108 52
55 6 110 36
56 6 112 35
57 6 114 22
58 6 115 47
59 6 25 11013
60 6 26 10163
61 6 27 9512
62 6 31 7607
63 6 33 6886
64 6 34 6377
65 6 39 4899
66 6 42 3786
67 6 30 7871
68 6 37 5875
69 6 41 4289
70 6 45 2992
71 6 47 2354
72 6 49 1889
73 6 56 894
74 6 60 551
75 6 62 474
76 6 64 378
77 6 36 5917
78 6 52 1349
79 6 53 1322
80 6 54 1068
81 6 55 935
82 6 61 484
83 6 63 363
84 6 72 171
85 6 74 170
86 6 79 123
87 6 81 129
88 6 83 89
89 6 59 633
90 6 66 330
91 6 69 239
92 6 71 190
93 6 76 146
94 6 58 667
95 6 78 109
96 6 85 77
97 6 88 84
98 6 91 74
99 6 93 55
100 6 65 297
101 6 75 155
102 6 89 60
103 6 96 82
104 6 98 60
105 6 100 62
106 6 104 44
107 6 107 44
108 6 87 76
109 6 68 216
110 6 80 78
111 6 94 48
112 6 109 46
113 6 111 44
114 6 113 41
115 6 116 24
116 6 118 26
117 6 120 32
118 6 122 16
119 6 86 87
120 6 102 53
121 6 117 28
122 6 123 12
123 6 125 15
124 6 127 6
125 6 129 4
126 6 119 19
127 6 121 24
128 6 126 15
129 6 132 4
130 6 136 4
131 6 139 2
132 6 142 2
133 6 143 2
134 6 124 15
135 6 128 6
136 6 130 6
137 6 131 4
138 6 133 3
139 6 135 4
140 4 2 12230534
141 4 3 15012854
142 4 5 1995991
143 4 19 7999
144 4 14 19463
145 4 9 70428
146 4 1 41374539
147 4 4 4712657
148 4 6 657358
149 4 7 270223
150 4 8 122163
151 4 10 43346
152 4 11 37038
153 4 12 30093
154 4 13 25297
155 4 15 15885
156 4 16 12425
157 4 79 16
158 4 80 20
159 4 30 3092
160 4 111 10
161 4 63 64
162 4 25 4224
163 4 26 4052
164 4 31 2873
165 4 37 1906
166 4 42 1117
167 4 29 3313
168 4 33 2578
169 4 18 10267
170 4 20 6482
171 4 22 6081
172 4 28 3563
173 4 34 2471
174 4 36 2139
175 4 38 1755
176 4 39 1622
177 4 43 972
178 4 48 499
179 4 50 333
180 4 17 10301
181 4 21 6252
182 4 24 5120
183 4 23 5371
184 4 32 2790
185 4 27 3703
186 4 40 1470
187 4 35 2209
188 4 41 1222
189 4 44 760
190 4 45 732
191 4 46 686
192 4 47 551
193 4 49 359
194 4 51 301
195 4 52 244
196 4 73 16
197 4 76 29
198 4 53 213
199 4 57 110
200 4 54 160
201 4 55 179
202 4 56 148
203 4 64 47
204 4 60 117
205 4 62 31
206 4 78 10
207 4 100 4
208 4 58 118
209 4 89 4
210 4 97 4
211 4 59 112
212 4 61 81
213 4 66 34
214 4 67 41
215 4 84 6
216 4 99 5
217 4 104 6
218 4 114 2
219 4 115 4
220 4 65 39
221 4 96 6
222 4 70 20
223 4 75 24
224 4 68 38
225 4 71 16
226 4 85 10
227 4 69 17
228 4 101 5
229 4 126 6
230 4 87 12
231 4 92 10
232 4 98 6
233 4 77 6
234 4 95 8
235 4 109 4
236 4 110 2
237 4 120 5
238 4 130 2
239 4 131 5
240 4 134 2
241 4 135 2
242 4 137 2
243 4 138 2
244 4 140 2
245 4 141 2
246 4 143 4
247 4 93 10
248 4 82 17
249 4 81 6
250 4 88 2
251 4 74 16
252 4 83 22
253 4 86 10
254 4 119 5
255 4 72 16
256 4 112 4
257 4 90 2
258 4 122 2
259 4 123 6
260 4 124 4
261 4 105 2
262 4 91 12
263 4 103 2
264 4 118 2
265 4 128 2
266 4 129 2
267 4 132 2
268 4 107 2
269 4 94 2
270 4 117 2
271 4 106 2
272 7 2 193731409
273 7 3 221061272
274 7 4 80804298
275 7 5 34556187
276 7 1 467883864
277 7 6 12611903
278 7 7 5437592
279 7 8 2795883
280 7 9 1665577
281 7 10 1052880
282 7 11 882586
283 7 12 717089
284 7 13 611134
285 7 14 489138
286 7 15 391883
287 7 16 301795
288 7 17 232464
289 7 19 153558
290 7 20 118651
291 7 23 84141
292 7 22 103748
293 7 21 102660
294 7 18 204175
295 7 24 83007
296 7 25 66490
297 7 26 60392
298 7 27 55926
299 7 28 50839
300 7 29 47655
301 7 30 45661
302 7 31 42856
303 7 32 40618
304 7 33 38372
305 7 34 36430
306 7 35 33928
307 7 36 32118
308 7 37 30064
309 7 38 27429
310 7 39 25306
311 7 40 22893
312 7 41 20964
313 7 42 19031
314 7 43 17024
315 7 44 15677
316 7 45 14051
317 7 46 12272
318 7 47 10861
319 7 48 9710
320 7 49 8694
321 7 50 7848
322 7 51 6949
323 7 52 6053
324 7 53 5183
325 7 54 4567
326 7 55 4174
327 7 56 3742
328 7 57 3186
329 7 58 2947
330 7 59 2663
331 7 60 2333
332 7 61 2152
333 7 62 1834
334 7 63 1670
335 7 64 1490
336 7 65 1421
337 7 66 1293
338 7 67 1166
339 7 68 1024
340 7 69 920
341 7 70 910
342 7 71 851
343 7 72 790
344 7 73 708
345 7 74 764
346 7 75 641
347 7 76 629
348 7 77 533
349 7 78 554
350 7 79 517
351 7 80 492
352 7 81 458
353 7 82 464
354 7 83 441
355 7 84 408
356 7 85 426
357 7 86 380
358 7 87 355
359 7 88 373
360 7 89 397
361 7 90 364
362 7 91 368
363 7 92 320
364 7 93 317
365 7 94 343
366 7 95 325
367 7 96 298
368 7 97 298
369 7 98 296
370 7 99 294
371 7 100 294
372 7 101 294
373 7 102 314
374 7 103 251
375 7 104 261
376 7 105 236
377 7 107 224
378 7 108 210
379 7 109 180
380 7 110 161
381 7 111 149
382 7 112 169
383 7 113 146
384 7 114 167
385 7 115 126
386 7 106 229
387 7 116 126
388 7 117 103
389 7 118 102
390 7 119 125
391 7 120 121
392 7 121 89
393 7 122 57
394 7 123 60
395 7 124 50
396 7 125 59
397 7 126 37
398 7 127 33
399 7 128 24
400 7 129 23
401 7 130 12
402 7 131 12
403 7 132 14
404 7 133 11
405 7 134 6
406 7 135 8
407 7 136 3
408 7 137 2
409 7 138 1
410 7 139 1
411 7 140 1
412 7 141 1
413 7 142 1
414 7 143 5
415 5 2 2195039
416 5 3 2443598
417 5 4 993197
418 5 5 473494
419 5 22 7929
420 5 18 17758
421 5 8 89614
422 5 1 4804686
423 5 6 215463
424 5 7 125858
425 5 9 82769
426 5 10 75842
427 5 11 67418
428 5 12 58772
429 5 13 51365
430 5 14 44099
431 5 15 36525
432 5 16 30367
433 5 17 23681
434 5 19 15301
435 5 31 1987
436 5 42 713
437 5 49 288
438 5 56 130
439 5 94 8
440 5 109 11
441 5 23 6317
442 5 24 5292
443 5 32 1881
444 5 30 2202
445 5 38 1018
446 5 27 3063
447 5 28 2664
448 5 20 10712
449 5 26 3429
450 5 34 1616
451 5 40 900
452 5 45 476
453 5 21 9379
454 5 37 1213
455 5 25 4199
456 5 35 1350
457 5 36 1289
458 5 39 958
459 5 29 2564
460 5 33 1648
461 5 41 779
462 5 43 645
463 5 46 464
464 5 47 398
465 5 62 70
466 5 73 21
467 5 44 575
468 5 54 188
469 5 60 84
470 5 76 17
471 5 48 332
472 5 52 209
473 5 55 163
474 5 59 70
475 5 71 31
476 5 72 10
477 5 75 20
478 5 90 16
479 5 51 234
480 5 57 104
481 5 53 177
482 5 67 30
483 5 68 32
484 5 99 6
485 5 101 7
486 5 104 4
487 5 50 238
488 5 83 14
489 5 95 16
490 5 64 48
491 5 85 12
492 5 91 6
493 5 113 2
494 5 84 19
495 5 89 9
496 5 61 87
497 5 69 29
498 5 111 2
499 5 115 6
500 5 66 33
501 5 86 6
502 5 93 8
503 5 70 30
504 5 87 16
505 5 58 73
506 5 118 4
507 5 124 2
508 5 128 1
509 5 65 32
510 5 78 13
511 5 100 11
512 5 96 6
513 5 97 8
514 5 120 2
515 5 122 2
516 5 77 8
517 5 92 10
518 5 63 51
519 5 105 6
520 5 110 6
521 5 116 4
522 5 98 6
523 5 79 14
524 5 106 10
525 5 131 2
526 5 134 2
527 5 107 4
528 5 74 14
529 5 102 4
530 5 81 6
531 5 88 2
532 5 82 5
533 5 125 2
534 5 129 2
535 5 112 6
536 5 108 5
537 5 103 8
538 5 80 7
539 5 119 2
540 8 2 1228513
541 8 1 2653400
542 8 3 1166871
543 8 7 35895
544 8 10 4297
545 8 4 468165
546 8 5 205458
547 8 6 78937
548 8 8 18685
549 8 9 9392
550 8 11 4182
551 8 12 2923
552 8 23 152
553 8 14 1174
554 8 22 140
555 8 41 40
556 8 13 1770
557 8 15 978
558 8 16 577
559 8 17 314
560 8 21 82
561 8 27 38
562 8 33 30
563 8 52 16
564 8 57 2
565 8 30 44
566 8 54 10
567 8 19 264
568 8 48 12
569 8 32 32
570 8 18 577
571 8 20 142
572 8 26 63
573 8 37 36
574 8 35 66
575 8 24 178
576 8 38 34
577 8 28 92
578 8 39 44
579 8 34 70
580 8 45 26
581 8 31 50
582 8 80 4
583 8 25 80
584 8 29 44
585 8 70 2
586 8 43 44
587 8 50 18
588 8 47 20
589 8 36 34
590 8 51 6
591 8 60 4
592 8 40 24
593 8 42 30
594 8 46 24
595 8 44 14
596 8 56 6
597 8 53 2
598 8 58 8
599 8 86 4
600 8 55 2
601 8 49 12
602 8 74 2
603 8 85 2
604 -1 2 4225
605 -1 3 2902
606 -1 1 18381
607 -1 5 963
608 -1 6 495
609 -1 7 430
610 -1 4 1397
611 -1 8 129
612 -1 9 55
613 -1 12 39
614 -1 14 9
615 -1 17 5
616 -1 11 32
617 -1 13 20
618 -1 10 68
619 -1 15 5
620 -1 25 1
621 -1 26 1
622 -1 23 1
623 -1 16 4
624 -1 19 2
625 -1 33 2

View File

@ -16,12 +16,25 @@ from pyspark.sql import Window
from pyspark.sql.types import *
from decimal import *
CHECKDIR = "/home/claudio/google_2019/thesis_queries/figure_8/"
if len(sys.argv) is not 4:
print(sys.argv[0] + " {cluster} {tmpdir} {maxram}")
sys.exit()
cluster=sys.argv[1]
if os.path.exists(CHECKDIR + cluster + "_figure8cd.json"):
print("already computed")
sys.exit()
if os.path.exists(CHECKDIR + cluster + "_figure8cd_working"):
print("already in execution")
sys.exit()
os.system("touch " + CHECKDIR + cluster + "_figure8cd_working")
spark = pyspark.sql.SparkSession.builder \
.appName("task_slowdown") \
.config("spark.driver.maxResultSize", "128g") \
@ -30,12 +43,15 @@ spark = pyspark.sql.SparkSession.builder \
.getOrCreate()
sc = spark.sparkContext
def tabid(x):
return Decimal(x.collection_id) + Decimal(x.instance_index) / Decimal(2**64)
#
# READING INSTANCE EVENTS DATA
#
dfepath = "/home/claudio/google_2019/instance_events/" + cluster + "/" + cluster + "_instance_events*.json.gz"
#dfepath = "/home/claudio/google_2019/instance_events/" + cluster + "/" + cluster + "_test.json"
df = spark.read.json(dfepath)
df = df.limit(10000)
# READING MACHINE EVENTS DATA, sort them and save them as broadcast variable
print("Starting to read machine events...")
@ -56,7 +72,7 @@ print("Converting to broadcast variable...")
dfm = sc.broadcast([tuple(r) for r in dfm.to_numpy()])
print("Done with machine events.")
df = dfe.rdd \
df = df.rdd \
.filter(lambda x: x.time is not None and x.type is not None and x.machine_id is not None and
x.instance_index is not None and x.collection_id is not None and x.resource_request is not None and
x.resource_request.cpus is not None and x.resource_request.memory is not None) \
@ -68,12 +84,12 @@ df = dfe.rdd \
def get_machine_time_resources(machine_id, time):
def aux(i, j):
if i == j:
return dfm.value[i] if dfm.value[i][1] == machine_id else None
return i if dfm.value[i][1] == machine_id else None
elif i + 1 == j:
if dfm.value[i][1] == machine_id:
return dfm.value[i]
return i
elif dfm.value[j][1] == machine_id:
return dfm.value[j]
return j
else:
return None
@ -170,20 +186,19 @@ def fold_resobjs(ro1, ro2):
ro1[k][kk] = [sum(x) for x in zip(ro1[k][kk], ro2[k][kk])]
return ro1
# TODO: partition by id and in the for-each-row
# function implement lookup to dfm.value to understand its memory capacity
import random
result = df.rdd \
.groupBy(lambda x: x.mid) \
.partitionBy(10000, lambda x: random.randint(0, 10000-1)) \
.partitionBy(1000, lambda x: random.randint(0, 1000-1)) \
.map(for_each_joined) \
.fold(None, fold_resobjs)
d = os.path.dirname(os.path.realpath(__file__))
with open(d + "/" + cluster + "_figure8.json", "w") as f:
with open(d + "/" + cluster + "_figure8cd.json", "w") as f:
json.dump(result, f)
os.system("rm " + CHECKDIR + cluster + "_figure8cd_working")
# vim: set ts=4 sw=4 et tw=120: