From ed8a9b6006ef5c47d0c696f0d3c840e5c2dc5f5b Mon Sep 17 00:00:00 2001 From: "Claudio Maggioni (maggicl)" Date: Mon, 19 Apr 2021 12:27:09 +0000 Subject: [PATCH] more results for 7c --- figure_7/f_figure7c.csv | 625 ++++++++++++++++++++++++++++++++++++ figure_8/figure8-cd-only.py | 35 +- 2 files changed, 650 insertions(+), 10 deletions(-) create mode 100644 figure_7/f_figure7c.csv diff --git a/figure_7/f_figure7c.csv b/figure_7/f_figure7c.csv new file mode 100644 index 00000000..26cf9579 --- /dev/null +++ b/figure_7/f_figure7c.csv @@ -0,0 +1,625 @@ +term,n_exec,count +6,1,35245280 +6,3,15198454 +6,4,5903899 +6,18,25111 +6,21,16003 +6,23,13437 +6,17,26546 +6,10,100248 +6,7,470015 +6,2,15600345 +6,5,2555041 +6,6,992293 +6,8,260317 +6,9,155453 +6,11,85147 +6,12,69196 +6,13,56955 +6,14,46034 +6,15,38905 +6,16,32620 +6,19,20696 +6,20,17279 +6,22,15868 +6,24,13132 +6,28,9021 +6,29,8540 +6,32,7072 +6,35,6284 +6,38,5117 +6,40,4652 +6,43,3633 +6,44,3261 +6,46,2671 +6,48,2147 +6,50,1716 +6,51,1659 +6,57,802 +6,67,264 +6,70,251 +6,73,178 +6,77,144 +6,82,112 +6,84,92 +6,90,87 +6,92,83 +6,95,74 +6,97,46 +6,99,55 +6,101,63 +6,103,74 +6,105,55 +6,106,53 +6,108,52 +6,110,36 +6,112,35 +6,114,22 +6,115,47 +6,25,11013 +6,26,10163 +6,27,9512 +6,31,7607 +6,33,6886 +6,34,6377 +6,39,4899 +6,42,3786 +6,30,7871 +6,37,5875 +6,41,4289 +6,45,2992 +6,47,2354 +6,49,1889 +6,56,894 +6,60,551 +6,62,474 +6,64,378 +6,36,5917 +6,52,1349 +6,53,1322 +6,54,1068 +6,55,935 +6,61,484 +6,63,363 +6,72,171 +6,74,170 +6,79,123 +6,81,129 +6,83,89 +6,59,633 +6,66,330 +6,69,239 +6,71,190 +6,76,146 +6,58,667 +6,78,109 +6,85,77 +6,88,84 +6,91,74 +6,93,55 +6,65,297 +6,75,155 +6,89,60 +6,96,82 +6,98,60 +6,100,62 +6,104,44 +6,107,44 +6,87,76 +6,68,216 +6,80,78 +6,94,48 +6,109,46 +6,111,44 +6,113,41 +6,116,24 +6,118,26 +6,120,32 +6,122,16 +6,86,87 +6,102,53 +6,117,28 +6,123,12 +6,125,15 +6,127,6 +6,129,4 +6,119,19 +6,121,24 +6,126,15 +6,132,4 +6,136,4 +6,139,2 +6,142,2 +6,143,2 +6,124,15 +6,128,6 +6,130,6 +6,131,4 +6,133,3 +6,135,4 +4,2,12230534 +4,3,15012854 +4,5,1995991 +4,19,7999 +4,14,19463 +4,9,70428 +4,1,41374539 +4,4,4712657 +4,6,657358 +4,7,270223 +4,8,122163 +4,10,43346 +4,11,37038 +4,12,30093 +4,13,25297 +4,15,15885 +4,16,12425 +4,79,16 +4,80,20 +4,30,3092 +4,111,10 +4,63,64 +4,25,4224 +4,26,4052 +4,31,2873 +4,37,1906 +4,42,1117 +4,29,3313 +4,33,2578 +4,18,10267 +4,20,6482 +4,22,6081 +4,28,3563 +4,34,2471 +4,36,2139 +4,38,1755 +4,39,1622 +4,43,972 +4,48,499 +4,50,333 +4,17,10301 +4,21,6252 +4,24,5120 +4,23,5371 +4,32,2790 +4,27,3703 +4,40,1470 +4,35,2209 +4,41,1222 +4,44,760 +4,45,732 +4,46,686 +4,47,551 +4,49,359 +4,51,301 +4,52,244 +4,73,16 +4,76,29 +4,53,213 +4,57,110 +4,54,160 +4,55,179 +4,56,148 +4,64,47 +4,60,117 +4,62,31 +4,78,10 +4,100,4 +4,58,118 +4,89,4 +4,97,4 +4,59,112 +4,61,81 +4,66,34 +4,67,41 +4,84,6 +4,99,5 +4,104,6 +4,114,2 +4,115,4 +4,65,39 +4,96,6 +4,70,20 +4,75,24 +4,68,38 +4,71,16 +4,85,10 +4,69,17 +4,101,5 +4,126,6 +4,87,12 +4,92,10 +4,98,6 +4,77,6 +4,95,8 +4,109,4 +4,110,2 +4,120,5 +4,130,2 +4,131,5 +4,134,2 +4,135,2 +4,137,2 +4,138,2 +4,140,2 +4,141,2 +4,143,4 +4,93,10 +4,82,17 +4,81,6 +4,88,2 +4,74,16 +4,83,22 +4,86,10 +4,119,5 +4,72,16 +4,112,4 +4,90,2 +4,122,2 +4,123,6 +4,124,4 +4,105,2 +4,91,12 +4,103,2 +4,118,2 +4,128,2 +4,129,2 +4,132,2 +4,107,2 +4,94,2 +4,117,2 +4,106,2 +7,2,193731409 +7,3,221061272 +7,4,80804298 +7,5,34556187 +7,1,467883864 +7,6,12611903 +7,7,5437592 +7,8,2795883 +7,9,1665577 +7,10,1052880 +7,11,882586 +7,12,717089 +7,13,611134 +7,14,489138 +7,15,391883 +7,16,301795 +7,17,232464 +7,19,153558 +7,20,118651 +7,23,84141 +7,22,103748 +7,21,102660 +7,18,204175 +7,24,83007 +7,25,66490 +7,26,60392 +7,27,55926 +7,28,50839 +7,29,47655 +7,30,45661 +7,31,42856 +7,32,40618 +7,33,38372 +7,34,36430 +7,35,33928 +7,36,32118 +7,37,30064 +7,38,27429 +7,39,25306 +7,40,22893 +7,41,20964 +7,42,19031 +7,43,17024 +7,44,15677 +7,45,14051 +7,46,12272 +7,47,10861 +7,48,9710 +7,49,8694 +7,50,7848 +7,51,6949 +7,52,6053 +7,53,5183 +7,54,4567 +7,55,4174 +7,56,3742 +7,57,3186 +7,58,2947 +7,59,2663 +7,60,2333 +7,61,2152 +7,62,1834 +7,63,1670 +7,64,1490 +7,65,1421 +7,66,1293 +7,67,1166 +7,68,1024 +7,69,920 +7,70,910 +7,71,851 +7,72,790 +7,73,708 +7,74,764 +7,75,641 +7,76,629 +7,77,533 +7,78,554 +7,79,517 +7,80,492 +7,81,458 +7,82,464 +7,83,441 +7,84,408 +7,85,426 +7,86,380 +7,87,355 +7,88,373 +7,89,397 +7,90,364 +7,91,368 +7,92,320 +7,93,317 +7,94,343 +7,95,325 +7,96,298 +7,97,298 +7,98,296 +7,99,294 +7,100,294 +7,101,294 +7,102,314 +7,103,251 +7,104,261 +7,105,236 +7,107,224 +7,108,210 +7,109,180 +7,110,161 +7,111,149 +7,112,169 +7,113,146 +7,114,167 +7,115,126 +7,106,229 +7,116,126 +7,117,103 +7,118,102 +7,119,125 +7,120,121 +7,121,89 +7,122,57 +7,123,60 +7,124,50 +7,125,59 +7,126,37 +7,127,33 +7,128,24 +7,129,23 +7,130,12 +7,131,12 +7,132,14 +7,133,11 +7,134,6 +7,135,8 +7,136,3 +7,137,2 +7,138,1 +7,139,1 +7,140,1 +7,141,1 +7,142,1 +7,143,5 +5,2,2195039 +5,3,2443598 +5,4,993197 +5,5,473494 +5,22,7929 +5,18,17758 +5,8,89614 +5,1,4804686 +5,6,215463 +5,7,125858 +5,9,82769 +5,10,75842 +5,11,67418 +5,12,58772 +5,13,51365 +5,14,44099 +5,15,36525 +5,16,30367 +5,17,23681 +5,19,15301 +5,31,1987 +5,42,713 +5,49,288 +5,56,130 +5,94,8 +5,109,11 +5,23,6317 +5,24,5292 +5,32,1881 +5,30,2202 +5,38,1018 +5,27,3063 +5,28,2664 +5,20,10712 +5,26,3429 +5,34,1616 +5,40,900 +5,45,476 +5,21,9379 +5,37,1213 +5,25,4199 +5,35,1350 +5,36,1289 +5,39,958 +5,29,2564 +5,33,1648 +5,41,779 +5,43,645 +5,46,464 +5,47,398 +5,62,70 +5,73,21 +5,44,575 +5,54,188 +5,60,84 +5,76,17 +5,48,332 +5,52,209 +5,55,163 +5,59,70 +5,71,31 +5,72,10 +5,75,20 +5,90,16 +5,51,234 +5,57,104 +5,53,177 +5,67,30 +5,68,32 +5,99,6 +5,101,7 +5,104,4 +5,50,238 +5,83,14 +5,95,16 +5,64,48 +5,85,12 +5,91,6 +5,113,2 +5,84,19 +5,89,9 +5,61,87 +5,69,29 +5,111,2 +5,115,6 +5,66,33 +5,86,6 +5,93,8 +5,70,30 +5,87,16 +5,58,73 +5,118,4 +5,124,2 +5,128,1 +5,65,32 +5,78,13 +5,100,11 +5,96,6 +5,97,8 +5,120,2 +5,122,2 +5,77,8 +5,92,10 +5,63,51 +5,105,6 +5,110,6 +5,116,4 +5,98,6 +5,79,14 +5,106,10 +5,131,2 +5,134,2 +5,107,4 +5,74,14 +5,102,4 +5,81,6 +5,88,2 +5,82,5 +5,125,2 +5,129,2 +5,112,6 +5,108,5 +5,103,8 +5,80,7 +5,119,2 +8,2,1228513 +8,1,2653400 +8,3,1166871 +8,7,35895 +8,10,4297 +8,4,468165 +8,5,205458 +8,6,78937 +8,8,18685 +8,9,9392 +8,11,4182 +8,12,2923 +8,23,152 +8,14,1174 +8,22,140 +8,41,40 +8,13,1770 +8,15,978 +8,16,577 +8,17,314 +8,21,82 +8,27,38 +8,33,30 +8,52,16 +8,57,2 +8,30,44 +8,54,10 +8,19,264 +8,48,12 +8,32,32 +8,18,577 +8,20,142 +8,26,63 +8,37,36 +8,35,66 +8,24,178 +8,38,34 +8,28,92 +8,39,44 +8,34,70 +8,45,26 +8,31,50 +8,80,4 +8,25,80 +8,29,44 +8,70,2 +8,43,44 +8,50,18 +8,47,20 +8,36,34 +8,51,6 +8,60,4 +8,40,24 +8,42,30 +8,46,24 +8,44,14 +8,56,6 +8,53,2 +8,58,8 +8,86,4 +8,55,2 +8,49,12 +8,74,2 +8,85,2 +-1,2,4225 +-1,3,2902 +-1,1,18381 +-1,5,963 +-1,6,495 +-1,7,430 +-1,4,1397 +-1,8,129 +-1,9,55 +-1,12,39 +-1,14,9 +-1,17,5 +-1,11,32 +-1,13,20 +-1,10,68 +-1,15,5 +-1,25,1 +-1,26,1 +-1,23,1 +-1,16,4 +-1,19,2 +-1,33,2 diff --git a/figure_8/figure8-cd-only.py b/figure_8/figure8-cd-only.py index a62a8f59..02835ed4 100755 --- a/figure_8/figure8-cd-only.py +++ b/figure_8/figure8-cd-only.py @@ -16,12 +16,25 @@ from pyspark.sql import Window from pyspark.sql.types import * from decimal import * +CHECKDIR = "/home/claudio/google_2019/thesis_queries/figure_8/" + if len(sys.argv) is not 4: print(sys.argv[0] + " {cluster} {tmpdir} {maxram}") sys.exit() cluster=sys.argv[1] +if os.path.exists(CHECKDIR + cluster + "_figure8cd.json"): + print("already computed") + sys.exit() + +if os.path.exists(CHECKDIR + cluster + "_figure8cd_working"): + print("already in execution") + sys.exit() + +os.system("touch " + CHECKDIR + cluster + "_figure8cd_working") + + spark = pyspark.sql.SparkSession.builder \ .appName("task_slowdown") \ .config("spark.driver.maxResultSize", "128g") \ @@ -30,12 +43,15 @@ spark = pyspark.sql.SparkSession.builder \ .getOrCreate() sc = spark.sparkContext +def tabid(x): + return Decimal(x.collection_id) + Decimal(x.instance_index) / Decimal(2**64) + # # READING INSTANCE EVENTS DATA # dfepath = "/home/claudio/google_2019/instance_events/" + cluster + "/" + cluster + "_instance_events*.json.gz" +#dfepath = "/home/claudio/google_2019/instance_events/" + cluster + "/" + cluster + "_test.json" df = spark.read.json(dfepath) -df = df.limit(10000) # READING MACHINE EVENTS DATA, sort them and save them as broadcast variable print("Starting to read machine events...") @@ -56,7 +72,7 @@ print("Converting to broadcast variable...") dfm = sc.broadcast([tuple(r) for r in dfm.to_numpy()]) print("Done with machine events.") -df = dfe.rdd \ +df = df.rdd \ .filter(lambda x: x.time is not None and x.type is not None and x.machine_id is not None and x.instance_index is not None and x.collection_id is not None and x.resource_request is not None and x.resource_request.cpus is not None and x.resource_request.memory is not None) \ @@ -68,12 +84,12 @@ df = dfe.rdd \ def get_machine_time_resources(machine_id, time): def aux(i, j): if i == j: - return dfm.value[i] if dfm.value[i][1] == machine_id else None + return i if dfm.value[i][1] == machine_id else None elif i + 1 == j: if dfm.value[i][1] == machine_id: - return dfm.value[i] + return i elif dfm.value[j][1] == machine_id: - return dfm.value[j] + return j else: return None @@ -170,20 +186,19 @@ def fold_resobjs(ro1, ro2): ro1[k][kk] = [sum(x) for x in zip(ro1[k][kk], ro2[k][kk])] return ro1 -# TODO: partition by id and in the for-each-row -# function implement lookup to dfm.value to understand its memory capacity - import random result = df.rdd \ .groupBy(lambda x: x.mid) \ - .partitionBy(10000, lambda x: random.randint(0, 10000-1)) \ + .partitionBy(1000, lambda x: random.randint(0, 1000-1)) \ .map(for_each_joined) \ .fold(None, fold_resobjs) d = os.path.dirname(os.path.realpath(__file__)) -with open(d + "/" + cluster + "_figure8.json", "w") as f: +with open(d + "/" + cluster + "_figure8cd.json", "w") as f: json.dump(result, f) +os.system("rm " + CHECKDIR + cluster + "_figure8cd_working") + # vim: set ts=4 sw=4 et tw=120: