From 9367c293ae935873c47e3dc66ccba471bbd84131 Mon Sep 17 00:00:00 2001 From: "Claudio Maggioni (maggicl)" Date: Fri, 16 Apr 2021 10:29:34 +0000 Subject: [PATCH] More query results --- figure_7/figure7c.py | 119 +++++++++++++++++++++ figure_8/a_figure8.json | 1 + figure_8/a_figure8abef.json | 1 + figure_8/b_figure8abef.json | 1 + figure_8/c_figure8abef.json | 1 + figure_8/d_figure8abef.json | 1 + figure_8/e_figure8ab.json | 1 + figure_8/f_figure8ab.json | 1 + figure_8/figure8-ab-only.py | 118 +++++++++++++++++++++ figure_8/figure8-abef-only.py | 157 ++++++++++++++++++++++++++++ figure_8/figure8-cd-only.py | 190 ++++++++++++++++++++++++++++++++++ figure_8/h_figure8ab.json | 1 + 12 files changed, 592 insertions(+) create mode 100755 figure_7/figure7c.py create mode 100644 figure_8/a_figure8.json create mode 100644 figure_8/a_figure8abef.json create mode 100644 figure_8/b_figure8abef.json create mode 100644 figure_8/c_figure8abef.json create mode 100644 figure_8/d_figure8abef.json create mode 100644 figure_8/e_figure8ab.json create mode 100644 figure_8/f_figure8ab.json create mode 100755 figure_8/figure8-ab-only.py create mode 100755 figure_8/figure8-abef-only.py create mode 100755 figure_8/figure8-cd-only.py create mode 100644 figure_8/h_figure8ab.json diff --git a/figure_7/figure7c.py b/figure_7/figure7c.py new file mode 100755 index 00000000..6275f4c2 --- /dev/null +++ b/figure_7/figure7c.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +# coding: utf-8 + +import os +import json +import pandas as pd +import findspark +findspark.init() +import pyspark +import pyspark.sql +import sys +import gzip +from pyspark import AccumulatorParam +from pyspark.sql.functions import lit +from pyspark.sql import Window +from pyspark.sql.types import * +from decimal import * +import random + +if len(sys.argv) is not 4: + print(sys.argv[0] + " {cluster} {tmpdir} {maxram}") + sys.exit() + +cluster=sys.argv[1] + +spark = pyspark.sql.SparkSession.builder \ + .appName("task_slowdown") \ + .config("spark.driver.maxResultSize", "128g") \ + .config("spark.local.dir", sys.argv[2]) \ + .config("spark.driver.memory", sys.argv[3]) \ + .getOrCreate() +sc = spark.sparkContext + +# READING INSTANCE EVENTS DATA +dfepath = "/home/claudio/google_2019/instance_events/" + cluster + "/" + cluster + "_instance_events*.json.gz" +#dfepath = "/home/claudio/google_2019/instance_events/" + cluster + "/" + cluster + "_test.json" +df = spark.read.json(dfepath) + +def tabid(x): + return Decimal(x.collection_id) + Decimal(x.instance_index) / Decimal(2**64) + +def bucket_sum_per_termination(bucket, last_term_by_id): + result = {-1: None, 4: None, 5: None, 6: None, 7: None, 8: None} + for tid, vs in bucket.items(): + term = last_term_by_id[tid] + if result[term] is None: + result[term] = vs + else: + result[term] = [sum(x) for x in zip(result[term], vs)] + return result + +def tally_event(bucket, term, nexec): + if term not in bucket: + bucket[term] = {} + if nexec not in bucket[term]: + bucket[term][nexec] = 0 + bucket[term][nexec] += 1 + +def for_each_joined(x): + machine_id = x[0] + ts = x[1] + + ts = sorted(ts, key=lambda x: x["time"]) + in_execution = set() + chum = {} + + for t in ts: + in_execution.add(t["id"]) + tally_event(chum, t["term"], len(in_execution)) + if t["end"]: + in_execution.remove(t["id"]) + + return chum + +def fold_resobjs(ro1, ro2): + if not ro1.keys(): + return ro2 + elif ro2.keys(): + for k in set(ro1.keys()).union(set(ro2.keys())): + if k not in ro1: + ro1[k] = ro2[k] + elif k in ro2: + for kk in set(ro1[k].keys()).union(set(ro2[k].keys())): + if kk not in ro1[k]: + ro1[k][kk] = ro2[k][kk] + elif kk in ro2[k]: + ro1[k][kk] += ro2[k][kk] + return ro1 + +def mark_next(data): + ts = data[1] + ts = sorted(ts, key=lambda z: z[1]) + last_term = -1 + for i in range(0, len(ts)): + t = ts[i] + ts[i] = {"id": t[0], "time": t[1], "type": t[2], "mid": t[3], "end": (i == len(ts) -1 or t[3] != ts[i+1][3])} + if ts[i]["type"] >= 4 or ts[i]["type"] <= 8: + last_term = ts[i]["type"] + for t in ts: + t["term"] = last_term + return ts + +result = df.rdd \ + .filter(lambda x: x.time is not None and x.type is not None and + x.instance_index is not None and x.collection_id is not None) \ + .map(lambda x: [tabid(x), int(x.time), int(x.type), x.machine_id]) \ + .groupBy(lambda x: x[0]) \ + .flatMap(mark_next) \ + .groupBy(lambda x: x["mid"]) \ + .partitionBy(10000, lambda x: random.randint(0, 10000-1)) \ + .map(for_each_joined) \ + .fold({}, fold_resobjs) + +d = os.path.dirname(os.path.realpath(__file__)) + +with open(d + "/" + cluster + "_figure7c.json", "w") as f: + json.dump(result, f) + +# vim: set ts=4 sw=4 et tw=120: diff --git a/figure_8/a_figure8.json b/figure_8/a_figure8.json new file mode 100644 index 00000000..6e7f52f4 --- /dev/null +++ b/figure_8/a_figure8.json @@ -0,0 +1 @@ +{"rcpu": {"-1": [11792934, 100581, 119504, 33602], "4": [149466039, 3954384, 522279, 319171], "5": [30461383, 705982, 138442, 333102], "6": [152111586, 740741, 53960, 66869], "7": [273451772, 9013332, 1598241, 1251452], "8": [3637379, 106001, 59062, 23583]}, "rram": {"-1": [11916974, 84471, 11523, 33653], "4": [152162293, 1463695, 481913, 153972], "5": [31343967, 108823, 112227, 73892], "6": [136880563, 16017698, 62739, 12156], "7": [278881639, 3888697, 1611197, 933264], "8": [3727840, 74738, 6063, 17384]}, "rscpu": {"-1": [17686, 33931, 24226, 19646, 19606, 23959, 11907567], "4": [345377, 76462, 78162, 75465, 80602, 92738, 153513067], "5": [98104, 17584, 19982, 19529, 20228, 20879, 31442603], "6": [385085, 39535, 69156, 95234, 123505, 157547, 152103094], "7": [856729, 139265, 154106, 162797, 168047, 182857, 283650996], "8": [3169, 2773, 2464, 2334, 2164, 2065, 3811056]}, "rsram": {"-1": [17686, 27788, 27824, 22687, 21497, 20384, 11908755], "4": [345377, 64842, 88600, 90874, 106968, 121787, 153443425], "5": [98104, 19833, 24948, 25351, 27876, 27829, 31414968], "6": [385085, 38938, 83458, 160369, 311025, 355386, 151638895], "7": [856729, 147165, 190726, 204662, 218966, 232164, 283464385], "8": [3169, 2997, 2561, 2433, 2460, 2717, 3809688]}, "ucpu": {"-1": [65359, 11980186, 1059, 17, 0, 0, 0], "4": [20079611, 134178471, 3727, 60, 3, 0, 1], "5": [12413418, 19219959, 2058, 1262, 989, 586, 637], "6": [7877670, 145094583, 807, 8, 13, 10, 65], "7": [21046407, 264204898, 61360, 1623, 252, 126, 131], "8": [224410, 3598907, 2570, 53, 47, 15, 23]}, "uram": {"-1": [11681, 12034890, 50, 0, 0, 0, 0], "4": [936757, 153325001, 114, 1, 0, 0, 0], "5": [349268, 31289585, 56, 0, 0, 0, 0], "6": [175056, 152797241, 859, 0, 0, 0, 0], "7": [2281373, 283033270, 154, 0, 0, 0, 0], "8": [27953, 3798067, 5, 0, 0, 0, 0]}} \ No newline at end of file diff --git a/figure_8/a_figure8abef.json b/figure_8/a_figure8abef.json new file mode 100644 index 00000000..5bf0f113 --- /dev/null +++ b/figure_8/a_figure8abef.json @@ -0,0 +1 @@ +{"-1": {"rcpu": [0, 6827, 614, 14, 123, 166, 262, 40, 98, 0, 0, 0, 881, 0, 0, 0, 0, 0, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "rram": [0, 8329, 305, 46, 3, 0, 2, 271, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "wucpu": [0, 7669, 358, 141, 456, 350, 73, 88, 17, 3, 1, 4, 0, 0, 1, 2, 2, 1, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "wuram": [0, 8824, 139, 16, 156, 32, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "ucpu": [9, 7460, 505, 135, 357, 392, 125, 151, 18, 3, 1, 4, 0, 0, 1, 2, 2, 1, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "uram": [9, 8661, 290, 18, 157, 32, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "4": {"rcpu": [0, 56119448, 1390283, 105619, 23250, 17146, 214, 7, 0, 0, 94, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "rram": [0, 56176572, 1096672, 268346, 85429, 10038, 1661, 1004, 16339, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "wucpu": [0, 56702268, 713777, 195195, 39161, 3932, 1307, 302, 74, 23, 8, 3, 4, 9, 2, 0, 0, 3, 2, 2, 2, 0, 0, 0, 0, 2, 2, 2, 0, 4, 0, 3, 2, 2, 0, 0, 0, 0, 0, 0, 5], "wuram": [0, 57632517, 21056, 1431, 450, 311, 172, 71, 46, 24, 17, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "ucpu": [139027, 56206725, 844997, 295754, 114008, 35748, 10973, 4382, 2535, 967, 447, 228, 102, 65, 44, 21, 11, 3, 10, 4, 6, 5, 5, 2, 2, 0, 2, 2, 3, 4, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 10], "uram": [139027, 57090007, 305753, 110827, 5291, 3335, 856, 406, 224, 199, 54, 35, 16, 21, 9, 11, 10, 5, 2, 1, 0, 2, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "5": {"rcpu": [0, 30198355, 479714, 103275, 4187, 9376, 2189, 382, 94, 3, 0, 0, 0, 0, 0, 0, 0, 966, 0, 0, 0, 0, 0, 0, 298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "rram": [0, 30606552, 123012, 66537, 715, 298, 262, 116, 1049, 0, 0, 0, 0, 298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "wucpu": [0, 30765109, 22108, 4311, 3518, 2075, 1020, 347, 134, 52, 27, 21, 12, 18, 16, 2, 9, 15, 3, 9, 3, 4, 6, 3, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 7], "wuram": [0, 30794496, 3255, 268, 213, 203, 172, 78, 97, 35, 15, 6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "ucpu": [100259, 29099717, 1576172, 11276, 3538, 2415, 1834, 1338, 871, 568, 375, 181, 85, 44, 33, 6, 17, 16, 16, 6, 0, 11, 0, 6, 7, 0, 1, 0, 6, 9, 0, 0, 3, 0, 0, 0, 4, 3, 0, 0, 22], "uram": [100259, 30388193, 269869, 36459, 1663, 1224, 362, 188, 163, 111, 71, 76, 17, 31, 28, 15, 38, 17, 9, 0, 3, 1, 2, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0]}, "6": {"rcpu": [0, 103335833, 240609, 56722, 54338, 7331, 6307, 336, 832, 125, 32, 0, 3463, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "rram": [0, 89959199, 13697698, 39420, 5358, 21, 403, 1160, 2667, 0, 0, 0, 1, 3, 2, 2, 2, 2, 0, 2, 3, 0, 1, 2, 0, 3, 0, 1, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0], "wucpu": [0, 96770974, 6731982, 180544, 16441, 3312, 841, 563, 455, 298, 307, 128, 7, 4, 0, 2, 3, 3, 1, 0, 0, 1, 1, 0, 1, 0, 4, 4, 0, 2, 0, 3, 0, 1, 3, 1, 0, 0, 0, 3, 67], "wuram": [0, 103554344, 133743, 15179, 291, 207, 134, 435, 765, 855, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "ucpu": [250503, 88982127, 12029787, 1864231, 427456, 113106, 24355, 8213, 3242, 1303, 661, 326, 142, 119, 98, 54, 45, 55, 13, 9, 5, 6, 5, 0, 2, 0, 5, 2, 1, 2, 0, 2, 1, 1, 3, 1, 0, 0, 0, 3, 72], "uram": [250503, 91773475, 9888120, 1642833, 100661, 37393, 5621, 3213, 2142, 265, 67, 67, 92, 102, 70, 73, 81, 156, 103, 164, 17, 29, 47, 82, 146, 260, 159, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "7": {"rcpu": [0, 398926915, 12139342, 1994046, 566048, 129576, 410719, 76357, 76460, 210417, 95971, 45570, 152691, 8392, 33978, 134, 0, 112, 409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "rram": [0, 406243962, 5915482, 1904089, 254873, 265449, 184846, 29904, 68304, 104, 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 1, 0], "wucpu": [0, 403108440, 10477395, 712694, 239199, 95364, 66733, 76130, 23756, 24228, 19244, 9680, 5368, 3222, 2012, 1224, 755, 528, 312, 247, 154, 110, 65, 42, 42, 28, 19, 16, 14, 8, 13, 6, 17, 3, 1, 0, 3, 6, 5, 6, 48], "wuram": [0, 414371333, 457163, 27417, 6874, 2373, 789, 453, 463, 139, 66, 29, 15, 13, 5, 1, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "ucpu": [1120191, 395058209, 13623326, 3489187, 839481, 268406, 126643, 78900, 54850, 53214, 42726, 30900, 23823, 12963, 9948, 9091, 8824, 4273, 3032, 2505, 1826, 1356, 921, 673, 445, 317, 187, 142, 123, 131, 106, 60, 58, 54, 37, 24, 25, 14, 6, 10, 130], "uram": [1120191, 403999193, 8144181, 1298340, 168120, 86588, 36094, 6294, 3203, 1779, 1287, 698, 519, 292, 142, 99, 29, 17, 24, 11, 8, 5, 7, 11, 3, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "8": {"rcpu": [0, 32333715, 370459, 231812, 9210, 7142, 48029, 5797, 10547, 285, 77, 2260, 4318, 0, 483, 11, 0, 0, 0, 0, 0, 12, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "rram": [0, 31918662, 804953, 7224, 245289, 37025, 1562, 828, 7148, 1443, 11, 12, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "wucpu": [0, 32387567, 566808, 47137, 7031, 1948, 2218, 1487, 1070, 964, 1110, 888, 631, 434, 349, 262, 242, 208, 185, 153, 211, 204, 164, 164, 168, 190, 185, 156, 136, 123, 117, 120, 121, 90, 140, 93, 85, 66, 75, 77, 804], "wuram": [0, 32889598, 53778, 30839, 26885, 15838, 5168, 1821, 244, 4, 2, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "ucpu": [96161, 30458267, 2189089, 185207, 54917, 17908, 6222, 2416, 1431, 1326, 1279, 1159, 1040, 896, 741, 585, 398, 351, 236, 200, 215, 201, 160, 163, 216, 175, 190, 171, 151, 114, 145, 149, 158, 137, 187, 135, 113, 88, 93, 73, 1318], "uram": [96161, 31828504, 860616, 138077, 32740, 22559, 15125, 10224, 6019, 4037, 4283, 1381, 1645, 648, 149, 777, 236, 263, 316, 193, 111, 56, 22, 37, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0]}} \ No newline at end of file diff --git a/figure_8/b_figure8abef.json b/figure_8/b_figure8abef.json new file mode 100644 index 00000000..7680eb25 --- /dev/null +++ b/figure_8/b_figure8abef.json @@ -0,0 +1 @@ +{"-1": {"rcpu": [0, 36617, 2407, 79, 62, 22, 13, 13, 6, 11, 12, 19, 24, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "rram": [0, 39028, 119, 25, 37, 25, 18, 4, 4, 4, 0, 2, 9, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "wucpu": [0, 37227, 1777, 172, 35, 10, 2, 5, 16, 8, 24, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "wuram": [0, 39233, 28, 6, 2, 5, 2, 2, 5, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "ucpu": [0, 36693, 1974, 455, 62, 18, 7, 11, 17, 9, 28, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "uram": [0, 39023, 193, 34, 13, 9, 3, 0, 1, 0, 1, 1, 0, 2, 0, 3, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "4": {"rcpu": [0, 63035516, 1083022, 124363, 17685, 9853, 24, 12, 7, 4, 3, 42, 36, 2, 4, 0, 2, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "rram": [0, 64170924, 56652, 6896, 3895, 49, 28830, 3246, 117, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "wucpu": [0, 63922778, 308176, 35882, 2529, 736, 283, 100, 40, 14, 2, 8, 9, 3, 0, 4, 0, 2, 2, 0, 8, 8, 4, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 17], "wuram": [0, 64264621, 5892, 26, 12, 6, 10, 8, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "ucpu": [0, 63636686, 513243, 95146, 19555, 4180, 898, 388, 214, 94, 70, 36, 28, 11, 0, 4, 4, 4, 0, 2, 5, 4, 0, 0, 0, 0, 0, 0, 4, 0, 8, 4, 0, 0, 0, 4, 0, 0, 0, 4, 17], "uram": [0, 64135281, 112972, 19965, 1000, 1173, 113, 14, 41, 34, 9, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "5": {"rcpu": [0, 5584934, 1145428, 112557, 25511, 85170, 966, 676, 29, 2, 4476, 2, 2, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "rram": [0, 6794781, 91543, 7237, 61042, 258, 3967, 384, 522, 1, 11, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0], "wucpu": [0, 6902144, 45340, 9694, 704, 698, 330, 259, 141, 75, 22, 31, 22, 6, 13, 1, 0, 9, 8, 4, 7, 8, 0, 4, 0, 0, 15, 0, 4, 12, 6, 5, 4, 0, 12, 0, 2, 0, 0, 1, 180], "wuram": [0, 6956236, 2463, 387, 516, 36, 41, 29, 17, 10, 3, 9, 3, 3, 2, 2, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], "ucpu": [0, 6884225, 49740, 17083, 5922, 1094, 433, 298, 187, 121, 115, 98, 59, 27, 17, 13, 9, 11, 4, 25, 7, 12, 0, 0, 2, 5, 7, 4, 11, 0, 2, 5, 6, 0, 4, 0, 4, 2, 0, 0, 209], "uram": [0, 6886178, 58540, 13183, 655, 402, 321, 345, 22, 24, 52, 15, 4, 8, 3, 2, 2, 1, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0]}, "6": {"rcpu": [0, 70946750, 1460974, 145467, 583748, 23631, 9171, 2160, 1143, 2611, 3272, 383, 5394, 156, 124, 92, 9382, 27306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "rram": [0, 64060185, 8218414, 836885, 88300, 401, 1753, 4818, 6745, 4260, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "wucpu": [0, 70892638, 2007753, 248911, 36834, 5218, 3350, 3056, 2532, 2074, 1916, 1704, 1284, 1389, 1651, 2139, 3664, 2916, 1265, 550, 244, 118, 80, 28, 46, 16, 14, 16, 10, 16, 6, 14, 26, 12, 8, 8, 4, 6, 5, 8, 235], "wuram": [0, 73074828, 104634, 38808, 1350, 966, 732, 317, 71, 20, 7, 3, 4, 8, 2, 4, 0, 4, 2, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "ucpu": [0, 68536378, 3837220, 612818, 139198, 47664, 15107, 4885, 3133, 2537, 2404, 2093, 1710, 1525, 1676, 1841, 2345, 2370, 1393, 723, 427, 410, 616, 509, 347, 288, 365, 509, 435, 160, 70, 70, 46, 28, 30, 20, 50, 14, 8, 24, 318], "uram": [0, 68778480, 3712314, 611275, 50982, 26067, 24347, 12995, 3180, 455, 475, 423, 225, 184, 88, 58, 76, 63, 34, 23, 16, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "7": {"rcpu": [0, 849246108, 37998493, 5614432, 3627821, 1130870, 530272, 450694, 117285, 1570717, 22163, 12146, 16837, 10294, 6619, 32381, 15393, 47062, 366, 188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "rram": [0, 882282897, 15523833, 1309360, 562724, 215422, 119893, 119043, 314387, 57, 2076, 21, 39, 58, 161, 5, 62, 6, 59, 17, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0], "wucpu": [0, 881806920, 16373913, 1418912, 455750, 231843, 75663, 22951, 12184, 7597, 5558, 4627, 4231, 5448, 5825, 5754, 4793, 3599, 1828, 969, 492, 291, 237, 165, 102, 61, 65, 44, 41, 26, 24, 13, 8, 2, 8, 4, 5, 0, 8, 4, 176], "wuram": [0, 898910626, 1414300, 80296, 24777, 11150, 3509, 2025, 1036, 735, 555, 288, 185, 151, 167, 123, 81, 31, 26, 35, 22, 7, 1, 8, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0], "ucpu": [0, 871556394, 22895815, 4223693, 945867, 394283, 196189, 96694, 49626, 24038, 13159, 8757, 6423, 6135, 5880, 5712, 4652, 3557, 2389, 1677, 1149, 1052, 1194, 1234, 1073, 719, 646, 679, 555, 248, 103, 69, 56, 58, 38, 21, 26, 18, 16, 10, 237], "uram": [0, 885730595, 11338333, 2202872, 789313, 252142, 62596, 44358, 17447, 5420, 2297, 1546, 955, 993, 549, 200, 145, 101, 70, 74, 34, 29, 17, 34, 0, 4, 2, 8, 0, 4, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0]}, "8": {"rcpu": [0, 16502145, 1096581, 300241, 77731, 6797, 67394, 16937, 211, 495, 28, 921, 13721, 1, 0, 0, 3, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "rram": [0, 16322178, 1715899, 29255, 2000, 155, 10, 11, 13697, 4, 0, 8, 0, 6, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0], "wucpu": [0, 17551519, 461581, 54665, 4467, 956, 583, 792, 1946, 2002, 1930, 1766, 651, 212, 64, 33, 13, 2, 4, 9, 0, 7, 0, 6, 0, 0, 0, 0, 0, 2, 7, 2, 0, 0, 0, 0, 0, 0, 0, 0, 17], "wuram": [0, 18021499, 48915, 2377, 270, 2177, 5237, 2531, 224, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0], "ucpu": [0, 17103008, 783044, 144151, 31331, 8628, 2519, 610, 656, 858, 1288, 1864, 1582, 1100, 995, 842, 421, 156, 84, 16, 19, 13, 4, 1, 5, 3, 0, 2, 0, 0, 0, 0, 6, 2, 0, 0, 0, 0, 0, 0, 28], "uram": [0, 17125672, 773425, 141779, 16955, 7471, 7352, 1153, 697, 1138, 1303, 1076, 701, 422, 330, 434, 608, 810, 629, 562, 356, 241, 102, 16, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]}} \ No newline at end of file diff --git a/figure_8/c_figure8abef.json b/figure_8/c_figure8abef.json new file mode 100644 index 00000000..27a3fcee --- /dev/null +++ b/figure_8/c_figure8abef.json @@ -0,0 +1 @@ +{"-1": {"rcpu": [0, 115406, 8826, 660, 200, 143, 62, 26, 11, 8, 5, 200, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "rram": [0, 123399, 1275, 506, 118, 172, 16, 11, 39, 15, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "wucpu": [0, 123341, 1249, 179, 138, 160, 142, 132, 119, 65, 25, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "wuram": [0, 124824, 481, 92, 110, 10, 4, 5, 5, 0, 20, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "ucpu": [0, 119701, 3740, 722, 447, 318, 181, 154, 144, 77, 41, 7, 4, 2, 4, 5, 3, 2, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "uram": [0, 119566, 4553, 855, 236, 87, 53, 134, 30, 1, 25, 4, 4, 1, 0, 0, 2, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "4": {"rcpu": [0, 95929301, 1764241, 166698, 328051, 37739, 7810, 7359, 7531, 10, 127, 18, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "rram": [0, 97945744, 231197, 15321, 13118, 10359, 5269, 2583, 25292, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "wucpu": [0, 97312878, 769859, 136043, 24588, 3493, 1160, 484, 172, 62, 34, 14, 8, 1, 3, 1, 1, 6, 8, 4, 0, 3, 0, 0, 0, 0, 0, 3, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60], "wuram": [0, 98237906, 6478, 2306, 1674, 443, 56, 20, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "ucpu": [231, 96433243, 1311897, 286687, 119822, 55072, 21167, 9541, 5401, 2666, 1280, 746, 431, 233, 157, 92, 38, 28, 26, 18, 8, 2, 3, 7, 2, 2, 0, 2, 5, 1, 10, 0, 0, 0, 0, 0, 2, 0, 0, 3, 66], "uram": [231, 97833849, 299274, 77857, 15191, 12676, 6282, 627, 517, 642, 588, 416, 366, 119, 167, 36, 14, 5, 19, 4, 8, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "5": {"rcpu": [0, 59753951, 6256415, 182544, 53939, 30817, 42094, 370236, 311, 890, 7750, 163, 50, 2982, 888, 0, 51, 0, 0, 0, 107, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "rram": [0, 66297761, 341483, 12871, 33010, 2251, 1147, 6848, 5049, 1538, 18, 0, 0, 0, 1339, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "wucpu": [0, 66610531, 33172, 35183, 9907, 8631, 4352, 986, 215, 85, 68, 20, 19, 15, 11, 7, 1, 0, 0, 7, 8, 10, 0, 0, 4, 0, 0, 6, 4, 7, 4, 0, 6, 0, 2, 0, 0, 0, 8, 0, 47], "wuram": [0, 66683161, 10398, 6707, 1131, 387, 255, 137, 201, 152, 170, 153, 119, 89, 90, 66, 70, 13, 10, 5, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "ucpu": [0, 65941821, 642027, 56795, 18774, 14445, 8786, 6975, 5265, 2837, 2160, 1718, 898, 315, 122, 97, 38, 28, 38, 8, 8, 12, 12, 0, 7, 7, 16, 2, 0, 8, 6, 14, 6, 0, 2, 0, 2, 0, 4, 1, 62], "uram": [0, 66463252, 175203, 39028, 10816, 4913, 4780, 2160, 559, 804, 439, 252, 268, 182, 172, 145, 151, 62, 39, 28, 15, 15, 5, 10, 11, 3, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "6": {"rcpu": [0, 142274582, 442959, 167933, 208055, 12561, 47101, 2976, 2658, 10507, 1540, 6726, 349, 19549, 281, 5, 60, 20, 43, 47, 20, 18, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "rram": [0, 123382535, 19476722, 70662, 246659, 11359, 2584, 4076, 2337, 150, 1, 18, 477, 144, 25, 25, 63, 15, 0, 117, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "wucpu": [0, 139520802, 3232068, 382343, 38122, 5814, 5168, 4546, 4051, 3034, 1216, 411, 160, 82, 43, 18, 23, 18, 15, 10, 12, 4, 5, 2, 2, 0, 4, 0, 0, 0, 0, 1, 0, 1, 2, 0, 2, 0, 0, 0, 18], "wuram": [0, 143077619, 97583, 14669, 6300, 391, 296, 152, 141, 100, 131, 151, 155, 105, 99, 75, 25, 0, 1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "ucpu": [311, 130892443, 9309337, 1775544, 656087, 282686, 134513, 70690, 38389, 17768, 9546, 5794, 2716, 1109, 430, 161, 124, 90, 57, 53, 37, 18, 20, 11, 7, 6, 8, 6, 3, 0, 0, 3, 0, 2, 0, 0, 7, 1, 0, 0, 20], "uram": [311, 127080930, 12050896, 3138744, 521653, 331667, 62438, 6164, 1971, 961, 997, 371, 259, 160, 151, 144, 41, 16, 10, 17, 4, 3, 3, 3, 43, 0, 0, 0, 0, 0, 15, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "7": {"rcpu": [0, 1730361316, 167990795, 11359471, 1864455, 836513, 654391, 342387, 163557, 700362, 767500, 92554, 38170, 73677, 38837, 184, 8324, 32, 2995, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "rram": [0, 1791317165, 70981623, 18749359, 16277270, 8505621, 3189553, 1928272, 4034985, 46244, 159878, 36491, 43759, 18099, 4569, 36, 1370, 3, 0, 0, 0, 1, 1050, 0, 0, 0, 0, 192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "wucpu": [0, 1843844927, 65430823, 3464808, 802623, 437707, 340168, 251061, 209503, 174337, 126325, 81008, 48476, 31341, 20740, 13383, 8197, 4417, 2366, 1306, 727, 433, 240, 135, 96, 92, 83, 33, 26, 37, 12, 24, 15, 10, 4, 2, 2, 2, 0, 9, 42], "wuram": [0, 1899314098, 9185689, 5283518, 891198, 361975, 166698, 64116, 18825, 3433, 4553, 626, 299, 260, 98, 95, 27, 8, 10, 6, 0, 0, 0, 2, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "ucpu": [0, 1752358508, 115425911, 28569123, 10754330, 4548937, 1394461, 697187, 386369, 295485, 230193, 174599, 131137, 99706, 74959, 51587, 34577, 21954, 14170, 9459, 6531, 4817, 3219, 2160, 1470, 968, 732, 650, 505, 395, 368, 292, 214, 152, 118, 78, 50, 20, 24, 2, 123], "uram": [0, 1838018042, 54024128, 12012486, 4493393, 2499363, 1991212, 1141052, 464230, 199520, 125967, 95643, 77561, 63447, 28782, 23112, 12910, 7007, 5809, 4713, 2876, 1380, 903, 880, 367, 375, 134, 85, 66, 52, 18, 19, 4, 0, 2, 0, 2, 0, 0, 0, 0]}, "8": {"rcpu": [0, 40865707, 1226220, 383303, 177431, 56905, 38763, 6191, 10168, 11245, 6032, 2912, 957, 80, 0, 31, 1552, 0, 0, 0, 0, 25, 0, 0, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0], "rram": [0, 37528797, 4984727, 56915, 54976, 92637, 46780, 15988, 3897, 944, 150, 29, 491, 25, 1645, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "wucpu": [0, 41460725, 1082488, 190548, 30256, 10446, 4273, 2463, 2047, 1716, 1374, 916, 482, 179, 54, 31, 9, 3, 2, 0, 0, 4, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11], "wuram": [0, 42663198, 87448, 17666, 15594, 3032, 786, 176, 65, 30, 17, 12, 2, 0, 4, 1, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "ucpu": [0, 39121207, 2590250, 596893, 230238, 114177, 56026, 33614, 19162, 10124, 6191, 3529, 1942, 1231, 910, 519, 370, 344, 229, 167, 106, 108, 81, 105, 161, 71, 45, 45, 50, 50, 20, 16, 6, 8, 12, 6, 6, 0, 2, 0, 13], "uram": [0, 38440754, 3147078, 889122, 168852, 93592, 25377, 5997, 6767, 2332, 1955, 1628, 2658, 794, 355, 277, 183, 57, 62, 38, 18, 14, 16, 86, 8, 2, 7, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0]}} \ No newline at end of file diff --git a/figure_8/d_figure8abef.json b/figure_8/d_figure8abef.json new file mode 100644 index 00000000..40fdfa79 --- /dev/null +++ b/figure_8/d_figure8abef.json @@ -0,0 +1 @@ +{"-1": {"rcpu": [0, 31317, 2871, 346, 126, 21, 8, 9, 17, 3, 2, 49, 7, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "rram": [0, 33608, 830, 114, 179, 39, 3, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "wucpu": [0, 34130, 302, 152, 69, 40, 39, 23, 14, 14, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "wuram": [0, 34609, 155, 6, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "ucpu": [5, 33423, 725, 140, 131, 108, 89, 40, 21, 18, 26, 20, 7, 12, 10, 9, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "uram": [5, 33909, 564, 164, 80, 30, 6, 12, 6, 0, 1, 2, 4, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "4": {"rcpu": [0, 86338952, 1289094, 232847, 18965, 41096, 28043, 572, 1786, 158, 103, 134, 0, 1, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "rram": [0, 87681621, 169299, 13838, 9126, 37745, 29689, 9613, 817, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "wucpu": [0, 86491213, 1196885, 190259, 38978, 17051, 10962, 3211, 1178, 788, 326, 178, 50, 34, 18, 28, 16, 41, 14, 24, 38, 26, 28, 8, 26, 34, 16, 32, 16, 16, 6, 14, 8, 14, 12, 6, 0, 18, 14, 4, 168], "wuram": [0, 87916356, 16627, 10098, 6095, 2346, 231, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "ucpu": [13991981, 72495717, 1073758, 198638, 92081, 44917, 16993, 9944, 8576, 6839, 4542, 2481, 1206, 934, 812, 603, 378, 263, 190, 106, 42, 48, 24, 18, 14, 19, 4, 18, 24, 27, 16, 14, 36, 16, 12, 16, 18, 20, 14, 0, 399], "uram": [13991981, 73572313, 261375, 85330, 16684, 10288, 3925, 1647, 1299, 1113, 1165, 1262, 1083, 863, 672, 325, 182, 66, 29, 26, 34, 14, 22, 16, 8, 12, 10, 10, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0]}, "5": {"rcpu": [0, 19067312, 1454226, 304826, 788803, 54770, 11722, 4516, 4083, 327, 3657, 116, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "rram": [0, 20491662, 705618, 36610, 397543, 49687, 4567, 1083, 2690, 94, 62, 141, 4655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "wucpu": [0, 21566620, 94202, 14657, 7148, 4375, 1325, 446, 329, 273, 323, 323, 280, 300, 281, 251, 182, 186, 235, 166, 148, 156, 142, 127, 124, 105, 98, 56, 108, 59, 86, 48, 84, 75, 29, 23, 44, 42, 64, 40, 852], "wuram": [0, 21637135, 43168, 10717, 2847, 188, 151, 27, 139, 36, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "ucpu": [180076, 21273260, 156957, 39293, 15622, 9549, 5405, 3615, 2325, 1127, 655, 423, 331, 280, 187, 200, 183, 230, 162, 170, 186, 178, 174, 129, 164, 128, 98, 88, 102, 111, 107, 88, 97, 73, 104, 94, 85, 87, 53, 58, 2158], "uram": [180076, 21369917, 78865, 28384, 14737, 10001, 5188, 2765, 1438, 773, 514, 918, 505, 64, 19, 23, 29, 18, 71, 14, 7, 0, 23, 60, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0]}, "6": {"rcpu": [0, 244568097, 520933, 117331, 128437, 14263, 4282, 9141, 2826, 2052, 455, 908, 154, 2100, 3193, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "rram": [0, 198845175, 46439037, 18890, 59056, 177, 917, 8691, 1030, 936, 0, 165, 103, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "wucpu": [0, 229414142, 14520987, 1280675, 140990, 9999, 2908, 1390, 653, 440, 314, 361, 347, 204, 127, 37, 7, 4, 2, 0, 4, 0, 2, 0, 2, 1, 12, 8, 6, 6, 10, 2, 0, 13, 2, 4, 4, 4, 2, 15, 494], "wuram": [0, 245188551, 161099, 22139, 479, 571, 1121, 61, 153, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "ucpu": [137189955, 96966304, 8592890, 1751992, 529414, 198989, 80499, 33651, 15062, 6751, 3108, 1567, 839, 476, 294, 285, 186, 232, 213, 235, 192, 163, 116, 73, 67, 14, 10, 2, 1, 2, 2, 2, 0, 3, 0, 0, 0, 8, 4, 0, 577], "uram": [137189955, 87908308, 14789062, 4502931, 562015, 323624, 70473, 20411, 1976, 1207, 428, 226, 332, 790, 403, 475, 1265, 140, 11, 14, 25, 14, 91, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "7": {"rcpu": [0, 1255736209, 40540974, 4818279, 2118006, 919367, 596973, 281659, 68949, 109296, 153854, 103410, 1518, 36160, 56115, 0, 317, 13, 0, 11, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "rram": [0, 1239868887, 38508088, 10675771, 5258697, 3357623, 1589708, 1162987, 4956534, 82794, 20713, 38728, 16555, 3164, 870, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "wucpu": [0, 1270486705, 31442540, 2164684, 539651, 294837, 202860, 125643, 100395, 78871, 49737, 27123, 13445, 6717, 3412, 1920, 1042, 535, 295, 221, 115, 70, 28, 23, 10, 28, 21, 15, 12, 11, 19, 0, 4, 4, 10, 12, 6, 2, 0, 8, 88], "wuram": [0, 1296936514, 5548951, 1277069, 808984, 299126, 459720, 188435, 17519, 4075, 548, 101, 46, 16, 13, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "ucpu": [147245588, 1082057265, 54995150, 14232123, 4009064, 1373777, 482446, 278968, 190656, 155481, 112435, 83276, 67808, 58591, 50384, 39727, 31363, 23431, 16229, 10869, 8063, 5520, 3731, 2505, 1967, 1276, 902, 673, 444, 321, 238, 151, 118, 105, 88, 38, 40, 18, 13, 10, 267], "uram": [147245588, 1116778865, 28056977, 7702553, 3104585, 1032437, 582519, 303165, 210094, 176791, 102932, 72725, 52915, 29217, 19978, 18558, 11899, 11577, 13138, 7329, 2597, 1504, 1085, 1059, 367, 265, 157, 106, 63, 46, 14, 8, 6, 0, 0, 0, 0, 0, 0, 0, 0]}, "8": {"rcpu": [0, 61555384, 1568512, 1279765, 86843, 25877, 8242, 15823, 7025, 2839, 1381, 3921, 372, 851, 48, 3, 2, 0, 0, 0, 0, 49, 0, 0, 412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0], "rram": [0, 63979744, 394910, 46730, 37409, 39799, 21392, 14728, 2077, 17774, 2, 57, 404, 18, 1887, 0, 422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "wucpu": [0, 63476264, 637812, 387950, 37783, 9703, 4679, 1855, 620, 157, 85, 27, 39, 43, 30, 21, 30, 14, 14, 16, 23, 30, 7, 14, 21, 14, 9, 14, 0, 7, 7, 7, 0, 0, 7, 10, 0, 2, 18, 2, 19], "wuram": [0, 64488175, 41804, 12828, 4307, 9363, 364, 354, 127, 23, 6, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "ucpu": [521620, 61039156, 1839638, 491737, 425429, 105456, 70541, 26315, 16383, 9421, 4676, 2653, 1457, 962, 569, 277, 209, 153, 88, 48, 30, 56, 66, 32, 13, 16, 28, 18, 11, 16, 9, 14, 0, 20, 14, 14, 7, 21, 21, 7, 152], "uram": [521620, 60880390, 2478201, 464129, 133591, 45222, 12897, 4659, 3908, 2096, 1244, 1221, 744, 692, 5841, 155, 226, 45, 23, 3, 20, 9, 0, 9, 2, 4, 2, 400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}} \ No newline at end of file diff --git a/figure_8/e_figure8ab.json b/figure_8/e_figure8ab.json new file mode 100644 index 00000000..5a92256f --- /dev/null +++ b/figure_8/e_figure8ab.json @@ -0,0 +1 @@ +{"-1": {"rcpu": [0, 12076, 1006, 490, 838, 219, 40, 11, 0, 0, 36, 55, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "rram": [0, 12973, 1572, 168, 19, 20, 3, 4, 32, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "4": {"rcpu": [0, 20347796, 902125, 766341, 26926, 104874, 13456, 2521, 0, 0, 11389, 956, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "rram": [0, 22005994, 148318, 19943, 1521, 48, 2, 560, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "5": {"rcpu": [0, 15679371, 716841, 595324, 1126, 40010, 2465, 117, 61, 0, 0, 995, 491604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "rram": [0, 16429871, 1078605, 6649, 7031, 4944, 763, 2, 14, 0, 0, 14, 0, 1, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "6": {"rcpu": [0, 10124332, 15166033, 3453386, 596140, 255252, 70171, 10774, 540, 565, 11824, 973367, 31434, 33764, 36, 0, 24, 3, 21, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "rram": [0, 28182669, 1112336, 1290233, 63083, 11158, 905, 14117, 51153, 1982, 28, 47, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "7": {"rcpu": [0, 1319563158, 65452181, 16179658, 4864588, 1982307, 338136, 223834, 618, 1525, 10678, 504897, 367755, 48, 0, 1829, 389, 0, 0, 0, 0, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "rram": [0, 1377875241, 25806542, 2397591, 2730903, 554881, 38638, 3962, 19351, 19349, 39671, 3808, 1724, 0, 0, 0, 11, 0, 0, 0, 0, 61, 0, 0, 9, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0]}, "8": {"rcpu": [0, 4, 1411, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "rram": [0, 1415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}} \ No newline at end of file diff --git a/figure_8/f_figure8ab.json b/figure_8/f_figure8ab.json new file mode 100644 index 00000000..8293cbaf --- /dev/null +++ b/figure_8/f_figure8ab.json @@ -0,0 +1 @@ +{"-1": {"rcpu": [0, 27311, 2244, 438, 53, 423, 13, 22, 1, 0, 10, 29, 4, 1, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "rram": [0, 30210, 221, 47, 26, 74, 0, 2, 21, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "4": {"rcpu": [0, 72654607, 2035364, 2266839, 32152, 211157, 68608, 61548, 24893, 24946, 10, 165, 75, 54, 1346, 21, 12, 14, 4, 4, 2, 0, 0, 2, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "rram": [0, 76307431, 605581, 279503, 81292, 8860, 25947, 69369, 1647, 1, 1126, 1097, 22, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "5": {"rcpu": [0, 10972613, 1084557, 407277, 19766, 120268, 566, 11370, 109, 3736, 77, 17, 3261, 80, 0, 0, 39, 3, 7, 11, 2, 2, 35, 21, 722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "rram": [0, 12340674, 259595, 4623, 3996, 5073, 1030, 1616, 3156, 1553, 368, 97, 2602, 152, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "6": {"rcpu": [0, 71728547, 3842669, 738222, 152925, 244881, 33316, 39901, 31102, 35512, 28782, 36532, 49808, 24386, 20584, 21132, 24488, 30146, 6415, 1838, 1066, 862, 1062, 411, 3347, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "rram": [0, 66248694, 10293282, 504401, 21727, 8463, 7719, 3685, 2322, 2341, 720, 48, 2347, 1372, 644, 0, 66, 7, 6, 0, 26, 38, 9, 3, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "7": {"rcpu": [0, 921464185, 80650116, 11069588, 8057696, 2113207, 278409, 300672, 209361, 216900, 121828, 281065, 290973, 27617, 72253, 1859, 2603, 552, 3511, 1317, 7703, 354, 751, 275, 4215, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "rram": [0, 992546443, 25626422, 4909105, 709766, 407857, 224788, 300932, 122026, 50524, 37312, 29839, 197998, 219, 7432, 6283, 9, 2, 5, 1, 0, 33, 0, 4, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "8": {"rcpu": [0, 5721037, 0, 22602, 20945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "rram": [0, 5028956, 735628, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}} \ No newline at end of file diff --git a/figure_8/figure8-ab-only.py b/figure_8/figure8-ab-only.py new file mode 100755 index 00000000..28124e15 --- /dev/null +++ b/figure_8/figure8-ab-only.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +# coding: utf-8 + +import os +import json +import pandas as pd +import findspark +findspark.init() +import pyspark +import pyspark.sql +import sys +import gzip +from pyspark import AccumulatorParam +from pyspark.sql.functions import lit +from pyspark.sql import Window +from pyspark.sql.types import * +from decimal import * + +if len(sys.argv) is not 5: + print(sys.argv[0] + " {cluster} {tmpdir} {maxram} {basedir}") + sys.exit() + +cluster=sys.argv[1] + +spark = pyspark.sql.SparkSession.builder \ + .appName("task_slowdown") \ + .config("spark.driver.maxResultSize", "128g") \ + .config("spark.local.dir", sys.argv[2]) \ + .config("spark.driver.memory", sys.argv[3]) \ + .getOrCreate() +sc = spark.sparkContext + +# READING INSTANCE EVENTS DATA +dfepath = sys.argv[4] + "/" + cluster + "/" + cluster + "_instance_events*.json.gz" +#dfepath = sys.argv[4] + "/" + cluster + "/" + cluster + "_test.json" +df = spark.read.json(dfepath) + +#df = df.limit(10000) + +def tabid(x): + return Decimal(x.collection_id) + Decimal(x.instance_index) / Decimal(2**64) + + +def increment_reserv_bucket(bucket, value): + if value < 0: + idx = 0 + else: + idx = 40 if value >= 1 else (int(value * 40) + 1) + bucket[idx] += 1 + +def for_each_joined(var): + task_id = var[0] + ts = var[1] + + term = -1 + ts = sorted(ts, key=lambda x: -1 if x["time"] is None else x["time"]) + + cpu_request = [0] * 42 + ram_request = [0] * 42 + + cut = False + for i in range(0, len(ts)): + if ts[i]["time"] is not None: + cut = True + ts = ts[i:len(ts)] + cpu_request[41] = i + ram_request[41] = i + break + + if not cut: + raise Exception('all times are none') + + for i, t in enumerate(ts): + increment_reserv_bucket(cpu_request, t["rcpu"]) + increment_reserv_bucket(ram_request, t["rram"]) + + if t["type"] >= 4 and t["type"] <= 8: + term = t["type"] + + res = {-1: None, 4: None, 5: None, 6: None, 7: None, 8: None} + res[term] = {'rcpu': cpu_request, 'rram': ram_request} + + return res + +def fold_resobjs(ro1, ro2): + if ro1 is None: + return ro2 + elif ro2 is None: + return ro1 + else: + for k in ro1.keys(): + if ro1[k] is None: + ro1[k] = ro2[k] + elif ro2[k] is not None: + for kk in ro1[k].keys(): + if ro1[k][kk] is None: + ro1[k][kk] = ro2[k][kk] + elif ro2[k][kk] is not None: + ro1[k][kk] = [sum(x) for x in zip(ro1[k][kk], ro2[k][kk])] + return ro1 + +result = df.rdd \ + .filter(lambda x: x.time is not None and x.type is not None and x.machine_id is not None and + x.instance_index is not None and x.collection_id is not None and x.resource_request is not None and + x.resource_request.cpus is not None and x.resource_request.memory is not None) \ + .map(lambda x: {"id": tabid(x), "time": int(x.time), "type": int(x.type), + "rcpu": Decimal(x.resource_request.cpus), "rram": Decimal(x.resource_request.memory), + "mid": x.machine_id}) \ + .groupBy(lambda x: x["id"]) \ + .map(for_each_joined) \ + .fold(None, fold_resobjs) + +d = os.path.dirname(os.path.realpath(__file__)) + +with open(d + "/" + cluster + "_figure8ab.json", "w") as f: + json.dump(result, f) + +# vim: set ts=4 sw=4 et tw=120: diff --git a/figure_8/figure8-abef-only.py b/figure_8/figure8-abef-only.py new file mode 100755 index 00000000..32a6ce8b --- /dev/null +++ b/figure_8/figure8-abef-only.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +# coding: utf-8 + +import os +import json +import pandas as pd +import findspark +findspark.init() +import pyspark +import pyspark.sql +import sys +import gzip +from pyspark import AccumulatorParam +from pyspark.sql.functions import lit +from pyspark.sql import Window +from pyspark.sql.types import * +from decimal import * + +if len(sys.argv) is not 5: + print(sys.argv[0] + " {cluster} {tmpdir} {maxram} {joindir}") + sys.exit() + +joindir=sys.argv[4] + +cluster=sys.argv[1] + +spark = pyspark.sql.SparkSession.builder \ + .appName("task_slowdown") \ + .config("spark.driver.maxResultSize", sys.argv[3]) \ + .config("spark.local.dir", sys.argv[2]) \ + .config("spark.driver.memory", sys.argv[3]) \ + .getOrCreate() +sc = spark.sparkContext + +df = spark.read.parquet(joindir + "/figure-8-join-" + cluster + ".parquet") + +# READING MACHINE EVENTS DATA, sort them and save them as broadcast variable +print("Starting to read machine events...") +dfm = pd.read_csv("~/google_2019/machine_events/" + cluster + "_machine_events.csv", converters={ + 'time': lambda x: -1 if x == '' else int(x), + 'machine_id': lambda x: str(x), + 'capacity.cpus': lambda x: -1 if x == '' else Decimal(x), + 'capacity.memory': lambda x: -1 if x == '' else Decimal(x)}) +print("Dropping remove events...") +dfm = dfm[(dfm.type!=2)&(dfm.time!=-1)&(dfm["capacity.cpus"]!=-1)&(dfm["capacity.memory"]!=-1)] +print("Dropping missing data events...") +dfm = dfm[dfm.missing_data_reason.isnull()] +print("Projecting on useful columns...") +dfm = dfm[["time", "machine_id", "capacity.cpus", "capacity.memory"]] +print("Sorting by time...") +dfm = dfm.sort_values(by=["machine_id", "time"]) +print("Converting to broadcast variable...") +dfm = sc.broadcast([tuple(r) for r in dfm.to_numpy()]) +print("Done with machine events.") + +def get_machine_time_resources(machine_id, time): + def aux(i, j): + if i == j: + return dfm.value[i] if dfm.value[i][1] == machine_id else None + elif i + 1 == j: + if dfm.value[i][1] == machine_id: + return dfm.value[i] + elif dfm.value[j][1] == machine_id: + return dfm.value[j] + else: + return None + + mid = (i + j) // 2 + + if dfm.value[mid][1] > machine_id: + return aux(i, mid - 1) + elif dfm.value[mid][1] < machine_id: + return aux(mid + 1, j) + elif dfm.value[mid][0] > time: + return aux(i, mid) + elif dfm.value[mid][0] < time: + return aux(mid, j) + else: + return dfm.value[mid] + + return aux(0, len(dfm.value)-1) + +def increment_reserv_bucket(bucket, value): + if value < 0: + idx = 0 + else: + idx = 40 if value >= 1 else (int(value * 40) + 1) + bucket[idx] += 1 + +def for_each_joined(x): + task_id = x[0] + ts = x[1] + + term = -1 + ts = sorted(ts, key=lambda x: x.time) + + cpu_util = [0] * 41 + cpu_util_wr = [0] * 41 + ram_util = [0] * 41 + ram_util_wr = [0] * 41 + cpu_request = [0] * 41 + ram_request = [0] * 41 + + for i, t in enumerate(ts): + machine_log = get_machine_time_resources(t.mid, t.time) + if machine_log is not None: + util_cpu = t.acpu / machine_log[2] + util_ram = t.aram / machine_log[3] + else: + util_cpu = -1 + util_ram = -1 + # 8a-b + increment_reserv_bucket(cpu_request, t.rcpu) + increment_reserv_bucket(ram_request, t.rram) + # 8e-f (wrong old version) + increment_reserv_bucket(cpu_util_wr, t.acpu) + increment_reserv_bucket(ram_util_wr, t.aram) + # 8e-f + increment_reserv_bucket(cpu_util, util_cpu) + increment_reserv_bucket(ram_util, util_ram) + + if t.type >= 4 and t.type <= 8: + term = t.type + + res = {-1: None, 4: None, 5: None, 6: None, 7: None, 8: None} + res[term] = {'rcpu': cpu_request, 'rram': ram_request, 'wucpu': cpu_util_wr, 'wuram': ram_util_wr, 'ucpu': cpu_util, 'uram': ram_util} + + return res + +def fold_resobjs(ro1, ro2): + if ro1 is None: + return ro2 + elif ro2 is None: + return ro1 + else: + for k in ro1.keys(): + if ro1[k] is None: + ro1[k] = ro2[k] + elif ro2[k] is not None: + for kk in ro1[k].keys(): + if ro1[k][kk] is None: + ro1[k][kk] = ro2[k][kk] + elif ro2[k][kk] is not None: + ro1[k][kk] = [sum(x) for x in zip(ro1[k][kk], ro2[k][kk])] + return ro1 + +result = df.rdd \ + .groupBy(lambda x: x.id) \ + .map(for_each_joined) \ + .fold(None, fold_resobjs) + +d = os.path.dirname(os.path.realpath(__file__)) + +with open(d + "/" + cluster + "_figure8abef.json", "w") as f: + json.dump(result, f) + +# vim: set ts=4 sw=4 et tw=120: diff --git a/figure_8/figure8-cd-only.py b/figure_8/figure8-cd-only.py new file mode 100755 index 00000000..18abb80a --- /dev/null +++ b/figure_8/figure8-cd-only.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 +# coding: utf-8 + +import os +import json +import pandas as pd +import findspark +findspark.init() +import pyspark +import pyspark.sql +import sys +import gzip +from pyspark import AccumulatorParam +from pyspark.sql.functions import lit +from pyspark.sql import Window +from pyspark.sql.types import * +from decimal import * + +if len(sys.argv) is not 4: + print(sys.argv[0] + " {cluster} {tmpdir} {maxram}") + sys.exit() + +cluster=sys.argv[1] + +spark = pyspark.sql.SparkSession.builder \ + .appName("task_slowdown") \ + .config("spark.driver.maxResultSize", "128g") \ + .config("spark.local.dir", sys.argv[2]) \ + .config("spark.driver.memory", sys.argv[3]) \ + .getOrCreate() +sc = spark.sparkContext + +# +# READING INSTANCE EVENTS DATA +# +dfepath = "/home/claudio/google_2019/instance_events/" + cluster + "/" + cluster + "_instance_events*.json.gz" +df = spark.read.json(dfepath) +df = df.limit(10000) + +# READING MACHINE EVENTS DATA, sort them and save them as broadcast variable +print("Starting to read machine events...") +dfm = pd.read_csv("~/google_2019/machine_events/" + cluster + "_machine_events.csv", converters={ + 'time': lambda x: -1 if x == '' else int(x), + 'machine_id': lambda x: str(x), + 'capacity.cpus': lambda x: -1 if x == '' else Decimal(x), + 'capacity.memory': lambda x: -1 if x == '' else Decimal(x)}) +print("Dropping remove events...") +dfm = dfm[(dfm.type!=2)&(dfm.time!=-1)&(dfm["capacity.cpus"]!=-1)&(dfm["capacity.memory"]!=-1)] +print("Dropping missing data events...") +dfm = dfm[dfm.missing_data_reason.isnull()] +print("Projecting on useful columns...") +dfm = dfm[["time", "machine_id", "capacity.cpus", "capacity.memory"]] +print("Sorting by time...") +dfm = dfm.sort_values(by=["machine_id", "time"]) +print("Converting to broadcast variable...") +dfm = sc.broadcast([tuple(r) for r in dfm.to_numpy()]) +print("Done with machine events.") + +df = dfe.rdd \ + .filter(lambda x: x.time is not None and x.type is not None and x.machine_id is not None and + x.instance_index is not None and x.collection_id is not None and x.resource_request is not None and + x.resource_request.cpus is not None and x.resource_request.memory is not None) \ + .map(lambda x: [tabid(x), int(x.time), int(x.type), + Decimal(x.resource_request.cpus), Decimal(x.resource_request.memory), x.machine_id]) \ + .toDF(["id", "time", "type", "rcpu", "rram", "mid"]) + + +def get_machine_time_resources(machine_id, time): + def aux(i, j): + if i == j: + return dfm.value[i] if dfm.value[i][1] == machine_id else None + elif i + 1 == j: + if dfm.value[i][1] == machine_id: + return dfm.value[i] + elif dfm.value[j][1] == machine_id: + return dfm.value[j] + else: + return None + + mid = (i + j) // 2 + + if dfm.value[mid][1] > machine_id: + return aux(i, mid - 1) + elif dfm.value[mid][1] < machine_id: + return aux(mid + 1, j) + elif dfm.value[mid][0] > time: + return aux(i, mid) + elif dfm.value[mid][0] < time: + return aux(mid, j) + else: + return mid + + return aux(0, len(dfm.value)-1) + +def increment_reserv_bucket(bucket, taskid, value): + if value < 0: + idx = 0 + else: + idx = 40 if value >= 1 else (int(value * 40) + 1) + + if taskid not in bucket: + bucket[taskid] = [0] * (len(ceils) + 1) + bucket[taskid][idx] += 1 + +def bucket_sum_per_termination(bucket, last_term_by_id): + result = {-1: None, 4: None, 5: None, 6: None, 7: None, 8: None} + for tid, vs in bucket.items(): + term = last_term_by_id[tid] + if result[term] is None: + result[term] = vs + else: + result[term] = [sum(x) for x in zip(result[term], vs)] + return result + +def for_each_joined(x): + machine_id = x[0] + ts = x[1] + + ts = sorted(ts, key=lambda x: x.time) + last_req_by_id = {} # map taskid -> last known req [cpu, ram] (data removed when task terminates) + + reserv_ceils = [0] + cpu_reservs_by_id = {} + ram_reservs_by_id = {} + + last_term_by_id = {} # map taskid -> last termination + start = get_machine_time_resources(machine_id, 0) + end = get_machine_time_resources(machine_id, 6_000_000_000_000) + machine_logs = None if start is None or end is None else dfm.value[start:(end+1)] + + for i, t in enumerate(ts): + if machine_logs is not None and len(machine_logs) > 1 and machine_logs[1][0] >= t.time: + machine_logs.pop(0) + if t.id not in last_term_by_id: + last_term_by_id[t.id] = -1 + if t.rcpu is not None and t.rram is not None: + last_req_by_id[t.id] = (t.rcpu, t.rram) + # 8b + tot_req = [sum(x) for x in zip(*last_req_by_id.values())] + if machine_logs is not None: + reserv_cpu = tot_req[0] / machine_logs[0][2] + reserv_ram = tot_req[1] / machine_logs[0][3] + else: + reserv_cpu = -1 + reserv_ram = -1 + increment_reserv_bucket(cpu_reservs_by_id, t.id, reserv_cpu) + increment_reserv_bucket(ram_reservs_by_id, t.id, reserv_ram) + if t.type >= 4 and t.type <= 8: + last_term_by_id[t.id] = t.type + + resobj = {'rscpu': cpu_reservs_by_id, 'rsram': ram_reservs_by_id} + + for k, v in resobj.items(): + resobj[k] = bucket_sum_per_termination(v, last_term_by_id) + + return resobj + +def fold_resobjs(ro1, ro2): + if ro1 is None: + return ro2 + elif ro2 is None: + return ro1 + else: + for k in ro1.keys(): + for kk in ro1[k].keys(): + if ro1[k][kk] is None: + ro1[k][kk] = ro2[k][kk] + elif ro2[k][kk] is None: + continue + else: + ro1[k][kk] = [sum(x) for x in zip(ro1[k][kk], ro2[k][kk])] + return ro1 + +# TODO: partition by id and in the for-each-row +# function implement lookup to dfm.value to understand its memory capacity + +import random + +result = df.rdd \ + .groupBy(lambda x: x.mid) \ + .partitionBy(10000, lambda x: random.randint(0, 10000-1)) \ + .map(for_each_joined) \ + .fold(None, fold_resobjs) + +d = os.path.dirname(os.path.realpath(__file__)) + +with open(d + "/" + cluster + "_figure8.json", "w") as f: + json.dump(result, f) + +# vim: set ts=4 sw=4 et tw=120: diff --git a/figure_8/h_figure8ab.json b/figure_8/h_figure8ab.json new file mode 100644 index 00000000..aec5b760 --- /dev/null +++ b/figure_8/h_figure8ab.json @@ -0,0 +1 @@ +{"-1": {"rcpu": [0, 37134, 7823, 1095, 705, 254, 405, 68, 86, 0, 5, 16, 44, 32, 0, 0, 0, 0, 0, 0, 8, 2, 0, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "rram": [0, 41623, 2064, 3246, 443, 170, 86, 8, 42, 0, 0, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "4": {"rcpu": [0, 8052123, 1800398, 247696, 45236, 86246, 21765, 5071, 636, 7, 2, 14, 35, 2, 162, 2368, 1233, 4, 2, 2, 3, 0, 2, 10, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "rram": [0, 9508643, 252810, 65861, 40368, 391949, 491, 1653, 897, 251, 33, 60, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0]}, "5": {"rcpu": [0, 3795007, 148577, 622218, 3839, 1547, 131, 2295, 60, 21, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "rram": [0, 3901983, 649474, 10598, 9401, 1142, 270, 729, 4, 10, 9, 9, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}, "6": {"rcpu": [0, 60804001, 3479863, 795860, 798587, 244544, 77010, 88312, 13604, 1556, 1233, 9895, 8786, 2826, 14, 12, 4, 2, 0, 0, 146, 27, 0, 333, 22448, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "rram": [0, 52755625, 13102031, 328036, 90326, 30961, 6605, 996, 9483, 2, 0, 22523, 1854, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 18, 234, 336, 16, 9, 4, 0, 2, 0, 0]}, "7": {"rcpu": [0, 1044640256, 52093234, 8435166, 16531024, 1822904, 118770, 256291, 125781, 93327, 18627, 152169, 502207, 222, 2065, 1849, 1332, 1320, 1481, 77, 169, 42, 4, 1668, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "rram": [0, 1096664639, 19126164, 5041851, 2122806, 1223639, 118066, 101040, 97793, 55379, 16593, 18182, 213667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 18, 0, 12, 35, 31, 1, 0, 0, 0, 0, 0]}, "8": {"rcpu": [0, 3687040, 71509, 887, 1004, 63718, 20, 10, 6, 0, 0, 8, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], "rram": [0, 3768719, 8504, 46889, 52, 20, 6, 6, 8, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}} \ No newline at end of file