From 344afc53919049c5156d8f6dd36ce80d3639e18c Mon Sep 17 00:00:00 2001 From: "Claudio Maggioni (maggicl)" Date: Wed, 10 Mar 2021 16:03:54 +0000 Subject: [PATCH] Added partial spatial_resource_waste results --- .gitignore | 1 + .../a_res_micros_requested.json | 3 + .../b_res_micros_requested.json | 3 + .../c_res_micros_requested.json | 3 + .../d_res_micros_requested.json | 3 + .../e_res_micros_requested.json | 3 + .../f_res_micros_requested.json | 3 + .../g_res_micros_requested.json | 3 + .../spatial_resource_waste.py | 91 +++++++++++++++++++ task_slowdown/.gitignore | 1 - thesis-dev.md | 2 +- 11 files changed, 114 insertions(+), 2 deletions(-) create mode 100644 spatial_resource_waste/a_res_micros_requested.json create mode 100644 spatial_resource_waste/b_res_micros_requested.json create mode 100644 spatial_resource_waste/c_res_micros_requested.json create mode 100644 spatial_resource_waste/d_res_micros_requested.json create mode 100644 spatial_resource_waste/e_res_micros_requested.json create mode 100644 spatial_resource_waste/f_res_micros_requested.json create mode 100644 spatial_resource_waste/g_res_micros_requested.json create mode 100644 spatial_resource_waste/spatial_resource_waste.py delete mode 100644 task_slowdown/.gitignore diff --git a/.gitignore b/.gitignore index e532dd83..fe443f32 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ **/.ipynb_checkpoints/ +task_slowdown/?_state_changes.json.gz diff --git a/spatial_resource_waste/a_res_micros_requested.json b/spatial_resource_waste/a_res_micros_requested.json new file mode 100644 index 00000000..a8e4f62f --- /dev/null +++ b/spatial_resource_waste/a_res_micros_requested.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f497c0c812bba8611403aef2c74710c10cc37c66b9e041c60b54a2f66ee9328 +size 344 diff --git a/spatial_resource_waste/b_res_micros_requested.json b/spatial_resource_waste/b_res_micros_requested.json new file mode 100644 index 00000000..bd2d0931 --- /dev/null +++ b/spatial_resource_waste/b_res_micros_requested.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3504b56b94ecf8f4affb2448baa01e832e7f3b7e04451a6c1462ad4f32b914e +size 315 diff --git a/spatial_resource_waste/c_res_micros_requested.json b/spatial_resource_waste/c_res_micros_requested.json new file mode 100644 index 00000000..8013fc10 --- /dev/null +++ b/spatial_resource_waste/c_res_micros_requested.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88babb6b9578b29ccd63d0a57fdc780c1bf9a9847d35c90aecacfcf9730d54f8 +size 313 diff --git a/spatial_resource_waste/d_res_micros_requested.json b/spatial_resource_waste/d_res_micros_requested.json new file mode 100644 index 00000000..df3efee2 --- /dev/null +++ b/spatial_resource_waste/d_res_micros_requested.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb1dd43dcaab429bd5e13815077abce6f0e1a7730f56575b51eac5e2b538d4be +size 311 diff --git a/spatial_resource_waste/e_res_micros_requested.json b/spatial_resource_waste/e_res_micros_requested.json new file mode 100644 index 00000000..bbf19fe4 --- /dev/null +++ b/spatial_resource_waste/e_res_micros_requested.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a8fef2a4fecca114595d9ef98a2da20db1b4aacc5cfc9c8c152ba1066740960 +size 293 diff --git a/spatial_resource_waste/f_res_micros_requested.json b/spatial_resource_waste/f_res_micros_requested.json new file mode 100644 index 00000000..7a135fbd --- /dev/null +++ b/spatial_resource_waste/f_res_micros_requested.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:160cd464b3da4d63674a25a1a0e4a82aa6b23ac12a519d67e71b979b1c80fa43 +size 313 diff --git a/spatial_resource_waste/g_res_micros_requested.json b/spatial_resource_waste/g_res_micros_requested.json new file mode 100644 index 00000000..39c3ba2b --- /dev/null +++ b/spatial_resource_waste/g_res_micros_requested.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3d33312f68da23311645a388e1f4bbec42c1d626fdb2724d54f6dfe15d1bf51 +size 307 diff --git a/spatial_resource_waste/spatial_resource_waste.py b/spatial_resource_waste/spatial_resource_waste.py new file mode 100644 index 00000000..a39c15f9 --- /dev/null +++ b/spatial_resource_waste/spatial_resource_waste.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +# coding: utf-8 + +import json +import pandas +from IPython import display +import findspark +findspark.init() +import pyspark +import pyspark.sql +import sys +import gzip + +from pyspark import AccumulatorParam +from pyspark.sql.functions import lit +from pyspark.sql import Window +from pyspark.sql.types import ByteType + +cluster=sys.argv[1] + +spark = pyspark.sql.SparkSession.builder \ + .appName("task_slowdown") \ + .config("spark.driver.maxResultSize", "32g") \ + .config("spark.local.dir", "/run/tmpfiles.d/spark") \ + .config("spark.driver.memory", "75g") \ + .getOrCreate() +sc = spark.sparkContext + +df = spark.read.json("/home/claudio/google_2019/instance_events/" + cluster + "/" + cluster + "_instance_events*.json.gz") +#df = spark.read.json("/home/claudio/google_2019/instance_events/" + cluster + "/" + cluster + "_test.json") + +try: + df["collection_type"] = df["collection_type"].cast(ByteType()) +except: + df = df.withColumn("collection_type", lit(None).cast(ByteType())) + +RUN = set([(3,1), (3,4), (3,5), (3,6), (3,7), (3,8), (3,10), (10,1), (10,4), (10,5), (10,6), (10,7), (10,8), (10,10)]) + +def is_res_none(tres): + return tres is None or tres["cpus"] is None or tres["memory"] is None + +def for_each_task(ts): + ts = sorted(ts, key=lambda x: x["time"]) + last_term = None + last_resources = None + prev = None + cpu = 0 + ram = 0 + + for i,t in enumerate(ts): + if t["type"] >= 4 and t["type"] <= 8: + last_term = t["type"] + if prev is not None: + if (prev["type"], t["type"]) in RUN: + if is_res_none(last_resources): + last_resources = t["res"] + if not is_res_none(last_resources): + delta = t["time"] - prev["time"] + cpu += round(delta * last_resources["cpus"]) + ram += round(delta * last_resources["memory"]) + prev = t + if not is_res_none(last_resources): + last_resources = t["res"] + + return [("cpu-" + str(last_term), cpu), ("ram-" + str(last_term), ram)] + +def cleanup(x): + return { + "time": int(x.time), + "type": 0 if x.type is None else int(x.type), + "id": x.collection_id + "-" + x.instance_index, + "res": x.resource_request + } + +df2 = df.rdd \ + .filter(lambda x: x.collection_type is None or x.collection_type == 0) \ + .filter(lambda x: x.time is not None and x.instance_index is not None and x.collection_id is not None) \ + .map(cleanup) \ + .groupBy(lambda x: x["id"]) \ + .mapValues(for_each_task) \ + .flatMap(lambda x: x[1]) \ + .groupBy(lambda x: x[0]) \ + .mapValues(lambda xs: sum(n for _, n in xs)) \ + .collect() + +result = {} +for pair in df2: + result[pair[0]] = pair[1] + +with open(cluster + "_res_micros_requested.json", "w") as out: + json.dump(result, out, separators=(',', ':')) diff --git a/task_slowdown/.gitignore b/task_slowdown/.gitignore deleted file mode 100644 index eab83f30..00000000 --- a/task_slowdown/.gitignore +++ /dev/null @@ -1 +0,0 @@ -task_slowdown/?_state_changes.json.gz diff --git a/thesis-dev.md b/thesis-dev.md index d43cfb9b..cb276db9 100644 --- a/thesis-dev.md +++ b/thesis-dev.md @@ -38,7 +38,7 @@ Google drive. - (%) total wasted time per unsuccessful event type - (mins.) avg. wasted time per number of events for each event type - breakdown of wasted time per *submission*, *scheduling*, *queue* -- *III-A-I: Average slowdown per task*: (Table II) +- [✅ **task_slowdown**] *III-A-I: Average slowdown per task*: (Table II) For FINISH type tasks, compute *slowdown*, i.e. mean (**ask Rosa**) of all *response time* for each task event over *response time* of last event (which is by def. FINISH). Response time is defined as *Queue time* + *Exec time*