{ "cells": [ { "cell_type": "code", "execution_count": 28, "id": "behind-rings", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "root\n", " |-- alloc_collection_id: string (nullable = true)\n", " |-- alloc_instance_index: string (nullable = true)\n", " |-- collection_id: string (nullable = true)\n", " |-- collection_type: string (nullable = true)\n", " |-- constraint: array (nullable = true)\n", " | |-- element: struct (containsNull = true)\n", " | | |-- name: string (nullable = true)\n", " | | |-- relation: string (nullable = true)\n", " | | |-- value: string (nullable = true)\n", " |-- instance_index: string (nullable = true)\n", " |-- machine_id: string (nullable = true)\n", " |-- missing_type: string (nullable = true)\n", " |-- priority: string (nullable = true)\n", " |-- resource_request: struct (nullable = true)\n", " | |-- cpus: double (nullable = true)\n", " | |-- memory: double (nullable = true)\n", " |-- scheduling_class: string (nullable = true)\n", " |-- time: string (nullable = true)\n", " |-- type: string (nullable = true)\n", "\n", "+-------------------+--------------------+-------------+---------------+--------------------+--------------+------------+------------+--------+--------------------+----------------+-------------+----+\n", "|alloc_collection_id|alloc_instance_index|collection_id|collection_type| constraint|instance_index| machine_id|missing_type|priority| resource_request|scheduling_class| time|type|\n", "+-------------------+--------------------+-------------+---------------+--------------------+--------------+------------+------------+--------+--------------------+----------------+-------------+----+\n", "| null| -1| 311844978975| null| []| 100|294975034872| null| 103|[0.00660705566406...| null| 678322664130| 3|\n", "| null| -1| 324883683214| null| []| null| null| null| null|[0.01055908203125...| null|1976760570767|null|\n", "| null| -1| 323371581333| null|[[9eCGRtl6XN5GQoO...| 571|153437873934| null| null|[0.00810241699218...| null|2240234044164| 7|\n", "| null| -1| 313182190373| null| []| 34| null| null| null|[0.00228881835937...| null| 968876915742|null|\n", "| null| -1| 299950283520| null|[[9eCGRtl6XN5GQoO...| 2135| null| null| 117|[0.01531982421875...| 1| 142387817674|null|\n", "| null| -1| 324794084626| null| []| 314| -1| null| 103|[0.0810546875, 0....| null|1950212179272| 2|\n", "| null| -1| 311869083179| null|[[5dEuieuWMFy+CNM...| 4729| 1994402429| null| 117|[0.01109313964843...| 1|1049052607582| 3|\n", "| null| -1| 314633575062| null| []| 708| null| null| null|[0.00589752197265...| null|1245121652306|null|\n", "| null| -1| 310024318230| null| []| 132| null| null| 103|[0.00607299804687...| null| 398893109200| 1|\n", "| null| -1| 314083275570| null| []| null| 1821717262| null| 103|[4.40597534179687...| null|1245124281990| 6|\n", "| null| -1| 326302449610| null| []| 2740| null| null| null|[0.00589752197265...| null|2353733113490|null|\n", "| null| -1| 312310995026| null| []| 101| null| null| null|[8.09669494628906...| null| 802162973717|null|\n", "| null| -1| 325098713157| null| []| 77| null| null| 103|[0.0810546875, 0....| null|2049169911483| 2|\n", "| null| -1| 322296137788| null| []| null| 1888361814| null| 200|[6.16073608398437...| null|1511423336811| 10|\n", "| null| -1| 325292207036| null| []| 13| -1| null| 103|[0.09716796875, 0...| null|2128029854228| 10|\n", "| null| -1| 311499868798| null|[[9eCGRtl6XN5GQoO...| 6571| null| null| null|[0.01620483398437...| null| 673538231587|null|\n", "| null| -1| 325505190758| null|[[9eCGRtl6XN5GQoO...| 306| null| null| null|[0.00242996215820...| null|2424201701414| 2|\n", "| null| -1| 310897142142| null| []| null| null| null| null|[0.00757598876953...| null| 498367858680|null|\n", "| null| -1| 321838026533| null| []| 667| 3459537598| null| 103|[0.01048278808593...| null|1416689475146| 10|\n", "| null| -1| 322571597197| null| []| 3741| 10368072540| null| 103|[0.0123291015625,...| null|1591142575136| 3|\n", "+-------------------+--------------------+-------------+---------------+--------------------+--------------+------------+------------+--------+--------------------+----------------+-------------+----+\n", "only showing top 20 rows\n", "\n" ] } ], "source": [ "#!/usr/bin/env python3\n", "# coding: utf-8\n", "\n", "# # Temporal impact: machine time waste\n", "\n", "# Inefficient version of the machine_time_waste.py script in the same dir. Please ignore, kept for future need\n", "\n", "import json\n", "import pandas\n", "from IPython import display\n", "import findspark\n", "findspark.init()\n", "import pyspark\n", "import pyspark.sql\n", "import sys\n", "\n", "from pyspark.sql.functions import col, lag, when, concat_ws, last, first\n", "from pyspark.sql import Window\n", "from pyspark.sql.types import LongType\n", "\n", "cluster=\"h\"\n", "\n", "spark = pyspark.sql.SparkSession.builder \\\n", " .appName(\"task_slowdown\") \\\n", " .config(\"spark.local.dir\", \"/run/tmpfiles.d/spark\") \\\n", " .config(\"spark.driver.memory\", \"124g\") \\\n", " .getOrCreate()\n", "sc = spark.sparkContext\n", "\n", "# df = spark.read.json(\"/home/claudio/google_2019/instance_events/\" + cluster + \"/\" + cluster + \"_instance_events000000000000.json.gz\")\n", "df = spark.read.json(\"/home/claudio/google_2019/instance_events/\" + cluster + \"/\" + cluster + \"_test.json\")\n", "\n", "df.printSchema()\n", "\n", "df.show()" ] }, { "cell_type": "code", "execution_count": 32, "id": "banned-temperature", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[(103, [[], [], [], [], [(62850578515, [3, 3, 5])], [], [], [], [], [], [(36278465040, [1, 5])], [], [], [], [], [], [], [], [], [], [], [(1937109982, [2, 5])], [], [(66934359137, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(99766245, [2, 5])], [], [], [], [], [(679575974, [2, 5])], [], [], [], [], [], [], [], [], [(27987724401, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(40033883995, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [(2072441674, [3, 5])], [], [], [], [], [], [(30503998595, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(11191193265, [3, 5])], [], [], [], [], [], [(13449203940, [2, 5])], [(9056832716, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [(2624004083, [3, 5])], [], [], [], [], [(5791004634, [2, 5])], [], [], [], [], [], [], [(31522009960, [2, 5])], [], [], [], [], [], [], [], [(27223865505, [3, 5])], [], [], [], [], [], [], [], [], [], [], [(62363966217, [2, 5])], [(22329997715, [3, 5])], [], [], [], [], [], [], [], [], [], [], [(35738940742, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(5461413747, [10, 5])], [], [], [], [], [], [], [(44714567682, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(32537907976, [10, 5])], [], [], [], [(3856023850, [2, 5])], [], [], [], [], [], [], [], [], [], [], [(8289226525, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(88907761383, [2, 5])], [], [], [], [], [], [], [(25322047766, [2, 5])], [], [], [], [], [(788099086, [3, 5])], [], [], [], [], [], [(36244532207, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(560020738, [2, 5])], [(71461045, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(124699375375, [2, 5])], [], [], [(7836906167, [2, 5])], [], [], [(5422253956, [2, 5])], [], [], [(5855268565, [2, 5])], [], [(5023716519, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(18332998966, [2, 5])], [], [], [], [], [], [(35660231933, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(1287188765, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(3907777878, [3, 5])], [], [], [], [], [], [], [], [], [], [], [(20506184461, [2, 5])], [], [(8408788368, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(13629775669, [3, 5])], [], [], [], [], [], [(780912697, [3, 5])], [], [], [], [(32298679126, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(30750739304, [3, 5])], [(6594675644, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(15402056687, [3, 5])], [], [], [], [], [], [(109413809124, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(305255719, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(3827846545, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(57934339021, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(40760757121, [3, 5])], [], [], [], [(4259078476, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(1034707355, [1, 5])], [], [], [], [], [(93784278917, [9, 2, 3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(4188472119, [1, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(154867489, [2, 5])], [], [], [], [], [(40095625660, [2, 5])], [], [], [], [], [(6313871106, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(75345872, [2, 5])], [], [(42854643, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(42240713762, [2, 5])], [(7940276639, [3, 5])], [], [], [(8067825463, [1, 5])], [], [], [], [(11564450554, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(14248092214, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(1151512362, [3, 5])], [], [], [], [], [], [], [], [], [], [], [(6814736882, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(28725095651, [10, 5])], [], [], [], [], [], [(5429365968, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [(78497996693, [2, 5])], [(4007939091, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(11207024590, [2, 2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(33028562460, [3, 3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(2244935788, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(4803409513, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(89924261, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(11515083156, [3, 5])], [], [], [], [], [], [], [], [(1028315320, [3, 5])], [], [], [], [(5669930326, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(92102821544, [9, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(12939585657, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(576460267, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [(16602852770, [3, 5])], [], [], [], [], [], [(3975575979, [9, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(15699214070, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(4273785458, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [(63081663, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(3673691412, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(4433586428, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(1737102869, [3, 3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [(7597739079, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(13564991758, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(3784960872, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(5558155540, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(1020015051, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(7386544705, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(7659182297, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(2612206009, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], []]), (360, [[(194634757478, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [(75332399743, [2, 2, 5])], [], [(327145123, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], []]), (118, [[], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], []]), (105, [[], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], []]), (119, [[(487900370258, [2, 5])], [(141154490783, [3, 3, 5]), (1211954116878, [3, 3, 3, 2, 3, 2, 3, 3, 3, 2, 5]), (106762922927, [2, 2, 3, 5])], [], [], [(1370779131465, [2, 2, 5])], [], []]), (0, [[], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], []]), (200, [[], [], [], [], [], [], [], [], [(70814003028, [2, 5])], [], [], [(1205243170887, [3, 2, 5])], [], [], [], [], [], [], [], [], [], []]), (115, [[], []]), (107, [[], []])]\n", "55444\n", "51766\n" ] } ], "source": [ "non = sc.accumulator(0)\n", "tot = sc.accumulator(0)\n", "\n", "\n", "def for_each_task(ts):\n", " global none\n", " global tot\n", " \n", " ts = sorted(ts, key=lambda x: x[\"time\"])\n", " \n", " last_term = None\n", " priority = None\n", " responding = False\n", " \n", " resp_burst_start = None\n", " resp_burst_type = None\n", " \n", " resp_time = []\n", " resp_time_last = 0\n", "\n", " for i,t in enumerate(ts):\n", " if t[\"priority\"] is not None and priority is None:\n", " priority = t[\"priority\"]\n", " if responding:\n", " resp_burst_type.append(t[\"type\"])\n", " if t[\"type\"] >= 4 and t[\"type\"] <= 8:\n", " last_term = t[\"type\"]\n", " if responding:\n", " # This response time interval has ended, so record the time delta and term\n", " resp_time.append((t[\"time\"] - resp_burst_start, resp_burst_type))\n", " responding = False\n", " if (not responding) and (t[\"type\"] < 4 or t[\"type\"] > 8):\n", " resp_burst_start = t[\"time\"]\n", " resp_burst_type = [t[\"type\"]]\n", " responding = True\n", " \n", " \n", " tot.add(1)\n", " if last_term != 6:\n", " non.add(1)\n", " return (priority, resp_time) if last_term == 5 else None\n", "\n", " \n", "def cleanup(x):\n", " return {\n", " \"time\": int(x.time),\n", " \"type\": 0 if x.type is None else int(x.type),\n", " \"id\": x.collection_id + \"-\" + x.instance_index,\n", " \"priority\": 0 if x.priority is None else int(x.priority)\n", " }\n", "\n", "\n", "df2 = df.limit(100000).rdd \\\n", " .filter(lambda x: x.collection_type is None or x.collection_type == 0) \\\n", " .filter(lambda x: x.type is not None and x.time is not None\n", " and x.instance_index is not None and x.collection_id is not None) \\\n", " .map(cleanup) \\\n", " .groupBy(lambda x: x[\"id\"]) \\\n", " .mapValues(for_each_task) \\\n", " .filter(lambda x: x[1] is not None) \\\n", " .map(lambda x: x[1]) \\\n", " .groupBy(lambda x: x[0]) \\\n", " .mapValues(lambda x: [e[1] for e in x])\n", "\n", "a = {\"val\": df2.collect(), \"tot\": tot.value, \"non\": non.value}\n", "with open(cluster + \"_state_changes.json\", \"w\") as out:\n", " json.dump(a, out)\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "id": "arctic-tomato", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.9" } }, "nbformat": 4, "nbformat_minor": 5 }