maggicl/bachelorThesis
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity
bachelorThesis/task_slowdown/task_slowdown.ipynb

219 lines
22 KiB
Text
Raw Normal View History

:Added task slowdown results
2021-02-27 12:07:15 +00:00
{
"cells": [
{
"cell_type": "code",
"execution_count": 28,
"id": "behind-rings",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"root\n",
" |-- alloc_collection_id: string (nullable = true)\n",
" |-- alloc_instance_index: string (nullable = true)\n",
" |-- collection_id: string (nullable = true)\n",
" |-- collection_type: string (nullable = true)\n",
" |-- constraint: array (nullable = true)\n",
" | |-- element: struct (containsNull = true)\n",
" | | |-- name: string (nullable = true)\n",
" | | |-- relation: string (nullable = true)\n",
" | | |-- value: string (nullable = true)\n",
" |-- instance_index: string (nullable = true)\n",
" |-- machine_id: string (nullable = true)\n",
" |-- missing_type: string (nullable = true)\n",
" |-- priority: string (nullable = true)\n",
" |-- resource_request: struct (nullable = true)\n",
" | |-- cpus: double (nullable = true)\n",
" | |-- memory: double (nullable = true)\n",
" |-- scheduling_class: string (nullable = true)\n",
" |-- time: string (nullable = true)\n",
" |-- type: string (nullable = true)\n",
"\n",
"+-------------------+--------------------+-------------+---------------+--------------------+--------------+------------+------------+--------+--------------------+----------------+-------------+----+\n",
"|alloc_collection_id|alloc_instance_index|collection_id|collection_type| constraint|instance_index| machine_id|missing_type|priority| resource_request|scheduling_class| time|type|\n",
"+-------------------+--------------------+-------------+---------------+--------------------+--------------+------------+------------+--------+--------------------+----------------+-------------+----+\n",
"| null| -1| 311844978975| null| []| 100|294975034872| null| 103|[0.00660705566406...| null| 678322664130| 3|\n",
"| null| -1| 324883683214| null| []| null| null| null| null|[0.01055908203125...| null|1976760570767|null|\n",
"| null| -1| 323371581333| null|[[9eCGRtl6XN5GQoO...| 571|153437873934| null| null|[0.00810241699218...| null|2240234044164| 7|\n",
"| null| -1| 313182190373| null| []| 34| null| null| null|[0.00228881835937...| null| 968876915742|null|\n",
"| null| -1| 299950283520| null|[[9eCGRtl6XN5GQoO...| 2135| null| null| 117|[0.01531982421875...| 1| 142387817674|null|\n",
"| null| -1| 324794084626| null| []| 314| -1| null| 103|[0.0810546875, 0....| null|1950212179272| 2|\n",
"| null| -1| 311869083179| null|[[5dEuieuWMFy+CNM...| 4729| 1994402429| null| 117|[0.01109313964843...| 1|1049052607582| 3|\n",
"| null| -1| 314633575062| null| []| 708| null| null| null|[0.00589752197265...| null|1245121652306|null|\n",
"| null| -1| 310024318230| null| []| 132| null| null| 103|[0.00607299804687...| null| 398893109200| 1|\n",
"| null| -1| 314083275570| null| []| null| 1821717262| null| 103|[4.40597534179687...| null|1245124281990| 6|\n",
"| null| -1| 326302449610| null| []| 2740| null| null| null|[0.00589752197265...| null|2353733113490|null|\n",
"| null| -1| 312310995026| null| []| 101| null| null| null|[8.09669494628906...| null| 802162973717|null|\n",
"| null| -1| 325098713157| null| []| 77| null| null| 103|[0.0810546875, 0....| null|2049169911483| 2|\n",
"| null| -1| 322296137788| null| []| null| 1888361814| null| 200|[6.16073608398437...| null|1511423336811| 10|\n",
"| null| -1| 325292207036| null| []| 13| -1| null| 103|[0.09716796875, 0...| null|2128029854228| 10|\n",
"| null| -1| 311499868798| null|[[9eCGRtl6XN5GQoO...| 6571| null| null| null|[0.01620483398437...| null| 673538231587|null|\n",
"| null| -1| 325505190758| null|[[9eCGRtl6XN5GQoO...| 306| null| null| null|[0.00242996215820...| null|2424201701414| 2|\n",
"| null| -1| 310897142142| null| []| null| null| null| null|[0.00757598876953...| null| 498367858680|null|\n",
"| null| -1| 321838026533| null| []| 667| 3459537598| null| 103|[0.01048278808593...| null|1416689475146| 10|\n",
"| null| -1| 322571597197| null| []| 3741| 10368072540| null| 103|[0.0123291015625,...| null|1591142575136| 3|\n",
"+-------------------+--------------------+-------------+---------------+--------------------+--------------+------------+------------+--------+--------------------+----------------+-------------+----+\n",
"only showing top 20 rows\n",
"\n"
]
}
],
"source": [
"#!/usr/bin/env python3\n",
"# coding: utf-8\n",
"\n",
"# # Temporal impact: machine time waste\n",
"\n",
"# Inefficient version of the machine_time_waste.py script in the same dir. Please ignore, kept for future need\n",
"\n",
"import json\n",
"import pandas\n",
"from IPython import display\n",
"import findspark\n",
"findspark.init()\n",
"import pyspark\n",
"import pyspark.sql\n",
"import sys\n",
"\n",
"from pyspark.sql.functions import col, lag, when, concat_ws, last, first\n",
"from pyspark.sql import Window\n",
"from pyspark.sql.types import LongType\n",
"\n",
"cluster=\"h\"\n",
"\n",
"spark = pyspark.sql.SparkSession.builder \\\n",
" .appName(\"task_slowdown\") \\\n",
" .config(\"spark.local.dir\", \"/run/tmpfiles.d/spark\") \\\n",
" .config(\"spark.driver.memory\", \"124g\") \\\n",
" .getOrCreate()\n",
"sc = spark.sparkContext\n",
"\n",
"# df = spark.read.json(\"/home/claudio/google_2019/instance_events/\" + cluster + \"/\" + cluster + \"_instance_events000000000000.json.gz\")\n",
"df = spark.read.json(\"/home/claudio/google_2019/instance_events/\" + cluster + \"/\" + cluster + \"_test.json\")\n",
"\n",
"df.printSchema()\n",
"\n",
"df.show()"
]
},
{
"cell_type": "code",
"execution_count": 32,
"id": "banned-temperature",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[(103, [[], [], [], [], [(62850578515, [3, 3, 5])], [], [], [], [], [], [(36278465040, [1, 5])], [], [], [], [], [], [], [], [], [], [], [(1937109982, [2, 5])], [], [(66934359137, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(99766245, [2, 5])], [], [], [], [], [(679575974, [2, 5])], [], [], [], [], [], [], [], [], [(27987724401, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(40033883995, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [(2072441674, [3, 5])], [], [], [], [], [], [(30503998595, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(11191193265, [3, 5])], [], [], [], [], [], [(13449203940, [2, 5])], [(9056832716, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [(2624004083, [3, 5])], [], [], [], [], [(5791004634, [2, 5])], [], [], [], [], [], [], [(31522009960, [2, 5])], [], [], [], [], [], [], [], [(27223865505, [3, 5])], [], [], [], [], [], [], [], [], [], [], [(62363966217, [2, 5])], [(22329997715, [3, 5])], [], [], [], [], [], [], [], [], [], [], [(35738940742, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(5461413747, [10, 5])], [], [], [], [], [], [], [(44714567682, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(32537907976, [10, 5])], [], [], [], [(3856023850, [2, 5])], [], [], [], [], [], [], [], [], [], [], [(8289226525, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(88907761383, [2, 5])], [], [], [], [], [], [], [(25322047766, [2, 5])], [], [], [], [], [(788099086, [3, 5])], [], [], [], [], [], [(36244532207, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(560020738, [2, 5])], [(71461045, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(124699375375, [2, 5])], [], [], [(7836906167, [2, 5])], [], [], [(5422253956, [2, 5])], [], [], [(5855268565, [2, 5])], [], [(5023716519, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(18332998966, [2, 5])], [], [], [], [], [], [(35660231933, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(1287188765, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(3907777878, [3, 5])], [], [], [], [], [], [], [], [], [], [], [(20506184461, [2, 5])], [], [(8408788368, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(13629775669, [3, 5])], [], [], [], [], [], [(780912697, [3, 5])], [], [], [], [(32298679126, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(30750739304, [3, 5])], [(6594675644, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(15402056687, [3, 5])], [], [], [], [], [], [(109413809124, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(305255719, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(3827846545, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(57934339021, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(40760757121, [3, 5])], [], [], [], [(4259078476, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(1034707355, [1, 5])], [], [], [], [], [(93784278917, [9, 2, 3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(4188472119, [1, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(154867489, [2, 5])], [], [], [], [], [(40095625660, [2, 5])], [], [], [], [], [(6313871106, [2, 5])], [], [], [], []
"55444\n",
"51766\n"
]
}
],
"source": [
"non = sc.accumulator(0)\n",
"tot = sc.accumulator(0)\n",
"\n",
"\n",
"def for_each_task(ts):\n",
" global none\n",
" global tot\n",
" \n",
" ts = sorted(ts, key=lambda x: x[\"time\"])\n",
" \n",
" last_term = None\n",
" priority = None\n",
" responding = False\n",
" \n",
" resp_burst_start = None\n",
" resp_burst_type = None\n",
" \n",
" resp_time = []\n",
" resp_time_last = 0\n",
"\n",
" for i,t in enumerate(ts):\n",
" if t[\"priority\"] is not None and priority is None:\n",
" priority = t[\"priority\"]\n",
" if responding:\n",
" resp_burst_type.append(t[\"type\"])\n",
" if t[\"type\"] >= 4 and t[\"type\"] <= 8:\n",
" last_term = t[\"type\"]\n",
" if responding:\n",
" # This response time interval has ended, so record the time delta and term\n",
" resp_time.append((t[\"time\"] - resp_burst_start, resp_burst_type))\n",
" responding = False\n",
" if (not responding) and (t[\"type\"] < 4 or t[\"type\"] > 8):\n",
" resp_burst_start = t[\"time\"]\n",
" resp_burst_type = [t[\"type\"]]\n",
" responding = True\n",
" \n",
" \n",
" tot.add(1)\n",
" if last_term != 6:\n",
" non.add(1)\n",
" return (priority, resp_time) if last_term == 5 else None\n",
"\n",
" \n",
"def cleanup(x):\n",
" return {\n",
" \"time\": int(x.time),\n",
" \"type\": 0 if x.type is None else int(x.type),\n",
" \"id\": x.collection_id + \"-\" + x.instance_index,\n",
" \"priority\": 0 if x.priority is None else int(x.priority)\n",
" }\n",
"\n",
"\n",
"df2 = df.limit(100000).rdd \\\n",
" .filter(lambda x: x.collection_type is None or x.collection_type == 0) \\\n",
" .filter(lambda x: x.type is not None and x.time is not None\n",
" and x.instance_index is not None and x.collection_id is not None) \\\n",
" .map(cleanup) \\\n",
" .groupBy(lambda x: x[\"id\"]) \\\n",
" .mapValues(for_each_task) \\\n",
" .filter(lambda x: x[1] is not None) \\\n",
" .map(lambda x: x[1]) \\\n",
" .groupBy(lambda x: x[0]) \\\n",
" .mapValues(lambda x: [e[1] for e in x])\n",
"\n",
"a = {\"val\": df2.collect(), \"tot\": tot.value, \"non\": non.value}\n",
"with open(cluster + \"_state_changes.json\", \"w\") as out:\n",
" json.dump(a, out)\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "arctic-tomato",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.9"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Reference in a new issue Copy permalink