219 lines
22 KiB
Plaintext
219 lines
22 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 28,
|
|
"id": "behind-rings",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"root\n",
|
|
" |-- alloc_collection_id: string (nullable = true)\n",
|
|
" |-- alloc_instance_index: string (nullable = true)\n",
|
|
" |-- collection_id: string (nullable = true)\n",
|
|
" |-- collection_type: string (nullable = true)\n",
|
|
" |-- constraint: array (nullable = true)\n",
|
|
" | |-- element: struct (containsNull = true)\n",
|
|
" | | |-- name: string (nullable = true)\n",
|
|
" | | |-- relation: string (nullable = true)\n",
|
|
" | | |-- value: string (nullable = true)\n",
|
|
" |-- instance_index: string (nullable = true)\n",
|
|
" |-- machine_id: string (nullable = true)\n",
|
|
" |-- missing_type: string (nullable = true)\n",
|
|
" |-- priority: string (nullable = true)\n",
|
|
" |-- resource_request: struct (nullable = true)\n",
|
|
" | |-- cpus: double (nullable = true)\n",
|
|
" | |-- memory: double (nullable = true)\n",
|
|
" |-- scheduling_class: string (nullable = true)\n",
|
|
" |-- time: string (nullable = true)\n",
|
|
" |-- type: string (nullable = true)\n",
|
|
"\n",
|
|
"+-------------------+--------------------+-------------+---------------+--------------------+--------------+------------+------------+--------+--------------------+----------------+-------------+----+\n",
|
|
"|alloc_collection_id|alloc_instance_index|collection_id|collection_type| constraint|instance_index| machine_id|missing_type|priority| resource_request|scheduling_class| time|type|\n",
|
|
"+-------------------+--------------------+-------------+---------------+--------------------+--------------+------------+------------+--------+--------------------+----------------+-------------+----+\n",
|
|
"| null| -1| 311844978975| null| []| 100|294975034872| null| 103|[0.00660705566406...| null| 678322664130| 3|\n",
|
|
"| null| -1| 324883683214| null| []| null| null| null| null|[0.01055908203125...| null|1976760570767|null|\n",
|
|
"| null| -1| 323371581333| null|[[9eCGRtl6XN5GQoO...| 571|153437873934| null| null|[0.00810241699218...| null|2240234044164| 7|\n",
|
|
"| null| -1| 313182190373| null| []| 34| null| null| null|[0.00228881835937...| null| 968876915742|null|\n",
|
|
"| null| -1| 299950283520| null|[[9eCGRtl6XN5GQoO...| 2135| null| null| 117|[0.01531982421875...| 1| 142387817674|null|\n",
|
|
"| null| -1| 324794084626| null| []| 314| -1| null| 103|[0.0810546875, 0....| null|1950212179272| 2|\n",
|
|
"| null| -1| 311869083179| null|[[5dEuieuWMFy+CNM...| 4729| 1994402429| null| 117|[0.01109313964843...| 1|1049052607582| 3|\n",
|
|
"| null| -1| 314633575062| null| []| 708| null| null| null|[0.00589752197265...| null|1245121652306|null|\n",
|
|
"| null| -1| 310024318230| null| []| 132| null| null| 103|[0.00607299804687...| null| 398893109200| 1|\n",
|
|
"| null| -1| 314083275570| null| []| null| 1821717262| null| 103|[4.40597534179687...| null|1245124281990| 6|\n",
|
|
"| null| -1| 326302449610| null| []| 2740| null| null| null|[0.00589752197265...| null|2353733113490|null|\n",
|
|
"| null| -1| 312310995026| null| []| 101| null| null| null|[8.09669494628906...| null| 802162973717|null|\n",
|
|
"| null| -1| 325098713157| null| []| 77| null| null| 103|[0.0810546875, 0....| null|2049169911483| 2|\n",
|
|
"| null| -1| 322296137788| null| []| null| 1888361814| null| 200|[6.16073608398437...| null|1511423336811| 10|\n",
|
|
"| null| -1| 325292207036| null| []| 13| -1| null| 103|[0.09716796875, 0...| null|2128029854228| 10|\n",
|
|
"| null| -1| 311499868798| null|[[9eCGRtl6XN5GQoO...| 6571| null| null| null|[0.01620483398437...| null| 673538231587|null|\n",
|
|
"| null| -1| 325505190758| null|[[9eCGRtl6XN5GQoO...| 306| null| null| null|[0.00242996215820...| null|2424201701414| 2|\n",
|
|
"| null| -1| 310897142142| null| []| null| null| null| null|[0.00757598876953...| null| 498367858680|null|\n",
|
|
"| null| -1| 321838026533| null| []| 667| 3459537598| null| 103|[0.01048278808593...| null|1416689475146| 10|\n",
|
|
"| null| -1| 322571597197| null| []| 3741| 10368072540| null| 103|[0.0123291015625,...| null|1591142575136| 3|\n",
|
|
"+-------------------+--------------------+-------------+---------------+--------------------+--------------+------------+------------+--------+--------------------+----------------+-------------+----+\n",
|
|
"only showing top 20 rows\n",
|
|
"\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"#!/usr/bin/env python3\n",
|
|
"# coding: utf-8\n",
|
|
"\n",
|
|
"# # Temporal impact: machine time waste\n",
|
|
"\n",
|
|
"# Inefficient version of the machine_time_waste.py script in the same dir. Please ignore, kept for future need\n",
|
|
"\n",
|
|
"import json\n",
|
|
"import pandas\n",
|
|
"from IPython import display\n",
|
|
"import findspark\n",
|
|
"findspark.init()\n",
|
|
"import pyspark\n",
|
|
"import pyspark.sql\n",
|
|
"import sys\n",
|
|
"\n",
|
|
"from pyspark.sql.functions import col, lag, when, concat_ws, last, first\n",
|
|
"from pyspark.sql import Window\n",
|
|
"from pyspark.sql.types import LongType\n",
|
|
"\n",
|
|
"cluster=\"h\"\n",
|
|
"\n",
|
|
"spark = pyspark.sql.SparkSession.builder \\\n",
|
|
" .appName(\"task_slowdown\") \\\n",
|
|
" .config(\"spark.local.dir\", \"/run/tmpfiles.d/spark\") \\\n",
|
|
" .config(\"spark.driver.memory\", \"124g\") \\\n",
|
|
" .getOrCreate()\n",
|
|
"sc = spark.sparkContext\n",
|
|
"\n",
|
|
"# df = spark.read.json(\"/home/claudio/google_2019/instance_events/\" + cluster + \"/\" + cluster + \"_instance_events000000000000.json.gz\")\n",
|
|
"df = spark.read.json(\"/home/claudio/google_2019/instance_events/\" + cluster + \"/\" + cluster + \"_test.json\")\n",
|
|
"\n",
|
|
"df.printSchema()\n",
|
|
"\n",
|
|
"df.show()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 32,
|
|
"id": "banned-temperature",
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"[(103, [[], [], [], [], [(62850578515, [3, 3, 5])], [], [], [], [], [], [(36278465040, [1, 5])], [], [], [], [], [], [], [], [], [], [], [(1937109982, [2, 5])], [], [(66934359137, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(99766245, [2, 5])], [], [], [], [], [(679575974, [2, 5])], [], [], [], [], [], [], [], [], [(27987724401, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(40033883995, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [(2072441674, [3, 5])], [], [], [], [], [], [(30503998595, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(11191193265, [3, 5])], [], [], [], [], [], [(13449203940, [2, 5])], [(9056832716, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [(2624004083, [3, 5])], [], [], [], [], [(5791004634, [2, 5])], [], [], [], [], [], [], [(31522009960, [2, 5])], [], [], [], [], [], [], [], [(27223865505, [3, 5])], [], [], [], [], [], [], [], [], [], [], [(62363966217, [2, 5])], [(22329997715, [3, 5])], [], [], [], [], [], [], [], [], [], [], [(35738940742, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(5461413747, [10, 5])], [], [], [], [], [], [], [(44714567682, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(32537907976, [10, 5])], [], [], [], [(3856023850, [2, 5])], [], [], [], [], [], [], [], [], [], [], [(8289226525, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(88907761383, [2, 5])], [], [], [], [], [], [], [(25322047766, [2, 5])], [], [], [], [], [(788099086, [3, 5])], [], [], [], [], [], [(36244532207, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(560020738, [2, 5])], [(71461045, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(124699375375, [2, 5])], [], [], [(7836906167, [2, 5])], [], [], [(5422253956, [2, 5])], [], [], [(5855268565, [2, 5])], [], [(5023716519, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(18332998966, [2, 5])], [], [], [], [], [], [(35660231933, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(1287188765, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(3907777878, [3, 5])], [], [], [], [], [], [], [], [], [], [], [(20506184461, [2, 5])], [], [(8408788368, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(13629775669, [3, 5])], [], [], [], [], [], [(780912697, [3, 5])], [], [], [], [(32298679126, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(30750739304, [3, 5])], [(6594675644, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(15402056687, [3, 5])], [], [], [], [], [], [(109413809124, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(305255719, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(3827846545, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(57934339021, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(40760757121, [3, 5])], [], [], [], [(4259078476, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(1034707355, [1, 5])], [], [], [], [], [(93784278917, [9, 2, 3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(4188472119, [1, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(154867489, [2, 5])], [], [], [], [], [(40095625660, [2, 5])], [], [], [], [], [(6313871106, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(75345872, [2, 5])], [], [(42854643, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(42240713762, [2, 5])], [(7940276639, [3, 5])], [], [], [(8067825463, [1, 5])], [], [], [], [(11564450554, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(14248092214, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(1151512362, [3, 5])], [], [], [], [], [], [], [], [], [], [], [(6814736882, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(28725095651, [10, 5])], [], [], [], [], [], [(5429365968, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [(78497996693, [2, 5])], [(4007939091, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(11207024590, [2, 2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(33028562460, [3, 3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(2244935788, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(4803409513, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(89924261, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(11515083156, [3, 5])], [], [], [], [], [], [], [], [(1028315320, [3, 5])], [], [], [], [(5669930326, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(92102821544, [9, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(12939585657, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(576460267, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [(16602852770, [3, 5])], [], [], [], [], [], [(3975575979, [9, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(15699214070, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(4273785458, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [(63081663, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(3673691412, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(4433586428, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(1737102869, [3, 3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [(7597739079, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(13564991758, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(3784960872, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(5558155540, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(1020015051, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(7386544705, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(7659182297, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(2612206009, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], []]), (360, [[(194634757478, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [(75332399743, [2, 2, 5])], [], [(327145123, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], []]), (118, [[], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], []]), (105, [[], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], []]), (119, [[(487900370258, [2, 5])], [(141154490783, [3, 3, 5]), (1211954116878, [3, 3, 3, 2, 3, 2, 3, 3, 3, 2, 5]), (106762922927, [2, 2, 3, 5])], [], [], [(1370779131465, [2, 2, 5])], [], []]), (0, [[], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], []]), (200, [[], [], [], [], [], [], [], [], [(70814003028, [2, 5])], [], [], [(1205243170887, [3, 2, 5])], [], [], [], [], [], [], [], [], [], []]), (115, [[], []]), (107, [[], []])]\n",
|
|
"55444\n",
|
|
"51766\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"non = sc.accumulator(0)\n",
|
|
"tot = sc.accumulator(0)\n",
|
|
"\n",
|
|
"\n",
|
|
"def for_each_task(ts):\n",
|
|
" global none\n",
|
|
" global tot\n",
|
|
" \n",
|
|
" ts = sorted(ts, key=lambda x: x[\"time\"])\n",
|
|
" \n",
|
|
" last_term = None\n",
|
|
" priority = None\n",
|
|
" responding = False\n",
|
|
" \n",
|
|
" resp_burst_start = None\n",
|
|
" resp_burst_type = None\n",
|
|
" \n",
|
|
" resp_time = []\n",
|
|
" resp_time_last = 0\n",
|
|
"\n",
|
|
" for i,t in enumerate(ts):\n",
|
|
" if t[\"priority\"] is not None and priority is None:\n",
|
|
" priority = t[\"priority\"]\n",
|
|
" if responding:\n",
|
|
" resp_burst_type.append(t[\"type\"])\n",
|
|
" if t[\"type\"] >= 4 and t[\"type\"] <= 8:\n",
|
|
" last_term = t[\"type\"]\n",
|
|
" if responding:\n",
|
|
" # This response time interval has ended, so record the time delta and term\n",
|
|
" resp_time.append((t[\"time\"] - resp_burst_start, resp_burst_type))\n",
|
|
" responding = False\n",
|
|
" if (not responding) and (t[\"type\"] < 4 or t[\"type\"] > 8):\n",
|
|
" resp_burst_start = t[\"time\"]\n",
|
|
" resp_burst_type = [t[\"type\"]]\n",
|
|
" responding = True\n",
|
|
" \n",
|
|
" \n",
|
|
" tot.add(1)\n",
|
|
" if last_term != 6:\n",
|
|
" non.add(1)\n",
|
|
" return (priority, resp_time) if last_term == 5 else None\n",
|
|
"\n",
|
|
" \n",
|
|
"def cleanup(x):\n",
|
|
" return {\n",
|
|
" \"time\": int(x.time),\n",
|
|
" \"type\": 0 if x.type is None else int(x.type),\n",
|
|
" \"id\": x.collection_id + \"-\" + x.instance_index,\n",
|
|
" \"priority\": 0 if x.priority is None else int(x.priority)\n",
|
|
" }\n",
|
|
"\n",
|
|
"\n",
|
|
"df2 = df.limit(100000).rdd \\\n",
|
|
" .filter(lambda x: x.collection_type is None or x.collection_type == 0) \\\n",
|
|
" .filter(lambda x: x.type is not None and x.time is not None\n",
|
|
" and x.instance_index is not None and x.collection_id is not None) \\\n",
|
|
" .map(cleanup) \\\n",
|
|
" .groupBy(lambda x: x[\"id\"]) \\\n",
|
|
" .mapValues(for_each_task) \\\n",
|
|
" .filter(lambda x: x[1] is not None) \\\n",
|
|
" .map(lambda x: x[1]) \\\n",
|
|
" .groupBy(lambda x: x[0]) \\\n",
|
|
" .mapValues(lambda x: [e[1] for e in x])\n",
|
|
"\n",
|
|
"a = {\"val\": df2.collect(), \"tot\": tot.value, \"non\": non.value}\n",
|
|
"with open(cluster + \"_state_changes.json\", \"w\") as out:\n",
|
|
" json.dump(a, out)\n",
|
|
"\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "arctic-tomato",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.6.9"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|