diff --git a/machine_time_waste/a_state_changes_jobs.json b/machine_time_waste/a_state_changes_jobs.json new file mode 100644 index 00000000..62298c63 --- /dev/null +++ b/machine_time_waste/a_state_changes_jobs.json @@ -0,0 +1 @@ +{"filtered": 1641409153, "total": 1717317922, "data": [[null, {"0-9": 62889218, "9-1": 574847866520, "1-2": 266421041095728, "2-3": 37061909979697, "0-1": 479829380, "3-10": 2379731446042535, "10-10": 18461076566741235014, "9-2": 878159684299, "0-2": 3722284816, "2-9": 798198410561, "9-3": 5523154138779, "9-9": 548089204516941, "3-0": 526538292}], [4, {"0-1": 9285704917, "1-2": 758162399152668, "2-3": 53797491275202, "3-4": 8927535243167509, "4-0": 550366515143, "0-2": 47308159753593, "3-7": 11976311248731724, "7-0": 222050015869005, "2-9": 17113125591728, "9-3": 36343719748541, "3-0": 1423384355499624, "3-8": 415101438960, "8-3": 1787112950, "7-3": 146514681528769, "7-4": 31027588610710, "3-10": 8522396702857751372728, "10-10": 3623996601500180, "10-4": 2929278134868108, "2-4": 12635140003148, "8-4": 118790870, "8-7": 3936, "2-8": 46124286807, "10-0": 1353838838897479, "10-7": 1392318415415307, "2-7": 34545611294399, "3-5": 737101930588296, "5-0": 576175871, "0-9": 127321109779240, "9-2": 141698154931043, "9-1": 127975786957, "1-7": 1309160853899, "9-7": 11235334125732, "10-5": 415569638001568, "0-7": 2580548901373, "9-4": 1148878808121, "4-7": 154512914737, "1-9": 41258429797, "7-5": 657977963210, "3-6": 33203620074, "6-0": 8230516984, "7-9": 5658086411, "10-8": 2122220944, "7-2": 177582628892, "8-0": 60898618, "4-3": 8636948691, "9-9": 9676250880138, "7-1": 2695174526, "4-4": 7816627553, "2-5": 140208329, "3-3": 18446741404792061490, "5-3": 16706705, "10-6": 1359500090, "0-4": 256496170, "0-3": 230889827, "8-5": 1}], [5, {"0-1": 565737682, "1-2": 927658944495370, "2-3": 1443774911889, "3-10": 184467937311562062251, "10-5": 1567935427234927, "5-0": 1941235, "0-2": 10424325791921, "3-5": 1453513291259688, "0-9": 5864944672934, "9-1": 1584295325840, "2-9": 1292501611609, "9-3": 1641580641463, "9-2": 8268347200644, "4-0": 71505, "3-7": 1294237212450449, "7-0": 92994581682912, "3-4": 528775033912806, "10-7": 890852070914291, "10-0": 1575519907727489, "3-8": 98233519157, "8-3": 221721789, "7-3": 72814792784424, "10-10": 903490383865973, "3-0": 1039525349309203, "7-5": 5264202172940, "5-5": 406732829307, "10-4": 100410004555724, "2-4": 111695073250, "2-7": 610928751822, "3-6": 309896274729, "6-0": 1702687423562, "1-7": 22821973044, "0-7": 81546438436, "2-8": 6328325972, "8-7": 297, "10-6": 2226569278212, "7-4": 533048828446, "8-4": 87, "9-7": 114819257353, "9-4": 8745198026, "3-3": 19205246901, "4-7": 3570003009, "2-5": 446627507, "7-2": 75421670111, "8-0": 13, "0-5": 3535042290, "2-2": 2400408, "10-8": 195485236, "5-3": 1817634, "8-5": 3, "5-7": 49063474, "7-9": 14352512, "4-3": 2061630}], [6, {"10-6": 10430833437502549, "6-0": 55340387329641713905, "0-2": 173884279320552, "2-3": 658647495862312, "3-6": 21783383407070368, "2-9": 39756182538586, "9-3": 103021245486826, "3-10": 2953535582984881, "10-0": 4485092928005981, "3-8": 1932367637783, "8-3": 4928289684, "6-3": 7009762116281543531082, "2-6": 119417372152212, "0-9": 1319086862917023, "9-2": 3774044292878245, "9-6": 7436646546114771, "3-0": 5934135672786786, "10-10": 1456873062908955, "0-1": 242287406579669, "1-2": 30871525600349902, "10-1": 104666414423854, "1-6": 38756575242032, "6-2": 1094283994606752, "8-6": 1623505305, "9-1": 229928632691182, "3-1": 19985596921246, "1-9": 870177509493017, "10-4": 131988185972550, "4-0": 3177464, "3-7": 112146981800663, "7-0": 66038468974, "3-5": 233795315757012, "5-0": 932029, "3-4": 371902828915342, "10-7": 52010197051345, "10-5": 231234638200094, "2-4": 1194871439702, "2-8": 196301674478, "10-8": 52146674625, "7-6": 7743752863175, "9-4": 63518971050, "0-6": 24102029798237, "2-7": 2424732639988, "7-3": 6800183529156, "6-6": 18446746705199169010, "9-9": 4138781220858, "9-7": 886502154008, "6-1": 18285071074, "7-4": 468580427384, "4-7": 188719662, "7-5": 400726209580, "8-0": 10675077616, "2-5": 946186060, "8-4": 9, "8-7": 44, "3-2": 119, "2-10": 66717105, "6-7": 131, "2-0": 0, "0-3": 472, "0-7": 1289638569, "6-9": 296646957883}], [7, {"0-1": 7779699570201, "1-2": 34872566915938892, "2-3": 1351843250553592, "3-7": 137314005345368051, "3-4": 21101237820191209, "4-0": 53972695977090, "0-2": 31253878968285585, "2-9": 444451340267722, "9-3": 656784070187405, "7-0": 40341971453228296, "3-10": 223814367834209840756728, "10-5": 48859125066076751, "5-0": 77853904304, "10-0": 93724355493816042, "2-7": 1192137524870731, "3-0": 44902650356266238, "0-9": 3552080320054946, "9-2": 8048694994398823, "1-7": 12292216136182571, "1-9": 82650180879414, "10-7": 280386402099433301, "10-4": 66834592619893400, "9-1": 79360824138530, "3-5": 8049280587789709, "0-7": 982783794115290191, "3-8": 14838693488624, "8-3": 131327228635, "7-3": 56926650518380549995372, "7-7": 3716672610058, "3-6": 113457352619788, "6-0": 43914320435005, "2-4": 103017039258099, "7-4": 674366379932643, "7-1": 67169245937410, "9-7": 16892428984866330, "2-8": 1279467345493, "8-7": 1065616944, "10-10": 6788414709838044178660, "4-7": 1029052176333, "7-5": 24087446000368, "9-4": 17078248746361, "10-6": 38869367339860, "9-9": 2697185885457770, "8-4": 1968654468, "8-0": 1244658952, "7-2": 3624200523370, "4-3": 81655323346, "5-7": 468181434680, "10-8": 35999135981, "0-5": 15721191939, "7-9": 301474857097, "8-6": 38, "6-7": 32486, "2-5": 9080375186, "3-2": 5255287, "7-6": 235596312138, "8-10": 37059720, "2-6": 3624252609, "5-2": 49569578, "8-5": 28, "2-10": 79945897, "5-3": 583965815, "3-3": 5771217177, "7-10": 3795081618, "9-8": 57032194, "0-10": 18895246, "9-5": 3065979, "8-9": 12522716, "5-4": 1791010978, "4-5": 702303004, "0-4": 826664768, "4-2": 9, "8-2": 1}], [8, {"0-9": 3202919185255, "9-1": 702028050285, "1-7": 464860736729548, "7-1": 516720664264048, "1-8": 3304177800000000, "0-1": 6598181609, "7-0": 47771349426, "4-0": 781, "0-2": 24929, "2-3": 1660692385, "3-4": 124283909374, "3-7": 110940681299, "3-5": 113321516132, "5-0": 261, "2-7": 270799289324, "3-0": 236864931239, "3-8": 553402241845522418064, "8-3": 182921704, "2-9": 159776270345, "9-3": 330648554353, "3-10": 224014654529, "10-4": 221775483292, "10-7": 203035001586, "9-2": 1108844970327, "7-4": 74631846843, "8-4": 30, "8-7": 30, "10-6": 172267124389, "6-0": 298522938, "10-0": 303949876583, "3-6": 13327947607, "2-6": 181736406600, "9-6": 248182189602, "6-3": 91617321393, "8-6": 21, "9-7": 24467972271, "2-8": 514508117, "7-3": 39258982849, "3-1": 3144671011, "1-6": 1195049638, "6-2": 272249832779, "10-1": 19714580924, "1-2": 137202887559, "8-0": 2, "4-7": 614717997, "10-5": 5299223143, "7-5": 270022684, "2-4": 12736329}]]} \ No newline at end of file diff --git a/machine_time_waste/machine_time_waste_inefficient.ipynb b/machine_time_waste/machine_time_waste_inefficient.ipynb deleted file mode 100644 index 40fa4661..00000000 --- a/machine_time_waste/machine_time_waste_inefficient.ipynb +++ /dev/null @@ -1,895 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "proper-gnome", - "metadata": {}, - "source": [ - "# Temporal impact: machine time waste" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "fantastic-harrison", - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } - }, - "outputs": [], - "source": [ - "import pandas\n", - "from IPython import display\n", - "import findspark\n", - "findspark.init()\n", - "import pyspark\n", - "import pyspark.sql\n", - "import sys\n", - "\n", - "from pyspark.sql.functions import col, lag, when, concat_ws, last, first\n", - "from pyspark.sql import Window\n", - "from pyspark.sql.types import LongType" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "failing-rebecca", - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } - }, - "outputs": [], - "source": [ - "cluster=\"b\"" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "transsexual-baptist", - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "ERROR:root:Exception while sending command.\n", - "Traceback (most recent call last):\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1207, in send_command\n", - " raise Py4JNetworkError(\"Answer from Java side is empty\")\n", - "py4j.protocol.Py4JNetworkError: Answer from Java side is empty\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1033, in send_command\n", - " response = connection.send_command(command)\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1212, in send_command\n", - " \"Error while receiving\", e, proto.ERROR_ON_RECEIVE)\n", - "py4j.protocol.Py4JNetworkError: Error while receiving\n", - "ERROR:py4j.java_gateway:An error occurred while trying to connect to the Java server (127.0.0.1:36135)\n", - "Traceback (most recent call last):\n", - " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3343, in run_code\n", - " exec(code_obj, self.user_global_ns, self.user_ns)\n", - " File \"\", line 5, in \n", - " df = spark.read.json(\"/home/claudio/google_2019/instance_events/\" + cluster + \"/\" + cluster + \"_instance_events*.json.gz\")\n", - " File \"/opt/spark/python/pyspark/sql/readwriter.py\", line 300, in json\n", - " return self._df(self._jreader.json(self._spark._sc._jvm.PythonUtils.toSeq(path)))\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1305, in __call__\n", - " answer, self.gateway_client, self.target_id, self.name)\n", - " File \"/opt/spark/python/pyspark/sql/utils.py\", line 128, in deco\n", - " return f(*a, **kw)\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/protocol.py\", line 336, in get_return_value\n", - " format(target_id, \".\", name))\n", - "py4j.protocol.Py4JError: An error occurred while calling o26.json\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2044, in showtraceback\n", - " stb = value._render_traceback_()\n", - "AttributeError: 'Py4JError' object has no attribute '_render_traceback_'\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 977, in _get_connection\n", - " connection = self.deque.pop()\n", - "IndexError: pop from an empty deque\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1115, in start\n", - " self.socket.connect((self.address, self.port))\n", - "ConnectionRefusedError: [Errno 111] Connection refused\n", - "ERROR:py4j.java_gateway:An error occurred while trying to connect to the Java server (127.0.0.1:36135)\n", - "Traceback (most recent call last):\n", - " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3343, in run_code\n", - " exec(code_obj, self.user_global_ns, self.user_ns)\n", - " File \"\", line 5, in \n", - " df = spark.read.json(\"/home/claudio/google_2019/instance_events/\" + cluster + \"/\" + cluster + \"_instance_events*.json.gz\")\n", - " File \"/opt/spark/python/pyspark/sql/readwriter.py\", line 300, in json\n", - " return self._df(self._jreader.json(self._spark._sc._jvm.PythonUtils.toSeq(path)))\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1305, in __call__\n", - " answer, self.gateway_client, self.target_id, self.name)\n", - " File \"/opt/spark/python/pyspark/sql/utils.py\", line 128, in deco\n", - " return f(*a, **kw)\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/protocol.py\", line 336, in get_return_value\n", - " format(target_id, \".\", name))\n", - "py4j.protocol.Py4JError: An error occurred while calling o26.json\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2044, in showtraceback\n", - " stb = value._render_traceback_()\n", - "AttributeError: 'Py4JError' object has no attribute '_render_traceback_'\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 977, in _get_connection\n", - " connection = self.deque.pop()\n", - "IndexError: pop from an empty deque\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1115, in start\n", - " self.socket.connect((self.address, self.port))\n", - "ConnectionRefusedError: [Errno 111] Connection refused\n", - "ERROR:py4j.java_gateway:An error occurred while trying to connect to the Java server (127.0.0.1:36135)\n", - "Traceback (most recent call last):\n", - " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3343, in run_code\n", - " exec(code_obj, self.user_global_ns, self.user_ns)\n", - " File \"\", line 5, in \n", - " df = spark.read.json(\"/home/claudio/google_2019/instance_events/\" + cluster + \"/\" + cluster + \"_instance_events*.json.gz\")\n", - " File \"/opt/spark/python/pyspark/sql/readwriter.py\", line 300, in json\n", - " return self._df(self._jreader.json(self._spark._sc._jvm.PythonUtils.toSeq(path)))\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1305, in __call__\n", - " answer, self.gateway_client, self.target_id, self.name)\n", - " File \"/opt/spark/python/pyspark/sql/utils.py\", line 128, in deco\n", - " return f(*a, **kw)\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/protocol.py\", line 336, in get_return_value\n", - " format(target_id, \".\", name))\n", - "py4j.protocol.Py4JError: An error occurred while calling o26.json\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2044, in showtraceback\n", - " stb = value._render_traceback_()\n", - "AttributeError: 'Py4JError' object has no attribute '_render_traceback_'\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 977, in _get_connection\n", - " connection = self.deque.pop()\n", - "IndexError: pop from an empty deque\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1115, in start\n", - " self.socket.connect((self.address, self.port))\n", - "ConnectionRefusedError: [Errno 111] Connection refused\n", - "ERROR:py4j.java_gateway:An error occurred while trying to connect to the Java server (127.0.0.1:36135)\n", - "Traceback (most recent call last):\n", - " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3343, in run_code\n", - " exec(code_obj, self.user_global_ns, self.user_ns)\n", - " File \"\", line 5, in \n", - " df = spark.read.json(\"/home/claudio/google_2019/instance_events/\" + cluster + \"/\" + cluster + \"_instance_events*.json.gz\")\n", - " File \"/opt/spark/python/pyspark/sql/readwriter.py\", line 300, in json\n", - " return self._df(self._jreader.json(self._spark._sc._jvm.PythonUtils.toSeq(path)))\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1305, in __call__\n", - " answer, self.gateway_client, self.target_id, self.name)\n", - " File \"/opt/spark/python/pyspark/sql/utils.py\", line 128, in deco\n", - " return f(*a, **kw)\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/protocol.py\", line 336, in get_return_value\n", - " format(target_id, \".\", name))\n", - "py4j.protocol.Py4JError: An error occurred while calling o26.json\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2044, in showtraceback\n", - " stb = value._render_traceback_()\n", - "AttributeError: 'Py4JError' object has no attribute '_render_traceback_'\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 977, in _get_connection\n", - " connection = self.deque.pop()\n", - "IndexError: pop from an empty deque\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1115, in start\n", - " self.socket.connect((self.address, self.port))\n", - "ConnectionRefusedError: [Errno 111] Connection refused\n", - "ERROR:py4j.java_gateway:An error occurred while trying to connect to the Java server (127.0.0.1:36135)\n", - "Traceback (most recent call last):\n", - " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3343, in run_code\n", - " exec(code_obj, self.user_global_ns, self.user_ns)\n", - " File \"\", line 5, in \n", - " df = spark.read.json(\"/home/claudio/google_2019/instance_events/\" + cluster + \"/\" + cluster + \"_instance_events*.json.gz\")\n", - " File \"/opt/spark/python/pyspark/sql/readwriter.py\", line 300, in json\n", - " return self._df(self._jreader.json(self._spark._sc._jvm.PythonUtils.toSeq(path)))\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1305, in __call__\n", - " answer, self.gateway_client, self.target_id, self.name)\n", - " File \"/opt/spark/python/pyspark/sql/utils.py\", line 128, in deco\n", - " return f(*a, **kw)\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/protocol.py\", line 336, in get_return_value\n", - " format(target_id, \".\", name))\n", - "py4j.protocol.Py4JError: An error occurred while calling o26.json\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2044, in showtraceback\n", - " stb = value._render_traceback_()\n", - "AttributeError: 'Py4JError' object has no attribute '_render_traceback_'\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 977, in _get_connection\n", - " connection = self.deque.pop()\n", - "IndexError: pop from an empty deque\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1115, in start\n", - " self.socket.connect((self.address, self.port))\n", - "ConnectionRefusedError: [Errno 111] Connection refused\n", - "ERROR:py4j.java_gateway:An error occurred while trying to connect to the Java server (127.0.0.1:36135)\n", - "Traceback (most recent call last):\n", - " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3343, in run_code\n", - " exec(code_obj, self.user_global_ns, self.user_ns)\n", - " File \"\", line 5, in \n", - " df = spark.read.json(\"/home/claudio/google_2019/instance_events/\" + cluster + \"/\" + cluster + \"_instance_events*.json.gz\")\n", - " File \"/opt/spark/python/pyspark/sql/readwriter.py\", line 300, in json\n", - " return self._df(self._jreader.json(self._spark._sc._jvm.PythonUtils.toSeq(path)))\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1305, in __call__\n", - " answer, self.gateway_client, self.target_id, self.name)\n", - " File \"/opt/spark/python/pyspark/sql/utils.py\", line 128, in deco\n", - " return f(*a, **kw)\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/protocol.py\", line 336, in get_return_value\n", - " format(target_id, \".\", name))\n", - "py4j.protocol.Py4JError: An error occurred while calling o26.json\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2044, in showtraceback\n", - " stb = value._render_traceback_()\n", - "AttributeError: 'Py4JError' object has no attribute '_render_traceback_'\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 977, in _get_connection\n", - " connection = self.deque.pop()\n", - "IndexError: pop from an empty deque\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1115, in start\n", - " self.socket.connect((self.address, self.port))\n", - "ConnectionRefusedError: [Errno 111] Connection refused\n", - "ERROR:py4j.java_gateway:An error occurred while trying to connect to the Java server (127.0.0.1:36135)\n", - "Traceback (most recent call last):\n", - " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3343, in run_code\n", - " exec(code_obj, self.user_global_ns, self.user_ns)\n", - " File \"\", line 5, in \n", - " df = spark.read.json(\"/home/claudio/google_2019/instance_events/\" + cluster + \"/\" + cluster + \"_instance_events*.json.gz\")\n", - " File \"/opt/spark/python/pyspark/sql/readwriter.py\", line 300, in json\n", - " return self._df(self._jreader.json(self._spark._sc._jvm.PythonUtils.toSeq(path)))\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1305, in __call__\n", - " answer, self.gateway_client, self.target_id, self.name)\n", - " File \"/opt/spark/python/pyspark/sql/utils.py\", line 128, in deco\n", - " return f(*a, **kw)\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/protocol.py\", line 336, in get_return_value\n", - " format(target_id, \".\", name))\n", - "py4j.protocol.Py4JError: An error occurred while calling o26.json\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2044, in showtraceback\n", - " stb = value._render_traceback_()\n", - "AttributeError: 'Py4JError' object has no attribute '_render_traceback_'\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 977, in _get_connection\n", - " connection = self.deque.pop()\n", - "IndexError: pop from an empty deque\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1115, in start\n", - " self.socket.connect((self.address, self.port))\n", - "ConnectionRefusedError: [Errno 111] Connection refused\n", - "ERROR:py4j.java_gateway:An error occurred while trying to connect to the Java server (127.0.0.1:36135)\n", - "Traceback (most recent call last):\n", - " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3343, in run_code\n", - " exec(code_obj, self.user_global_ns, self.user_ns)\n", - " File \"\", line 5, in \n", - " df = spark.read.json(\"/home/claudio/google_2019/instance_events/\" + cluster + \"/\" + cluster + \"_instance_events*.json.gz\")\n", - " File \"/opt/spark/python/pyspark/sql/readwriter.py\", line 300, in json\n", - " return self._df(self._jreader.json(self._spark._sc._jvm.PythonUtils.toSeq(path)))\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1305, in __call__\n", - " answer, self.gateway_client, self.target_id, self.name)\n", - " File \"/opt/spark/python/pyspark/sql/utils.py\", line 128, in deco\n", - " return f(*a, **kw)\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/protocol.py\", line 336, in get_return_value\n", - " format(target_id, \".\", name))\n", - "py4j.protocol.Py4JError: An error occurred while calling o26.json\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2044, in showtraceback\n", - " stb = value._render_traceback_()\n", - "AttributeError: 'Py4JError' object has no attribute '_render_traceback_'\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 977, in _get_connection\n", - " connection = self.deque.pop()\n", - "IndexError: pop from an empty deque\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1115, in start\n", - " self.socket.connect((self.address, self.port))\n", - "ConnectionRefusedError: [Errno 111] Connection refused\n", - "ERROR:py4j.java_gateway:An error occurred while trying to connect to the Java server (127.0.0.1:36135)\n", - "Traceback (most recent call last):\n", - " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3343, in run_code\n", - " exec(code_obj, self.user_global_ns, self.user_ns)\n", - " File \"\", line 5, in \n", - " df = spark.read.json(\"/home/claudio/google_2019/instance_events/\" + cluster + \"/\" + cluster + \"_instance_events*.json.gz\")\n", - " File \"/opt/spark/python/pyspark/sql/readwriter.py\", line 300, in json\n", - " return self._df(self._jreader.json(self._spark._sc._jvm.PythonUtils.toSeq(path)))\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1305, in __call__\n", - " answer, self.gateway_client, self.target_id, self.name)\n", - " File \"/opt/spark/python/pyspark/sql/utils.py\", line 128, in deco\n", - " return f(*a, **kw)\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/protocol.py\", line 336, in get_return_value\n", - " format(target_id, \".\", name))\n", - "py4j.protocol.Py4JError: An error occurred while calling o26.json\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2044, in showtraceback\n", - " stb = value._render_traceback_()\n", - "AttributeError: 'Py4JError' object has no attribute '_render_traceback_'\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 977, in _get_connection\n", - " connection = self.deque.pop()\n", - "IndexError: pop from an empty deque\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1115, in start\n", - " self.socket.connect((self.address, self.port))\n", - "ConnectionRefusedError: [Errno 111] Connection refused\n", - "ERROR:py4j.java_gateway:An error occurred while trying to connect to the Java server (127.0.0.1:36135)\n", - "Traceback (most recent call last):\n", - " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3343, in run_code\n", - " exec(code_obj, self.user_global_ns, self.user_ns)\n", - " File \"\", line 5, in \n", - " df = spark.read.json(\"/home/claudio/google_2019/instance_events/\" + cluster + \"/\" + cluster + \"_instance_events*.json.gz\")\n", - " File \"/opt/spark/python/pyspark/sql/readwriter.py\", line 300, in json\n", - " return self._df(self._jreader.json(self._spark._sc._jvm.PythonUtils.toSeq(path)))\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1305, in __call__\n", - " answer, self.gateway_client, self.target_id, self.name)\n", - " File \"/opt/spark/python/pyspark/sql/utils.py\", line 128, in deco\n", - " return f(*a, **kw)\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/protocol.py\", line 336, in get_return_value\n", - " format(target_id, \".\", name))\n", - "py4j.protocol.Py4JError: An error occurred while calling o26.json\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2044, in showtraceback\n", - " stb = value._render_traceback_()\n", - "AttributeError: 'Py4JError' object has no attribute '_render_traceback_'\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 977, in _get_connection\n", - " connection = self.deque.pop()\n", - "IndexError: pop from an empty deque\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1115, in start\n", - " self.socket.connect((self.address, self.port))\n", - "ConnectionRefusedError: [Errno 111] Connection refused\n", - "ERROR:py4j.java_gateway:An error occurred while trying to connect to the Java server (127.0.0.1:36135)\n", - "Traceback (most recent call last):\n", - " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3343, in run_code\n", - " exec(code_obj, self.user_global_ns, self.user_ns)\n", - " File \"\", line 5, in \n", - " df = spark.read.json(\"/home/claudio/google_2019/instance_events/\" + cluster + \"/\" + cluster + \"_instance_events*.json.gz\")\n", - " File \"/opt/spark/python/pyspark/sql/readwriter.py\", line 300, in json\n", - " return self._df(self._jreader.json(self._spark._sc._jvm.PythonUtils.toSeq(path)))\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1305, in __call__\n", - " answer, self.gateway_client, self.target_id, self.name)\n", - " File \"/opt/spark/python/pyspark/sql/utils.py\", line 128, in deco\n", - " return f(*a, **kw)\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/protocol.py\", line 336, in get_return_value\n", - " format(target_id, \".\", name))\n", - "py4j.protocol.Py4JError: An error occurred while calling o26.json\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2044, in showtraceback\n", - " stb = value._render_traceback_()\n", - "AttributeError: 'Py4JError' object has no attribute '_render_traceback_'\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 977, in _get_connection\n", - " connection = self.deque.pop()\n", - "IndexError: pop from an empty deque\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1115, in start\n", - " self.socket.connect((self.address, self.port))\n", - "ConnectionRefusedError: [Errno 111] Connection refused\n", - "ERROR:py4j.java_gateway:An error occurred while trying to connect to the Java server (127.0.0.1:36135)\n", - "Traceback (most recent call last):\n", - " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3343, in run_code\n", - " exec(code_obj, self.user_global_ns, self.user_ns)\n", - " File \"\", line 5, in \n", - " df = spark.read.json(\"/home/claudio/google_2019/instance_events/\" + cluster + \"/\" + cluster + \"_instance_events*.json.gz\")\n", - " File \"/opt/spark/python/pyspark/sql/readwriter.py\", line 300, in json\n", - " return self._df(self._jreader.json(self._spark._sc._jvm.PythonUtils.toSeq(path)))\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1305, in __call__\n", - " answer, self.gateway_client, self.target_id, self.name)\n", - " File \"/opt/spark/python/pyspark/sql/utils.py\", line 128, in deco\n", - " return f(*a, **kw)\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/protocol.py\", line 336, in get_return_value\n", - " format(target_id, \".\", name))\n", - "py4j.protocol.Py4JError: An error occurred while calling o26.json\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2044, in showtraceback\n", - " stb = value._render_traceback_()\n", - "AttributeError: 'Py4JError' object has no attribute '_render_traceback_'\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 977, in _get_connection\n", - " connection = self.deque.pop()\n", - "IndexError: pop from an empty deque\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1115, in start\n", - " self.socket.connect((self.address, self.port))\n", - "ConnectionRefusedError: [Errno 111] Connection refused\n", - "ERROR:py4j.java_gateway:An error occurred while trying to connect to the Java server (127.0.0.1:36135)\n", - "Traceback (most recent call last):\n", - " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3343, in run_code\n", - " exec(code_obj, self.user_global_ns, self.user_ns)\n", - " File \"\", line 5, in \n", - " df = spark.read.json(\"/home/claudio/google_2019/instance_events/\" + cluster + \"/\" + cluster + \"_instance_events*.json.gz\")\n", - " File \"/opt/spark/python/pyspark/sql/readwriter.py\", line 300, in json\n", - " return self._df(self._jreader.json(self._spark._sc._jvm.PythonUtils.toSeq(path)))\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1305, in __call__\n", - " answer, self.gateway_client, self.target_id, self.name)\n", - " File \"/opt/spark/python/pyspark/sql/utils.py\", line 128, in deco\n", - " return f(*a, **kw)\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/protocol.py\", line 336, in get_return_value\n", - " format(target_id, \".\", name))\n", - "py4j.protocol.Py4JError: An error occurred while calling o26.json\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2044, in showtraceback\n", - " stb = value._render_traceback_()\n", - "AttributeError: 'Py4JError' object has no attribute '_render_traceback_'\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 977, in _get_connection\n", - " connection = self.deque.pop()\n", - "IndexError: pop from an empty deque\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1115, in start\n", - " self.socket.connect((self.address, self.port))\n", - "ConnectionRefusedError: [Errno 111] Connection refused\n", - "ERROR:py4j.java_gateway:An error occurred while trying to connect to the Java server (127.0.0.1:36135)\n", - "Traceback (most recent call last):\n", - " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3343, in run_code\n", - " exec(code_obj, self.user_global_ns, self.user_ns)\n", - " File \"\", line 5, in \n", - " df = spark.read.json(\"/home/claudio/google_2019/instance_events/\" + cluster + \"/\" + cluster + \"_instance_events*.json.gz\")\n", - " File \"/opt/spark/python/pyspark/sql/readwriter.py\", line 300, in json\n", - " return self._df(self._jreader.json(self._spark._sc._jvm.PythonUtils.toSeq(path)))\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1305, in __call__\n", - " answer, self.gateway_client, self.target_id, self.name)\n", - " File \"/opt/spark/python/pyspark/sql/utils.py\", line 128, in deco\n", - " return f(*a, **kw)\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/protocol.py\", line 336, in get_return_value\n", - " format(target_id, \".\", name))\n", - "py4j.protocol.Py4JError: An error occurred while calling o26.json\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2044, in showtraceback\n", - " stb = value._render_traceback_()\n", - "AttributeError: 'Py4JError' object has no attribute '_render_traceback_'\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 977, in _get_connection\n", - " connection = self.deque.pop()\n", - "IndexError: pop from an empty deque\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1115, in start\n", - " self.socket.connect((self.address, self.port))\n", - "ConnectionRefusedError: [Errno 111] Connection refused\n", - "ERROR:py4j.java_gateway:An error occurred while trying to connect to the Java server (127.0.0.1:36135)\n", - "Traceback (most recent call last):\n", - " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3343, in run_code\n", - " exec(code_obj, self.user_global_ns, self.user_ns)\n", - " File \"\", line 5, in \n", - " df = spark.read.json(\"/home/claudio/google_2019/instance_events/\" + cluster + \"/\" + cluster + \"_instance_events*.json.gz\")\n", - " File \"/opt/spark/python/pyspark/sql/readwriter.py\", line 300, in json\n", - " return self._df(self._jreader.json(self._spark._sc._jvm.PythonUtils.toSeq(path)))\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1305, in __call__\n", - " answer, self.gateway_client, self.target_id, self.name)\n", - " File \"/opt/spark/python/pyspark/sql/utils.py\", line 128, in deco\n", - " return f(*a, **kw)\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/protocol.py\", line 336, in get_return_value\n", - " format(target_id, \".\", name))\n", - "py4j.protocol.Py4JError: An error occurred while calling o26.json\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2044, in showtraceback\n", - " stb = value._render_traceback_()\n", - "AttributeError: 'Py4JError' object has no attribute '_render_traceback_'\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 977, in _get_connection\n", - " connection = self.deque.pop()\n", - "IndexError: pop from an empty deque\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1115, in start\n", - " self.socket.connect((self.address, self.port))\n", - "ConnectionRefusedError: [Errno 111] Connection refused\n", - "ERROR:py4j.java_gateway:An error occurred while trying to connect to the Java server (127.0.0.1:36135)\n", - "Traceback (most recent call last):\n", - " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3343, in run_code\n", - " exec(code_obj, self.user_global_ns, self.user_ns)\n", - " File \"\", line 5, in \n", - " df = spark.read.json(\"/home/claudio/google_2019/instance_events/\" + cluster + \"/\" + cluster + \"_instance_events*.json.gz\")\n", - " File \"/opt/spark/python/pyspark/sql/readwriter.py\", line 300, in json\n", - " return self._df(self._jreader.json(self._spark._sc._jvm.PythonUtils.toSeq(path)))\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1305, in __call__\n", - " answer, self.gateway_client, self.target_id, self.name)\n", - " File \"/opt/spark/python/pyspark/sql/utils.py\", line 128, in deco\n", - " return f(*a, **kw)\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/protocol.py\", line 336, in get_return_value\n", - " format(target_id, \".\", name))\n", - "py4j.protocol.Py4JError: An error occurred while calling o26.json\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2044, in showtraceback\n", - " stb = value._render_traceback_()\n", - "AttributeError: 'Py4JError' object has no attribute '_render_traceback_'\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 977, in _get_connection\n", - " connection = self.deque.pop()\n", - "IndexError: pop from an empty deque\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1115, in start\n", - " self.socket.connect((self.address, self.port))\n", - "ConnectionRefusedError: [Errno 111] Connection refused\n", - "ERROR:py4j.java_gateway:An error occurred while trying to connect to the Java server (127.0.0.1:36135)\n", - "Traceback (most recent call last):\n", - " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3343, in run_code\n", - " exec(code_obj, self.user_global_ns, self.user_ns)\n", - " File \"\", line 5, in \n", - " df = spark.read.json(\"/home/claudio/google_2019/instance_events/\" + cluster + \"/\" + cluster + \"_instance_events*.json.gz\")\n", - " File \"/opt/spark/python/pyspark/sql/readwriter.py\", line 300, in json\n", - " return self._df(self._jreader.json(self._spark._sc._jvm.PythonUtils.toSeq(path)))\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1305, in __call__\n", - " answer, self.gateway_client, self.target_id, self.name)\n", - " File \"/opt/spark/python/pyspark/sql/utils.py\", line 128, in deco\n", - " return f(*a, **kw)\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/protocol.py\", line 336, in get_return_value\n", - " format(target_id, \".\", name))\n", - "py4j.protocol.Py4JError: An error occurred while calling o26.json\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2044, in showtraceback\n", - " stb = value._render_traceback_()\n", - "AttributeError: 'Py4JError' object has no attribute '_render_traceback_'\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 977, in _get_connection\n", - " connection = self.deque.pop()\n", - "IndexError: pop from an empty deque\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1115, in start\n", - " self.socket.connect((self.address, self.port))\n", - "ConnectionRefusedError: [Errno 111] Connection refused\n" - ] - }, - { - "ename": "Py4JError", - "evalue": "An error occurred while calling o26.json", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mPy4JError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mgetOrCreate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mspark\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjson\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"/home/claudio/google_2019/instance_events/\"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mcluster\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\"/\"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mcluster\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\"_instance_events*.json.gz\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;32m/opt/spark/python/pyspark/sql/readwriter.py\u001b[0m in \u001b[0;36mjson\u001b[0;34m(self, path, schema, primitivesAsString, prefersDecimal, allowComments, allowUnquotedFieldNames, allowSingleQuotes, allowNumericLeadingZero, allowBackslashEscapingAnyCharacter, mode, columnNameOfCorruptRecord, dateFormat, timestampFormat, multiLine, allowUnquotedControlChars, lineSep, samplingRatio, dropFieldIfAllNull, encoding, locale, pathGlobFilter, recursiveFileLookup)\u001b[0m\n\u001b[1;32m 298\u001b[0m \u001b[0mpath\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 299\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 300\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_df\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jreader\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjson\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_spark\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jvm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPythonUtils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoSeq\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 301\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mRDD\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 302\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args)\u001b[0m\n\u001b[1;32m 1303\u001b[0m \u001b[0manswer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgateway_client\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend_command\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcommand\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1304\u001b[0m return_value = get_return_value(\n\u001b[0;32m-> 1305\u001b[0;31m answer, self.gateway_client, self.target_id, self.name)\n\u001b[0m\u001b[1;32m 1306\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1307\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mtemp_arg\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtemp_args\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/opt/spark/python/pyspark/sql/utils.py\u001b[0m in \u001b[0;36mdeco\u001b[0;34m(*a, **kw)\u001b[0m\n\u001b[1;32m 126\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mdeco\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkw\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 127\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 128\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkw\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 129\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mpy4j\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprotocol\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPy4JJavaError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 130\u001b[0m \u001b[0mconverted\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconvert_exception\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjava_exception\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/protocol.py\u001b[0m in \u001b[0;36mget_return_value\u001b[0;34m(answer, gateway_client, target_id, name)\u001b[0m\n\u001b[1;32m 334\u001b[0m raise Py4JError(\n\u001b[1;32m 335\u001b[0m \u001b[0;34m\"An error occurred while calling {0}{1}{2}\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 336\u001b[0;31m format(target_id, \".\", name))\n\u001b[0m\u001b[1;32m 337\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 338\u001b[0m \u001b[0mtype\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0manswer\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mPy4JError\u001b[0m: An error occurred while calling o26.json" - ] - } - ], - "source": [ - "spark = pyspark.sql.SparkSession.builder \\\n", - " .appName(\"machine_time_waste\") \\\n", - " .getOrCreate()\n", - "\n", - "df = spark.read.json(\"/home/claudio/google_2019/instance_events/\" + cluster + \"/\" + cluster + \"_instance_events*.json.gz\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "juvenile-absolute", - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } - }, - "outputs": [], - "source": [ - "df.printSchema()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "lucky-western", - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } - }, - "outputs": [], - "source": [ - "df.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "normal-settlement", - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } - }, - "outputs": [], - "source": [ - "# .filter(df.collection_type == 0) \\\n", - "df2 = df \\\n", - " .withColumn(\"time\", col(\"time\").cast(LongType())) \\\n", - " .withColumn(\"type\", col(\"type\").cast(LongType())) \\\n", - " .withColumn(\"type\", when(col(\"type\").isNull(), 0).otherwise(col(\"type\"))) \\\n", - " .withColumn(\"id\", concat_ws(\"-\", \"collection_id\", \"instance_index\")) \\\n", - " .where(col(\"time\").isNotNull()) \\\n", - " .where(col(\"type\").isNotNull()) \\\n", - " .where((col(\"instance_index\").isNotNull()) & (col(\"collection_id\").isNotNull())) \\\n", - " .select(\"machine_id\", \"id\", \"time\", \"type\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "typical-homeless", - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } - }, - "outputs": [], - "source": [ - "df2.show()\n", - "print(\"Total: \" + str(df.count()))\n", - "print(\"Filtered: \" + str(df2.count()))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "collect-saying", - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } - }, - "outputs": [], - "source": [ - "# my_window = Window.partitionBy(\"machine_id\", \"id\").orderBy(df2.time.asc())\n", - "\n", - "w2 = Window.partitionBy(\"id\").orderBy(df2.time.asc()).rowsBetween(Window.currentRow, Window.unboundedFollowing)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "cooperative-appraisal", - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } - }, - "outputs": [], - "source": [ - "# .withColumn(\"prev_time\", lag(df2.time).over(my_window)) \\\n", - "# .withColumn(\"prev_type\", lag(df2.type).over(my_window)) \\\n", - "\n", - "df3 = df2 \\\n", - " .withColumn(\"t3_time\", when((df2.type != 3), None).otherwise(df2.time)) \\\n", - " .withColumn(\"t45678_time\", when((df2.type < 4) | (df2.type > 8), None).otherwise(df2.time)) \\\n", - " .withColumn(\"t45678_type\", when((df2.type < 4) | (df2.type > 8), None).otherwise(df2.type)) \\\n", - " .withColumn(\"t01_time\", when((df2.type != 0) & (df2.type != 1), None).otherwise(df2.time)) \\\n", - " .withColumn(\"t01_type\", when((df2.type != 0) & (df2.type != 1), None).otherwise(df2.type)) \\\n", - " .withColumn(\"next_time\", when(df2.type == 3, first(col(\"t45678_time\"), ignorenulls=True).over(w2)) \\\n", - " .when((df2.type == 0) | (df2.type == 1), first(col(\"t3_time\"), ignorenulls=True).over(w2)) \\\n", - " .when((df2.type >= 4) | (df2.type <= 8), first(col(\"t01_time\"), ignorenulls=True).over(w2)) \\\n", - " .otherwise(None)) \\\n", - " .withColumn(\"next_type\", when(df2.type == 3, first(col(\"t45678_type\"), ignorenulls=True).over(w2)) \\\n", - " .when((df2.type == 0) | (df2.type == 1), 3) \\\n", - " .when((df2.type >= 4) | (df2.type <= 8), first(col(\"t01_type\"), ignorenulls=True).over(w2)) \\\n", - " .otherwise(None)) \\\n", - " .withColumn(\"last_term_type\", last(col(\"t45678_type\"), ignorenulls=True).over(w2)) \\\n", - " .withColumn(\"time_delta\", col(\"next_time\") - col(\"time\")) \\\n", - " .select(\"machine_id\", \"id\", \"time\", \"type\", \"last_term_type\", \"time_delta\", \"t01_time\", \"t01_type\", \"t3_time\", \"t45678_time\", \"t45678_type\", \"next_time\", \"next_type\") \\" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ideal-angle", - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } - }, - "outputs": [], - "source": [ - "df4 = df3.where(df3.next_type.isNotNull()).groupby(\"type\", \"next_type\", \"last_term_type\").sum(\"time_delta\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "working-difficulty", - "metadata": { - "collapsed": false, - "jupyter": { - "outputs_hidden": false - } - }, - "outputs": [], - "source": [ - "# df3.orderBy(df3.machine_id, df3.time).show(n=100)\n", - "# df3.printSchema()\n", - "df4.show(n=1000000)\n", - "df4.write.csv(\"/home/claudio/google_2019/thesis_queries/machine_time_waste/\" + cluster + \"_state_change.csv\")" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.9" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/machine_time_waste/machine_time_waste_inefficient.py b/machine_time_waste/machine_time_waste_inefficient.py deleted file mode 100755 index 0a21b034..00000000 --- a/machine_time_waste/machine_time_waste_inefficient.py +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env python3 -# coding: utf-8 - -# # Temporal impact: machine time waste - -# Inefficient version of the machine_time_waste.py script in the same dir. Please ignore, kept for future need - -import pandas -from IPython import display -import findspark -findspark.init() -import pyspark -import pyspark.sql -import sys - -from pyspark.sql.functions import col, lag, when, concat_ws, last, first -from pyspark.sql import Window -from pyspark.sql.types import LongType - -cluster="b" - -spark = pyspark.sql.SparkSession.builder \ - .appName("machine_time_waste") \ - .config("spark.local.dir", "/run/tmpfiles.d/spark") \ - .config("spark.driver.memory", "124g") \ - .getOrCreate() - -df = spark.read.json("/home/claudio/google_2019/instance_events/" + cluster + "/" + cluster + "_instance_events*.json.gz") - -df.printSchema() - -df.show() - -# .filter(df.collection_type == 0) \ -df2 = df \ - .withColumn("time", col("time").cast(LongType())) \ - .withColumn("type", col("type").cast(LongType())) \ - .withColumn("type", when(col("type").isNull(), 0).otherwise(col("type"))) \ - .withColumn("id", concat_ws("-", "collection_id", "instance_index")) \ - .where(col("time").isNotNull()) \ - .where(col("type").isNotNull()) \ - .where((col("instance_index").isNotNull()) & (col("collection_id").isNotNull())) \ - .select("time", "type", "id") - -df2.show() -print("Total: " + str(df.count())) -print("Filtered: " + str(df2.count())) - -# my_window = Window.partitionBy("machine_id", "id").orderBy(df2.time.asc()) - -w2 = Window.partitionBy("id").orderBy(df2.time.asc()).rowsBetween(Window.currentRow, Window.unboundedFollowing) - -# .withColumn("prev_time", lag(df2.time).over(my_window)) \ -# .withColumn("prev_type", lag(df2.type).over(my_window)) \ -df3 = df2 \ - .withColumn("t3_time", when((df2.type != 3), None).otherwise(df2.time)) \ - .withColumn("t45678_time", when((df2.type < 4) | (df2.type > 8), None).otherwise(df2.time)) \ - .withColumn("t45678_type", when((df2.type < 4) | (df2.type > 8), None).otherwise(df2.type)) \ - .withColumn("t01_time", when((df2.type != 0) & (df2.type != 1), None).otherwise(df2.time)) \ - .withColumn("t01_type", when((df2.type != 0) & (df2.type != 1), None).otherwise(df2.type)) \ - .withColumn("next_time", when(df2.type == 3, first(col("t45678_time"), True).over(w2)) \ - .when((df2.type == 0) | (df2.type == 1), first(col("t3_time"), True).over(w2)) \ - .when((df2.type >= 4) | (df2.type <= 8), first(col("t01_time"), True).over(w2)) \ - .otherwise(None)) \ - .withColumn("next_type", when(df2.type == 3, first(col("t45678_type"), True).over(w2)) \ - .when((df2.type == 0) | (df2.type == 1), 3) \ - .when((df2.type >= 4) | (df2.type <= 8), first(col("t01_type"), True).over(w2)) \ - .otherwise(None)) \ - .withColumn("last_term_type", last(col("t45678_type"), True).over(w2)) \ - .withColumn("time_delta", col("next_time") - col("time")) \ - .select("id", "time", "type", "last_term_type", "time_delta", "t01_time", \ - "t01_type", "t3_time", "t45678_time", "t45678_type", "next_time", "next_type") - -df4 = df3.where(df3.next_type.isNotNull()).groupby("type", "next_type", "last_term_type").sum("time_delta") - -# df3.orderBy(df3.machine_id, df3.time).show(n=100) -# df3.printSchema() -df4.show(n=1000000) -df4.write.csv("/home/claudio/google_2019/thesis_queries/machine_time_waste/" + cluster + "_state_change.csv") - -# vim: set ts=2 sw=2 et tw=120: diff --git a/task_slowdown/.gitignore b/task_slowdown/.gitignore new file mode 100644 index 00000000..eab83f30 --- /dev/null +++ b/task_slowdown/.gitignore @@ -0,0 +1 @@ +task_slowdown/?_state_changes.json.gz diff --git a/task_slowdown/a_slowdown_table.csv b/task_slowdown/a_slowdown_table.csv new file mode 100644 index 00000000..a622242d --- /dev/null +++ b/task_slowdown/a_slowdown_table.csv @@ -0,0 +1,26 @@ +,priority,n_fsh,n_non,finished%,c_zero_end,s_last,m_last,s_all,m_all,s_slow,m_slow +0,-1,499210,4201399,0.10620113266174659,0.0,390649944354691.0,782600570.8581482,428760224746533.0,592714553.7453593,631784.7046286245,1.265672957712968 +1,24,0,4,0.0,,,,,,, +2,25,171,51172,0.0033305416512474924,0.0,986562190939.0,5769369537.654971,81858305998487.0,1203020192.7941773,929847.5905649044,5437.705207981897 +3,100,0,9528,0.0,,,,,,, +4,101,125339,27667,0.8191770257375528,0.0,7934233077571009.0,63305219515.77804,2.4435921623968336e+17,6345535958.256735,16491986771215.066,131585350.7952021 +5,102,0,8992,0.0,,,,,,, +6,103,1406882,7978164,0.14990677722836948,0.0,4323773334171624.0,3074344398.783867,4888365760155938.0,3032797273.2650743,16505180966495.656,11735724.038591769 +7,105,24095559,17680282,0.5767821406635476,0.0,4.0141745605863256e+16,1666128633.975603,4.330222470396991e+16,1749707987.6882613,16507785342461.914,685174.3347848131 +8,107,5056370,4320033,0.5392654304641129,0.0,5167495901308944.0,1021994587.1937487,5251141006134509.0,1030654424.6556122,16507790953107.875,3264806.2664798116 +9,114,0,54127,0.0,,,,,,, +10,115,5171,120690,0.041085006475397465,0.0,10550552314593.0,2040725786.1882012,10596168447023.0,2042044410.6808634,16507790958401.475,3192996316.9055076 +11,116,171154,1140843,0.13045304219445625,0.0,760319854963877.0,4443119013.597687,785219224506580.0,4443446611.511143,16507791187685.025,96467401.73842806 +12,117,0,89560,0.0,,,,,,, +13,118,70767,523560,0.11907081455158523,0.0,128245552414526.0,1816612165.2009463,128693641215270.0,1814017270.1746447,16507791258726.13,233834654.353308 +14,119,158153,585586,0.21264583408964705,0.0,355813050795329.0,2250101502.5126414,535471126096282.0,2877438302.871586,17403987765646.334,110059872.54727907 +15,170,0,109,0.0,,,,,,, +16,200,1812271,4847612,0.27211754320608933,0.0,8227216998822171.0,4545546812.495018,3.386947908315813e+16,16844687840.384369,165162147163583.97,91252275.42822097 +17,205,0,21334,0.0,,,,,,, +18,210,0,1169,0.0,,,,,,, +19,214,0,1390,0.0,,,,,,, +20,215,0,1169,0.0,,,,,,, +21,360,805,129798,0.006163717525631111,0.0,395919732038355.0,514181470179.6818,1157676429161870.0,400580079294.7647,165285702535957.12,214656756540.20407 +22,400,0,75,0.0,,,,,,, +23,450,112,4971,0.02203423175290183,0.0,13736332570286.0,686816628514.3,15693068016609.0,653877834025.375,165285702535990.84,8264285126799.542 +24,500,0,75,0.0,,,,,,, diff --git a/task_slowdown/a_state_changes.json.gz b/task_slowdown/a_state_changes.json.gz deleted file mode 100644 index f03613f9..00000000 Binary files a/task_slowdown/a_state_changes.json.gz and /dev/null differ diff --git a/task_slowdown/b_slowdown_table.csv b/task_slowdown/b_slowdown_table.csv new file mode 100644 index 00000000..b923d4e6 --- /dev/null +++ b/task_slowdown/b_slowdown_table.csv @@ -0,0 +1,24 @@ +,priority,n_fsh,n_non,finished%,c_zero_end,s_last,m_last,s_all,m_all,s_slow,m_slow +0,0,28120567,34102646,0.4519304877425729,0.0,3.799639631137341e+16,1351418653.3085494,4.469884375416512e+16,1466778246.0122163,1456542764472.7542,51804.888156219255 +1,25,19,104988,0.00018094031826449667,0.0,203222731195.0,10695933220.789474,27126548952735.0,4121323146.87557,1456543059817.196,76660161043.01031 +2,80,0,1,0.0,,,,,,, +3,100,0,87231,0.0,,,,,,, +4,101,25076,12644,0.6647932131495228,0.0,152191120001561.0,6069194448.93767,6.596179176874172e+16,5401534266.648366,1758002110737.1948,70106959.27329697 +5,103,16512,15505605,0.0010637724222797701,0.0,320710989163979.0,19429964204.77275,527606225086647.0,14896556132.098001,1767936252845.3188,107108703.06829752 +6,105,10945,2351495,0.004632921894312659,0.0,1.0226301871488184e+16,934420858140.3677,2.462585154304219e+16,392431341519.6678,1768020014445.6218,161551536.40767744 +7,107,0,501380,0.0,,,,,,, +8,114,1999,293319,0.00676897446142802,0.0,65864722282875.0,32948835559.21711,66090092132932.0,30470305271.061317,1768020016461.5698,884452234.3479589 +9,115,196,4564,0.041176470588235294,0.0,4886710126627.0,25584869772.91623,28913941115452.0,107088670797.97037,2151127269026.9988,11262446434.69633 +10,116,215967,2380902,0.0831643798743795,0.0,6325603475353640.0,29289947331.06587,7019216941178176.0,29017499012.708668,2285560716882.638,10583014.455502689 +11,117,0,1849,0.0,,,,,,, +12,118,394,126176,0.0031129019514892944,12.0,1068751378088.0,2775977605.4233766,1068751378088.0,2775977605.4233766,2285560717255.638,5936521343.521138 +13,119,108,54995,0.001959965882075386,0.0,20852738589032.0,193080912861.4074,53282074942406.0,304468999670.8914,2285561567862.601,21162607109.838898 +14,170,0,14,0.0,,,,,,, +15,199,0,14474,0.0,,,,,,, +16,200,91396,204224,0.3091671740748258,68.0,1.6689308702771376e+16,182604366742.21384,1.6201186903133786e+17,466328562061.3029,5237700121655813.0,57307760970.45618 +17,205,0,35822,0.0,,,,,,, +18,210,0,1869,0.0,,,,,,, +19,214,0,400,0.0,,,,,,, +20,215,0,1869,0.0,,,,,,, +21,360,1203,33139,0.03502999242909557,0.0,1261039006121058.0,1048245225370.7881,2032979622033016.0,495124116423.04333,5237700140362286.0,4353865453335.2334 +22,450,174,28215,0.006129134523935327,0.0,274809884969137.0,1579367154995.0403,290615463635764.0,1529555071767.179,5237700140362653.0,30101724944612.95 diff --git a/task_slowdown/b_state_changes.json.gz b/task_slowdown/b_state_changes.json.gz deleted file mode 100644 index e41fff72..00000000 Binary files a/task_slowdown/b_state_changes.json.gz and /dev/null differ diff --git a/task_slowdown/c_slowdown_table.csv b/task_slowdown/c_slowdown_table.csv new file mode 100644 index 00000000..dae2ccba --- /dev/null +++ b/task_slowdown/c_slowdown_table.csv @@ -0,0 +1,24 @@ +,priority,n_fsh,n_non,finished%,c_zero_end,s_last,m_last,s_all,m_all,s_slow,m_slow +0,0,106013713,102314553,0.508878199946233,4.0,9.890266864180749e+16,933028622.4118965,1.0936524434036358e+17,1001823443.2076209,6123674401667.88,57769.55839058094 +1,3,0,3,0.0,,,,,,, +2,10,0,3,0.0,,,,,,, +3,25,903,3116,0.22468275690470266,0.0,154666893037724.0,171281166154.73312,1266916422470409.0,4551424874.873935,15646632159631.086,17327388881.09755 +4,100,0,827,0.0,,,,,,, +5,101,25060,22557,0.5262826301530966,0.0,157151661002382.0,6271016001.691221,6.623793903991674e+16,2497980285.401496,16298829897442.27,650392254.4869221 +6,103,2437,45665340,5.3363666026485154e-05,219.0,5831404975264.0,3343695513.3394494,16294921810434.0,7444002654.378255,16709002329334.33,9580849959.480694 +7,105,8972,38135622,0.00023521026334688476,8.0,1.0778395208878152e+16,1202140888788.5513,1.4791091149243492e+16,863763790542.1333,16709168004960.305,1863614544.3854902 +8,107,27,11026707,2.448594479562126e-06,0.0,189897833827.0,7033253104.703704,2793068197304.0,93102273243.46666,17143029732233.47,634927027119.7582 +9,114,223,1003339,0.00022220849334669905,0.0,702101441201.0,3148436956.0582957,710011356178.0,3141643168.9292035,17143029732510.385,76874572791.52638 +10,115,353,124899,0.0028183182703669404,1.0,5184592226157.0,14728955187.946022,10269344080150.0,27167576931.613758,17154514855201.568,48734417202.27718 +11,116,536,3873404,0.00013836042891732966,0.0,1525621648629.0,2846309045.949627,1559366613321.0,2850761633.1279707,17154514855821.39,32004691895.189163 +12,117,259,19,0.9316546762589928,0.0,553079186614.0,2143717777.5736434,553079186614.0,2143717777.5736434,17154514856079.39,66490367659.22244 +13,118,98,2368621,4.1372573108080784e-05,0.0,109189718829.0,1114180804.377551,120109722380.0,1112127059.074074,17154514856185.732,175046069961.0789 +14,119,1523,67207,0.022159173577768077,0.0,873806372198378.0,573740231253.0387,1786103338848500.0,242446496382.31302,49098665689622.64,32238125863.179672 +15,170,0,30,0.0,,,,,,, +16,200,40086,1071316,0.03606795740875039,96.0,1.4133034700908504e+16,352603031308.5301,5.850686106232538e+16,357993398166.3426,250440847510464.97,6248212352.439124 +17,205,0,2560,0.0,,,,,,, +18,210,0,160,0.0,,,,,,, +19,214,0,2560,0.0,,,,,,, +20,215,0,160,0.0,,,,,,, +21,360,1764,38626,0.04367417677642981,3.0,1357017724420526.0,769284424274.6746,2796928614584114.0,442062369935.8486,252051867201690.6,142886546032.7044 +22,450,472,30733,0.015125781124819741,0.0,656162472642778.0,1390174730175.3772,699478629122954.0,1319770998345.1963,252051867202432.4,534008193225.4924 diff --git a/task_slowdown/c_state_changes.json.gz b/task_slowdown/c_state_changes.json.gz deleted file mode 100644 index cd0749c3..00000000 Binary files a/task_slowdown/c_state_changes.json.gz and /dev/null differ diff --git a/task_slowdown/d_slowdown_table.csv b/task_slowdown/d_slowdown_table.csv new file mode 100644 index 00000000..00c57dfb --- /dev/null +++ b/task_slowdown/d_slowdown_table.csv @@ -0,0 +1,24 @@ +,priority,n_fsh,n_non,finished%,c_zero_end,s_last,m_last,s_all,m_all,s_slow,m_slow +0,0,22214009,61540068,0.26522898700202974,0.0,3.1055625498764056e+16,1398282144.4098547,3.465814625449232e+16,1469082312.5360591,5049183094792.711,227339.8925932346 +1,5,0,1,0.0,,,,,,, +2,25,785,4033,0.16293067662930677,0.0,16824467821138.0,21432443084.252228,1104970471364140.0,4037395212.59606,5082576480847.372,6474619720.824677 +3,100,0,389,0.0,,,,,,, +4,101,25099,30289,0.45314869646854916,0.0,210602264772631.0,8390862774.318937,6.6540641585327544e+16,3317182700.392372,5082722548327.359,202506974.31480774 +5,103,1409,31032032,4.540263517667925e-05,149.0,8107930234531.0,6790561335.453099,8640791772999.0,6646762902.306923,5082722555518.349,4256886562.4106774 +6,105,8438,16308736,0.0005171238598056257,3.0,6967671696826425.0,825749193745.7247,2.018562059248311e+16,924080781563.9587,5082735553839.623,602362592.3014486 +7,107,21,6004332,3.497462590890309e-06,0.0,6311162556240.0,300531550297.1429,9790846227466.0,174836539776.17856,5082735553873.485,242035026374.9279 +8,114,0,1271161,0.0,,,,,,, +9,115,969,17705,0.05189032879940023,1.0,12194814321934.0,12597948679.683884,26664716586105.0,26141879005.985294,7303039756501.865,7544462558.369696 +10,116,2205,1745653,0.0012615441300151385,0.0,20436333384618.0,9268178405.722448,26128064845391.0,10955163457.187002,7683329684843.638,3484503258.432489 +11,117,6,1,0.8571428571428571,0.0,65815182581.0,10969197096.833334,65815182581.0,10969197096.833334,7683329684849.638,1280554947474.9397 +12,118,684,1264690,0.0005405516471809915,0.0,16443832326929.0,24040690536.445908,33689279361478.0,30598800509.970936,7683329686398.121,11232938138.00895 +13,119,413,93059,0.004418435467305718,0.0,76007488187599.0,184484194630.09467,229579544718504.0,172746083309.6343,8050015119135.113,19538871648.386196 +14,197,0,9,0.0,,,,,,, +15,199,0,2056,0.0,,,,,,, +16,200,83851,1200482,0.06528758507334156,104.0,2.3440935680172868e+16,279564636964.1836,1.2926151685985552e+17,349363949014.5856,767029108363450.2,9147852165.387966 +17,205,0,1160,0.0,,,,,,, +18,210,0,116,0.0,,,,,,, +19,214,0,1160,0.0,,,,,,, +20,215,0,116,0.0,,,,,,, +21,360,630,38869,0.01594977088027545,1.0,409573341242607.0,650116414670.8048,1014392629034870.0,390151011167.2577,767029130560754.0,1217506556445.6414 +22,450,169,27484,0.006111452645282609,0.0,158644914383813.0,938727304046.2307,211037058203755.0,523665156833.139,767029130561366.4,4538633908647.139 diff --git a/task_slowdown/d_state_changes.json.gz b/task_slowdown/d_state_changes.json.gz deleted file mode 100644 index 042193cf..00000000 Binary files a/task_slowdown/d_state_changes.json.gz and /dev/null differ diff --git a/task_slowdown/e_slowdown_table.csv b/task_slowdown/e_slowdown_table.csv new file mode 100644 index 00000000..63a9f78a --- /dev/null +++ b/task_slowdown/e_slowdown_table.csv @@ -0,0 +1,23 @@ +,priority,n_fsh,n_non,finished%,c_zero_end,s_last,m_last,s_all,m_all,s_slow,m_slow +0,0,18722018,25015686,0.42805214466676167,1.0,1.500353270905454e+16,802192523.3827926,2.1598252363190628e+16,1126540642.2870827,555025694753.4033,29675.5085119268 +1,25,560,9918,0.0534453139912197,0.0,17929535963182.0,32247366840.255394,47981881861148.0,38946332679.50325,555025701244.8347,998247664.1094149 +2,100,0,2801,0.0,,,,,,, +3,101,1000,6281160,0.0001591809186649178,0.0,30603262218491.0,30603262218.491,34352364938395.0,27725879692.00565,555025751516.257,555025751.5162569 +4,103,2901,13390514,0.0002165989779305726,63.0,218657914919168.0,76293759567.05095,691624969429776.0,48552121406.09168,555026057152.8004,193658777.79232395 +5,105,1819,447535,0.004048033399057314,10.0,194046126324950.0,106677364664.62341,2862241027748842.0,64190200218.63292,558184748019.9266,306863522.82568806 +6,107,0,17636,0.0,,,,,,, +7,114,0,26682,0.0,,,,,,, +8,115,35,128050,0.000273256040910333,0.0,2353296659913.0,67237047426.08572,2353296659913.0,65369351664.25,558184748054.9266,15948135658.712189 +9,116,0,22414,0.0,,,,,,, +10,117,0,1927223,0.0,,,,,,, +11,118,0,231,0.0,,,,,,, +12,119,32,6951,0.004582557639982815,0.0,1987923999531.0,62122624985.34375,20497271238102.0,83322240805.29268,558184748263.1056,17443273383.22205 +13,170,0,25,0.0,,,,,,, +14,200,52049,2604518,0.019592579445577695,55.0,1.20565574443766e+16,231638599096.55518,1.0291142081922765e+17,414148798615.74414,20960917278040.887,402715081.52012306 +15,201,0,52,0.0,,,,,,, +16,205,0,48240,0.0,,,,,,, +17,210,0,2691,0.0,,,,,,, +18,215,0,2691,0.0,,,,,,, +19,220,0,360,0.0,,,,,,, +20,360,3670,6207,0.3715703148729371,0.0,2244218737366886.0,611503743151.7401,6448185707562959.0,439279631280.26154,20971123506308.477,5714202590.274789 +21,450,82,14869,0.005484582971038727,0.0,65910927328054.0,803791796683.5853,73377545680348.0,824466805397.1686,20971123506484.12,255745408615.66 diff --git a/task_slowdown/e_state_changes.json.gz b/task_slowdown/e_state_changes.json.gz deleted file mode 100644 index cf9c4991..00000000 Binary files a/task_slowdown/e_state_changes.json.gz and /dev/null differ diff --git a/task_slowdown/f_slowdown_table.csv b/task_slowdown/f_slowdown_table.csv new file mode 100644 index 00000000..bcc6ce8d --- /dev/null +++ b/task_slowdown/f_slowdown_table.csv @@ -0,0 +1,18 @@ +,priority,n_fsh,n_non,finished%,c_zero_end,s_last,m_last,s_all,m_all,s_slow,m_slow +0,0,12923639,15663283,0.4520822143776095,0.0,3.78338056722563e+16,2928982128.368198,4.116929832921998e+16,2972515519.4539537,1201620091124.0388,93025.8986494546 +1,25,41,6291,0.0064750473783954515,0.0,7565244176289.0,184518150641.19513,16877753924299.0,34096472574.341415,1201620091466.3787,29307807108.936066 +2,100,0,1,0.0,,,,,,, +3,101,24996,37034,0.4029663066258262,0.0,203958763846899.0,8159656098.851776,6.6053822975088104e+16,10082616647.838951,1202133084785.5625,48093018.2743464 +4,103,7762,13279173,0.0005841828834114113,0.0,358824272376313.0,46443731863.35918,418872350566253.0,47234139666.92072,1202133235297.3857,155595810.93675715 +5,105,12351,5541856,0.002223719785740791,0.0,1.372628292784802e+16,1111529915608.391,2.128195689989448e+16,1173594182193.365,1202168661283.2175,97349474.55528525 +6,107,135,221684,0.0006086043125250768,0.0,10820342510697.0,80150685264.42223,10958054196969.0,78834922280.35252,1202168661421.0103,8904953047.56304 +7,114,46,661054,6.958100136136742e-05,0.0,31149185194.0,677156199.8695652,31149185194.0,677156199.8695652,1202168661467.0103,26134101336.239353 +8,115,284,7503,0.036471041479388726,0.0,31792318431347.0,121344726837.20229,161956898216762.0,252662867732.85803,1202171165553.6995,4588439563.182059 +9,116,0,3286187,0.0,,,,,,, +10,117,2,2330310,8.582541736900466e-07,0.0,31750539045.0,15875269522.5,31750539045.0,15875269522.5,1202171165555.6995,601085582777.8497 +11,118,28,1344923,2.081860231339283e-05,0.0,841251531517.0,30044697554.17857,841251531517.0,25492470652.030304,1202171165583.6995,42934684485.132126 +12,119,9624,21070,0.31354662148954193,0.0,1483986210888747.0,154196405952.69608,1.1291352785335876e+16,68833343200.92097,1242061274616.297,129058735.9326992 +13,200,91296,2407549,0.036535279299036154,4.0,2.7130266058773024e+16,297168178877.2019,1.612418735061216e+17,492371666990.7219,313966009267630.25,3438989761.5189085 +14,201,0,2710,0.0,,,,,,, +15,360,3433,42804,0.07424789670610117,0.0,3307182681726832.0,963350620951.5968,7181626757621634.0,569428065146.0223,314099638070996.44,91494214410.42715 +16,450,257,25634,0.009926229191610984,0.0,286756266724416.0,1115783139005.5098,292793287192594.0,1113282460808.3423,314099638071341.06,1222177580044.1287 diff --git a/task_slowdown/f_state_changes.json.gz b/task_slowdown/f_state_changes.json.gz deleted file mode 100644 index 94cd3ba9..00000000 Binary files a/task_slowdown/f_state_changes.json.gz and /dev/null differ diff --git a/task_slowdown/g_slowdown_table.csv b/task_slowdown/g_slowdown_table.csv new file mode 100644 index 00000000..abba8ff7 --- /dev/null +++ b/task_slowdown/g_slowdown_table.csv @@ -0,0 +1,18 @@ +,priority,n_fsh,n_non,finished%,c_zero_end,s_last,m_last,s_all,m_all,s_slow,m_slow +0,0,5492965,10849211,0.3361220072528897,0.0,1.6527216940835638e+16,3009921472.2279587,1.8824305818044464e+16,3316599522.012279,11162954215411.947,2032986.9031638177 +1,25,16,6841,0.002333381945457197,0.0,987330663809.0,61708166488.0625,8582428925651.0,12156414908.854107,11162954217604.191,697684638600.262 +2,50,0,114,0.0,,,,,,, +3,100,0,352,0.0,,,,,,, +4,101,99978,3658,0.9647033849241576,0.0,1.3392322932850584e+16,133952698922.26875,2.5952343687590653e+17,7448011529.683315,20715452279317.676,207200106.81667644 +5,103,1955,6006297,0.0003253858193697601,0.0,230231524187399.0,118310135759.19785,292689255555383.0,112746246361.85786,20715452288533.508,10645145060.91136 +6,105,260,132200,0.0019628567114600634,8.0,2150573680378.0,8271437232.223077,2152161004117.0,8214354977.545801,20715452288785.64,79674816495.32939 +7,107,0,23092,0.0,,,,,,, +8,114,0,120,0.0,,,,,,, +9,115,130,1573,0.07633587786259542,0.0,4406301766057.0,34424232547.32031,7940457046678.0,56315298203.39007,21102853071855.844,164866039623.87378 +10,117,0,1839528,0.0,,,,,,, +11,118,1425,1485,0.4896907216494845,0.0,867149560067822.0,608526007065.1383,3362027382573537.0,474728520555.4274,21121295768754.938,14821961942.985922 +12,119,122,141831,0.0008594393919114073,0.0,15128747139790.0,124006124096.63934,47898774339924.0,231395045120.4058,21121296641155.156,173125382304.55048 +13,170,0,260,0.0,,,,,,, +14,200,51527,141118,0.2674712554179968,43.0,5974724112527930.0,115953269402.99124,8.707510143782531e+16,399049986883.15314,191818188297781.62,3722673322.680956 +15,360,1713,104101,0.016188784092842155,1.0,1347435910813181.0,786594226977.9224,2855922335929930.0,488025006139.7693,191818197237788.03,111977931837.58788 +16,450,643,22848,0.027372185092162955,0.0,774652144131685.0,1204746724932.636,803257942748745.0,1188251394598.735,191818197238602.72,298317569577.9202 diff --git a/task_slowdown/g_state_changes.json.gz b/task_slowdown/g_state_changes.json.gz deleted file mode 100644 index 24393151..00000000 Binary files a/task_slowdown/g_state_changes.json.gz and /dev/null differ diff --git a/task_slowdown/h_slowdown_table.csv b/task_slowdown/h_slowdown_table.csv new file mode 100644 index 00000000..59116656 --- /dev/null +++ b/task_slowdown/h_slowdown_table.csv @@ -0,0 +1,18 @@ +,priority,n_fsh,n_non,finished%,c_zero_end,s_last,m_last,s_all,m_all,s_slow,m_slow +0,0,6626426,17257424,0.2774437957029541,0.0,3.7506706117482584e+16,5663231755.760943,4.2099702663376344e+16,6211184824.091121,955621700576.83,144291.72063921622 +1,19,0,10,0.0,,,,,,, +2,25,79,7497,0.01042766631467793,0.0,24084767122845.0,304870469909.43036,73800261137123.0,283847158219.70386,955623663255.364,12096502066.523596 +3,101,25000,0,1.0,0.0,851579182155103.0,34063167286.20412,6.5093086443320904e+16,12249648080.349836,1012710576185.66,40508423.0474264 +4,103,24112,4986108,0.004812563120980716,0.0,6553604470468504.0,272635180566.95667,8271087007558414.0,92893899318.92467,1012712234179.6411,42129637.83091942 +5,105,10000,690645,0.014272563138251183,0.0,6117631322603891.0,611763132260.3892,2.572798470086729e+16,393761531411.06055,1012750158700.6492,101275015.87006491 +6,107,0,146547,0.0,,,,,,, +7,115,23,426,0.051224944320712694,0.0,664857822015.0,28906861826.739132,664857822015.0,28906861826.739132,1012750158723.6492,44032615596.6804 +8,116,134,12809,0.010353086610523062,0.0,84843310887.0,633159036.4701493,6231571054261.0,42105209826.08784,1012750568167.0226,7557840060.947929 +9,117,1,2013415,4.966683487168077e-07,0.0,655786059.0,655786059.0,655786059.0,655786059.0,1012750568168.0226,1012750568168.0226 +10,118,506,49926,0.010033312182741116,0.0,100029587052190.0,197686930933.18182,194769739643864.0,139121242602.76,1012750618649.7072,2001483436.0666149 +11,119,3970,2729917,0.0014521448765073318,0.0,555431711526397.0,139907232122.51813,4055258837608037.0,15558074672.488085,1246603528452.2136,314005926.56227046 +12,200,57852,2082619,0.027027696240687214,21.0,1.7285546693220572e+16,298799424256.18964,1.0022405625008838e+17,470783023233.28876,17132799512734.742,296159023.5563482 +13,201,0,363,0.0,,,,,,, +14,220,0,360,0.0,,,,,,, +15,360,1862,40210,0.044257463396082906,1.0,1561694630087215.0,838718920562.4141,3152187766522115.0,397301205762.80756,17132832150876.488,9201306203.478243 +16,450,91,16906,0.005353885979878802,0.0,133822527771016.0,1470577228252.923,141139695870169.0,1411396958701.69,17132832151004.443,188272880780.26862 diff --git a/task_slowdown/h_state_changes.json.gz b/task_slowdown/h_state_changes.json.gz deleted file mode 100644 index f0a008ad..00000000 Binary files a/task_slowdown/h_state_changes.json.gz and /dev/null differ diff --git a/task_slowdown/task_slowdown.ipynb b/task_slowdown/task_slowdown.ipynb deleted file mode 100644 index 0f7c9375..00000000 --- a/task_slowdown/task_slowdown.ipynb +++ /dev/null @@ -1,218 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 28, - "id": "behind-rings", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "root\n", - " |-- alloc_collection_id: string (nullable = true)\n", - " |-- alloc_instance_index: string (nullable = true)\n", - " |-- collection_id: string (nullable = true)\n", - " |-- collection_type: string (nullable = true)\n", - " |-- constraint: array (nullable = true)\n", - " | |-- element: struct (containsNull = true)\n", - " | | |-- name: string (nullable = true)\n", - " | | |-- relation: string (nullable = true)\n", - " | | |-- value: string (nullable = true)\n", - " |-- instance_index: string (nullable = true)\n", - " |-- machine_id: string (nullable = true)\n", - " |-- missing_type: string (nullable = true)\n", - " |-- priority: string (nullable = true)\n", - " |-- resource_request: struct (nullable = true)\n", - " | |-- cpus: double (nullable = true)\n", - " | |-- memory: double (nullable = true)\n", - " |-- scheduling_class: string (nullable = true)\n", - " |-- time: string (nullable = true)\n", - " |-- type: string (nullable = true)\n", - "\n", - "+-------------------+--------------------+-------------+---------------+--------------------+--------------+------------+------------+--------+--------------------+----------------+-------------+----+\n", - "|alloc_collection_id|alloc_instance_index|collection_id|collection_type| constraint|instance_index| machine_id|missing_type|priority| resource_request|scheduling_class| time|type|\n", - "+-------------------+--------------------+-------------+---------------+--------------------+--------------+------------+------------+--------+--------------------+----------------+-------------+----+\n", - "| null| -1| 311844978975| null| []| 100|294975034872| null| 103|[0.00660705566406...| null| 678322664130| 3|\n", - "| null| -1| 324883683214| null| []| null| null| null| null|[0.01055908203125...| null|1976760570767|null|\n", - "| null| -1| 323371581333| null|[[9eCGRtl6XN5GQoO...| 571|153437873934| null| null|[0.00810241699218...| null|2240234044164| 7|\n", - "| null| -1| 313182190373| null| []| 34| null| null| null|[0.00228881835937...| null| 968876915742|null|\n", - "| null| -1| 299950283520| null|[[9eCGRtl6XN5GQoO...| 2135| null| null| 117|[0.01531982421875...| 1| 142387817674|null|\n", - "| null| -1| 324794084626| null| []| 314| -1| null| 103|[0.0810546875, 0....| null|1950212179272| 2|\n", - "| null| -1| 311869083179| null|[[5dEuieuWMFy+CNM...| 4729| 1994402429| null| 117|[0.01109313964843...| 1|1049052607582| 3|\n", - "| null| -1| 314633575062| null| []| 708| null| null| null|[0.00589752197265...| null|1245121652306|null|\n", - "| null| -1| 310024318230| null| []| 132| null| null| 103|[0.00607299804687...| null| 398893109200| 1|\n", - "| null| -1| 314083275570| null| []| null| 1821717262| null| 103|[4.40597534179687...| null|1245124281990| 6|\n", - "| null| -1| 326302449610| null| []| 2740| null| null| null|[0.00589752197265...| null|2353733113490|null|\n", - "| null| -1| 312310995026| null| []| 101| null| null| null|[8.09669494628906...| null| 802162973717|null|\n", - "| null| -1| 325098713157| null| []| 77| null| null| 103|[0.0810546875, 0....| null|2049169911483| 2|\n", - "| null| -1| 322296137788| null| []| null| 1888361814| null| 200|[6.16073608398437...| null|1511423336811| 10|\n", - "| null| -1| 325292207036| null| []| 13| -1| null| 103|[0.09716796875, 0...| null|2128029854228| 10|\n", - "| null| -1| 311499868798| null|[[9eCGRtl6XN5GQoO...| 6571| null| null| null|[0.01620483398437...| null| 673538231587|null|\n", - "| null| -1| 325505190758| null|[[9eCGRtl6XN5GQoO...| 306| null| null| null|[0.00242996215820...| null|2424201701414| 2|\n", - "| null| -1| 310897142142| null| []| null| null| null| null|[0.00757598876953...| null| 498367858680|null|\n", - "| null| -1| 321838026533| null| []| 667| 3459537598| null| 103|[0.01048278808593...| null|1416689475146| 10|\n", - "| null| -1| 322571597197| null| []| 3741| 10368072540| null| 103|[0.0123291015625,...| null|1591142575136| 3|\n", - "+-------------------+--------------------+-------------+---------------+--------------------+--------------+------------+------------+--------+--------------------+----------------+-------------+----+\n", - "only showing top 20 rows\n", - "\n" - ] - } - ], - "source": [ - "#!/usr/bin/env python3\n", - "# coding: utf-8\n", - "\n", - "# # Temporal impact: machine time waste\n", - "\n", - "# Inefficient version of the machine_time_waste.py script in the same dir. Please ignore, kept for future need\n", - "\n", - "import json\n", - "import pandas\n", - "from IPython import display\n", - "import findspark\n", - "findspark.init()\n", - "import pyspark\n", - "import pyspark.sql\n", - "import sys\n", - "\n", - "from pyspark.sql.functions import col, lag, when, concat_ws, last, first\n", - "from pyspark.sql import Window\n", - "from pyspark.sql.types import LongType\n", - "\n", - "cluster=\"h\"\n", - "\n", - "spark = pyspark.sql.SparkSession.builder \\\n", - " .appName(\"task_slowdown\") \\\n", - " .config(\"spark.local.dir\", \"/run/tmpfiles.d/spark\") \\\n", - " .config(\"spark.driver.memory\", \"124g\") \\\n", - " .getOrCreate()\n", - "sc = spark.sparkContext\n", - "\n", - "# df = spark.read.json(\"/home/claudio/google_2019/instance_events/\" + cluster + \"/\" + cluster + \"_instance_events000000000000.json.gz\")\n", - "df = spark.read.json(\"/home/claudio/google_2019/instance_events/\" + cluster + \"/\" + cluster + \"_test.json\")\n", - "\n", - "df.printSchema()\n", - "\n", - "df.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "id": "banned-temperature", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[(103, [[], [], [], [], [(62850578515, [3, 3, 5])], [], [], [], [], [], [(36278465040, [1, 5])], [], [], [], [], [], [], [], [], [], [], [(1937109982, [2, 5])], [], [(66934359137, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(99766245, [2, 5])], [], [], [], [], [(679575974, [2, 5])], [], [], [], [], [], [], [], [], [(27987724401, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(40033883995, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [(2072441674, [3, 5])], [], [], [], [], [], [(30503998595, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(11191193265, [3, 5])], [], [], [], [], [], [(13449203940, [2, 5])], [(9056832716, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [(2624004083, [3, 5])], [], [], [], [], [(5791004634, [2, 5])], [], [], [], [], [], [], [(31522009960, [2, 5])], [], [], [], [], [], [], [], [(27223865505, [3, 5])], [], [], [], [], [], [], [], [], [], [], [(62363966217, [2, 5])], [(22329997715, [3, 5])], [], [], [], [], [], [], [], [], [], [], [(35738940742, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(5461413747, [10, 5])], [], [], [], [], [], [], [(44714567682, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(32537907976, [10, 5])], [], [], [], [(3856023850, [2, 5])], [], [], [], [], [], [], [], [], [], [], [(8289226525, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(88907761383, [2, 5])], [], [], [], [], [], [], [(25322047766, [2, 5])], [], [], [], [], [(788099086, [3, 5])], [], [], [], [], [], [(36244532207, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(560020738, [2, 5])], [(71461045, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(124699375375, [2, 5])], [], [], [(7836906167, [2, 5])], [], [], [(5422253956, [2, 5])], [], [], [(5855268565, [2, 5])], [], [(5023716519, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(18332998966, [2, 5])], [], [], [], [], [], [(35660231933, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(1287188765, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(3907777878, [3, 5])], [], [], [], [], [], [], [], [], [], [], [(20506184461, [2, 5])], [], [(8408788368, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(13629775669, [3, 5])], [], [], [], [], [], [(780912697, [3, 5])], [], [], [], [(32298679126, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(30750739304, [3, 5])], [(6594675644, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(15402056687, [3, 5])], [], [], [], [], [], [(109413809124, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(305255719, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(3827846545, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(57934339021, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(40760757121, [3, 5])], [], [], [], [(4259078476, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(1034707355, [1, 5])], [], [], [], [], [(93784278917, [9, 2, 3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(4188472119, [1, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(154867489, [2, 5])], [], [], [], [], [(40095625660, [2, 5])], [], [], [], [], [(6313871106, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(75345872, [2, 5])], [], [(42854643, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(42240713762, [2, 5])], [(7940276639, [3, 5])], [], [], [(8067825463, [1, 5])], [], [], [], [(11564450554, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(14248092214, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(1151512362, [3, 5])], [], [], [], [], [], [], [], [], [], [], [(6814736882, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(28725095651, [10, 5])], [], [], [], [], [], [(5429365968, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [(78497996693, [2, 5])], [(4007939091, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(11207024590, [2, 2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(33028562460, [3, 3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(2244935788, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(4803409513, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(89924261, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(11515083156, [3, 5])], [], [], [], [], [], [], [], [(1028315320, [3, 5])], [], [], [], [(5669930326, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(92102821544, [9, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(12939585657, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(576460267, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [(16602852770, [3, 5])], [], [], [], [], [], [(3975575979, [9, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(15699214070, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(4273785458, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [(63081663, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(3673691412, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(4433586428, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(1737102869, [3, 3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [(7597739079, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(13564991758, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(3784960872, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(5558155540, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(1020015051, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(7386544705, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(7659182297, [2, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [(2612206009, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], []]), (360, [[(194634757478, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [(75332399743, [2, 2, 5])], [], [(327145123, [3, 5])], [], [], [], [], [], [], [], [], [], [], [], [], [], [], []]), (118, [[], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], []]), (105, [[], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], []]), (119, [[(487900370258, [2, 5])], [(141154490783, [3, 3, 5]), (1211954116878, [3, 3, 3, 2, 3, 2, 3, 3, 3, 2, 5]), (106762922927, [2, 2, 3, 5])], [], [], [(1370779131465, [2, 2, 5])], [], []]), (0, [[], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], []]), (200, [[], [], [], [], [], [], [], [], [(70814003028, [2, 5])], [], [], [(1205243170887, [3, 2, 5])], [], [], [], [], [], [], [], [], [], []]), (115, [[], []]), (107, [[], []])]\n", - "55444\n", - "51766\n" - ] - } - ], - "source": [ - "non = sc.accumulator(0)\n", - "tot = sc.accumulator(0)\n", - "\n", - "\n", - "def for_each_task(ts):\n", - " global none\n", - " global tot\n", - " \n", - " ts = sorted(ts, key=lambda x: x[\"time\"])\n", - " \n", - " last_term = None\n", - " priority = None\n", - " responding = False\n", - " \n", - " resp_burst_start = None\n", - " resp_burst_type = None\n", - " \n", - " resp_time = []\n", - " resp_time_last = 0\n", - "\n", - " for i,t in enumerate(ts):\n", - " if t[\"priority\"] is not None and priority is None:\n", - " priority = t[\"priority\"]\n", - " if responding:\n", - " resp_burst_type.append(t[\"type\"])\n", - " if t[\"type\"] >= 4 and t[\"type\"] <= 8:\n", - " last_term = t[\"type\"]\n", - " if responding:\n", - " # This response time interval has ended, so record the time delta and term\n", - " resp_time.append((t[\"time\"] - resp_burst_start, resp_burst_type))\n", - " responding = False\n", - " if (not responding) and (t[\"type\"] < 4 or t[\"type\"] > 8):\n", - " resp_burst_start = t[\"time\"]\n", - " resp_burst_type = [t[\"type\"]]\n", - " responding = True\n", - " \n", - " \n", - " tot.add(1)\n", - " if last_term != 6:\n", - " non.add(1)\n", - " return (priority, resp_time) if last_term == 5 else None\n", - "\n", - " \n", - "def cleanup(x):\n", - " return {\n", - " \"time\": int(x.time),\n", - " \"type\": 0 if x.type is None else int(x.type),\n", - " \"id\": x.collection_id + \"-\" + x.instance_index,\n", - " \"priority\": 0 if x.priority is None else int(x.priority)\n", - " }\n", - "\n", - "\n", - "df2 = df.limit(100000).rdd \\\n", - " .filter(lambda x: x.collection_type is None or x.collection_type == 0) \\\n", - " .filter(lambda x: x.type is not None and x.time is not None\n", - " and x.instance_index is not None and x.collection_id is not None) \\\n", - " .map(cleanup) \\\n", - " .groupBy(lambda x: x[\"id\"]) \\\n", - " .mapValues(for_each_task) \\\n", - " .filter(lambda x: x[1] is not None) \\\n", - " .map(lambda x: x[1]) \\\n", - " .groupBy(lambda x: x[0]) \\\n", - " .mapValues(lambda x: [e[1] for e in x])\n", - "\n", - "a = {\"val\": df2.collect(), \"tot\": tot.value, \"non\": non.value}\n", - "with open(cluster + \"_state_changes.json\", \"w\") as out:\n", - " json.dump(a, out)\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "arctic-tomato", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.9" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/task_slowdown/task_slowdown.py b/task_slowdown/task_slowdown.py index 7efc1259..133134b1 100755 --- a/task_slowdown/task_slowdown.py +++ b/task_slowdown/task_slowdown.py @@ -9,39 +9,54 @@ findspark.init() import pyspark import pyspark.sql import sys +import gzip -from pyspark.sql.functions import col, lag, when, concat_ws, last, first +from pyspark import AccumulatorParam +from pyspark.sql.functions import lit from pyspark.sql import Window -from pyspark.sql.types import LongType +from pyspark.sql.types import ByteType cluster=sys.argv[1] spark = pyspark.sql.SparkSession.builder \ .appName("task_slowdown") \ + .config("spark.driver.maxResultSize", "32g") \ .config("spark.local.dir", "/run/tmpfiles.d/spark") \ - .config("spark.driver.memory", "124g") \ + .config("spark.driver.memory", "75g") \ .getOrCreate() sc = spark.sparkContext df = spark.read.json("/home/claudio/google_2019/instance_events/" + cluster + "/" + cluster + "_instance_events*.json.gz") -# df = spark.read.json("/home/claudio/google_2019/instance_events/" + cluster + "/" + cluster + "_test.json") +#df = spark.read.json("/home/claudio/google_2019/instance_events/" + cluster + "/" + cluster + "_test.json") -df.printSchema() +try: + df["collection_type"] = df["collection_type"].cast(ByteType()) +except: + df = df.withColumn("collection_type", lit(None).cast(ByteType())) -df.show() -non = sc.accumulator(0) -tot = sc.accumulator(0) +class NonPriorityAcc(pyspark.AccumulatorParam): + def zero(self, value): + return {} + + def addInPlace(self, v1, v2): + for key in v2: + if key in v1: + v1[key] += v2[key] + else: + v1[key] = v2[key] + return v1 + +non = sc.accumulator({}, NonPriorityAcc()) def for_each_task(ts): - global none - global tot + global non ts = sorted(ts, key=lambda x: x["time"]) last_term = None - priority = None + priority = -1 responding = False resp_burst_start = None @@ -51,13 +66,14 @@ def for_each_task(ts): resp_time_last = 0 for i,t in enumerate(ts): - if t["priority"] is not None and priority is None: + if t["priority"] is not -1 and priority is -1: priority = t["priority"] - if responding: - resp_burst_type.append(t["type"]) + #if responding: + # resp_burst_type.append(t["type"]) if t["type"] >= 4 and t["type"] <= 8: last_term = t["type"] if responding: + resp_burst_type.append(t["type"]) # This response time interval has ended, so record the time delta and term resp_time.append((t["time"] - resp_burst_start, resp_burst_type)) responding = False @@ -65,12 +81,10 @@ def for_each_task(ts): resp_burst_start = t["time"] resp_burst_type = [t["type"]] responding = True - - - tot.add(1) + if last_term != 6: - non.add(1) - return (priority, resp_time) if last_term == 5 else None + non.add({priority: 1}) + return (priority, resp_time) if last_term == 6 else None def cleanup(x): @@ -78,14 +92,13 @@ def cleanup(x): "time": int(x.time), "type": 0 if x.type is None else int(x.type), "id": x.collection_id + "-" + x.instance_index, - "priority": 0 if x.priority is None else int(x.priority) + "priority": -1 if x.priority is None else int(x.priority) } df2 = df.rdd \ .filter(lambda x: x.collection_type is None or x.collection_type == 0) \ - .filter(lambda x: x.type is not None and x.time is not None - and x.instance_index is not None and x.collection_id is not None) \ + .filter(lambda x: x.time is not None and x.instance_index is not None and x.collection_id is not None) \ .map(cleanup) \ .groupBy(lambda x: x["id"]) \ .mapValues(for_each_task) \ @@ -94,6 +107,12 @@ df2 = df.rdd \ .groupBy(lambda x: x[0]) \ .mapValues(lambda x: [e[1] for e in x]) -a = {"val": df2.collect(), "tot": tot.value, "non": non.value} -with open(cluster + "_state_changes.json", "w") as out: - json.dump(a, out) +val = df2.collect() +val2 = {} +for e in val: + val2[e[0]] = e[1] + +a = {"val": val2, "non": non.value} + +with gzip.open(cluster + "_state_changes.json.gz", "wt") as out: + json.dump(a, out, separators=(',', ':')) diff --git a/task_slowdown/task_slowdown_table.ipynb b/task_slowdown/task_slowdown_table.ipynb new file mode 100644 index 00000000..ef2e4a69 --- /dev/null +++ b/task_slowdown/task_slowdown_table.ipynb @@ -0,0 +1,3762 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "handy-organization", + "metadata": {}, + "source": [ + "# Task slowdown" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "guided-excellence", + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import sys\n", + "import gzip\n", + "import pandas\n", + "import seaborn as sns\n", + "import matplotlib as mpl\n", + "import matplotlib.pyplot as plt\n", + "from IPython.display import display, HTML" + ] + }, + { + "cell_type": "markdown", + "id": "august-lobby", + "metadata": {}, + "source": [ + "### Legend for columns:\n", + "- *n_fsh*: number of jobs that terminated with status 6 (FINISH)\n", + "- *n_non*: number of jobs that did not terminate with status 6 or did not terminate at all\n", + "- *finished%*: `n_fsh` / (`n_fsh` + `n_non`)\n", + "- *c_zero_end*: count of number of **Finished** jobs that have a last execution time of `0` \n", + "- *s_last*: sum of execution times for last events\n", + "- *m_last*: mean execution time for last event\n", + "- *s_all*: sum of all execution times for all events\n", + "- *m_all*: mean execution time for all events\n", + "- *s_slow*: sum of **slowdown** values computed for each job: `job_slowdown` = sum(`exec_time`) / last(`exec_time`)\n", + "- *m_slow*: mean job-wise **slowdown** value, i.e. `s_slow` / `n_fsh`\n", + "- *m_slow_2*: priority-wise mean **slowdown**, i.e. `s_all` / `s_last`" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "revised-mediterranean", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Cluster a:\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Unnamed: 0priorityn_fshn_nonfinished%c_zero_ends_lastm_lasts_allm_alls_slowm_slowm_slow_2
00-149921042013990.1062010.03.906499e+147.826006e+084.287602e+145.927146e+086.317847e+051.265673e+001.097556
1124040.000000NaNNaNNaNNaNNaNNaNNaNNaN
2225171511720.0033310.09.865622e+115.769370e+098.185831e+131.203020e+099.298476e+055.437705e+0382.973285
33100095280.000000NaNNaNNaNNaNNaNNaNNaNNaN
44101125339276670.8191770.07.934233e+156.330522e+102.443592e+176.345536e+091.649199e+131.315854e+0830.798089
55102089920.000000NaNNaNNaNNaNNaNNaNNaNNaN
66103140688279781640.1499070.04.323773e+153.074344e+094.888366e+153.032797e+091.650518e+131.173572e+071.130579
7710524095559176802820.5767820.04.014175e+161.666129e+094.330222e+161.749708e+091.650779e+136.851743e+051.078733
88107505637043200330.5392650.05.167496e+151.021995e+095.251141e+151.030654e+091.650779e+133.264806e+061.016187
991140541270.000000NaNNaNNaNNaNNaNNaNNaNNaN
101011551711206900.0410850.01.055055e+132.040726e+091.059617e+132.042044e+091.650779e+133.192996e+091.004324
111111617115411408430.1304530.07.603199e+144.443119e+097.852192e+144.443447e+091.650779e+139.646740e+071.032749
12121170895600.000000NaNNaNNaNNaNNaNNaNNaNNaN
1313118707675235600.1190710.01.282456e+141.816612e+091.286936e+141.814017e+091.650779e+132.338347e+081.003494
14141191581535855860.2126460.03.558131e+142.250102e+095.354711e+142.877438e+091.740399e+131.100599e+081.504923
151517001090.000000NaNNaNNaNNaNNaNNaNNaNNaN
1616200181227148476120.2721180.08.227217e+154.545547e+093.386948e+161.684469e+101.651621e+149.125228e+074.116760
17172050213340.000000NaNNaNNaNNaNNaNNaNNaNNaN
1818210011690.000000NaNNaNNaNNaNNaNNaNNaNNaN
1919214013900.000000NaNNaNNaNNaNNaNNaNNaNNaN
2020215011690.000000NaNNaNNaNNaNNaNNaNNaNNaN
21213608051297980.0061640.03.959197e+145.141815e+111.157676e+154.005801e+111.652857e+142.146568e+112.924018
22224000750.000000NaNNaNNaNNaNNaNNaNNaNNaN
232345011249710.0220340.01.373633e+136.868166e+111.569307e+136.538778e+111.652857e+148.264285e+121.142450
24245000750.000000NaNNaNNaNNaNNaNNaNNaNNaN
\n", + "
" + ], + "text/plain": [ + " Unnamed: 0 priority n_fsh n_non finished% c_zero_end \\\n", + "0 0 -1 499210 4201399 0.106201 0.0 \n", + "1 1 24 0 4 0.000000 NaN \n", + "2 2 25 171 51172 0.003331 0.0 \n", + "3 3 100 0 9528 0.000000 NaN \n", + "4 4 101 125339 27667 0.819177 0.0 \n", + "5 5 102 0 8992 0.000000 NaN \n", + "6 6 103 1406882 7978164 0.149907 0.0 \n", + "7 7 105 24095559 17680282 0.576782 0.0 \n", + "8 8 107 5056370 4320033 0.539265 0.0 \n", + "9 9 114 0 54127 0.000000 NaN \n", + "10 10 115 5171 120690 0.041085 0.0 \n", + "11 11 116 171154 1140843 0.130453 0.0 \n", + "12 12 117 0 89560 0.000000 NaN \n", + "13 13 118 70767 523560 0.119071 0.0 \n", + "14 14 119 158153 585586 0.212646 0.0 \n", + "15 15 170 0 109 0.000000 NaN \n", + "16 16 200 1812271 4847612 0.272118 0.0 \n", + "17 17 205 0 21334 0.000000 NaN \n", + "18 18 210 0 1169 0.000000 NaN \n", + "19 19 214 0 1390 0.000000 NaN \n", + "20 20 215 0 1169 0.000000 NaN \n", + "21 21 360 805 129798 0.006164 0.0 \n", + "22 22 400 0 75 0.000000 NaN \n", + "23 23 450 112 4971 0.022034 0.0 \n", + "24 24 500 0 75 0.000000 NaN \n", + "\n", + " s_last m_last s_all m_all s_slow \\\n", + "0 3.906499e+14 7.826006e+08 4.287602e+14 5.927146e+08 6.317847e+05 \n", + "1 NaN NaN NaN NaN NaN \n", + "2 9.865622e+11 5.769370e+09 8.185831e+13 1.203020e+09 9.298476e+05 \n", + "3 NaN NaN NaN NaN NaN \n", + "4 7.934233e+15 6.330522e+10 2.443592e+17 6.345536e+09 1.649199e+13 \n", + "5 NaN NaN NaN NaN NaN \n", + "6 4.323773e+15 3.074344e+09 4.888366e+15 3.032797e+09 1.650518e+13 \n", + "7 4.014175e+16 1.666129e+09 4.330222e+16 1.749708e+09 1.650779e+13 \n", + "8 5.167496e+15 1.021995e+09 5.251141e+15 1.030654e+09 1.650779e+13 \n", + "9 NaN NaN NaN NaN NaN \n", + "10 1.055055e+13 2.040726e+09 1.059617e+13 2.042044e+09 1.650779e+13 \n", + "11 7.603199e+14 4.443119e+09 7.852192e+14 4.443447e+09 1.650779e+13 \n", + "12 NaN NaN NaN NaN NaN \n", + "13 1.282456e+14 1.816612e+09 1.286936e+14 1.814017e+09 1.650779e+13 \n", + "14 3.558131e+14 2.250102e+09 5.354711e+14 2.877438e+09 1.740399e+13 \n", + "15 NaN NaN NaN NaN NaN \n", + "16 8.227217e+15 4.545547e+09 3.386948e+16 1.684469e+10 1.651621e+14 \n", + "17 NaN NaN NaN NaN NaN \n", + "18 NaN NaN NaN NaN NaN \n", + "19 NaN NaN NaN NaN NaN \n", + "20 NaN NaN NaN NaN NaN \n", + "21 3.959197e+14 5.141815e+11 1.157676e+15 4.005801e+11 1.652857e+14 \n", + "22 NaN NaN NaN NaN NaN \n", + "23 1.373633e+13 6.868166e+11 1.569307e+13 6.538778e+11 1.652857e+14 \n", + "24 NaN NaN NaN NaN NaN \n", + "\n", + " m_slow m_slow_2 \n", + "0 1.265673e+00 1.097556 \n", + "1 NaN NaN \n", + "2 5.437705e+03 82.973285 \n", + "3 NaN NaN \n", + "4 1.315854e+08 30.798089 \n", + "5 NaN NaN \n", + "6 1.173572e+07 1.130579 \n", + "7 6.851743e+05 1.078733 \n", + "8 3.264806e+06 1.016187 \n", + "9 NaN NaN \n", + "10 3.192996e+09 1.004324 \n", + "11 9.646740e+07 1.032749 \n", + "12 NaN NaN \n", + "13 2.338347e+08 1.003494 \n", + "14 1.100599e+08 1.504923 \n", + "15 NaN NaN \n", + "16 9.125228e+07 4.116760 \n", + "17 NaN NaN \n", + "18 NaN NaN \n", + "19 NaN NaN \n", + "20 NaN NaN \n", + "21 2.146568e+11 2.924018 \n", + "22 NaN NaN \n", + "23 8.264285e+12 1.142450 \n", + "24 NaN NaN " + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Cluster b:\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Unnamed: 0priorityn_fshn_nonfinished%c_zero_ends_lastm_lasts_allm_alls_slowm_slowm_slow_2
00028120567341026460.4519300.03.799640e+161.351419e+094.469884e+161.466778e+091.456543e+125.180489e+041.176397
1125191049880.0001810.02.032227e+111.069593e+102.712655e+134.121323e+091.456543e+127.666016e+10133.481864
2280010.000000NaNNaNNaNNaNNaNNaNNaNNaN
331000872310.000000NaNNaNNaNNaNNaNNaNNaNNaN
4410125076126440.6647930.01.521911e+146.069194e+096.596179e+165.401534e+091.758002e+127.010696e+07433.414195
5510316512155056050.0010640.03.207110e+141.942996e+105.276062e+141.489656e+101.767936e+121.071087e+081.645114
661051094523514950.0046330.01.022630e+169.344209e+112.462585e+163.924313e+111.768020e+121.615515e+082.408090
7710705013800.000000NaNNaNNaNNaNNaNNaNNaNNaN
8811419992933190.0067690.06.586472e+133.294884e+106.609009e+133.047031e+101.768020e+128.844522e+081.003422
9911519645640.0411760.04.886710e+122.558487e+102.891394e+131.070887e+112.151127e+121.126245e+105.916852
101011621596723809020.0831640.06.325603e+152.928995e+107.019217e+152.901750e+102.285561e+121.058301e+071.109652
1111117018490.000000NaNNaNNaNNaNNaNNaNNaNNaN
12121183941261760.00311312.01.068751e+122.775978e+091.068751e+122.775978e+092.285561e+125.936521e+091.000000
1313119108549950.0019600.02.085274e+131.930809e+115.328207e+133.044690e+112.285562e+122.116261e+102.555160
14141700140.000000NaNNaNNaNNaNNaNNaNNaNNaN
15151990144740.000000NaNNaNNaNNaNNaNNaNNaNNaN
1616200913962042240.30916768.01.668931e+161.826044e+111.620119e+174.663286e+115.237700e+155.730776e+109.707524
17172050358220.000000NaNNaNNaNNaNNaNNaNNaNNaN
1818210018690.000000NaNNaNNaNNaNNaNNaNNaNNaN
191921404000.000000NaNNaNNaNNaNNaNNaNNaNNaN
2020215018690.000000NaNNaNNaNNaNNaNNaNNaNNaN
21213601203331390.0350300.01.261039e+151.048245e+122.032980e+154.951241e+115.237700e+154.353865e+121.612147
2222450174282150.0061290.02.748099e+141.579367e+122.906155e+141.529555e+125.237700e+153.010172e+131.057515
\n", + "
" + ], + "text/plain": [ + " Unnamed: 0 priority n_fsh n_non finished% c_zero_end \\\n", + "0 0 0 28120567 34102646 0.451930 0.0 \n", + "1 1 25 19 104988 0.000181 0.0 \n", + "2 2 80 0 1 0.000000 NaN \n", + "3 3 100 0 87231 0.000000 NaN \n", + "4 4 101 25076 12644 0.664793 0.0 \n", + "5 5 103 16512 15505605 0.001064 0.0 \n", + "6 6 105 10945 2351495 0.004633 0.0 \n", + "7 7 107 0 501380 0.000000 NaN \n", + "8 8 114 1999 293319 0.006769 0.0 \n", + "9 9 115 196 4564 0.041176 0.0 \n", + "10 10 116 215967 2380902 0.083164 0.0 \n", + "11 11 117 0 1849 0.000000 NaN \n", + "12 12 118 394 126176 0.003113 12.0 \n", + "13 13 119 108 54995 0.001960 0.0 \n", + "14 14 170 0 14 0.000000 NaN \n", + "15 15 199 0 14474 0.000000 NaN \n", + "16 16 200 91396 204224 0.309167 68.0 \n", + "17 17 205 0 35822 0.000000 NaN \n", + "18 18 210 0 1869 0.000000 NaN \n", + "19 19 214 0 400 0.000000 NaN \n", + "20 20 215 0 1869 0.000000 NaN \n", + "21 21 360 1203 33139 0.035030 0.0 \n", + "22 22 450 174 28215 0.006129 0.0 \n", + "\n", + " s_last m_last s_all m_all s_slow \\\n", + "0 3.799640e+16 1.351419e+09 4.469884e+16 1.466778e+09 1.456543e+12 \n", + "1 2.032227e+11 1.069593e+10 2.712655e+13 4.121323e+09 1.456543e+12 \n", + "2 NaN NaN NaN NaN NaN \n", + "3 NaN NaN NaN NaN NaN \n", + "4 1.521911e+14 6.069194e+09 6.596179e+16 5.401534e+09 1.758002e+12 \n", + "5 3.207110e+14 1.942996e+10 5.276062e+14 1.489656e+10 1.767936e+12 \n", + "6 1.022630e+16 9.344209e+11 2.462585e+16 3.924313e+11 1.768020e+12 \n", + "7 NaN NaN NaN NaN NaN \n", + "8 6.586472e+13 3.294884e+10 6.609009e+13 3.047031e+10 1.768020e+12 \n", + "9 4.886710e+12 2.558487e+10 2.891394e+13 1.070887e+11 2.151127e+12 \n", + "10 6.325603e+15 2.928995e+10 7.019217e+15 2.901750e+10 2.285561e+12 \n", + "11 NaN NaN NaN NaN NaN \n", + "12 1.068751e+12 2.775978e+09 1.068751e+12 2.775978e+09 2.285561e+12 \n", + "13 2.085274e+13 1.930809e+11 5.328207e+13 3.044690e+11 2.285562e+12 \n", + "14 NaN NaN NaN NaN NaN \n", + "15 NaN NaN NaN NaN NaN \n", + "16 1.668931e+16 1.826044e+11 1.620119e+17 4.663286e+11 5.237700e+15 \n", + "17 NaN NaN NaN NaN NaN \n", + "18 NaN NaN NaN NaN NaN \n", + "19 NaN NaN NaN NaN NaN \n", + "20 NaN NaN NaN NaN NaN \n", + "21 1.261039e+15 1.048245e+12 2.032980e+15 4.951241e+11 5.237700e+15 \n", + "22 2.748099e+14 1.579367e+12 2.906155e+14 1.529555e+12 5.237700e+15 \n", + "\n", + " m_slow m_slow_2 \n", + "0 5.180489e+04 1.176397 \n", + "1 7.666016e+10 133.481864 \n", + "2 NaN NaN \n", + "3 NaN NaN \n", + "4 7.010696e+07 433.414195 \n", + "5 1.071087e+08 1.645114 \n", + "6 1.615515e+08 2.408090 \n", + "7 NaN NaN \n", + "8 8.844522e+08 1.003422 \n", + "9 1.126245e+10 5.916852 \n", + "10 1.058301e+07 1.109652 \n", + "11 NaN NaN \n", + "12 5.936521e+09 1.000000 \n", + "13 2.116261e+10 2.555160 \n", + "14 NaN NaN \n", + "15 NaN NaN \n", + "16 5.730776e+10 9.707524 \n", + "17 NaN NaN \n", + "18 NaN NaN \n", + "19 NaN NaN \n", + "20 NaN NaN \n", + "21 4.353865e+12 1.612147 \n", + "22 3.010172e+13 1.057515 " + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Cluster c:\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Unnamed: 0priorityn_fshn_nonfinished%c_zero_ends_lastm_lasts_allm_alls_slowm_slowm_slow_2
0001060137131023145530.5088784.09.890267e+169.330286e+081.093652e+171.001823e+096.123674e+125.776956e+041.105787
113030.000000NaNNaNNaNNaNNaNNaNNaNNaN
2210030.000000NaNNaNNaNNaNNaNNaNNaNNaN
332590331160.2246830.01.546669e+141.712812e+111.266916e+154.551425e+091.564663e+131.732739e+108.191258
4410008270.000000NaNNaNNaNNaNNaNNaNNaNNaN
5510125060225570.5262830.01.571517e+146.271016e+096.623794e+162.497980e+091.629883e+136.503923e+08421.490544
661032437456653400.000053219.05.831405e+123.343696e+091.629492e+137.444003e+091.670900e+139.580850e+092.794339
771058972381356220.0002358.01.077840e+161.202141e+121.479109e+168.637638e+111.670917e+131.863615e+091.372291
8810727110267070.0000020.01.898978e+117.033253e+092.793068e+129.310227e+101.714303e+136.349270e+1114.708268
9911422310033390.0002220.07.021014e+113.148437e+097.100114e+113.141643e+091.714303e+137.687457e+101.011266
10101153531248990.0028181.05.184592e+121.472896e+101.026934e+132.716758e+101.715451e+134.873442e+101.980743
111111653638734040.0001380.01.525622e+122.846309e+091.559367e+122.850762e+091.715451e+133.200469e+101.022119
1212117259190.9316550.05.530792e+112.143718e+095.530792e+112.143718e+091.715451e+136.649037e+101.000000
13131189823686210.0000410.01.091897e+111.114181e+091.201097e+111.112127e+091.715451e+131.750461e+111.100009
14141191523672070.0221590.08.738064e+145.737402e+111.786103e+152.424465e+114.909867e+133.223813e+102.044049
15151700300.000000NaNNaNNaNNaNNaNNaNNaNNaN
16162004008610713160.03606896.01.413303e+163.526030e+115.850686e+163.579934e+112.504408e+146.248212e+094.139724
1717205025600.000000NaNNaNNaNNaNNaNNaNNaNNaN
181821001600.000000NaNNaNNaNNaNNaNNaNNaNNaN
1919214025600.000000NaNNaNNaNNaNNaNNaNNaNNaN
202021501600.000000NaNNaNNaNNaNNaNNaNNaNNaN
21213601764386260.0436743.01.357018e+157.692844e+112.796929e+154.420624e+112.520519e+141.428865e+112.061085
2222450472307330.0151260.06.561625e+141.390175e+126.994786e+141.319771e+122.520519e+145.340082e+111.066014
\n", + "
" + ], + "text/plain": [ + " Unnamed: 0 priority n_fsh n_non finished% c_zero_end \\\n", + "0 0 0 106013713 102314553 0.508878 4.0 \n", + "1 1 3 0 3 0.000000 NaN \n", + "2 2 10 0 3 0.000000 NaN \n", + "3 3 25 903 3116 0.224683 0.0 \n", + "4 4 100 0 827 0.000000 NaN \n", + "5 5 101 25060 22557 0.526283 0.0 \n", + "6 6 103 2437 45665340 0.000053 219.0 \n", + "7 7 105 8972 38135622 0.000235 8.0 \n", + "8 8 107 27 11026707 0.000002 0.0 \n", + "9 9 114 223 1003339 0.000222 0.0 \n", + "10 10 115 353 124899 0.002818 1.0 \n", + "11 11 116 536 3873404 0.000138 0.0 \n", + "12 12 117 259 19 0.931655 0.0 \n", + "13 13 118 98 2368621 0.000041 0.0 \n", + "14 14 119 1523 67207 0.022159 0.0 \n", + "15 15 170 0 30 0.000000 NaN \n", + "16 16 200 40086 1071316 0.036068 96.0 \n", + "17 17 205 0 2560 0.000000 NaN \n", + "18 18 210 0 160 0.000000 NaN \n", + "19 19 214 0 2560 0.000000 NaN \n", + "20 20 215 0 160 0.000000 NaN \n", + "21 21 360 1764 38626 0.043674 3.0 \n", + "22 22 450 472 30733 0.015126 0.0 \n", + "\n", + " s_last m_last s_all m_all s_slow \\\n", + "0 9.890267e+16 9.330286e+08 1.093652e+17 1.001823e+09 6.123674e+12 \n", + "1 NaN NaN NaN NaN NaN \n", + "2 NaN NaN NaN NaN NaN \n", + "3 1.546669e+14 1.712812e+11 1.266916e+15 4.551425e+09 1.564663e+13 \n", + "4 NaN NaN NaN NaN NaN \n", + "5 1.571517e+14 6.271016e+09 6.623794e+16 2.497980e+09 1.629883e+13 \n", + "6 5.831405e+12 3.343696e+09 1.629492e+13 7.444003e+09 1.670900e+13 \n", + "7 1.077840e+16 1.202141e+12 1.479109e+16 8.637638e+11 1.670917e+13 \n", + "8 1.898978e+11 7.033253e+09 2.793068e+12 9.310227e+10 1.714303e+13 \n", + "9 7.021014e+11 3.148437e+09 7.100114e+11 3.141643e+09 1.714303e+13 \n", + "10 5.184592e+12 1.472896e+10 1.026934e+13 2.716758e+10 1.715451e+13 \n", + "11 1.525622e+12 2.846309e+09 1.559367e+12 2.850762e+09 1.715451e+13 \n", + "12 5.530792e+11 2.143718e+09 5.530792e+11 2.143718e+09 1.715451e+13 \n", + "13 1.091897e+11 1.114181e+09 1.201097e+11 1.112127e+09 1.715451e+13 \n", + "14 8.738064e+14 5.737402e+11 1.786103e+15 2.424465e+11 4.909867e+13 \n", + "15 NaN NaN NaN NaN NaN \n", + "16 1.413303e+16 3.526030e+11 5.850686e+16 3.579934e+11 2.504408e+14 \n", + "17 NaN NaN NaN NaN NaN \n", + "18 NaN NaN NaN NaN NaN \n", + "19 NaN NaN NaN NaN NaN \n", + "20 NaN NaN NaN NaN NaN \n", + "21 1.357018e+15 7.692844e+11 2.796929e+15 4.420624e+11 2.520519e+14 \n", + "22 6.561625e+14 1.390175e+12 6.994786e+14 1.319771e+12 2.520519e+14 \n", + "\n", + " m_slow m_slow_2 \n", + "0 5.776956e+04 1.105787 \n", + "1 NaN NaN \n", + "2 NaN NaN \n", + "3 1.732739e+10 8.191258 \n", + "4 NaN NaN \n", + "5 6.503923e+08 421.490544 \n", + "6 9.580850e+09 2.794339 \n", + "7 1.863615e+09 1.372291 \n", + "8 6.349270e+11 14.708268 \n", + "9 7.687457e+10 1.011266 \n", + "10 4.873442e+10 1.980743 \n", + "11 3.200469e+10 1.022119 \n", + "12 6.649037e+10 1.000000 \n", + "13 1.750461e+11 1.100009 \n", + "14 3.223813e+10 2.044049 \n", + "15 NaN NaN \n", + "16 6.248212e+09 4.139724 \n", + "17 NaN NaN \n", + "18 NaN NaN \n", + "19 NaN NaN \n", + "20 NaN NaN \n", + "21 1.428865e+11 2.061085 \n", + "22 5.340082e+11 1.066014 " + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Cluster d:\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Unnamed: 0priorityn_fshn_nonfinished%c_zero_ends_lastm_lasts_allm_alls_slowm_slowm_slow_2
00022214009615400680.2652290.03.105563e+161.398282e+093.465815e+161.469082e+095.049183e+122.273399e+051.116002
115010.000000NaNNaNNaNNaNNaNNaNNaNNaN
222578540330.1629310.01.682447e+132.143244e+101.104970e+154.037395e+095.082576e+126.474620e+0965.676400
3310003890.000000NaNNaNNaNNaNNaNNaNNaNNaN
4410125099302890.4531490.02.106023e+148.390863e+096.654064e+163.317183e+095.082723e+122.025070e+08315.954065
551031409310320320.000045149.08.107930e+126.790561e+098.640792e+126.646763e+095.082723e+124.256887e+091.065721
661058438163087360.0005173.06.967672e+158.257492e+112.018562e+169.240808e+115.082736e+126.023626e+082.897040
771072160043320.0000030.06.311163e+123.005316e+119.790846e+121.748365e+115.082736e+122.420350e+111.551354
88114012711610.000000NaNNaNNaNNaNNaNNaNNaNNaN
99115969177050.0518901.01.219481e+131.259795e+102.666472e+132.614188e+107.303040e+127.544463e+092.186562
1010116220517456530.0012620.02.043633e+139.268178e+092.612806e+131.095516e+107.683330e+123.484503e+091.278510
1111117610.8571430.06.581518e+101.096920e+106.581518e+101.096920e+107.683330e+121.280555e+121.000000
121211868412646900.0005410.01.644383e+132.404069e+103.368928e+133.059880e+107.683330e+121.123294e+102.048749
1313119413930590.0044180.07.600749e+131.844842e+112.295795e+141.727461e+118.050015e+121.953887e+103.020486
1414197090.000000NaNNaNNaNNaNNaNNaNNaNNaN
1515199020560.000000NaNNaNNaNNaNNaNNaNNaNNaN
16162008385112004820.065288104.02.344094e+162.795646e+111.292615e+173.493639e+117.670291e+149.147852e+095.514350
1717205011600.000000NaNNaNNaNNaNNaNNaNNaNNaN
181821001160.000000NaNNaNNaNNaNNaNNaNNaNNaN
1919214011600.000000NaNNaNNaNNaNNaNNaNNaNNaN
202021501160.000000NaNNaNNaNNaNNaNNaNNaNNaN
2121360630388690.0159501.04.095733e+146.501164e+111.014393e+153.901510e+117.670291e+141.217507e+122.476706
2222450169274840.0061110.01.586449e+149.387273e+112.110371e+145.236652e+117.670291e+144.538634e+121.330248
\n", + "
" + ], + "text/plain": [ + " Unnamed: 0 priority n_fsh n_non finished% c_zero_end \\\n", + "0 0 0 22214009 61540068 0.265229 0.0 \n", + "1 1 5 0 1 0.000000 NaN \n", + "2 2 25 785 4033 0.162931 0.0 \n", + "3 3 100 0 389 0.000000 NaN \n", + "4 4 101 25099 30289 0.453149 0.0 \n", + "5 5 103 1409 31032032 0.000045 149.0 \n", + "6 6 105 8438 16308736 0.000517 3.0 \n", + "7 7 107 21 6004332 0.000003 0.0 \n", + "8 8 114 0 1271161 0.000000 NaN \n", + "9 9 115 969 17705 0.051890 1.0 \n", + "10 10 116 2205 1745653 0.001262 0.0 \n", + "11 11 117 6 1 0.857143 0.0 \n", + "12 12 118 684 1264690 0.000541 0.0 \n", + "13 13 119 413 93059 0.004418 0.0 \n", + "14 14 197 0 9 0.000000 NaN \n", + "15 15 199 0 2056 0.000000 NaN \n", + "16 16 200 83851 1200482 0.065288 104.0 \n", + "17 17 205 0 1160 0.000000 NaN \n", + "18 18 210 0 116 0.000000 NaN \n", + "19 19 214 0 1160 0.000000 NaN \n", + "20 20 215 0 116 0.000000 NaN \n", + "21 21 360 630 38869 0.015950 1.0 \n", + "22 22 450 169 27484 0.006111 0.0 \n", + "\n", + " s_last m_last s_all m_all s_slow \\\n", + "0 3.105563e+16 1.398282e+09 3.465815e+16 1.469082e+09 5.049183e+12 \n", + "1 NaN NaN NaN NaN NaN \n", + "2 1.682447e+13 2.143244e+10 1.104970e+15 4.037395e+09 5.082576e+12 \n", + "3 NaN NaN NaN NaN NaN \n", + "4 2.106023e+14 8.390863e+09 6.654064e+16 3.317183e+09 5.082723e+12 \n", + "5 8.107930e+12 6.790561e+09 8.640792e+12 6.646763e+09 5.082723e+12 \n", + "6 6.967672e+15 8.257492e+11 2.018562e+16 9.240808e+11 5.082736e+12 \n", + "7 6.311163e+12 3.005316e+11 9.790846e+12 1.748365e+11 5.082736e+12 \n", + "8 NaN NaN NaN NaN NaN \n", + "9 1.219481e+13 1.259795e+10 2.666472e+13 2.614188e+10 7.303040e+12 \n", + "10 2.043633e+13 9.268178e+09 2.612806e+13 1.095516e+10 7.683330e+12 \n", + "11 6.581518e+10 1.096920e+10 6.581518e+10 1.096920e+10 7.683330e+12 \n", + "12 1.644383e+13 2.404069e+10 3.368928e+13 3.059880e+10 7.683330e+12 \n", + "13 7.600749e+13 1.844842e+11 2.295795e+14 1.727461e+11 8.050015e+12 \n", + "14 NaN NaN NaN NaN NaN \n", + "15 NaN NaN NaN NaN NaN \n", + "16 2.344094e+16 2.795646e+11 1.292615e+17 3.493639e+11 7.670291e+14 \n", + "17 NaN NaN NaN NaN NaN \n", + "18 NaN NaN NaN NaN NaN \n", + "19 NaN NaN NaN NaN NaN \n", + "20 NaN NaN NaN NaN NaN \n", + "21 4.095733e+14 6.501164e+11 1.014393e+15 3.901510e+11 7.670291e+14 \n", + "22 1.586449e+14 9.387273e+11 2.110371e+14 5.236652e+11 7.670291e+14 \n", + "\n", + " m_slow m_slow_2 \n", + "0 2.273399e+05 1.116002 \n", + "1 NaN NaN \n", + "2 6.474620e+09 65.676400 \n", + "3 NaN NaN \n", + "4 2.025070e+08 315.954065 \n", + "5 4.256887e+09 1.065721 \n", + "6 6.023626e+08 2.897040 \n", + "7 2.420350e+11 1.551354 \n", + "8 NaN NaN \n", + "9 7.544463e+09 2.186562 \n", + "10 3.484503e+09 1.278510 \n", + "11 1.280555e+12 1.000000 \n", + "12 1.123294e+10 2.048749 \n", + "13 1.953887e+10 3.020486 \n", + "14 NaN NaN \n", + "15 NaN NaN \n", + "16 9.147852e+09 5.514350 \n", + "17 NaN NaN \n", + "18 NaN NaN \n", + "19 NaN NaN \n", + "20 NaN NaN \n", + "21 1.217507e+12 2.476706 \n", + "22 4.538634e+12 1.330248 " + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Cluster e:\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Unnamed: 0priorityn_fshn_nonfinished%c_zero_ends_lastm_lasts_allm_alls_slowm_slowm_slow_2
00018722018250156860.4280521.01.500353e+168.021925e+082.159825e+161.126541e+095.550257e+112.967551e+041.439544
112556099180.0534450.01.792954e+133.224737e+104.798188e+133.894633e+105.550257e+119.982477e+082.676136
22100028010.000000NaNNaNNaNNaNNaNNaNNaNNaN
33101100062811600.0001590.03.060326e+133.060326e+103.435236e+132.772588e+105.550258e+115.550258e+081.122507
441032901133905140.00021763.02.186579e+147.629376e+106.916250e+144.855212e+105.550261e+111.936588e+083.163046
5510518194475350.00404810.01.940461e+141.066774e+112.862241e+156.419020e+105.581847e+113.068635e+0814.750313
661070176360.000000NaNNaNNaNNaNNaNNaNNaNNaN
771140266820.000000NaNNaNNaNNaNNaNNaNNaNNaN
88115351280500.0002730.02.353297e+126.723705e+102.353297e+126.536935e+105.581847e+111.594814e+101.000000
991160224140.000000NaNNaNNaNNaNNaNNaNNaNNaN
1010117019272230.000000NaNNaNNaNNaNNaNNaNNaNNaN
111111802310.000000NaNNaNNaNNaNNaNNaNNaNNaN
12121193269510.0045830.01.987924e+126.212262e+102.049727e+138.332224e+105.581847e+111.744327e+1010.310893
13131700250.000000NaNNaNNaNNaNNaNNaNNaNNaN
14142005204926045180.01959355.01.205656e+162.316386e+111.029114e+174.141488e+112.096092e+134.027151e+088.535722
15152010520.000000NaNNaNNaNNaNNaNNaNNaNNaN
16162050482400.000000NaNNaNNaNNaNNaNNaNNaNNaN
1717210026910.000000NaNNaNNaNNaNNaNNaNNaNNaN
1818215026910.000000NaNNaNNaNNaNNaNNaNNaNNaN
191922003600.000000NaNNaNNaNNaNNaNNaNNaNNaN
2020360367062070.3715700.02.244219e+156.115037e+116.448186e+154.392796e+112.097112e+135.714203e+092.873243
212145082148690.0054850.06.591093e+138.037918e+117.337755e+138.244668e+112.097112e+132.557454e+111.113283
\n", + "
" + ], + "text/plain": [ + " Unnamed: 0 priority n_fsh n_non finished% c_zero_end \\\n", + "0 0 0 18722018 25015686 0.428052 1.0 \n", + "1 1 25 560 9918 0.053445 0.0 \n", + "2 2 100 0 2801 0.000000 NaN \n", + "3 3 101 1000 6281160 0.000159 0.0 \n", + "4 4 103 2901 13390514 0.000217 63.0 \n", + "5 5 105 1819 447535 0.004048 10.0 \n", + "6 6 107 0 17636 0.000000 NaN \n", + "7 7 114 0 26682 0.000000 NaN \n", + "8 8 115 35 128050 0.000273 0.0 \n", + "9 9 116 0 22414 0.000000 NaN \n", + "10 10 117 0 1927223 0.000000 NaN \n", + "11 11 118 0 231 0.000000 NaN \n", + "12 12 119 32 6951 0.004583 0.0 \n", + "13 13 170 0 25 0.000000 NaN \n", + "14 14 200 52049 2604518 0.019593 55.0 \n", + "15 15 201 0 52 0.000000 NaN \n", + "16 16 205 0 48240 0.000000 NaN \n", + "17 17 210 0 2691 0.000000 NaN \n", + "18 18 215 0 2691 0.000000 NaN \n", + "19 19 220 0 360 0.000000 NaN \n", + "20 20 360 3670 6207 0.371570 0.0 \n", + "21 21 450 82 14869 0.005485 0.0 \n", + "\n", + " s_last m_last s_all m_all s_slow \\\n", + "0 1.500353e+16 8.021925e+08 2.159825e+16 1.126541e+09 5.550257e+11 \n", + "1 1.792954e+13 3.224737e+10 4.798188e+13 3.894633e+10 5.550257e+11 \n", + "2 NaN NaN NaN NaN NaN \n", + "3 3.060326e+13 3.060326e+10 3.435236e+13 2.772588e+10 5.550258e+11 \n", + "4 2.186579e+14 7.629376e+10 6.916250e+14 4.855212e+10 5.550261e+11 \n", + "5 1.940461e+14 1.066774e+11 2.862241e+15 6.419020e+10 5.581847e+11 \n", + "6 NaN NaN NaN NaN NaN \n", + "7 NaN NaN NaN NaN NaN \n", + "8 2.353297e+12 6.723705e+10 2.353297e+12 6.536935e+10 5.581847e+11 \n", + "9 NaN NaN NaN NaN NaN \n", + "10 NaN NaN NaN NaN NaN \n", + "11 NaN NaN NaN NaN NaN \n", + "12 1.987924e+12 6.212262e+10 2.049727e+13 8.332224e+10 5.581847e+11 \n", + "13 NaN NaN NaN NaN NaN \n", + "14 1.205656e+16 2.316386e+11 1.029114e+17 4.141488e+11 2.096092e+13 \n", + "15 NaN NaN NaN NaN NaN \n", + "16 NaN NaN NaN NaN NaN \n", + "17 NaN NaN NaN NaN NaN \n", + "18 NaN NaN NaN NaN NaN \n", + "19 NaN NaN NaN NaN NaN \n", + "20 2.244219e+15 6.115037e+11 6.448186e+15 4.392796e+11 2.097112e+13 \n", + "21 6.591093e+13 8.037918e+11 7.337755e+13 8.244668e+11 2.097112e+13 \n", + "\n", + " m_slow m_slow_2 \n", + "0 2.967551e+04 1.439544 \n", + "1 9.982477e+08 2.676136 \n", + "2 NaN NaN \n", + "3 5.550258e+08 1.122507 \n", + "4 1.936588e+08 3.163046 \n", + "5 3.068635e+08 14.750313 \n", + "6 NaN NaN \n", + "7 NaN NaN \n", + "8 1.594814e+10 1.000000 \n", + "9 NaN NaN \n", + "10 NaN NaN \n", + "11 NaN NaN \n", + "12 1.744327e+10 10.310893 \n", + "13 NaN NaN \n", + "14 4.027151e+08 8.535722 \n", + "15 NaN NaN \n", + "16 NaN NaN \n", + "17 NaN NaN \n", + "18 NaN NaN \n", + "19 NaN NaN \n", + "20 5.714203e+09 2.873243 \n", + "21 2.557454e+11 1.113283 " + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Cluster f:\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Unnamed: 0priorityn_fshn_nonfinished%c_zero_ends_lastm_lasts_allm_alls_slowm_slowm_slow_2
00012923639156632834.520822e-010.03.783381e+162.928982e+094.116930e+162.972516e+091.201620e+129.302590e+041.088162
11254162916.475047e-030.07.565244e+121.845182e+111.687775e+133.409647e+101.201620e+122.930781e+102.230960
22100010.000000e+00NaNNaNNaNNaNNaNNaNNaNNaN
3310124996370344.029663e-010.02.039588e+148.159656e+096.605382e+161.008262e+101.202133e+124.809302e+07323.858714
441037762132791735.841829e-040.03.588243e+144.644373e+104.188724e+144.723414e+101.202133e+121.555958e+081.167347
551051235155418562.223720e-030.01.372628e+161.111530e+122.128196e+161.173594e+121.202169e+129.734947e+071.550453
661071352216846.086043e-040.01.082034e+138.015069e+101.095805e+137.883492e+101.202169e+128.904953e+091.012727
77114466610546.958100e-050.03.114919e+106.771562e+083.114919e+106.771562e+081.202169e+122.613410e+101.000000
8811528475033.647104e-020.03.179232e+131.213447e+111.619569e+142.526629e+111.202171e+124.588440e+095.094215
99116032861870.000000e+00NaNNaNNaNNaNNaNNaNNaNNaN
1010117223303108.582542e-070.03.175054e+101.587527e+103.175054e+101.587527e+101.202171e+126.010856e+111.000000
11111182813449232.081860e-050.08.412515e+113.004470e+108.412515e+112.549247e+101.202171e+124.293468e+101.000000
12121199624210703.135466e-010.01.483986e+151.541964e+111.129135e+166.883334e+101.242061e+121.290587e+087.608799
13132009129624075493.653528e-024.02.713027e+162.971682e+111.612419e+174.923717e+113.139660e+143.438990e+095.943247
1414201027100.000000e+00NaNNaNNaNNaNNaNNaNNaNNaN
15153603433428047.424790e-020.03.307183e+159.633506e+117.181627e+155.694281e+113.140996e+149.149421e+102.171524
1616450257256349.926229e-030.02.867563e+141.115783e+122.927933e+141.113282e+123.140996e+141.222178e+121.021053
\n", + "
" + ], + "text/plain": [ + " Unnamed: 0 priority n_fsh n_non finished% c_zero_end \\\n", + "0 0 0 12923639 15663283 4.520822e-01 0.0 \n", + "1 1 25 41 6291 6.475047e-03 0.0 \n", + "2 2 100 0 1 0.000000e+00 NaN \n", + "3 3 101 24996 37034 4.029663e-01 0.0 \n", + "4 4 103 7762 13279173 5.841829e-04 0.0 \n", + "5 5 105 12351 5541856 2.223720e-03 0.0 \n", + "6 6 107 135 221684 6.086043e-04 0.0 \n", + "7 7 114 46 661054 6.958100e-05 0.0 \n", + "8 8 115 284 7503 3.647104e-02 0.0 \n", + "9 9 116 0 3286187 0.000000e+00 NaN \n", + "10 10 117 2 2330310 8.582542e-07 0.0 \n", + "11 11 118 28 1344923 2.081860e-05 0.0 \n", + "12 12 119 9624 21070 3.135466e-01 0.0 \n", + "13 13 200 91296 2407549 3.653528e-02 4.0 \n", + "14 14 201 0 2710 0.000000e+00 NaN \n", + "15 15 360 3433 42804 7.424790e-02 0.0 \n", + "16 16 450 257 25634 9.926229e-03 0.0 \n", + "\n", + " s_last m_last s_all m_all s_slow \\\n", + "0 3.783381e+16 2.928982e+09 4.116930e+16 2.972516e+09 1.201620e+12 \n", + "1 7.565244e+12 1.845182e+11 1.687775e+13 3.409647e+10 1.201620e+12 \n", + "2 NaN NaN NaN NaN NaN \n", + "3 2.039588e+14 8.159656e+09 6.605382e+16 1.008262e+10 1.202133e+12 \n", + "4 3.588243e+14 4.644373e+10 4.188724e+14 4.723414e+10 1.202133e+12 \n", + "5 1.372628e+16 1.111530e+12 2.128196e+16 1.173594e+12 1.202169e+12 \n", + "6 1.082034e+13 8.015069e+10 1.095805e+13 7.883492e+10 1.202169e+12 \n", + "7 3.114919e+10 6.771562e+08 3.114919e+10 6.771562e+08 1.202169e+12 \n", + "8 3.179232e+13 1.213447e+11 1.619569e+14 2.526629e+11 1.202171e+12 \n", + "9 NaN NaN NaN NaN NaN \n", + "10 3.175054e+10 1.587527e+10 3.175054e+10 1.587527e+10 1.202171e+12 \n", + "11 8.412515e+11 3.004470e+10 8.412515e+11 2.549247e+10 1.202171e+12 \n", + "12 1.483986e+15 1.541964e+11 1.129135e+16 6.883334e+10 1.242061e+12 \n", + "13 2.713027e+16 2.971682e+11 1.612419e+17 4.923717e+11 3.139660e+14 \n", + "14 NaN NaN NaN NaN NaN \n", + "15 3.307183e+15 9.633506e+11 7.181627e+15 5.694281e+11 3.140996e+14 \n", + "16 2.867563e+14 1.115783e+12 2.927933e+14 1.113282e+12 3.140996e+14 \n", + "\n", + " m_slow m_slow_2 \n", + "0 9.302590e+04 1.088162 \n", + "1 2.930781e+10 2.230960 \n", + "2 NaN NaN \n", + "3 4.809302e+07 323.858714 \n", + "4 1.555958e+08 1.167347 \n", + "5 9.734947e+07 1.550453 \n", + "6 8.904953e+09 1.012727 \n", + "7 2.613410e+10 1.000000 \n", + "8 4.588440e+09 5.094215 \n", + "9 NaN NaN \n", + "10 6.010856e+11 1.000000 \n", + "11 4.293468e+10 1.000000 \n", + "12 1.290587e+08 7.608799 \n", + "13 3.438990e+09 5.943247 \n", + "14 NaN NaN \n", + "15 9.149421e+10 2.171524 \n", + "16 1.222178e+12 1.021053 " + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Cluster g:\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Unnamed: 0priorityn_fshn_nonfinished%c_zero_ends_lastm_lasts_allm_alls_slowm_slowm_slow_2
0005492965108492110.3361220.01.652722e+163.009921e+091.882431e+163.316600e+091.116295e+132.032987e+061.138988
11251668410.0023330.09.873307e+116.170817e+108.582429e+121.215641e+101.116295e+136.976846e+118.692558
225001140.000000NaNNaNNaNNaNNaNNaNNaNNaN
3310003520.000000NaNNaNNaNNaNNaNNaNNaNNaN
441019997836580.9647030.01.339232e+161.339527e+112.595234e+177.448012e+092.071545e+132.072001e+0819.378523
55103195560062970.0003250.02.302315e+141.183101e+112.926893e+141.127462e+112.071545e+131.064515e+101.271282
661052601322000.0019638.02.150574e+128.271437e+092.152161e+128.214355e+092.071545e+137.967482e+101.000738
771070230920.000000NaNNaNNaNNaNNaNNaNNaNNaN
8811401200.000000NaNNaNNaNNaNNaNNaNNaNNaN
9911513015730.0763360.04.406302e+123.442423e+107.940457e+125.631530e+102.110285e+131.648660e+111.802068
1010117018395280.000000NaNNaNNaNNaNNaNNaNNaNNaN
1111118142514850.4896910.08.671496e+146.085260e+113.362027e+154.747285e+112.112130e+131.482196e+103.877102
12121191221418310.0008590.01.512875e+131.240061e+114.789877e+132.313950e+112.112130e+131.731254e+113.166077
131317002600.000000NaNNaNNaNNaNNaNNaNNaNNaN
1414200515271411180.26747143.05.974724e+151.159533e+118.707510e+163.990500e+111.918182e+143.722673e+0914.573912
151536017131041010.0161891.01.347436e+157.865942e+112.855922e+154.880250e+111.918182e+141.119779e+112.119524
1616450643228480.0273720.07.746521e+141.204747e+128.032579e+141.188251e+121.918182e+142.983176e+111.036927
\n", + "
" + ], + "text/plain": [ + " Unnamed: 0 priority n_fsh n_non finished% c_zero_end \\\n", + "0 0 0 5492965 10849211 0.336122 0.0 \n", + "1 1 25 16 6841 0.002333 0.0 \n", + "2 2 50 0 114 0.000000 NaN \n", + "3 3 100 0 352 0.000000 NaN \n", + "4 4 101 99978 3658 0.964703 0.0 \n", + "5 5 103 1955 6006297 0.000325 0.0 \n", + "6 6 105 260 132200 0.001963 8.0 \n", + "7 7 107 0 23092 0.000000 NaN \n", + "8 8 114 0 120 0.000000 NaN \n", + "9 9 115 130 1573 0.076336 0.0 \n", + "10 10 117 0 1839528 0.000000 NaN \n", + "11 11 118 1425 1485 0.489691 0.0 \n", + "12 12 119 122 141831 0.000859 0.0 \n", + "13 13 170 0 260 0.000000 NaN \n", + "14 14 200 51527 141118 0.267471 43.0 \n", + "15 15 360 1713 104101 0.016189 1.0 \n", + "16 16 450 643 22848 0.027372 0.0 \n", + "\n", + " s_last m_last s_all m_all s_slow \\\n", + "0 1.652722e+16 3.009921e+09 1.882431e+16 3.316600e+09 1.116295e+13 \n", + "1 9.873307e+11 6.170817e+10 8.582429e+12 1.215641e+10 1.116295e+13 \n", + "2 NaN NaN NaN NaN NaN \n", + "3 NaN NaN NaN NaN NaN \n", + "4 1.339232e+16 1.339527e+11 2.595234e+17 7.448012e+09 2.071545e+13 \n", + "5 2.302315e+14 1.183101e+11 2.926893e+14 1.127462e+11 2.071545e+13 \n", + "6 2.150574e+12 8.271437e+09 2.152161e+12 8.214355e+09 2.071545e+13 \n", + "7 NaN NaN NaN NaN NaN \n", + "8 NaN NaN NaN NaN NaN \n", + "9 4.406302e+12 3.442423e+10 7.940457e+12 5.631530e+10 2.110285e+13 \n", + "10 NaN NaN NaN NaN NaN \n", + "11 8.671496e+14 6.085260e+11 3.362027e+15 4.747285e+11 2.112130e+13 \n", + "12 1.512875e+13 1.240061e+11 4.789877e+13 2.313950e+11 2.112130e+13 \n", + "13 NaN NaN NaN NaN NaN \n", + "14 5.974724e+15 1.159533e+11 8.707510e+16 3.990500e+11 1.918182e+14 \n", + "15 1.347436e+15 7.865942e+11 2.855922e+15 4.880250e+11 1.918182e+14 \n", + "16 7.746521e+14 1.204747e+12 8.032579e+14 1.188251e+12 1.918182e+14 \n", + "\n", + " m_slow m_slow_2 \n", + "0 2.032987e+06 1.138988 \n", + "1 6.976846e+11 8.692558 \n", + "2 NaN NaN \n", + "3 NaN NaN \n", + "4 2.072001e+08 19.378523 \n", + "5 1.064515e+10 1.271282 \n", + "6 7.967482e+10 1.000738 \n", + "7 NaN NaN \n", + "8 NaN NaN \n", + "9 1.648660e+11 1.802068 \n", + "10 NaN NaN \n", + "11 1.482196e+10 3.877102 \n", + "12 1.731254e+11 3.166077 \n", + "13 NaN NaN \n", + "14 3.722673e+09 14.573912 \n", + "15 1.119779e+11 2.119524 \n", + "16 2.983176e+11 1.036927 " + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Cluster h:\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Unnamed: 0priorityn_fshn_nonfinished%c_zero_ends_lastm_lasts_allm_alls_slowm_slowm_slow_2
0006626426172574242.774438e-010.03.750671e+165.663232e+094.209970e+166.211185e+099.556217e+111.442917e+051.122458
11190100.000000e+00NaNNaNNaNNaNNaNNaNNaNNaN
22257974971.042767e-020.02.408477e+133.048705e+117.380026e+132.838472e+119.556237e+111.209650e+103.064188
331012500001.000000e+000.08.515792e+143.406317e+106.509309e+161.224965e+101.012711e+124.050842e+0776.438090
441032411249861084.812563e-030.06.553604e+152.726352e+118.271087e+159.289390e+101.012712e+124.212964e+071.262067
55105100006906451.427256e-020.06.117631e+156.117631e+112.572798e+163.937615e+111.012750e+121.012750e+084.205547
6610701465470.000000e+00NaNNaNNaNNaNNaNNaNNaNNaN
77115234265.122494e-020.06.648578e+112.890686e+106.648578e+112.890686e+101.012750e+124.403262e+101.000000
88116134128091.035309e-020.08.484331e+106.331590e+086.231571e+124.210521e+101.012751e+127.557840e+0973.447995
99117120134154.966683e-070.06.557861e+086.557861e+086.557861e+086.557861e+081.012751e+121.012751e+121.000000
1010118506499261.003331e-020.01.000296e+141.976869e+111.947697e+141.391212e+111.012751e+122.001483e+091.947121
1111119397027299171.452145e-030.05.554317e+141.399072e+114.055259e+151.555807e+101.246604e+123.140059e+087.301093
12122005785220826192.702770e-0221.01.728555e+162.987994e+111.002241e+174.707830e+111.713280e+132.961590e+085.798142
131320103630.000000e+00NaNNaNNaNNaNNaNNaNNaNNaN
141422003600.000000e+00NaNNaNNaNNaNNaNNaNNaNNaN
15153601862402104.425746e-021.01.561695e+158.387189e+113.152188e+153.973012e+111.713283e+139.201306e+092.018441
161645091169065.353886e-030.01.338225e+141.470577e+121.411397e+141.411397e+121.713283e+131.882729e+111.054678
\n", + "
" + ], + "text/plain": [ + " Unnamed: 0 priority n_fsh n_non finished% c_zero_end \\\n", + "0 0 0 6626426 17257424 2.774438e-01 0.0 \n", + "1 1 19 0 10 0.000000e+00 NaN \n", + "2 2 25 79 7497 1.042767e-02 0.0 \n", + "3 3 101 25000 0 1.000000e+00 0.0 \n", + "4 4 103 24112 4986108 4.812563e-03 0.0 \n", + "5 5 105 10000 690645 1.427256e-02 0.0 \n", + "6 6 107 0 146547 0.000000e+00 NaN \n", + "7 7 115 23 426 5.122494e-02 0.0 \n", + "8 8 116 134 12809 1.035309e-02 0.0 \n", + "9 9 117 1 2013415 4.966683e-07 0.0 \n", + "10 10 118 506 49926 1.003331e-02 0.0 \n", + "11 11 119 3970 2729917 1.452145e-03 0.0 \n", + "12 12 200 57852 2082619 2.702770e-02 21.0 \n", + "13 13 201 0 363 0.000000e+00 NaN \n", + "14 14 220 0 360 0.000000e+00 NaN \n", + "15 15 360 1862 40210 4.425746e-02 1.0 \n", + "16 16 450 91 16906 5.353886e-03 0.0 \n", + "\n", + " s_last m_last s_all m_all s_slow \\\n", + "0 3.750671e+16 5.663232e+09 4.209970e+16 6.211185e+09 9.556217e+11 \n", + "1 NaN NaN NaN NaN NaN \n", + "2 2.408477e+13 3.048705e+11 7.380026e+13 2.838472e+11 9.556237e+11 \n", + "3 8.515792e+14 3.406317e+10 6.509309e+16 1.224965e+10 1.012711e+12 \n", + "4 6.553604e+15 2.726352e+11 8.271087e+15 9.289390e+10 1.012712e+12 \n", + "5 6.117631e+15 6.117631e+11 2.572798e+16 3.937615e+11 1.012750e+12 \n", + "6 NaN NaN NaN NaN NaN \n", + "7 6.648578e+11 2.890686e+10 6.648578e+11 2.890686e+10 1.012750e+12 \n", + "8 8.484331e+10 6.331590e+08 6.231571e+12 4.210521e+10 1.012751e+12 \n", + "9 6.557861e+08 6.557861e+08 6.557861e+08 6.557861e+08 1.012751e+12 \n", + "10 1.000296e+14 1.976869e+11 1.947697e+14 1.391212e+11 1.012751e+12 \n", + "11 5.554317e+14 1.399072e+11 4.055259e+15 1.555807e+10 1.246604e+12 \n", + "12 1.728555e+16 2.987994e+11 1.002241e+17 4.707830e+11 1.713280e+13 \n", + "13 NaN NaN NaN NaN NaN \n", + "14 NaN NaN NaN NaN NaN \n", + "15 1.561695e+15 8.387189e+11 3.152188e+15 3.973012e+11 1.713283e+13 \n", + "16 1.338225e+14 1.470577e+12 1.411397e+14 1.411397e+12 1.713283e+13 \n", + "\n", + " m_slow m_slow_2 \n", + "0 1.442917e+05 1.122458 \n", + "1 NaN NaN \n", + "2 1.209650e+10 3.064188 \n", + "3 4.050842e+07 76.438090 \n", + "4 4.212964e+07 1.262067 \n", + "5 1.012750e+08 4.205547 \n", + "6 NaN NaN \n", + "7 4.403262e+10 1.000000 \n", + "8 7.557840e+09 73.447995 \n", + "9 1.012751e+12 1.000000 \n", + "10 2.001483e+09 1.947121 \n", + "11 3.140059e+08 7.301093 \n", + "12 2.961590e+08 5.798142 \n", + "13 NaN NaN \n", + "14 NaN NaN \n", + "15 9.201306e+09 2.018441 \n", + "16 1.882729e+11 1.054678 " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "CLUSTERS = \"abcdefgh\"\n", + "DIR = \"/home/claudio/google_2019/thesis_queries/task_slowdown/\"\n", + "\n", + "df = {}\n", + "\n", + "for cluster in CLUSTERS:\n", + " print(\"Cluster \" + cluster + \":\")\n", + " df[cluster] = pandas.read_csv(DIR + \"/\" + cluster + \"_slowdown_table.csv\")\n", + " df[cluster][\"m_slow_2\"] = (df[cluster][\"s_all\"] / df[cluster][\"s_last\"]) \n", + " display(df[cluster])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "physical-function", + "metadata": {}, + "outputs": [], + "source": [ + "wc" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "checked-playing", + "metadata": {}, + "outputs": [], + "source": [ + "tc" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "talented-realtor", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/task_slowdown/task_slowdown_table.py b/task_slowdown/task_slowdown_table.py new file mode 100644 index 00000000..0a3768ad --- /dev/null +++ b/task_slowdown/task_slowdown_table.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python +# coding: utf-8 + +# # Task slowdown + +import json +import sys +import gzip +import pandas +import seaborn as sns +import matplotlib as mpl +import matplotlib.pyplot as plt +from IPython.display import display, HTML + +columns = ["priority", "n_fsh", "n_non", "finished%", "c_zero_end", "s_last", "m_last", "s_all", "m_all", "s_slow", "m_slow"] +CLUSTERS = "abcdefgh" +DIR = "/home/claudio/google_2019/thesis_queries/task_slowdown/" + +df = {} + +for cluster in CLUSTERS: + wc = 0 + tc = {} + s_slowdown = 0 + + # Loading + print("Loading cluster " + cluster + "...") + data = None + + with gzip.open(DIR + cluster + "_state_changes.json.gz", "r") as f: + data = json.load(f) + + print("Done loading") + + # Computation + priorities = sorted(set(data["val"].keys()).union(set(data["non"].keys())), key=lambda x: int(x)) + table = {} + for c in columns: + table[c] = [] + + def add(col, val): + table[col].append(val) + + def empty_row(): + add("m_last", None) + add("m_all", None) + add("m_slow", None) + add("s_last", None) + add("s_all", None) + add("s_slow", None) + add("c_zero_end", None) + + for priority in priorities: + print("Priority " + priority) + add("priority", priority) + + n_not_finished = 0 if priority not in data["non"] else data["non"][priority] + n_finished = 0 if priority not in data["val"] else len(data["val"][priority]) + + add("n_fsh", n_finished) + add("n_non", n_not_finished) + add("finished%", n_finished / (n_not_finished + n_finished)) + + if n_finished == 0: + empty_row() + else: + s = 0 + c = 0 + ss = 0 + sum_zero_end = 0 + for task in data["val"][priority]: + + idx_last = len(task) - 1 + t = 0 + + while idx_last >= 0 and task[idx_last][1][-1] != 6: + idx_last -= 1 + t += 1 + + if idx_last < 0: + #print("weird") + wc += 1 + n_finished -= 1 + n_not_finished += 1 + continue + + if t in tc: + tc[t] += 1 + else: + tc[1] = 1 + + last_slice = task[idx_last] + + s += last_slice[0] + + local_sum = 0 + for exec_slice in task: + local_sum += exec_slice[0] + c += 1 + + if last_slice[0] > 0: + s_slowdown += (local_sum / last_slice[0]) + else: + sum_zero_end += 1 + + ss += local_sum + + if n_finished == 0: + empty_row() + continue + + add("c_zero_end", sum_zero_end) + add("s_last", s) + add("m_last", s / n_finished) + add("s_all", ss) + add("m_all", ss / c) + add("s_slow", s_slowdown) + add("m_slow", s_slowdown / n_finished) + + df[cluster] = pandas.DataFrame(table, columns=columns) + df[cluster].to_csv(DIR + "/" + cluster + "_slowdown_table.csv") + + print("Done csv") + + + +