{ "cells": [ { "cell_type": "markdown", "id": "proper-gnome", "metadata": {}, "source": [ "# Temporal impact: machine time waste" ] }, { "cell_type": "code", "execution_count": 1, "id": "fantastic-harrison", "metadata": { "collapsed": false, "jupyter": { "outputs_hidden": false } }, "outputs": [], "source": [ "import pandas\n", "from IPython import display\n", "import findspark\n", "findspark.init()\n", "import pyspark\n", "import pyspark.sql\n", "import sys\n", "\n", "from pyspark.sql.functions import col, lag, when, concat_ws, last, first\n", "from pyspark.sql import Window\n", "from pyspark.sql.types import LongType" ] }, { "cell_type": "code", "execution_count": 2, "id": "failing-rebecca", "metadata": { "collapsed": false, "jupyter": { "outputs_hidden": false } }, "outputs": [], "source": [ "cluster=\"b\"" ] }, { "cell_type": "code", "execution_count": 3, "id": "transsexual-baptist", "metadata": { "collapsed": false, "jupyter": { "outputs_hidden": false } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "ERROR:root:Exception while sending command.\n", "Traceback (most recent call last):\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1207, in send_command\n", " raise Py4JNetworkError(\"Answer from Java side is empty\")\n", "py4j.protocol.Py4JNetworkError: Answer from Java side is empty\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1033, in send_command\n", " response = connection.send_command(command)\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1212, in send_command\n", " \"Error while receiving\", e, proto.ERROR_ON_RECEIVE)\n", "py4j.protocol.Py4JNetworkError: Error while receiving\n", "ERROR:py4j.java_gateway:An error occurred while trying to connect to the Java server (127.0.0.1:36135)\n", "Traceback (most recent call last):\n", " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3343, in run_code\n", " exec(code_obj, self.user_global_ns, self.user_ns)\n", " File \"\", line 5, in \n", " df = spark.read.json(\"/home/claudio/google_2019/instance_events/\" + cluster + \"/\" + cluster + \"_instance_events*.json.gz\")\n", " File \"/opt/spark/python/pyspark/sql/readwriter.py\", line 300, in json\n", " return self._df(self._jreader.json(self._spark._sc._jvm.PythonUtils.toSeq(path)))\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1305, in __call__\n", " answer, self.gateway_client, self.target_id, self.name)\n", " File \"/opt/spark/python/pyspark/sql/utils.py\", line 128, in deco\n", " return f(*a, **kw)\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/protocol.py\", line 336, in get_return_value\n", " format(target_id, \".\", name))\n", "py4j.protocol.Py4JError: An error occurred while calling o26.json\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2044, in showtraceback\n", " stb = value._render_traceback_()\n", "AttributeError: 'Py4JError' object has no attribute '_render_traceback_'\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 977, in _get_connection\n", " connection = self.deque.pop()\n", "IndexError: pop from an empty deque\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1115, in start\n", " self.socket.connect((self.address, self.port))\n", "ConnectionRefusedError: [Errno 111] Connection refused\n", "ERROR:py4j.java_gateway:An error occurred while trying to connect to the Java server (127.0.0.1:36135)\n", "Traceback (most recent call last):\n", " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3343, in run_code\n", " exec(code_obj, self.user_global_ns, self.user_ns)\n", " File \"\", line 5, in \n", " df = spark.read.json(\"/home/claudio/google_2019/instance_events/\" + cluster + \"/\" + cluster + \"_instance_events*.json.gz\")\n", " File \"/opt/spark/python/pyspark/sql/readwriter.py\", line 300, in json\n", " return self._df(self._jreader.json(self._spark._sc._jvm.PythonUtils.toSeq(path)))\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1305, in __call__\n", " answer, self.gateway_client, self.target_id, self.name)\n", " File \"/opt/spark/python/pyspark/sql/utils.py\", line 128, in deco\n", " return f(*a, **kw)\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/protocol.py\", line 336, in get_return_value\n", " format(target_id, \".\", name))\n", "py4j.protocol.Py4JError: An error occurred while calling o26.json\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2044, in showtraceback\n", " stb = value._render_traceback_()\n", "AttributeError: 'Py4JError' object has no attribute '_render_traceback_'\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 977, in _get_connection\n", " connection = self.deque.pop()\n", "IndexError: pop from an empty deque\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1115, in start\n", " self.socket.connect((self.address, self.port))\n", "ConnectionRefusedError: [Errno 111] Connection refused\n", "ERROR:py4j.java_gateway:An error occurred while trying to connect to the Java server (127.0.0.1:36135)\n", "Traceback (most recent call last):\n", " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3343, in run_code\n", " exec(code_obj, self.user_global_ns, self.user_ns)\n", " File \"\", line 5, in \n", " df = spark.read.json(\"/home/claudio/google_2019/instance_events/\" + cluster + \"/\" + cluster + \"_instance_events*.json.gz\")\n", " File \"/opt/spark/python/pyspark/sql/readwriter.py\", line 300, in json\n", " return self._df(self._jreader.json(self._spark._sc._jvm.PythonUtils.toSeq(path)))\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1305, in __call__\n", " answer, self.gateway_client, self.target_id, self.name)\n", " File \"/opt/spark/python/pyspark/sql/utils.py\", line 128, in deco\n", " return f(*a, **kw)\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/protocol.py\", line 336, in get_return_value\n", " format(target_id, \".\", name))\n", "py4j.protocol.Py4JError: An error occurred while calling o26.json\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2044, in showtraceback\n", " stb = value._render_traceback_()\n", "AttributeError: 'Py4JError' object has no attribute '_render_traceback_'\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 977, in _get_connection\n", " connection = self.deque.pop()\n", "IndexError: pop from an empty deque\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1115, in start\n", " self.socket.connect((self.address, self.port))\n", "ConnectionRefusedError: [Errno 111] Connection refused\n", "ERROR:py4j.java_gateway:An error occurred while trying to connect to the Java server (127.0.0.1:36135)\n", "Traceback (most recent call last):\n", " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3343, in run_code\n", " exec(code_obj, self.user_global_ns, self.user_ns)\n", " File \"\", line 5, in \n", " df = spark.read.json(\"/home/claudio/google_2019/instance_events/\" + cluster + \"/\" + cluster + \"_instance_events*.json.gz\")\n", " File \"/opt/spark/python/pyspark/sql/readwriter.py\", line 300, in json\n", " return self._df(self._jreader.json(self._spark._sc._jvm.PythonUtils.toSeq(path)))\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1305, in __call__\n", " answer, self.gateway_client, self.target_id, self.name)\n", " File \"/opt/spark/python/pyspark/sql/utils.py\", line 128, in deco\n", " return f(*a, **kw)\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/protocol.py\", line 336, in get_return_value\n", " format(target_id, \".\", name))\n", "py4j.protocol.Py4JError: An error occurred while calling o26.json\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2044, in showtraceback\n", " stb = value._render_traceback_()\n", "AttributeError: 'Py4JError' object has no attribute '_render_traceback_'\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 977, in _get_connection\n", " connection = self.deque.pop()\n", "IndexError: pop from an empty deque\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1115, in start\n", " self.socket.connect((self.address, self.port))\n", "ConnectionRefusedError: [Errno 111] Connection refused\n", "ERROR:py4j.java_gateway:An error occurred while trying to connect to the Java server (127.0.0.1:36135)\n", "Traceback (most recent call last):\n", " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3343, in run_code\n", " exec(code_obj, self.user_global_ns, self.user_ns)\n", " File \"\", line 5, in \n", " df = spark.read.json(\"/home/claudio/google_2019/instance_events/\" + cluster + \"/\" + cluster + \"_instance_events*.json.gz\")\n", " File \"/opt/spark/python/pyspark/sql/readwriter.py\", line 300, in json\n", " return self._df(self._jreader.json(self._spark._sc._jvm.PythonUtils.toSeq(path)))\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1305, in __call__\n", " answer, self.gateway_client, self.target_id, self.name)\n", " File \"/opt/spark/python/pyspark/sql/utils.py\", line 128, in deco\n", " return f(*a, **kw)\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/protocol.py\", line 336, in get_return_value\n", " format(target_id, \".\", name))\n", "py4j.protocol.Py4JError: An error occurred while calling o26.json\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2044, in showtraceback\n", " stb = value._render_traceback_()\n", "AttributeError: 'Py4JError' object has no attribute '_render_traceback_'\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 977, in _get_connection\n", " connection = self.deque.pop()\n", "IndexError: pop from an empty deque\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1115, in start\n", " self.socket.connect((self.address, self.port))\n", "ConnectionRefusedError: [Errno 111] Connection refused\n", "ERROR:py4j.java_gateway:An error occurred while trying to connect to the Java server (127.0.0.1:36135)\n", "Traceback (most recent call last):\n", " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3343, in run_code\n", " exec(code_obj, self.user_global_ns, self.user_ns)\n", " File \"\", line 5, in \n", " df = spark.read.json(\"/home/claudio/google_2019/instance_events/\" + cluster + \"/\" + cluster + \"_instance_events*.json.gz\")\n", " File \"/opt/spark/python/pyspark/sql/readwriter.py\", line 300, in json\n", " return self._df(self._jreader.json(self._spark._sc._jvm.PythonUtils.toSeq(path)))\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1305, in __call__\n", " answer, self.gateway_client, self.target_id, self.name)\n", " File \"/opt/spark/python/pyspark/sql/utils.py\", line 128, in deco\n", " return f(*a, **kw)\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/protocol.py\", line 336, in get_return_value\n", " format(target_id, \".\", name))\n", "py4j.protocol.Py4JError: An error occurred while calling o26.json\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2044, in showtraceback\n", " stb = value._render_traceback_()\n", "AttributeError: 'Py4JError' object has no attribute '_render_traceback_'\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 977, in _get_connection\n", " connection = self.deque.pop()\n", "IndexError: pop from an empty deque\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1115, in start\n", " self.socket.connect((self.address, self.port))\n", "ConnectionRefusedError: [Errno 111] Connection refused\n", "ERROR:py4j.java_gateway:An error occurred while trying to connect to the Java server (127.0.0.1:36135)\n", "Traceback (most recent call last):\n", " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3343, in run_code\n", " exec(code_obj, self.user_global_ns, self.user_ns)\n", " File \"\", line 5, in \n", " df = spark.read.json(\"/home/claudio/google_2019/instance_events/\" + cluster + \"/\" + cluster + \"_instance_events*.json.gz\")\n", " File \"/opt/spark/python/pyspark/sql/readwriter.py\", line 300, in json\n", " return self._df(self._jreader.json(self._spark._sc._jvm.PythonUtils.toSeq(path)))\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1305, in __call__\n", " answer, self.gateway_client, self.target_id, self.name)\n", " File \"/opt/spark/python/pyspark/sql/utils.py\", line 128, in deco\n", " return f(*a, **kw)\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/protocol.py\", line 336, in get_return_value\n", " format(target_id, \".\", name))\n", "py4j.protocol.Py4JError: An error occurred while calling o26.json\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2044, in showtraceback\n", " stb = value._render_traceback_()\n", "AttributeError: 'Py4JError' object has no attribute '_render_traceback_'\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 977, in _get_connection\n", " connection = self.deque.pop()\n", "IndexError: pop from an empty deque\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1115, in start\n", " self.socket.connect((self.address, self.port))\n", "ConnectionRefusedError: [Errno 111] Connection refused\n", "ERROR:py4j.java_gateway:An error occurred while trying to connect to the Java server (127.0.0.1:36135)\n", "Traceback (most recent call last):\n", " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3343, in run_code\n", " exec(code_obj, self.user_global_ns, self.user_ns)\n", " File \"\", line 5, in \n", " df = spark.read.json(\"/home/claudio/google_2019/instance_events/\" + cluster + \"/\" + cluster + \"_instance_events*.json.gz\")\n", " File \"/opt/spark/python/pyspark/sql/readwriter.py\", line 300, in json\n", " return self._df(self._jreader.json(self._spark._sc._jvm.PythonUtils.toSeq(path)))\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1305, in __call__\n", " answer, self.gateway_client, self.target_id, self.name)\n", " File \"/opt/spark/python/pyspark/sql/utils.py\", line 128, in deco\n", " return f(*a, **kw)\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/protocol.py\", line 336, in get_return_value\n", " format(target_id, \".\", name))\n", "py4j.protocol.Py4JError: An error occurred while calling o26.json\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2044, in showtraceback\n", " stb = value._render_traceback_()\n", "AttributeError: 'Py4JError' object has no attribute '_render_traceback_'\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 977, in _get_connection\n", " connection = self.deque.pop()\n", "IndexError: pop from an empty deque\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1115, in start\n", " self.socket.connect((self.address, self.port))\n", "ConnectionRefusedError: [Errno 111] Connection refused\n", "ERROR:py4j.java_gateway:An error occurred while trying to connect to the Java server (127.0.0.1:36135)\n", "Traceback (most recent call last):\n", " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3343, in run_code\n", " exec(code_obj, self.user_global_ns, self.user_ns)\n", " File \"\", line 5, in \n", " df = spark.read.json(\"/home/claudio/google_2019/instance_events/\" + cluster + \"/\" + cluster + \"_instance_events*.json.gz\")\n", " File \"/opt/spark/python/pyspark/sql/readwriter.py\", line 300, in json\n", " return self._df(self._jreader.json(self._spark._sc._jvm.PythonUtils.toSeq(path)))\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1305, in __call__\n", " answer, self.gateway_client, self.target_id, self.name)\n", " File \"/opt/spark/python/pyspark/sql/utils.py\", line 128, in deco\n", " return f(*a, **kw)\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/protocol.py\", line 336, in get_return_value\n", " format(target_id, \".\", name))\n", "py4j.protocol.Py4JError: An error occurred while calling o26.json\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2044, in showtraceback\n", " stb = value._render_traceback_()\n", "AttributeError: 'Py4JError' object has no attribute '_render_traceback_'\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 977, in _get_connection\n", " connection = self.deque.pop()\n", "IndexError: pop from an empty deque\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1115, in start\n", " self.socket.connect((self.address, self.port))\n", "ConnectionRefusedError: [Errno 111] Connection refused\n", "ERROR:py4j.java_gateway:An error occurred while trying to connect to the Java server (127.0.0.1:36135)\n", "Traceback (most recent call last):\n", " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3343, in run_code\n", " exec(code_obj, self.user_global_ns, self.user_ns)\n", " File \"\", line 5, in \n", " df = spark.read.json(\"/home/claudio/google_2019/instance_events/\" + cluster + \"/\" + cluster + \"_instance_events*.json.gz\")\n", " File \"/opt/spark/python/pyspark/sql/readwriter.py\", line 300, in json\n", " return self._df(self._jreader.json(self._spark._sc._jvm.PythonUtils.toSeq(path)))\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1305, in __call__\n", " answer, self.gateway_client, self.target_id, self.name)\n", " File \"/opt/spark/python/pyspark/sql/utils.py\", line 128, in deco\n", " return f(*a, **kw)\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/protocol.py\", line 336, in get_return_value\n", " format(target_id, \".\", name))\n", "py4j.protocol.Py4JError: An error occurred while calling o26.json\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2044, in showtraceback\n", " stb = value._render_traceback_()\n", "AttributeError: 'Py4JError' object has no attribute '_render_traceback_'\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 977, in _get_connection\n", " connection = self.deque.pop()\n", "IndexError: pop from an empty deque\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1115, in start\n", " self.socket.connect((self.address, self.port))\n", "ConnectionRefusedError: [Errno 111] Connection refused\n", "ERROR:py4j.java_gateway:An error occurred while trying to connect to the Java server (127.0.0.1:36135)\n", "Traceback (most recent call last):\n", " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3343, in run_code\n", " exec(code_obj, self.user_global_ns, self.user_ns)\n", " File \"\", line 5, in \n", " df = spark.read.json(\"/home/claudio/google_2019/instance_events/\" + cluster + \"/\" + cluster + \"_instance_events*.json.gz\")\n", " File \"/opt/spark/python/pyspark/sql/readwriter.py\", line 300, in json\n", " return self._df(self._jreader.json(self._spark._sc._jvm.PythonUtils.toSeq(path)))\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1305, in __call__\n", " answer, self.gateway_client, self.target_id, self.name)\n", " File \"/opt/spark/python/pyspark/sql/utils.py\", line 128, in deco\n", " return f(*a, **kw)\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/protocol.py\", line 336, in get_return_value\n", " format(target_id, \".\", name))\n", "py4j.protocol.Py4JError: An error occurred while calling o26.json\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2044, in showtraceback\n", " stb = value._render_traceback_()\n", "AttributeError: 'Py4JError' object has no attribute '_render_traceback_'\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 977, in _get_connection\n", " connection = self.deque.pop()\n", "IndexError: pop from an empty deque\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1115, in start\n", " self.socket.connect((self.address, self.port))\n", "ConnectionRefusedError: [Errno 111] Connection refused\n", "ERROR:py4j.java_gateway:An error occurred while trying to connect to the Java server (127.0.0.1:36135)\n", "Traceback (most recent call last):\n", " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3343, in run_code\n", " exec(code_obj, self.user_global_ns, self.user_ns)\n", " File \"\", line 5, in \n", " df = spark.read.json(\"/home/claudio/google_2019/instance_events/\" + cluster + \"/\" + cluster + \"_instance_events*.json.gz\")\n", " File \"/opt/spark/python/pyspark/sql/readwriter.py\", line 300, in json\n", " return self._df(self._jreader.json(self._spark._sc._jvm.PythonUtils.toSeq(path)))\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1305, in __call__\n", " answer, self.gateway_client, self.target_id, self.name)\n", " File \"/opt/spark/python/pyspark/sql/utils.py\", line 128, in deco\n", " return f(*a, **kw)\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/protocol.py\", line 336, in get_return_value\n", " format(target_id, \".\", name))\n", "py4j.protocol.Py4JError: An error occurred while calling o26.json\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2044, in showtraceback\n", " stb = value._render_traceback_()\n", "AttributeError: 'Py4JError' object has no attribute '_render_traceback_'\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 977, in _get_connection\n", " connection = self.deque.pop()\n", "IndexError: pop from an empty deque\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1115, in start\n", " self.socket.connect((self.address, self.port))\n", "ConnectionRefusedError: [Errno 111] Connection refused\n", "ERROR:py4j.java_gateway:An error occurred while trying to connect to the Java server (127.0.0.1:36135)\n", "Traceback (most recent call last):\n", " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3343, in run_code\n", " exec(code_obj, self.user_global_ns, self.user_ns)\n", " File \"\", line 5, in \n", " df = spark.read.json(\"/home/claudio/google_2019/instance_events/\" + cluster + \"/\" + cluster + \"_instance_events*.json.gz\")\n", " File \"/opt/spark/python/pyspark/sql/readwriter.py\", line 300, in json\n", " return self._df(self._jreader.json(self._spark._sc._jvm.PythonUtils.toSeq(path)))\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1305, in __call__\n", " answer, self.gateway_client, self.target_id, self.name)\n", " File \"/opt/spark/python/pyspark/sql/utils.py\", line 128, in deco\n", " return f(*a, **kw)\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/protocol.py\", line 336, in get_return_value\n", " format(target_id, \".\", name))\n", "py4j.protocol.Py4JError: An error occurred while calling o26.json\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2044, in showtraceback\n", " stb = value._render_traceback_()\n", "AttributeError: 'Py4JError' object has no attribute '_render_traceback_'\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 977, in _get_connection\n", " connection = self.deque.pop()\n", "IndexError: pop from an empty deque\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1115, in start\n", " self.socket.connect((self.address, self.port))\n", "ConnectionRefusedError: [Errno 111] Connection refused\n", "ERROR:py4j.java_gateway:An error occurred while trying to connect to the Java server (127.0.0.1:36135)\n", "Traceback (most recent call last):\n", " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3343, in run_code\n", " exec(code_obj, self.user_global_ns, self.user_ns)\n", " File \"\", line 5, in \n", " df = spark.read.json(\"/home/claudio/google_2019/instance_events/\" + cluster + \"/\" + cluster + \"_instance_events*.json.gz\")\n", " File \"/opt/spark/python/pyspark/sql/readwriter.py\", line 300, in json\n", " return self._df(self._jreader.json(self._spark._sc._jvm.PythonUtils.toSeq(path)))\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1305, in __call__\n", " answer, self.gateway_client, self.target_id, self.name)\n", " File \"/opt/spark/python/pyspark/sql/utils.py\", line 128, in deco\n", " return f(*a, **kw)\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/protocol.py\", line 336, in get_return_value\n", " format(target_id, \".\", name))\n", "py4j.protocol.Py4JError: An error occurred while calling o26.json\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2044, in showtraceback\n", " stb = value._render_traceback_()\n", "AttributeError: 'Py4JError' object has no attribute '_render_traceback_'\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 977, in _get_connection\n", " connection = self.deque.pop()\n", "IndexError: pop from an empty deque\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1115, in start\n", " self.socket.connect((self.address, self.port))\n", "ConnectionRefusedError: [Errno 111] Connection refused\n", "ERROR:py4j.java_gateway:An error occurred while trying to connect to the Java server (127.0.0.1:36135)\n", "Traceback (most recent call last):\n", " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3343, in run_code\n", " exec(code_obj, self.user_global_ns, self.user_ns)\n", " File \"\", line 5, in \n", " df = spark.read.json(\"/home/claudio/google_2019/instance_events/\" + cluster + \"/\" + cluster + \"_instance_events*.json.gz\")\n", " File \"/opt/spark/python/pyspark/sql/readwriter.py\", line 300, in json\n", " return self._df(self._jreader.json(self._spark._sc._jvm.PythonUtils.toSeq(path)))\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1305, in __call__\n", " answer, self.gateway_client, self.target_id, self.name)\n", " File \"/opt/spark/python/pyspark/sql/utils.py\", line 128, in deco\n", " return f(*a, **kw)\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/protocol.py\", line 336, in get_return_value\n", " format(target_id, \".\", name))\n", "py4j.protocol.Py4JError: An error occurred while calling o26.json\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2044, in showtraceback\n", " stb = value._render_traceback_()\n", "AttributeError: 'Py4JError' object has no attribute '_render_traceback_'\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 977, in _get_connection\n", " connection = self.deque.pop()\n", "IndexError: pop from an empty deque\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1115, in start\n", " self.socket.connect((self.address, self.port))\n", "ConnectionRefusedError: [Errno 111] Connection refused\n", "ERROR:py4j.java_gateway:An error occurred while trying to connect to the Java server (127.0.0.1:36135)\n", "Traceback (most recent call last):\n", " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3343, in run_code\n", " exec(code_obj, self.user_global_ns, self.user_ns)\n", " File \"\", line 5, in \n", " df = spark.read.json(\"/home/claudio/google_2019/instance_events/\" + cluster + \"/\" + cluster + \"_instance_events*.json.gz\")\n", " File \"/opt/spark/python/pyspark/sql/readwriter.py\", line 300, in json\n", " return self._df(self._jreader.json(self._spark._sc._jvm.PythonUtils.toSeq(path)))\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1305, in __call__\n", " answer, self.gateway_client, self.target_id, self.name)\n", " File \"/opt/spark/python/pyspark/sql/utils.py\", line 128, in deco\n", " return f(*a, **kw)\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/protocol.py\", line 336, in get_return_value\n", " format(target_id, \".\", name))\n", "py4j.protocol.Py4JError: An error occurred while calling o26.json\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2044, in showtraceback\n", " stb = value._render_traceback_()\n", "AttributeError: 'Py4JError' object has no attribute '_render_traceback_'\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 977, in _get_connection\n", " connection = self.deque.pop()\n", "IndexError: pop from an empty deque\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1115, in start\n", " self.socket.connect((self.address, self.port))\n", "ConnectionRefusedError: [Errno 111] Connection refused\n", "ERROR:py4j.java_gateway:An error occurred while trying to connect to the Java server (127.0.0.1:36135)\n", "Traceback (most recent call last):\n", " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3343, in run_code\n", " exec(code_obj, self.user_global_ns, self.user_ns)\n", " File \"\", line 5, in \n", " df = spark.read.json(\"/home/claudio/google_2019/instance_events/\" + cluster + \"/\" + cluster + \"_instance_events*.json.gz\")\n", " File \"/opt/spark/python/pyspark/sql/readwriter.py\", line 300, in json\n", " return self._df(self._jreader.json(self._spark._sc._jvm.PythonUtils.toSeq(path)))\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1305, in __call__\n", " answer, self.gateway_client, self.target_id, self.name)\n", " File \"/opt/spark/python/pyspark/sql/utils.py\", line 128, in deco\n", " return f(*a, **kw)\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/protocol.py\", line 336, in get_return_value\n", " format(target_id, \".\", name))\n", "py4j.protocol.Py4JError: An error occurred while calling o26.json\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/home/claudio/python-venv/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2044, in showtraceback\n", " stb = value._render_traceback_()\n", "AttributeError: 'Py4JError' object has no attribute '_render_traceback_'\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 977, in _get_connection\n", " connection = self.deque.pop()\n", "IndexError: pop from an empty deque\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\", line 1115, in start\n", " self.socket.connect((self.address, self.port))\n", "ConnectionRefusedError: [Errno 111] Connection refused\n" ] }, { "ename": "Py4JError", "evalue": "An error occurred while calling o26.json", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mPy4JError\u001b[0m Traceback (most recent call last)", "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mgetOrCreate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mspark\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjson\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"/home/claudio/google_2019/instance_events/\"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mcluster\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\"/\"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mcluster\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\"_instance_events*.json.gz\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", "\u001b[0;32m/opt/spark/python/pyspark/sql/readwriter.py\u001b[0m in \u001b[0;36mjson\u001b[0;34m(self, path, schema, primitivesAsString, prefersDecimal, allowComments, allowUnquotedFieldNames, allowSingleQuotes, allowNumericLeadingZero, allowBackslashEscapingAnyCharacter, mode, columnNameOfCorruptRecord, dateFormat, timestampFormat, multiLine, allowUnquotedControlChars, lineSep, samplingRatio, dropFieldIfAllNull, encoding, locale, pathGlobFilter, recursiveFileLookup)\u001b[0m\n\u001b[1;32m 298\u001b[0m \u001b[0mpath\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 299\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 300\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_df\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jreader\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjson\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_spark\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jvm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPythonUtils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoSeq\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 301\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mRDD\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 302\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args)\u001b[0m\n\u001b[1;32m 1303\u001b[0m \u001b[0manswer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgateway_client\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend_command\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcommand\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1304\u001b[0m return_value = get_return_value(\n\u001b[0;32m-> 1305\u001b[0;31m answer, self.gateway_client, self.target_id, self.name)\n\u001b[0m\u001b[1;32m 1306\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1307\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mtemp_arg\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtemp_args\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/opt/spark/python/pyspark/sql/utils.py\u001b[0m in \u001b[0;36mdeco\u001b[0;34m(*a, **kw)\u001b[0m\n\u001b[1;32m 126\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mdeco\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkw\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 127\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 128\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkw\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 129\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mpy4j\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprotocol\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPy4JJavaError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 130\u001b[0m \u001b[0mconverted\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconvert_exception\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjava_exception\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/protocol.py\u001b[0m in \u001b[0;36mget_return_value\u001b[0;34m(answer, gateway_client, target_id, name)\u001b[0m\n\u001b[1;32m 334\u001b[0m raise Py4JError(\n\u001b[1;32m 335\u001b[0m \u001b[0;34m\"An error occurred while calling {0}{1}{2}\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 336\u001b[0;31m format(target_id, \".\", name))\n\u001b[0m\u001b[1;32m 337\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 338\u001b[0m \u001b[0mtype\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0manswer\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mPy4JError\u001b[0m: An error occurred while calling o26.json" ] } ], "source": [ "spark = pyspark.sql.SparkSession.builder \\\n", " .appName(\"machine_time_waste\") \\\n", " .getOrCreate()\n", "\n", "df = spark.read.json(\"/home/claudio/google_2019/instance_events/\" + cluster + \"/\" + cluster + \"_instance_events*.json.gz\")" ] }, { "cell_type": "code", "execution_count": null, "id": "juvenile-absolute", "metadata": { "collapsed": false, "jupyter": { "outputs_hidden": false } }, "outputs": [], "source": [ "df.printSchema()" ] }, { "cell_type": "code", "execution_count": null, "id": "lucky-western", "metadata": { "collapsed": false, "jupyter": { "outputs_hidden": false } }, "outputs": [], "source": [ "df.show()" ] }, { "cell_type": "code", "execution_count": null, "id": "normal-settlement", "metadata": { "collapsed": false, "jupyter": { "outputs_hidden": false } }, "outputs": [], "source": [ "# .filter(df.collection_type == 0) \\\n", "df2 = df \\\n", " .withColumn(\"time\", col(\"time\").cast(LongType())) \\\n", " .withColumn(\"type\", col(\"type\").cast(LongType())) \\\n", " .withColumn(\"type\", when(col(\"type\").isNull(), 0).otherwise(col(\"type\"))) \\\n", " .withColumn(\"id\", concat_ws(\"-\", \"collection_id\", \"instance_index\")) \\\n", " .where(col(\"time\").isNotNull()) \\\n", " .where(col(\"type\").isNotNull()) \\\n", " .where((col(\"instance_index\").isNotNull()) & (col(\"collection_id\").isNotNull())) \\\n", " .select(\"machine_id\", \"id\", \"time\", \"type\")" ] }, { "cell_type": "code", "execution_count": null, "id": "typical-homeless", "metadata": { "collapsed": false, "jupyter": { "outputs_hidden": false } }, "outputs": [], "source": [ "df2.show()\n", "print(\"Total: \" + str(df.count()))\n", "print(\"Filtered: \" + str(df2.count()))" ] }, { "cell_type": "code", "execution_count": null, "id": "collect-saying", "metadata": { "collapsed": false, "jupyter": { "outputs_hidden": false } }, "outputs": [], "source": [ "# my_window = Window.partitionBy(\"machine_id\", \"id\").orderBy(df2.time.asc())\n", "\n", "w2 = Window.partitionBy(\"id\").orderBy(df2.time.asc()).rowsBetween(Window.currentRow, Window.unboundedFollowing)" ] }, { "cell_type": "code", "execution_count": null, "id": "cooperative-appraisal", "metadata": { "collapsed": false, "jupyter": { "outputs_hidden": false } }, "outputs": [], "source": [ "# .withColumn(\"prev_time\", lag(df2.time).over(my_window)) \\\n", "# .withColumn(\"prev_type\", lag(df2.type).over(my_window)) \\\n", "\n", "df3 = df2 \\\n", " .withColumn(\"t3_time\", when((df2.type != 3), None).otherwise(df2.time)) \\\n", " .withColumn(\"t45678_time\", when((df2.type < 4) | (df2.type > 8), None).otherwise(df2.time)) \\\n", " .withColumn(\"t45678_type\", when((df2.type < 4) | (df2.type > 8), None).otherwise(df2.type)) \\\n", " .withColumn(\"t01_time\", when((df2.type != 0) & (df2.type != 1), None).otherwise(df2.time)) \\\n", " .withColumn(\"t01_type\", when((df2.type != 0) & (df2.type != 1), None).otherwise(df2.type)) \\\n", " .withColumn(\"next_time\", when(df2.type == 3, first(col(\"t45678_time\"), ignorenulls=True).over(w2)) \\\n", " .when((df2.type == 0) | (df2.type == 1), first(col(\"t3_time\"), ignorenulls=True).over(w2)) \\\n", " .when((df2.type >= 4) | (df2.type <= 8), first(col(\"t01_time\"), ignorenulls=True).over(w2)) \\\n", " .otherwise(None)) \\\n", " .withColumn(\"next_type\", when(df2.type == 3, first(col(\"t45678_type\"), ignorenulls=True).over(w2)) \\\n", " .when((df2.type == 0) | (df2.type == 1), 3) \\\n", " .when((df2.type >= 4) | (df2.type <= 8), first(col(\"t01_type\"), ignorenulls=True).over(w2)) \\\n", " .otherwise(None)) \\\n", " .withColumn(\"last_term_type\", last(col(\"t45678_type\"), ignorenulls=True).over(w2)) \\\n", " .withColumn(\"time_delta\", col(\"next_time\") - col(\"time\")) \\\n", " .select(\"machine_id\", \"id\", \"time\", \"type\", \"last_term_type\", \"time_delta\", \"t01_time\", \"t01_type\", \"t3_time\", \"t45678_time\", \"t45678_type\", \"next_time\", \"next_type\") \\" ] }, { "cell_type": "code", "execution_count": null, "id": "ideal-angle", "metadata": { "collapsed": false, "jupyter": { "outputs_hidden": false } }, "outputs": [], "source": [ "df4 = df3.where(df3.next_type.isNotNull()).groupby(\"type\", \"next_type\", \"last_term_type\").sum(\"time_delta\")" ] }, { "cell_type": "code", "execution_count": null, "id": "working-difficulty", "metadata": { "collapsed": false, "jupyter": { "outputs_hidden": false } }, "outputs": [], "source": [ "# df3.orderBy(df3.machine_id, df3.time).show(n=100)\n", "# df3.printSchema()\n", "df4.show(n=1000000)\n", "df4.write.csv(\"/home/claudio/google_2019/thesis_queries/machine_time_waste/\" + cluster + \"_state_change.csv\")" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.9" } }, "nbformat": 4, "nbformat_minor": 5 }