diff --git a/figure_9/figure9a.py b/figure_9/figure9a.py index d55a6ca9..f565fdd8 100755 --- a/figure_9/figure9a.py +++ b/figure_9/figure9a.py @@ -31,8 +31,8 @@ spark = pyspark.sql.SparkSession.builder \ sc = spark.sparkContext # READING INSTANCE EVENTS DATA -#dfepath = sys.argv[4] + "/" + cluster + "/" + cluster + "_instance_events*.json.gz" -dfepath = sys.argv[4] + "/" + cluster + "/" + cluster + "_test.json" +dfepath = sys.argv[4] + "/" + cluster + "/" + cluster + "_instance_events*.json.gz" +#dfepath = sys.argv[4] + "/" + cluster + "/" + cluster + "_test.json" df = spark.read.json(dfepath) try: @@ -41,11 +41,12 @@ except: df = df.withColumn("collection_type", lit(None).cast(ByteType())) df = df.rdd.filter(lambda x: x.collection_id is not None and x.instance_index is not None and - (x.collection_type == 0)) \ + (x.collection_type == 0 or x.collection_type is None)) \ .map(lambda x: (x.collection_id, x.instance_index)) \ .groupBy(lambda x: x[0]) \ - .mapValues(lambda x: len(x)) + .map(lambda x: [x[0], len(x[1])]) \ + .toDF(["jobid", "task_count"]) -df.write.parquet("/home/claudio/raid0/figure-9a-task-count-" + cluster + ".parquet") +df.write.parquet("/home/claudio/google_2019/thesis_queries/figure_9/figure-9a-task-count-" + cluster + ".parquet") # vim: set ts=4 sw=4 et tw=120: