maggicl
/
bachelorThesis
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Corrected fig9a taskcount

This commit is contained in:
Claudio Maggioni 2021-04-20 16:46:36 +00:00
parent 637ee6818e
commit a106980034
1 changed files with 6 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
figure_9/figure9a.py
View File

@ -31,8 +31,8 @@ spark = pyspark.sql.SparkSession.builder \
sc = spark.sparkContext
# READING INSTANCE EVENTS DATA
#dfepath = sys.argv[4] + "/" + cluster + "/" + cluster + "_instance_events*.json.gz"
dfepath = sys.argv[4] + "/" + cluster + "/" + cluster + "_test.json"
dfepath = sys.argv[4] + "/" + cluster + "/" + cluster + "_instance_events*.json.gz"
#dfepath = sys.argv[4] + "/" + cluster + "/" + cluster + "_test.json"
df = spark.read.json(dfepath)
try:
@ -41,11 +41,12 @@ except:
df = df.withColumn("collection_type", lit(None).cast(ByteType()))
df = df.rdd.filter(lambda x: x.collection_id is not None and x.instance_index is not None and
(x.collection_type == 0)) \
(x.collection_type == 0 or x.collection_type is None)) \
.map(lambda x: (x.collection_id, x.instance_index)) \
.groupBy(lambda x: x[0]) \
.mapValues(lambda x: len(x))
.map(lambda x: [x[0], len(x[1])]) \
.toDF(["jobid", "task_count"])
df.write.parquet("/home/claudio/raid0/figure-9a-task-count-" + cluster + ".parquet")
df.write.parquet("/home/claudio/google_2019/thesis_queries/figure_9/figure-9a-task-count-" + cluster + ".parquet")
# vim: set ts=4 sw=4 et tw=120: