diff --git a/evaluate_classifiers.py b/evaluate_classifiers.py index bedd127..a22ae6d 100755 --- a/evaluate_classifiers.py +++ b/evaluate_classifiers.py @@ -160,7 +160,7 @@ def main(): ax.set(ylabel="Metric value", ylim=[0, 1], xlabel="Metric") ax.set_title("Distribution of metrics for each classifier") sns.despine(offset=10, trim=True) - f.savefig(OUT_DIR + '/boxplot.png') + f.savefig(OUT_DIR + '/boxplot.svg') if __name__ == '__main__': diff --git a/metric_stats.py b/metric_stats.py new file mode 100755 index 0000000..d301d58 --- /dev/null +++ b/metric_stats.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 + +import pandas as pd +import os + +def main(): + dfs = pd.read_csv(os.path.dirname(__file__) + '/metrics/feature_vectors_labeled.csv') + metrics = ['MTH', 'FLD', 'RFC', 'INT', 'SZ', 'CPX', 'EX', 'RET', 'BCM', + 'NML', 'WRD', 'DCM'] + df = dfs.agg(dict([(m, ['min', 'max', 'mean']) for m in metrics])).reset_index() + df = pd.melt(df, id_vars=['index'], value_vars=metrics, var_name='metric') \ + .pivot(index='metric', columns=['index'], values=['value']) \ + .reset_index() + df.columns = sorted([c[1] for c in df.columns]) + df = df.reindex([df.columns[0]] + list(reversed(sorted(df.columns[1:]))), axis=1) + print(df.to_markdown(index=False)) + + print() + print(dfs.groupby('buggy').count().loc[:, 'class_name']) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/models/boxplot.png b/models/boxplot.png deleted file mode 100644 index 2f2f59a..0000000 Binary files a/models/boxplot.png and /dev/null differ diff --git a/models/boxplot.svg b/models/boxplot.svg new file mode 100644 index 0000000..f26cd11 --- /dev/null +++ b/models/boxplot.svg @@ -0,0 +1,2230 @@ + + + + + + + + 2023-05-24T14:01:57.307239 + image/svg+xml + + + Matplotlib v3.7.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/report/build.sh b/report/build.sh new file mode 100755 index 0000000..237654a --- /dev/null +++ b/report/build.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -e + +SCRIPT_DIR=$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd) + +cd "$SCRIPT_DIR" +pandoc main.md -o main.pdf \ No newline at end of file diff --git a/report/main.md b/report/main.md new file mode 100644 index 0000000..ef73d9c --- /dev/null +++ b/report/main.md @@ -0,0 +1,172 @@ +--- +author: Claudio Maggioni +title: Information Modelling & Analysis -- Project 2 +geometry: margin=2cm,bottom=3cm +--- + + + +# Code Repository + +The code and result files, part of this submission, can be found at + +- Repository: [https://github.com/infoMA2023/project-02-bug-prediction-maggicl](https://github.com/infoMA2023/project-02-bug-prediction-maggicl) +- Commit ID: **TBD** + +# Data Pre-Processing + +I use the sources of the [Closure]{.smallcaps} repository that were already downloaded using the command: + +```shell +defects4j checkout -p Closure -v 1f -w ./resources +``` + +and used the code in the following subfolder for the project: + +``` +./resources/defects4j-checkout-closure-1f/src/com/google/javascript/jscomp/ +``` + +relative to the root folder of the repository. The resulting csv of extracted, labelled feature vectors can be found in +the repository at the following path: + +``` +./metrics/feature_vectors_labeled.csv +``` + +relative to the root folder of the repository. + +## Feature Vector Extraction + +I extracted **291** feature vectors in total. Aggregate metrics +about the extracted feature vectors, i.e. the distribution of the values of each +code metric, can be found in Table [1](#tab:metrics){reference-type="ref" +reference="tab:metrics"}. + +::: {#tab:metrics} +| **Metric** | **Min** | **Average** | **Max** | +|:----|------:|-----------:|-------:| +| BCM | 0 | 13.4124 | 221 | +| CPX | 0 | 5.8247 | 96 | +| DCM | 0 | 4.8652 | 176.2 | +| EX | 0 | 0.1134 | 2 | +| FLD | 0 | 6.5773 | 167 | +| INT | 0 | 0.6667 | 3 | +| MTH | 0 | 11.6529 | 209 | +| NML | 0 | 13.5622 | 28 | +| RET | 0 | 3.6735 | 86 | +| RFC | 0 | 107.2710 | 882 | +| SZ | 0 | 18.9966 | 347 | +| WRD | 0 | 314.4740 | 3133 | + + : Distribution of values for each extracted code metric. +::: + +## Feature Vector Labelling + +After feature vectors are labeled, I determine that the dataset contains +**75** buggy classes and **216** non-buggy classes. + +# Classifiers + +In every subsection below, describe in a concise way which different +hyperparameters you tried for the corresponding classifier, and report +the corresponding precision, recall and F1 values (for example in a +table or an [itemize]{.smallcaps}-environment). Furthermore, for every +type of classifiers, explicitly mention which hyperparameter +configuration you chose (based on above reported results) to be used in +further steps, and (in one or two sentences), explain why these +hyperparameters may outperform the other ones you tested.. + +## Decision Tree (DT) + +## Naive Bayes (NB) + +## Support Vector Machine (SVP) + +## Multi-Layer Perceptron (MLP) + +## Random Forest (RF) + +# Evaluation + +## Output Distributions + +Add a boxplot showing mean and standard deviation for **Precision** +values on all 6 classifiers (5 trained + 1 biased)\ +Add a boxplot showing mean and standard deviation for **Recall** values +on all 6 classifiers (5 trained + 1 biased)\ +Add a boxplot showing mean and standard deviation for **F1** values on +all 6 classifiers (5 trained + 1 biased) + +![Precision, Recall and F1 score distribution for each classifier for 20-times cross validation.](../models/boxplot.svg){#fig:boxplot} + +## Comparison and Significance + +For every combination of two classifiers and every performance metric +(precision, recall, f1) compare which algorithm performs better, by how +much, and report the corresponding p-value in the following +subsubsections: + +::: {#tab:precision} +| | DecisionTreeClassifier | GaussianNB | MLPClassifier | RandomForestClassifier | SVC | +|:-----------------------|:-------------------------|:-------------|:----------------|:-------------------------|------:| +| BiasedClassifier | 0 | 0 | 0 | 0 | 0 | +| DecisionTreeClassifier | -- | 0.0893 | 0.4012 | 0 | 0 | +| GaussianNB | -- | -- | 0.0348 | 0 | 0 | +| MLPClassifier | -- | -- | -- | 0 | 0 | +| RandomForestClassifier | -- | -- | -- | -- | 0 | + + : Pairwise Wilcoxon test for precision for each combination of classifiers. +::: + +::: {#tab:recall} +| | DecisionTreeClassifier | GaussianNB | MLPClassifier | RandomForestClassifier | SVC | +|:-----------------------|:-------------------------|:-------------|:----------------|:-------------------------|------:| +| BiasedClassifier | 0 | 0 | 0 | 0 | 0 | +| DecisionTreeClassifier | -- | 0 | 0.0118 | 0.3276 | 0 | +| GaussianNB | -- | -- | 0 | 0 | 0 | +| MLPClassifier | -- | -- | -- | 0.0001 | 0 | +| RandomForestClassifier | -- | -- | -- | -- | 0 | + : Pairwise Wilcoxon test for recall for each combination of classifiers. +::: + +::: {#tab:f1} +| | DecisionTreeClassifier | GaussianNB | MLPClassifier | RandomForestClassifier | SVC | +|:-----------------------|:-------------------------|:-------------|:----------------|:-------------------------|------:| +| BiasedClassifier | 0 | 0 | 0 | 0 | 0 | +| DecisionTreeClassifier | -- | 0 | 0.4711 | 0 | 0 | +| GaussianNB | -- | -- | 0 | 0 | 0 | +| MLPClassifier | -- | -- | -- | 0 | 0 | +| RandomForestClassifier | -- | -- | -- | -- | 0 | + : Pairwise Wilcoxon test for the F1 score metric for each combination of classifiers. +::: + +### F1 Values + +- + +- \... + +### Precision + +(same as for F1 above) + +### Recall + +(same as for F1 above) + +## Practical Usefulness + +Discuss the practical usefulness of the obtained classifiers in a +realistic bug prediction scenario (1 paragraph). diff --git a/report/main.pdf b/report/main.pdf new file mode 100644 index 0000000..7209dbe Binary files /dev/null and b/report/main.pdf differ