accuracy

report.tex

@@ -8,8 +8,12 @@
 \usepackage{xcolor}
 \usepackage{lmodern}
 \usepackage{booktabs}
+\usepackage{graphicx}
 \usepackage{float}
+\usepackage{tikz}
 \usepackage{listings}
+\usepackage{pgfplots}
+\usepackage{subcaption}
 \setlength{\parindent}{0cm}
 \setlength{\parskip}{0.3em}
 \hypersetup{pdfborder={0 0 0}}
@@ -178,6 +182,124 @@ when referring to a class by its fully-qualified domain name the prefix
 \textit{com.fasterxml.jackson.core} is omitted as all classes in the Jackson
 core project reside in this package or in a sub-package of this package.
 
+\section{\textit{Pattern4J} accuracy}
+
+As it would be very hard to check each class in the Jackson project for design
+patterns manually to get a true number of false positives and false negatives,
+we opt for a more statistical approach. We select 20 classes at random from the project
+and we review the reported results for this subset. The classes were selected
+from the \textit{target/classes} directory generated by Maven using the
+following command:
+
+\begin{verbatim}
+find . -name '*.class' | shuf -n 20 | sed 's#\.class##;s#/#.#'
+\end{verbatim}
+
+The selected classes and the analysis results for both \textit{Pattern4J} and
+our manual inspection are shown in Figure \ref{fig:conf}. Using those results,
+we can say that \textit{Pattern4J} shows a false positive
+rate\footnote{\url{https://en.wikipedia.org/wiki/False_positive_rate}} of 45.4\%, a
+false negative rate of 17.2\%, and an accuracy of 75.0\%. 
+
+\begin{figure}[h]
+  \begin{subfigure}{\linewidth}
+    \resizebox{\textwidth}{!}{
+    \begin{tabular}{l|rrrr|p{7cm}}
+      \toprule
+        Class & \multicolumn{1}{p{1.5cm}}{True positives} & 
+         \multicolumn{1}{p{1.5cm}}{False positives} & 
+         \multicolumn{1}{p{1.6cm}}{False negatives} & 
+         \multicolumn{1}{p{1.6cm}|}{True\ \ \ negatives} &
+         Notes\\
+      \midrule
+        util.DefaultPrettyPrinter\$Indenter & ~ & 1 & ~ & ~ & state and adapter, false positives \\ 
+        ObjectCodec & ~ & ~ & ~ & 1 & ~ \\ 
+        type.WritableTypeId & ~ & ~ & ~ & 1 & ~ \\ 
+        util.DefaultPrettyPrinter\$NopIndenter & 1 & ~ & ~ & ~ & variation of
+        singleton, true positive \\ 
+        json.PackageVersion & ~ & ~ & 1 & ~ & variation of singleton, false negative \\ 
+        io.UTF32Reader & ~ & ~ & ~ & 1 & ~ \\ 
+        io.NumberInput & ~ & ~ & ~ & 1 & ~ \\ 
+        json.JsonReadFeature & ~ & ~ & ~ & 1 & ~ \\ 
+        io.SerializedString & 1 & ~ & ~ & ~ & variation of singleton, true
+        positive \\ 
+        type.TypeReference & ~ & ~ & ~ & 1 & ~ \\ 
+        JsonPointer & ~ & 1 & ~ & ~ & singleton, false positive \\ 
+        json.UTF8DataInputJsonParser & ~ & ~ & ~ & 1 & ~ \\ 
+        format.InputAccessor\$Std & ~ & ~ & ~ & 1 & ~ \\ 
+        JsonStreamContext & 1 & ~ & ~ & ~ & template method (barely), true
+        positive \\ 
+        filter.TokenFilter & ~ & 0.5 & 0.5 & ~ & singleton, false positive;
+        strategy, false negative \\ 
+        util.VersionUtil & ~ & ~ & ~ & 1 & ~ \\ 
+        json.UTF8JsonGenerator & ~ & ~ & ~ & 1 & ~ \\ 
+        JsonParser\$Feature & ~ & ~ & ~ & 1 & ~ \\ 
+        exc.StreamReadException & ~ & ~ & ~ & 1 & ~ \\ 
+        util.JsonGeneratorDelegate & ~ & ~ & 1 & ~ & decorator pattern (base
+        class), false negative \\
+      \midrule
+        Total & 3 & 2,5 & 2,5 & 12 & ~ \\ 
+        Percentage & 15,0\% & 12,5\% & 12,5\% & 60,0\% \\ 
+      \bottomrule
+    \end{tabular}
+  }
+    \caption{Table of classes analyzed manually against the \textit{Pattern4J}
+    tool results. Units are classes, the value $0.5$ represents a pattern inside
+    a class with two patterns.}
+    \label{fig:conf:tab}
+  \end{subfigure}
+  \begin{subfigure}{\linewidth}
+    \centering
+  \vspace{1cm}
+  \begin{tikzpicture}
+    \begin{axis}[
+            colormap={parula}{color=(white) rgb255=(90,96,191)},
+            xlabel=Pattern4J,
+            xlabel style={yshift=-40pt},
+            ylabel=Manual detection, 
+            ylabel style={yshift=40pt},
+            xticklabels={Has pattern, No pattern},
+            xtick={0,...,1},
+            xtick style={draw=none},
+            yticklabels={Has pattern, No pattern},
+            ytick={0,...,1},
+            ytick style={draw=none},
+            enlargelimits=false,
+            colorbar,
+            xticklabel style={
+              rotate=45
+            },
+            nodes near coords={\pgfmathprintnumber\pgfplotspointmeta},
+            nodes near coords style={
+                yshift=-7pt
+            },
+        ]
+        \addplot[
+            matrix plot,
+            mesh/cols=2,
+            point meta=explicit,draw=gray
+        ] table [meta=C] {
+            x y C
+            0 0 3
+            1 0 2.5
+            
+            0 1 2.5
+            1 1 12
+        };
+    \end{axis}
+\end{tikzpicture}
+    \caption{Confusion matrix comparing the \textit{Pattern4J} detection results
+    with our manual interpretation of the classes. Units are classes, the value 
+    $0.5$ represents a pattern inside a class with two patterns.}
+    \label{fig:conf:mat}
+  \end{subfigure}
+    \caption{Results of the statistical analysis on the effectiveness of the
+    \textit{Pattern4J} tool as a class-by-class table (Figure
+    \ref{fig:conf:tab}) and as a confusion matrix (Figure \ref{fig:conf:mat}).}
+    \label{fig:conf}
+\end{figure}
+
+
 \section{Structural Patterns}
 
 \subsection{Singleton Pattern}
@@ -238,7 +360,7 @@ namely:
 
 \begin{description}
   \item[sym.Name1, JsonLocation, DefaultIndenter,
-    util.DefaultPrettyPrinter\$FixedSpaceIndenter] are not singletons and this
+    util.DefaultPrettyPrinter\$FixedSpaceIndenter] are not singletons and thus
     false positives. All these classes were detected because of ``default''
     instances they include in themselves as \textit{static final} fields and
     because their constructor, even if \textit{public}, is never used in Jackson