bug-2022: part 3 done

bug-2022/.gitignore

@@ -1,3 +1,5 @@
 defects4j/
 sources/
 env/
+*.iml
+.idea/

bug-2022/extract_feature_vectors.py

@@ -24,13 +24,14 @@ def count(node: javalang.tree.Node, the_filter) -> int:
 
 
 def metrics(clazz: javalang.tree.ClassDeclaration) -> dict[str, str]:
-    m = {}
-
-    m['MTH'] = len(clazz.methods)  # MTH: #Methods
-    m['FLD'] = len(clazz.fields)  # FLD: #Fields
+    # Method metrics
+    m = {
+        'MTH': len(clazz.methods),
+        'FLD': len(clazz.fields),
+        'RFC': 0
+    }
 
     # RFC: #Public methods + #Method invocations
-    m['RFC'] = 0
     for method in clazz.methods:
         if 'public' in method.modifiers:
             m['RFC'] += 1
@@ -81,8 +82,8 @@ def metrics(clazz: javalang.tree.ClassDeclaration) -> dict[str, str]:
             # BCM: #block comments
             m['BCM'] += 1
 
-            # #Words (longest alphanumeric substrings) in block comments
-            m['WRD'] += len(re.findall('\w+', node.documentation))
+            # #Words (the longest alphanumeric substrings) in block comments
+            m['WRD'] += len(re.findall('\\w+', node.documentation))
 
     # #Words in comments / #Statements
     s = count(clazz, javalang.tree.Statement)

bug-2022/label_feature_vectors.py

@@ -1,10 +1,11 @@
 #!/usr/bin/env python3
-import javalang
-import os
-import pandas as pd
 import glob
+import os
 import re
 
+import javalang
+import pandas as pd
+
 # God class if:
 # |M(C)| > E(M) + 6*V(M)
 # (number of methods greater than average across all classes plus 6 times the
@@ -22,24 +23,20 @@ def clean_output():
         os.remove(f)
 
 
-def get_is_buggy_df() -> pd.DataFrame:
-    lines: list[str] = set()
+def get_is_buggy_set() -> set[str]:
+    lines: set[str] = set()
 
     filelist = glob.glob(LABEL_DIR + '/*.src')
     for f in filelist:
-        with open(f, 'r') as fobj:
-            for line in fobj.readlines():
+        with open(f, 'r') as file_obj:
+            for line in file_obj.readlines():
                 lines.add(line.strip().strip('\n'))
 
     return lines
 
 
-def is_class_buggy(fqdn: str, se) -> bool:
-    clazz_path = fqdn.replace('.', '/')
-
-
 def main():
-    is_buggy_set = get_is_buggy_df()
+    is_buggy_set = get_is_buggy_set()
     df = pd.read_csv(IN_DIR + '/feature_vectors.csv')
     df['buggy'] = df['class_name'].apply(
         lambda fqdn: 1 if fqdn in is_buggy_set else 0)

bug-2022/models/models.csv

@@ -0,0 +1,130 @@
+mean_fit_time,std_fit_time,mean_score_time,std_score_time,params,split0_test_precision,split1_test_precision,split2_test_precision,split3_test_precision,split4_test_precision,mean_test_precision,std_test_precision,rank_test_precision,split0_test_recall,split1_test_recall,split2_test_recall,split3_test_recall,split4_test_recall,mean_test_recall,std_test_recall,rank_test_recall,split0_test_f1,split1_test_f1,split2_test_f1,split3_test_f1,split4_test_f1,mean_test_f1,std_test_f1,rank_test_f1,classifier,param_gamma,param_kernel,param_criterion,param_splitter,param_class_weight,param_max_features,param_activation,param_hidden_layer_sizes,param_learning_rate,param_max_iter,param_solver
+0.0007423877716064454,7.04728217174239e-05,0.00283050537109375,9.592939743102134e-05,{},0.42857142857142855,0.5,0.625,0.5714285714285714,0.3333333333333333,0.4916666666666667,0.10320817524615832,1,0.4,0.4,0.6666666666666666,0.5333333333333333,0.5333333333333333,0.5066666666666666,0.09977753031397174,1,0.4137931034482759,0.4444444444444445,0.6451612903225806,0.5517241379310344,0.4102564102564102,0.4930758772805491,0.09173813813446069,1,GaussianNB,,,,,,,,,,,
+0.006285238265991211,0.0006688574506527966,0.0031412124633789064,1.9395845971956088e-05,"{'gamma': 'scale', 'kernel': 'linear'}",0.7142857142857143,1.0,0.6666666666666666,0.6666666666666666,0.42857142857142855,0.6952380952380952,0.18220120447341898,1,0.3333333333333333,0.13333333333333333,0.26666666666666666,0.26666666666666666,0.2,0.24,0.06798692684790378,3,0.4545454545454545,0.23529411764705882,0.3809523809523809,0.3809523809523809,0.27272727272727276,0.34489432136490955,0.07980479420970431,1,SVC,scale,linear,,,,,,,,,
+0.004173183441162109,0.00029746753766620143,0.003175973892211914,9.203908307825466e-05,"{'gamma': 'scale', 'kernel': 'poly'}",0.25,0.75,0.8,0.75,0.5,0.61,0.2083266665599966,3,0.13333333333333333,0.2,0.26666666666666666,0.2,0.26666666666666666,0.21333333333333332,0.049888765156985884,5,0.1739130434782609,0.31578947368421056,0.4,0.31578947368421056,0.3478260869565218,0.31066361556064076,0.07498707444994873,4,SVC,scale,poly,,,,,,,,,
+0.004427480697631836,0.00015539635021160435,0.004806804656982422,0.00014942971890440543,"{'gamma': 'scale', 'kernel': 'rbf'}",0.6666666666666666,1.0,0.75,0.5,0.0,0.5833333333333333,0.33333333333333337,5,0.13333333333333333,0.13333333333333333,0.2,0.2,0.0,0.13333333333333336,0.07302967433402216,7,0.2222222222222222,0.23529411764705882,0.31578947368421056,0.28571428571428575,0.0,0.2118040198535555,0.11117281004727667,7,SVC,scale,rbf,,,,,,,,,
+0.0033874034881591795,0.00039109939540946353,0.003376340866088867,6.714192295872801e-05,"{'gamma': 'scale', 'kernel': 'sigmoid'}",0.42857142857142855,0.625,0.35294117647058826,0.26666666666666666,0.1875,0.3721358543417367,0.15013849214244676,8,0.2,0.3333333333333333,0.4,0.26666666666666666,0.2,0.27999999999999997,0.077746025264604,1,0.27272727272727276,0.43478260869565216,0.37500000000000006,0.26666666666666666,0.19354838709677422,0.3085449870372732,0.08557233927288813,5,SVC,scale,sigmoid,,,,,,,,,
+0.0063323974609375,0.0007743547697311502,0.003120231628417969,2.7272584440366456e-05,"{'gamma': 'auto', 'kernel': 'linear'}",0.7142857142857143,1.0,0.6666666666666666,0.6666666666666666,0.42857142857142855,0.6952380952380952,0.18220120447341898,1,0.3333333333333333,0.13333333333333333,0.26666666666666666,0.26666666666666666,0.2,0.24,0.06798692684790378,3,0.4545454545454545,0.23529411764705882,0.3809523809523809,0.3809523809523809,0.27272727272727276,0.34489432136490955,0.07980479420970431,1,SVC,auto,linear,,,,,,,,,
+0.0041983604431152345,0.0002696866527637961,0.003224611282348633,2.1358366160400856e-05,"{'gamma': 'auto', 'kernel': 'poly'}",0.25,0.75,0.8,0.6,0.5714285714285714,0.5942857142857143,0.19269305188537017,4,0.13333333333333333,0.2,0.26666666666666666,0.2,0.26666666666666666,0.21333333333333332,0.049888765156985884,5,0.1739130434782609,0.31578947368421056,0.4,0.3,0.36363636363636365,0.31066777615976704,0.0769776689956566,3,SVC,auto,poly,,,,,,,,,
+0.004366445541381836,0.0001752091663583595,0.004625844955444336,5.7817640981632584e-05,"{'gamma': 'auto', 'kernel': 'rbf'}",0.6666666666666666,1.0,0.75,0.5,0.0,0.5833333333333333,0.33333333333333337,5,0.13333333333333333,0.13333333333333333,0.2,0.2,0.0,0.13333333333333336,0.07302967433402216,7,0.2222222222222222,0.23529411764705882,0.31578947368421056,0.28571428571428575,0.0,0.2118040198535555,0.11117281004727667,7,SVC,auto,rbf,,,,,,,,,
+0.0033074378967285155,0.0001491025342117052,0.003362894058227539,5.046955851323005e-05,"{'gamma': 'auto', 'kernel': 'sigmoid'}",0.42857142857142855,0.5,0.5555555555555556,0.26666666666666666,0.2,0.39015873015873015,0.1358730158730159,7,0.2,0.3333333333333333,0.3333333333333333,0.26666666666666666,0.13333333333333333,0.2533333333333333,0.077746025264604,2,0.27272727272727276,0.4,0.4166666666666667,0.26666666666666666,0.16,0.3032121212121212,0.09488354468685073,6,SVC,auto,sigmoid,,,,,,,,,
+0.0016054153442382813,0.00014808558545298584,0.002702760696411133,6.182858956883495e-05,"{'criterion': 'gini', 'splitter': 'best'}",0.47058823529411764,0.46153846153846156,0.5263157894736842,0.5,0.375,0.4666884972612527,0.051212462440304515,3,0.5333333333333333,0.4,0.6666666666666666,0.4,0.4,0.48,0.10666666666666665,2,0.5,0.42857142857142855,0.5882352941176471,0.4444444444444445,0.38709677419354843,0.46966958826541366,0.06945703235093705,3,DecisionTreeClassifier,,,gini,best,,,,,,,
+0.000792837142944336,8.122128438731559e-05,0.002693319320678711,0.00010676304108566751,"{'criterion': 'gini', 'splitter': 'random'}",0.42857142857142855,0.5,0.3157894736842105,0.3333333333333333,0.3333333333333333,0.3822055137844611,0.07101184332702314,4,0.4,0.6,0.4,0.3333333333333333,0.3333333333333333,0.4133333333333333,0.09797958971132713,4,0.4137931034482759,0.5454545454545454,0.35294117647058826,0.3333333333333333,0.3333333333333333,0.3957710984080152,0.08045033028348736,4,DecisionTreeClassifier,,,gini,random,,,,,,,
+0.0016440391540527345,6.233678874052185e-05,0.0026673793792724608,0.00010989564511655804,"{'criterion': 'entropy', 'splitter': 'best'}",0.5625,0.5384615384615384,0.5,0.5,0.5,0.5201923076923076,0.02587235393668021,1,0.6,0.4666666666666667,0.5333333333333333,0.4,0.5333333333333333,0.5066666666666666,0.06798692684790378,1,0.5806451612903225,0.5,0.5161290322580646,0.4444444444444445,0.5161290322580646,0.5114695340501793,0.04348011522328528,1,DecisionTreeClassifier,,,entropy,best,,,,,,,
+0.0008202552795410156,0.00010772706513364226,0.002626943588256836,2.302703858685773e-05,"{'criterion': 'entropy', 'splitter': 'random'}",0.6,0.6363636363636364,0.4117647058823529,0.46153846153846156,0.42105263157894735,0.5061438870726797,0.09370565471727985,2,0.4,0.4666666666666667,0.4666666666666667,0.4,0.5333333333333333,0.4533333333333333,0.04988876515698587,3,0.48,0.5384615384615385,0.43749999999999994,0.42857142857142855,0.47058823529411764,0.47102424046541697,0.038870707370047254,2,DecisionTreeClassifier,,,entropy,random,,,,,,,
+0.13302092552185057,0.002269015176868814,0.010986566543579102,0.0001189285787967791,"{'class_weight': 'balanced', 'criterion': 'gini', 'max_features': 'sqrt'}",0.7777777777777778,0.75,0.8888888888888888,0.7,0.5,0.7233333333333333,0.12767628819753976,1,0.4666666666666667,0.4,0.5333333333333333,0.4666666666666667,0.26666666666666666,0.42666666666666664,0.09043106644167023,4,0.5833333333333334,0.5217391304347827,0.6666666666666667,0.56,0.3478260869565218,0.535913043478261,0.10535936443040032,2,RandomForestClassifier,,,gini,,balanced,sqrt,,,,,
+0.13147373199462892,0.0005338610432091862,0.011038064956665039,0.0004474278184982239,"{'class_weight': 'balanced', 'criterion': 'gini', 'max_features': 'log2'}",0.7777777777777778,0.75,0.8888888888888888,0.5454545454545454,0.5,0.6924242424242424,0.14685076238695957,2,0.4666666666666667,0.4,0.5333333333333333,0.4,0.3333333333333333,0.42666666666666664,0.06798692684790379,4,0.5833333333333334,0.5217391304347827,0.6666666666666667,0.4615384615384615,0.4,0.5266555183946489,0.09286712943843352,4,RandomForestClassifier,,,gini,,balanced,log2,,,,,
+0.13478245735168456,0.0016788268082505702,0.010595464706420898,0.00017647907495442738,"{'class_weight': 'balanced', 'criterion': 'entropy', 'max_features': 'sqrt'}",0.7,0.6666666666666666,0.8888888888888888,0.5714285714285714,0.5,0.6653968253968253,0.13216189862331665,5,0.4666666666666667,0.4,0.5333333333333333,0.26666666666666666,0.3333333333333333,0.39999999999999997,0.09428090415820635,8,0.56,0.5,0.6666666666666667,0.36363636363636365,0.4,0.49806060606060604,0.10950252312694084,7,RandomForestClassifier,,,entropy,,balanced,sqrt,,,,,
+0.13531460762023925,0.0014786637286605619,0.01061854362487793,0.0001862373336032819,"{'class_weight': 'balanced', 'criterion': 'entropy', 'max_features': 'log2'}",0.7,0.6666666666666666,0.7777777777777778,0.6,0.5,0.6488888888888888,0.09391354048495751,8,0.4666666666666667,0.4,0.4666666666666667,0.4,0.3333333333333333,0.4133333333333334,0.0498887651569859,6,0.56,0.5,0.5833333333333334,0.48,0.4,0.5046666666666667,0.06455316499685444,6,RandomForestClassifier,,,entropy,,balanced,log2,,,,,
+0.1650102138519287,0.005489192947411782,0.011378049850463867,0.001332666081590959,"{'class_weight': 'balanced_subsample', 'criterion': 'gini', 'max_features': 'sqrt'}",0.7,0.6666666666666666,0.875,0.5454545454545454,0.5,0.6574242424242425,0.13157274259209153,6,0.4666666666666667,0.4,0.4666666666666667,0.4,0.26666666666666666,0.4,0.07302967433402216,7,0.56,0.5,0.608695652173913,0.4615384615384615,0.3478260869565218,0.49561204013377935,0.08939201708694904,8,RandomForestClassifier,,,gini,,balanced_subsample,sqrt,,,,,
+0.1865290641784668,0.0124469621948864,0.011737585067749023,0.0011538109720151258,"{'class_weight': 'balanced_subsample', 'criterion': 'gini', 'max_features': 'log2'}",0.7,0.75,0.8888888888888888,0.5833333333333334,0.5,0.6844444444444444,0.13459587431535947,4,0.4666666666666667,0.4,0.5333333333333333,0.4666666666666667,0.3333333333333333,0.44000000000000006,0.06798692684790379,1,0.56,0.5217391304347827,0.6666666666666667,0.5185185185185186,0.4,0.5333848631239936,0.08558102321066285,3,RandomForestClassifier,,,gini,,balanced_subsample,log2,,,,,
+0.18512792587280275,0.021370909818433374,0.011930274963378906,0.001625654009975366,"{'class_weight': 'balanced_subsample', 'criterion': 'entropy', 'max_features': 'sqrt'}",0.8,0.75,0.8,0.5454545454545454,0.5555555555555556,0.6902020202020202,0.11555820439148168,3,0.5333333333333333,0.4,0.5333333333333333,0.4,0.3333333333333333,0.44000000000000006,0.07999999999999999,1,0.64,0.5217391304347827,0.64,0.4615384615384615,0.4166666666666667,0.5359888517279823,0.09123632712230355,1,RandomForestClassifier,,,entropy,,balanced_subsample,sqrt,,,,,
+0.16613612174987794,0.0008845499016503403,0.010677242279052734,7.291986152094173e-05,"{'class_weight': 'balanced_subsample', 'criterion': 'entropy', 'max_features': 'log2'}",0.6363636363636364,0.75,0.875,0.5384615384615384,0.45454545454545453,0.6508741258741259,0.14929865215346982,7,0.4666666666666667,0.4,0.4666666666666667,0.4666666666666667,0.3333333333333333,0.42666666666666675,0.053333333333333344,3,0.5384615384615385,0.5217391304347827,0.608695652173913,0.5,0.3846153846153846,0.5107023411371238,0.07281144617816618,5,RandomForestClassifier,,,entropy,,balanced_subsample,log2,,,,,
+0.009339666366577149,0.0007928127185369816,0.0029808998107910155,0.0002166133444215715,"{'activation': 'identity', 'hidden_layer_sizes': (10,), 'learning_rate': 'constant', 'max_iter': 2000, 'solver': 'lbfgs'}",0.625,0.6666666666666666,0.625,0.5454545454545454,0.5,0.5924242424242424,0.060634463041708296,29,0.3333333333333333,0.26666666666666666,0.3333333333333333,0.4,0.2,0.3066666666666667,0.06798692684790379,70,0.43478260869565216,0.3809523809523809,0.43478260869565216,0.4615384615384615,0.28571428571428575,0.3995540691192865,0.06265729645188609,68,MLPClassifier,,,,,,,identity,"(10,)",constant,2000,lbfgs
+0.20124430656433107,0.03403957283245018,0.003308010101318359,0.00046020431863945995,"{'activation': 'identity', 'hidden_layer_sizes': (10,), 'learning_rate': 'constant', 'max_iter': 2000, 'solver': 'sgd'}",0.5,0.75,0.625,0.625,0.5,0.6,0.09354143466934853,26,0.26666666666666666,0.2,0.3333333333333333,0.3333333333333333,0.3333333333333333,0.29333333333333333,0.05333333333333333,88,0.3478260869565218,0.31578947368421056,0.43478260869565216,0.43478260869565216,0.4,0.38663615560640735,0.04762400234150762,81,MLPClassifier,,,,,,,identity,"(10,)",constant,2000,sgd
+0.1948117733001709,0.037206202870292304,0.003155517578125,0.0004683599672825669,"{'activation': 'identity', 'hidden_layer_sizes': (10,), 'learning_rate': 'constant', 'max_iter': 2000, 'solver': 'adam'}",0.5555555555555556,0.6666666666666666,0.625,0.5454545454545454,0.5714285714285714,0.5928210678210679,0.04599695836129538,28,0.3333333333333333,0.26666666666666666,0.3333333333333333,0.4,0.26666666666666666,0.32,0.0498887651569859,65,0.4166666666666667,0.3809523809523809,0.43478260869565216,0.4615384615384615,0.36363636363636365,0.41151529629790495,0.035487196256680016,57,MLPClassifier,,,,,,,identity,"(10,)",constant,2000,adam
+0.009259462356567383,0.0015381071981169772,0.0028306007385253905,7.03237979495271e-05,"{'activation': 'identity', 'hidden_layer_sizes': (10,), 'learning_rate': 'invscaling', 'max_iter': 2000, 'solver': 'lbfgs'}",0.625,0.6666666666666666,0.625,0.5454545454545454,0.5,0.5924242424242424,0.060634463041708296,29,0.3333333333333333,0.26666666666666666,0.3333333333333333,0.4,0.2,0.3066666666666667,0.06798692684790379,70,0.43478260869565216,0.3809523809523809,0.43478260869565216,0.4615384615384615,0.28571428571428575,0.3995540691192865,0.06265729645188609,68,MLPClassifier,,,,,,,identity,"(10,)",invscaling,2000,lbfgs
+0.19477643966674804,0.17760258797410255,0.0029500484466552734,0.00018221323614782308,"{'activation': 'identity', 'hidden_layer_sizes': (10,), 'learning_rate': 'invscaling', 'max_iter': 2000, 'solver': 'sgd'}",0.16417910447761194,0.08571428571428572,0.06504065040650407,0.23076923076923078,0.14285714285714285,0.1377120828449551,0.05894321955158484,100,0.7333333333333333,0.8,0.5333333333333333,0.6,0.6,0.6533333333333333,0.09797958971132714,3,0.2682926829268293,0.15483870967741936,0.11594202898550725,0.33333333333333337,0.23076923076923073,0.220635197138464,0.07792899551670066,91,MLPClassifier,,,,,,,identity,"(10,)",invscaling,2000,sgd
+0.19190340042114257,0.01524357095429908,0.0027803421020507813,2.1146316760778618e-05,"{'activation': 'identity', 'hidden_layer_sizes': (10,), 'learning_rate': 'invscaling', 'max_iter': 2000, 'solver': 'adam'}",0.625,0.6666666666666666,0.625,0.5454545454545454,0.5714285714285714,0.6067099567099566,0.043036867895746025,23,0.3333333333333333,0.26666666666666666,0.3333333333333333,0.4,0.26666666666666666,0.32,0.0498887651569859,65,0.43478260869565216,0.3809523809523809,0.43478260869565216,0.4615384615384615,0.36363636363636365,0.415138484703702,0.03673118356170765,52,MLPClassifier,,,,,,,identity,"(10,)",invscaling,2000,adam
+0.008767652511596679,0.0006995990659209224,0.0027564048767089845,5.5725802527311884e-05,"{'activation': 'identity', 'hidden_layer_sizes': (10,), 'learning_rate': 'adaptive', 'max_iter': 2000, 'solver': 'lbfgs'}",0.625,0.6666666666666666,0.625,0.5454545454545454,0.5,0.5924242424242424,0.060634463041708296,29,0.3333333333333333,0.26666666666666666,0.3333333333333333,0.4,0.2,0.3066666666666667,0.06798692684790379,70,0.43478260869565216,0.3809523809523809,0.43478260869565216,0.4615384615384615,0.28571428571428575,0.3995540691192865,0.06265729645188609,68,MLPClassifier,,,,,,,identity,"(10,)",adaptive,2000,lbfgs
+0.20451765060424804,0.018346647257236334,0.0027574539184570313,1.6485468125299855e-05,"{'activation': 'identity', 'hidden_layer_sizes': (10,), 'learning_rate': 'adaptive', 'max_iter': 2000, 'solver': 'sgd'}",0.4444444444444444,0.6,0.7142857142857143,0.625,0.4444444444444444,0.5656349206349207,0.10599669128736988,42,0.26666666666666666,0.2,0.3333333333333333,0.3333333333333333,0.26666666666666666,0.27999999999999997,0.04988876515698587,92,0.33333333333333337,0.3,0.4545454545454545,0.43478260869565216,0.33333333333333337,0.3711989459815547,0.06152466465616364,84,MLPClassifier,,,,,,,identity,"(10,)",adaptive,2000,sgd
+0.17567076683044433,0.02696213488957541,0.0029958248138427734,0.00025045612888551,"{'activation': 'identity', 'hidden_layer_sizes': (10,), 'learning_rate': 'adaptive', 'max_iter': 2000, 'solver': 'adam'}",0.625,0.6666666666666666,0.625,0.5454545454545454,0.5714285714285714,0.6067099567099566,0.043036867895746025,23,0.3333333333333333,0.26666666666666666,0.3333333333333333,0.4,0.26666666666666666,0.32,0.0498887651569859,65,0.43478260869565216,0.3809523809523809,0.43478260869565216,0.4615384615384615,0.36363636363636365,0.415138484703702,0.03673118356170765,52,MLPClassifier,,,,,,,identity,"(10,)",adaptive,2000,adam
+0.016279983520507812,0.002330063450827653,0.00318603515625,0.0002234373852180452,"{'activation': 'identity', 'hidden_layer_sizes': (50,), 'learning_rate': 'constant', 'max_iter': 2000, 'solver': 'lbfgs'}",0.625,0.6666666666666666,0.625,0.5454545454545454,0.5,0.5924242424242424,0.060634463041708296,29,0.3333333333333333,0.26666666666666666,0.3333333333333333,0.4,0.2,0.3066666666666667,0.06798692684790379,70,0.43478260869565216,0.3809523809523809,0.43478260869565216,0.4615384615384615,0.28571428571428575,0.3995540691192865,0.06265729645188609,68,MLPClassifier,,,,,,,identity,"(50,)",constant,2000,lbfgs
+0.21524033546447754,0.048984556054449846,0.002980613708496094,0.00017999403326231518,"{'activation': 'identity', 'hidden_layer_sizes': (50,), 'learning_rate': 'constant', 'max_iter': 2000, 'solver': 'sgd'}",0.625,1.0,0.625,0.75,0.6666666666666666,0.7333333333333333,0.14092945437739804,2,0.3333333333333333,0.26666666666666666,0.3333333333333333,0.4,0.26666666666666666,0.32,0.0498887651569859,65,0.43478260869565216,0.4210526315789474,0.43478260869565216,0.5217391304347827,0.3809523809523809,0.43866187207148305,0.04598347299147871,29,MLPClassifier,,,,,,,identity,"(50,)",constant,2000,sgd
+0.14207496643066406,0.014091808612454165,0.0030428886413574217,6.580135093073586e-05,"{'activation': 'identity', 'hidden_layer_sizes': (50,), 'learning_rate': 'constant', 'max_iter': 2000, 'solver': 'adam'}",0.625,0.6666666666666666,0.7142857142857143,0.5454545454545454,0.5,0.6102813852813853,0.07820145923100053,18,0.3333333333333333,0.26666666666666666,0.3333333333333333,0.4,0.2,0.3066666666666667,0.06798692684790379,70,0.43478260869565216,0.3809523809523809,0.4545454545454545,0.4615384615384615,0.28571428571428575,0.40350663828924693,0.06532162344658447,62,MLPClassifier,,,,,,,identity,"(50,)",constant,2000,adam
+0.017014217376708985,0.0020943577392293303,0.0031023502349853517,1.9828285784603526e-05,"{'activation': 'identity', 'hidden_layer_sizes': (50,), 'learning_rate': 'invscaling', 'max_iter': 2000, 'solver': 'lbfgs'}",0.625,0.6666666666666666,0.625,0.5454545454545454,0.5,0.5924242424242424,0.060634463041708296,29,0.3333333333333333,0.26666666666666666,0.3333333333333333,0.4,0.2,0.3066666666666667,0.06798692684790379,70,0.43478260869565216,0.3809523809523809,0.43478260869565216,0.4615384615384615,0.28571428571428575,0.3995540691192865,0.06265729645188609,68,MLPClassifier,,,,,,,identity,"(50,)",invscaling,2000,lbfgs
+0.09009013175964356,0.027300231088963112,0.002983283996582031,7.960487527103751e-05,"{'activation': 'identity', 'hidden_layer_sizes': (50,), 'learning_rate': 'invscaling', 'max_iter': 2000, 'solver': 'sgd'}",0.19444444444444445,0.14285714285714285,0.22580645161290322,0.18309859154929578,0.12698412698412698,0.17463815148958267,0.03567352459567122,98,0.4666666666666667,0.26666666666666666,0.4666666666666667,0.8666666666666667,0.5333333333333333,0.52,0.19504985117770393,8,0.27450980392156865,0.18604651162790697,0.3043478260869565,0.3023255813953489,0.2051282051282051,0.25447158563199723,0.04959018938594115,90,MLPClassifier,,,,,,,identity,"(50,)",invscaling,2000,sgd
+0.1322352409362793,0.009411635379816752,0.002937173843383789,2.7633361298646475e-05,"{'activation': 'identity', 'hidden_layer_sizes': (50,), 'learning_rate': 'invscaling', 'max_iter': 2000, 'solver': 'adam'}",0.625,0.6666666666666666,0.7142857142857143,0.5454545454545454,0.5,0.6102813852813853,0.07820145923100053,18,0.3333333333333333,0.26666666666666666,0.3333333333333333,0.4,0.2,0.3066666666666667,0.06798692684790379,70,0.43478260869565216,0.3809523809523809,0.4545454545454545,0.4615384615384615,0.28571428571428575,0.40350663828924693,0.06532162344658447,62,MLPClassifier,,,,,,,identity,"(50,)",invscaling,2000,adam
+0.012340450286865234,0.0014134151924749146,0.002906465530395508,3.43668948574475e-05,"{'activation': 'identity', 'hidden_layer_sizes': (50,), 'learning_rate': 'adaptive', 'max_iter': 2000, 'solver': 'lbfgs'}",0.625,0.6666666666666666,0.625,0.5454545454545454,0.5,0.5924242424242424,0.060634463041708296,29,0.3333333333333333,0.26666666666666666,0.3333333333333333,0.4,0.2,0.3066666666666667,0.06798692684790379,70,0.43478260869565216,0.3809523809523809,0.43478260869565216,0.4615384615384615,0.28571428571428575,0.3995540691192865,0.06265729645188609,68,MLPClassifier,,,,,,,identity,"(50,)",adaptive,2000,lbfgs
+0.245164155960083,0.029563487507211734,0.003102016448974609,0.00013962353827944504,"{'activation': 'identity', 'hidden_layer_sizes': (50,), 'learning_rate': 'adaptive', 'max_iter': 2000, 'solver': 'sgd'}",0.625,0.8,0.7142857142857143,0.6666666666666666,0.5714285714285714,0.6754761904761905,0.07805602964648112,6,0.3333333333333333,0.26666666666666666,0.3333333333333333,0.4,0.26666666666666666,0.32,0.0498887651569859,65,0.43478260869565216,0.4,0.4545454545454545,0.5,0.36363636363636365,0.4305928853754941,0.046535421671757275,36,MLPClassifier,,,,,,,identity,"(50,)",adaptive,2000,sgd
+0.15197362899780273,0.012604312329159885,0.003071117401123047,0.0001239754053057335,"{'activation': 'identity', 'hidden_layer_sizes': (50,), 'learning_rate': 'adaptive', 'max_iter': 2000, 'solver': 'adam'}",0.625,0.6666666666666666,0.7142857142857143,0.5454545454545454,0.5,0.6102813852813853,0.07820145923100053,18,0.3333333333333333,0.26666666666666666,0.3333333333333333,0.4,0.2,0.3066666666666667,0.06798692684790379,70,0.43478260869565216,0.3809523809523809,0.4545454545454545,0.4615384615384615,0.28571428571428575,0.40350663828924693,0.06532162344658447,62,MLPClassifier,,,,,,,identity,"(50,)",adaptive,2000,adam
+0.025763463973999024,0.0018713299059961905,0.0031167030334472655,8.380901070137193e-05,"{'activation': 'identity', 'hidden_layer_sizes': (100, 20), 'learning_rate': 'constant', 'max_iter': 2000, 'solver': 'lbfgs'}",0.625,0.6666666666666666,0.625,0.5454545454545454,0.5,0.5924242424242424,0.060634463041708296,29,0.3333333333333333,0.26666666666666666,0.3333333333333333,0.4,0.2,0.3066666666666667,0.06798692684790379,70,0.43478260869565216,0.3809523809523809,0.43478260869565216,0.4615384615384615,0.28571428571428575,0.3995540691192865,0.06265729645188609,68,MLPClassifier,,,,,,,identity,"(100, 20)",constant,2000,lbfgs
+0.20890002250671386,0.02574473217810246,0.0032124042510986326,6.734713383152298e-05,"{'activation': 'identity', 'hidden_layer_sizes': (100, 20), 'learning_rate': 'constant', 'max_iter': 2000, 'solver': 'sgd'}",0.5,1.0,0.8333333333333334,0.6666666666666666,0.8,0.76,0.16786237749352229,1,0.26666666666666666,0.2,0.3333333333333333,0.4,0.26666666666666666,0.29333333333333333,0.0679869268479038,88,0.3478260869565218,0.33333333333333337,0.47619047619047616,0.5,0.4,0.4114699792960662,0.06680381453717062,58,MLPClassifier,,,,,,,identity,"(100, 20)",constant,2000,sgd
+0.09978580474853516,0.008466979925096322,0.0031958103179931642,8.643985845965268e-05,"{'activation': 'identity', 'hidden_layer_sizes': (100, 20), 'learning_rate': 'constant', 'max_iter': 2000, 'solver': 'adam'}",0.625,0.6666666666666666,0.7142857142857143,0.5454545454545454,0.5,0.6102813852813853,0.07820145923100053,18,0.3333333333333333,0.26666666666666666,0.3333333333333333,0.4,0.2,0.3066666666666667,0.06798692684790379,70,0.43478260869565216,0.3809523809523809,0.4545454545454545,0.4615384615384615,0.28571428571428575,0.40350663828924693,0.06532162344658447,62,MLPClassifier,,,,,,,identity,"(100, 20)",constant,2000,adam
+0.0354578971862793,0.01466165534260449,0.0032912731170654298,0.00011690765500418746,"{'activation': 'identity', 'hidden_layer_sizes': (100, 20), 'learning_rate': 'invscaling', 'max_iter': 2000, 'solver': 'lbfgs'}",0.625,0.6666666666666666,0.625,0.5454545454545454,0.5,0.5924242424242424,0.060634463041708296,29,0.3333333333333333,0.26666666666666666,0.3333333333333333,0.4,0.2,0.3066666666666667,0.06798692684790379,70,0.43478260869565216,0.3809523809523809,0.43478260869565216,0.4615384615384615,0.28571428571428575,0.3995540691192865,0.06265729645188609,68,MLPClassifier,,,,,,,identity,"(100, 20)",invscaling,2000,lbfgs
+0.654927921295166,0.4815234409946257,0.003370189666748047,0.00010615262667017042,"{'activation': 'identity', 'hidden_layer_sizes': (100, 20), 'learning_rate': 'invscaling', 'max_iter': 2000, 'solver': 'sgd'}",0.1016949152542373,0.4,0.2857142857142857,0.41379310344827586,0.21951219512195122,0.28414289990775005,0.11636056658376659,94,0.8,0.8,0.5333333333333333,0.8,0.6,0.7066666666666668,0.11623730516108466,1,0.1804511278195489,0.5333333333333333,0.37209302325581395,0.5454545454545454,0.32142857142857145,0.3905521202583626,0.13685208542624966,79,MLPClassifier,,,,,,,identity,"(100, 20)",invscaling,2000,sgd
+0.1157381534576416,0.03318473945610166,0.003713560104370117,0.00044057504381849556,"{'activation': 'identity', 'hidden_layer_sizes': (100, 20), 'learning_rate': 'invscaling', 'max_iter': 2000, 'solver': 'adam'}",0.625,0.6666666666666666,0.625,0.5454545454545454,0.75,0.6424242424242423,0.06658914225256284,12,0.3333333333333333,0.26666666666666666,0.3333333333333333,0.4,0.2,0.3066666666666667,0.06798692684790379,70,0.43478260869565216,0.3809523809523809,0.43478260869565216,0.4615384615384615,0.31578947368421056,0.4055691067132714,0.05197266929784708,60,MLPClassifier,,,,,,,identity,"(100, 20)",invscaling,2000,adam
+0.029738426208496094,0.005510867445197849,0.0034429550170898436,5.839637610226398e-05,"{'activation': 'identity', 'hidden_layer_sizes': (100, 20), 'learning_rate': 'adaptive', 'max_iter': 2000, 'solver': 'lbfgs'}",0.625,0.6666666666666666,0.625,0.5454545454545454,0.5,0.5924242424242424,0.060634463041708296,29,0.3333333333333333,0.26666666666666666,0.3333333333333333,0.4,0.2,0.3066666666666667,0.06798692684790379,70,0.43478260869565216,0.3809523809523809,0.43478260869565216,0.4615384615384615,0.28571428571428575,0.3995540691192865,0.06265729645188609,68,MLPClassifier,,,,,,,identity,"(100, 20)",adaptive,2000,lbfgs
+0.31555943489074706,0.02251984217573906,0.003392744064331055,8.6942648899661e-05,"{'activation': 'identity', 'hidden_layer_sizes': (100, 20), 'learning_rate': 'adaptive', 'max_iter': 2000, 'solver': 'sgd'}",0.5714285714285714,0.6,0.7142857142857143,0.7777777777777778,0.5714285714285714,0.646984126984127,0.08397770523218948,11,0.26666666666666666,0.2,0.3333333333333333,0.4666666666666667,0.26666666666666666,0.30666666666666664,0.09043106644167025,85,0.36363636363636365,0.3,0.4545454545454545,0.5833333333333334,0.36363636363636365,0.4130303030303031,0.09836869607405324,55,MLPClassifier,,,,,,,identity,"(100, 20)",adaptive,2000,sgd
+0.11446304321289062,0.026316232071720312,0.0035532474517822265,0.00012207539741534973,"{'activation': 'identity', 'hidden_layer_sizes': (100, 20), 'learning_rate': 'adaptive', 'max_iter': 2000, 'solver': 'adam'}",0.625,0.6666666666666666,0.625,0.5454545454545454,0.75,0.6424242424242423,0.06658914225256284,12,0.3333333333333333,0.26666666666666666,0.3333333333333333,0.4,0.2,0.3066666666666667,0.06798692684790379,70,0.43478260869565216,0.3809523809523809,0.43478260869565216,0.4615384615384615,0.31578947368421056,0.4055691067132714,0.05197266929784708,60,MLPClassifier,,,,,,,identity,"(100, 20)",adaptive,2000,adam
+0.07306756973266601,0.046031009644749515,0.003125619888305664,9.097053264977707e-05,"{'activation': 'logistic', 'hidden_layer_sizes': (10,), 'learning_rate': 'constant', 'max_iter': 2000, 'solver': 'lbfgs'}",0.4,0.38461538461538464,0.48,0.42857142857142855,0.23076923076923078,0.3847912087912088,0.08358891018002283,89,0.5333333333333333,0.3333333333333333,0.8,0.6,0.2,0.49333333333333335,0.20912516188477495,10,0.4571428571428572,0.3571428571428571,0.6,0.5,0.21428571428571427,0.4257142857142857,0.1313664449436874,38,MLPClassifier,,,,,,,logistic,"(10,)",constant,2000,lbfgs
+0.3306027889251709,0.039530208965049315,0.0028197765350341797,6.028153401802405e-05,"{'activation': 'logistic', 'hidden_layer_sizes': (10,), 'learning_rate': 'constant', 'max_iter': 2000, 'solver': 'sgd'}",0.0,0.0,0.0,0.0,0.0,0.0,0.0,105,0.0,0.0,0.0,0.0,0.0,0.0,0.0,105,0.0,0.0,0.0,0.0,0.0,0.0,0.0,105,MLPClassifier,,,,,,,logistic,"(10,)",constant,2000,sgd
+0.29283480644226073,0.031344503140914734,0.0029973983764648438,0.00012414929540425514,"{'activation': 'logistic', 'hidden_layer_sizes': (10,), 'learning_rate': 'constant', 'max_iter': 2000, 'solver': 'adam'}",0.5714285714285714,0.0,0.625,0.5833333333333334,0.0,0.3559523809523809,0.2911779189754371,91,0.26666666666666666,0.0,0.3333333333333333,0.4666666666666667,0.0,0.21333333333333332,0.18571184369578825,96,0.36363636363636365,0.0,0.43478260869565216,0.5185185185185186,0.0,0.26338749817010687,0.22057375404520516,89,MLPClassifier,,,,,,,logistic,"(10,)",constant,2000,adam
+0.0623227596282959,0.016465558418518624,0.0028204917907714844,5.604003216649613e-05,"{'activation': 'logistic', 'hidden_layer_sizes': (10,), 'learning_rate': 'invscaling', 'max_iter': 2000, 'solver': 'lbfgs'}",0.3333333333333333,0.38461538461538464,0.3684210526315789,0.6153846153846154,0.45454545454545453,0.43125996810207334,0.10015804698486312,75,0.4666666666666667,0.3333333333333333,0.4666666666666667,0.5333333333333333,0.3333333333333333,0.42666666666666664,0.08,31,0.3888888888888889,0.3571428571428571,0.4117647058823529,0.5714285714285715,0.3846153846153846,0.422768081591611,0.07632969747648866,42,MLPClassifier,,,,,,,logistic,"(10,)",invscaling,2000,lbfgs
+0.08247060775756836,0.06628108340985492,0.0029076099395751952,4.565615565741273e-05,"{'activation': 'logistic', 'hidden_layer_sizes': (10,), 'learning_rate': 'invscaling', 'max_iter': 2000, 'solver': 'sgd'}",0.09027777777777778,0.058394160583941604,0.10067114093959731,0.0,0.0603448275862069,0.061937581377504713,0.03508507389421089,103,0.8666666666666667,0.5333333333333333,1.0,0.0,0.4666666666666667,0.5733333333333334,0.3492213560989012,6,0.16352201257861634,0.10526315789473684,0.18292682926829268,0.0,0.10687022900763361,0.11171644574985588,0.06371880486501004,101,MLPClassifier,,,,,,,logistic,"(10,)",invscaling,2000,sgd
+0.32681841850280763,0.03737471096500944,0.003072023391723633,0.00021893857520252248,"{'activation': 'logistic', 'hidden_layer_sizes': (10,), 'learning_rate': 'invscaling', 'max_iter': 2000, 'solver': 'adam'}",0.0,0.0,0.6666666666666666,0.5714285714285714,0.75,0.39761904761904765,0.32953619574849435,86,0.0,0.0,0.13333333333333333,0.26666666666666666,0.2,0.12000000000000002,0.10666666666666666,99,0.0,0.0,0.2222222222222222,0.36363636363636365,0.31578947368421056,0.18032961190855928,0.15410598174136725,92,MLPClassifier,,,,,,,logistic,"(10,)",invscaling,2000,adam
+0.04625625610351562,0.008712070330845386,0.002765989303588867,4.690938710562138e-05,"{'activation': 'logistic', 'hidden_layer_sizes': (10,), 'learning_rate': 'adaptive', 'max_iter': 2000, 'solver': 'lbfgs'}",0.4666666666666667,0.35714285714285715,0.5882352941176471,0.5238095238095238,0.4,0.4671708683473389,0.08304562059873168,63,0.4666666666666667,0.3333333333333333,0.6666666666666666,0.7333333333333333,0.26666666666666666,0.49333333333333335,0.18184242262647807,10,0.4666666666666667,0.3448275862068965,0.625,0.611111111111111,0.32,0.47352107279693484,0.12810511324432666,9,MLPClassifier,,,,,,,logistic,"(10,)",adaptive,2000,lbfgs
+0.37090563774108887,0.02666214108353507,0.0027956485748291014,1.3399378682498446e-05,"{'activation': 'logistic', 'hidden_layer_sizes': (10,), 'learning_rate': 'adaptive', 'max_iter': 2000, 'solver': 'sgd'}",0.0,0.0,0.0,0.0,0.0,0.0,0.0,105,0.0,0.0,0.0,0.0,0.0,0.0,0.0,105,0.0,0.0,0.0,0.0,0.0,0.0,0.0,105,MLPClassifier,,,,,,,logistic,"(10,)",adaptive,2000,sgd
+0.2881649971008301,0.023747046755495084,0.0028429508209228517,7.194608417777043e-05,"{'activation': 'logistic', 'hidden_layer_sizes': (10,), 'learning_rate': 'adaptive', 'max_iter': 2000, 'solver': 'adam'}",0.0,0.0,0.7142857142857143,1.0,0.0,0.34285714285714286,0.4295227536677974,92,0.0,0.0,0.3333333333333333,0.06666666666666667,0.0,0.07999999999999999,0.12927146286443542,102,0.0,0.0,0.4545454545454545,0.125,0.0,0.11590909090909089,0.17610336934373194,100,MLPClassifier,,,,,,,logistic,"(10,)",adaptive,2000,adam
+0.056029653549194335,0.011364247078958481,0.002864170074462891,2.592757926214735e-05,"{'activation': 'logistic', 'hidden_layer_sizes': (50,), 'learning_rate': 'constant', 'max_iter': 2000, 'solver': 'lbfgs'}",0.36363636363636365,0.6428571428571429,0.5625,0.35714285714285715,0.4166666666666667,0.46856060606060607,0.11429775574436161,62,0.26666666666666666,0.6,0.6,0.3333333333333333,0.3333333333333333,0.42666666666666664,0.14360439485692011,31,0.30769230769230765,0.6206896551724138,0.5806451612903225,0.3448275862068965,0.3703703703703704,0.44484501614646216,0.12940157733718807,26,MLPClassifier,,,,,,,logistic,"(50,)",constant,2000,lbfgs
+0.6753923416137695,0.043415481124972705,0.0031158447265625,0.0001222258746336521,"{'activation': 'logistic', 'hidden_layer_sizes': (50,), 'learning_rate': 'constant', 'max_iter': 2000, 'solver': 'sgd'}",0.3333333333333333,1.0,1.0,0.0,1.0,0.6666666666666666,0.4216370213557839,8,0.06666666666666667,0.06666666666666667,0.06666666666666667,0.0,0.2,0.08,0.06531972647421809,101,0.1111111111111111,0.125,0.125,0.0,0.33333333333333337,0.1388888888888889,0.10794088459847473,98,MLPClassifier,,,,,,,logistic,"(50,)",constant,2000,sgd
+0.30585317611694335,0.05308676493926892,0.003101825714111328,5.041501654839186e-05,"{'activation': 'logistic', 'hidden_layer_sizes': (50,), 'learning_rate': 'constant', 'max_iter': 2000, 'solver': 'adam'}",0.5714285714285714,0.7142857142857143,0.6666666666666666,0.6153846153846154,0.625,0.6385531135531135,0.04848086020727936,14,0.26666666666666666,0.3333333333333333,0.4,0.5333333333333333,0.3333333333333333,0.3733333333333333,0.09043106644167025,54,0.36363636363636365,0.4545454545454545,0.5,0.5714285714285715,0.43478260869565216,0.46487859966120837,0.06906365492916805,13,MLPClassifier,,,,,,,logistic,"(50,)",constant,2000,adam
+0.05492725372314453,0.0017188900888596406,0.0031635284423828123,3.219530197389177e-05,"{'activation': 'logistic', 'hidden_layer_sizes': (50,), 'learning_rate': 'invscaling', 'max_iter': 2000, 'solver': 'lbfgs'}",0.4,0.3684210526315789,0.5,0.4166666666666667,0.38461538461538464,0.41394062078272603,0.04591293836418872,80,0.4,0.4666666666666667,0.6,0.3333333333333333,0.3333333333333333,0.42666666666666664,0.09977753031397177,31,0.4000000000000001,0.4117647058823529,0.5454545454545454,0.3703703703703704,0.3571428571428571,0.4169464957700252,0.0671925057567224,49,MLPClassifier,,,,,,,logistic,"(50,)",invscaling,2000,lbfgs
+0.5075193881988526,0.24815237529890488,0.0033550262451171875,8.14424744708026e-05,"{'activation': 'logistic', 'hidden_layer_sizes': (50,), 'learning_rate': 'invscaling', 'max_iter': 2000, 'solver': 'sgd'}",0.0,0.0,0.0,0.0,1.0,0.2,0.4,95,0.0,0.0,0.0,0.0,0.06666666666666667,0.013333333333333332,0.026666666666666665,104,0.0,0.0,0.0,0.0,0.125,0.025,0.05,104,MLPClassifier,,,,,,,logistic,"(50,)",invscaling,2000,sgd
+0.3095374584197998,0.043261470983007796,0.003397369384765625,5.208026705752229e-05,"{'activation': 'logistic', 'hidden_layer_sizes': (50,), 'learning_rate': 'invscaling', 'max_iter': 2000, 'solver': 'adam'}",0.5714285714285714,0.7142857142857143,0.6666666666666666,0.5833333333333334,0.7142857142857143,0.65,0.06190476190476191,9,0.26666666666666666,0.3333333333333333,0.4,0.4666666666666667,0.3333333333333333,0.36,0.0679869268479038,56,0.36363636363636365,0.4545454545454545,0.5,0.5185185185185186,0.4545454545454545,0.45824915824915824,0.053580862853959896,16,MLPClassifier,,,,,,,logistic,"(50,)",invscaling,2000,adam
+0.06613998413085938,0.016564219154995202,0.0034026145935058595,5.563784511800277e-05,"{'activation': 'logistic', 'hidden_layer_sizes': (50,), 'learning_rate': 'adaptive', 'max_iter': 2000, 'solver': 'lbfgs'}",0.4,0.4444444444444444,0.5263157894736842,0.3684210526315789,0.5384615384615384,0.45552856500224925,0.06735385246045535,65,0.4,0.26666666666666666,0.6666666666666666,0.4666666666666667,0.4666666666666667,0.4533333333333334,0.12927146286443542,16,0.4000000000000001,0.33333333333333337,0.5882352941176471,0.4117647058823529,0.5,0.44666666666666666,0.08846071423135912,24,MLPClassifier,,,,,,,logistic,"(50,)",adaptive,2000,lbfgs
+0.8062046051025391,0.09699132694897834,0.0033336162567138674,8.535217304817459e-05,"{'activation': 'logistic', 'hidden_layer_sizes': (50,), 'learning_rate': 'adaptive', 'max_iter': 2000, 'solver': 'sgd'}",0.5,0.0,1.0,0.0,0.75,0.45,0.4,67,0.06666666666666667,0.0,0.06666666666666667,0.0,0.2,0.06666666666666668,0.07302967433402216,103,0.11764705882352941,0.0,0.125,0.0,0.31578947368421056,0.11168730650154798,0.1156015138064585,102,MLPClassifier,,,,,,,logistic,"(50,)",adaptive,2000,sgd
+0.29743061065673826,0.04428889124731218,0.003319549560546875,4.4365693854108093e-05,"{'activation': 'logistic', 'hidden_layer_sizes': (50,), 'learning_rate': 'adaptive', 'max_iter': 2000, 'solver': 'adam'}",0.5714285714285714,0.7142857142857143,0.6666666666666666,0.5833333333333334,0.7142857142857143,0.65,0.06190476190476191,9,0.26666666666666666,0.3333333333333333,0.4,0.4666666666666667,0.3333333333333333,0.36,0.0679869268479038,56,0.36363636363636365,0.4545454545454545,0.5,0.5185185185185186,0.4545454545454545,0.45824915824915824,0.053580862853959896,16,MLPClassifier,,,,,,,logistic,"(50,)",adaptive,2000,adam
+0.3580580234527588,0.06301164433652041,0.003572416305541992,0.0001459300429175471,"{'activation': 'logistic', 'hidden_layer_sizes': (100, 20), 'learning_rate': 'constant', 'max_iter': 2000, 'solver': 'lbfgs'}",0.3888888888888889,0.3,0.3684210526315789,0.5,0.5454545454545454,0.4205528973950027,0.08962767825148864,78,0.4666666666666667,0.4,0.4666666666666667,0.4666666666666667,0.4,0.44000000000000006,0.03265986323710904,24,0.42424242424242425,0.34285714285714286,0.4117647058823529,0.4827586206896552,0.4615384615384615,0.4246322710420074,0.048153166507795296,40,MLPClassifier,,,,,,,logistic,"(100, 20)",constant,2000,lbfgs
+0.2549830436706543,0.020565032362827355,0.003612184524536133,0.00011226681029727733,"{'activation': 'logistic', 'hidden_layer_sizes': (100, 20), 'learning_rate': 'constant', 'max_iter': 2000, 'solver': 'sgd'}",0.0,0.0,0.0,0.0,0.0,0.0,0.0,105,0.0,0.0,0.0,0.0,0.0,0.0,0.0,105,0.0,0.0,0.0,0.0,0.0,0.0,0.0,105,MLPClassifier,,,,,,,logistic,"(100, 20)",constant,2000,sgd
+1.1718737602233886,0.24094826769673328,0.003545999526977539,0.00011167349109499017,"{'activation': 'logistic', 'hidden_layer_sizes': (100, 20), 'learning_rate': 'constant', 'max_iter': 2000, 'solver': 'adam'}",0.6,0.6363636363636364,0.5833333333333334,0.55,0.3333333333333333,0.5406060606060608,0.10730610677386535,47,0.4,0.4666666666666667,0.4666666666666667,0.7333333333333333,0.2,0.4533333333333334,0.17074997966487596,16,0.48,0.5384615384615385,0.5185185185185186,0.6285714285714286,0.25,0.4831102971102972,0.12635388984541385,6,MLPClassifier,,,,,,,logistic,"(100, 20)",constant,2000,adam
+0.3377342700958252,0.07612657036083671,0.0035915374755859375,6.52516414274953e-05,"{'activation': 'logistic', 'hidden_layer_sizes': (100, 20), 'learning_rate': 'invscaling', 'max_iter': 2000, 'solver': 'lbfgs'}",0.5625,0.375,0.3333333333333333,0.47368421052631576,0.3,0.40890350877192977,0.09644552346591491,81,0.6,0.4,0.4,0.6,0.2,0.44000000000000006,0.14966629547095764,24,0.5806451612903225,0.38709677419354843,0.3636363636363636,0.5294117647058824,0.24,0.42015801276522335,0.12201406397172566,47,MLPClassifier,,,,,,,logistic,"(100, 20)",invscaling,2000,lbfgs
+0.29044718742370607,0.234180550229874,0.0035352230072021483,0.0001671404638852201,"{'activation': 'logistic', 'hidden_layer_sizes': (100, 20), 'learning_rate': 'invscaling', 'max_iter': 2000, 'solver': 'sgd'}",0.0,0.0,0.10067114093959731,0.0,0.0,0.020134228187919462,0.040268456375838924,104,0.0,0.0,1.0,0.0,0.0,0.2,0.4000000000000001,97,0.0,0.0,0.18292682926829268,0.0,0.0,0.036585365853658534,0.07317073170731707,103,MLPClassifier,,,,,,,logistic,"(100, 20)",invscaling,2000,sgd
+1.1548531532287598,0.2776065482281074,0.0036094188690185547,0.00011349165130518803,"{'activation': 'logistic', 'hidden_layer_sizes': (100, 20), 'learning_rate': 'invscaling', 'max_iter': 2000, 'solver': 'adam'}",0.5,0.5833333333333334,0.5833333333333334,0.4444444444444444,0.3333333333333333,0.48888888888888904,0.09395296958493204,57,0.3333333333333333,0.4666666666666667,0.4666666666666667,0.5333333333333333,0.2,0.39999999999999997,0.11925695879998878,46,0.4,0.5185185185185186,0.5185185185185186,0.4848484848484848,0.25,0.4343771043771044,0.10186620373603875,34,MLPClassifier,,,,,,,logistic,"(100, 20)",invscaling,2000,adam
+0.3444072246551514,0.09869928584054863,0.003534078598022461,0.0001186712520928932,"{'activation': 'logistic', 'hidden_layer_sizes': (100, 20), 'learning_rate': 'adaptive', 'max_iter': 2000, 'solver': 'lbfgs'}",0.4166666666666667,0.4166666666666667,0.6666666666666666,0.45,0.2727272727272727,0.4445454545454545,0.1268412599833342,71,0.3333333333333333,0.3333333333333333,0.5333333333333333,0.6,0.2,0.39999999999999997,0.1460593486680443,46,0.3703703703703704,0.3703703703703704,0.5925925925925926,0.5142857142857143,0.23076923076923075,0.4156776556776557,0.1259516335201831,51,MLPClassifier,,,,,,,logistic,"(100, 20)",adaptive,2000,lbfgs
+0.36882634162902833,0.010158098420913704,0.0036492347717285156,0.00017554764998244988,"{'activation': 'logistic', 'hidden_layer_sizes': (100, 20), 'learning_rate': 'adaptive', 'max_iter': 2000, 'solver': 'sgd'}",0.0,0.0,0.0,0.0,0.0,0.0,0.0,105,0.0,0.0,0.0,0.0,0.0,0.0,0.0,105,0.0,0.0,0.0,0.0,0.0,0.0,0.0,105,MLPClassifier,,,,,,,logistic,"(100, 20)",adaptive,2000,sgd
+1.0779379844665526,0.23107617716201515,0.0036743640899658202,0.00010460753113904743,"{'activation': 'logistic', 'hidden_layer_sizes': (100, 20), 'learning_rate': 'adaptive', 'max_iter': 2000, 'solver': 'adam'}",0.5555555555555556,0.6363636363636364,0.6363636363636364,0.5555555555555556,0.25,0.5267676767676768,0.14302473880633854,50,0.3333333333333333,0.4666666666666667,0.4666666666666667,0.6666666666666666,0.13333333333333333,0.4133333333333333,0.17587874611030557,42,0.4166666666666667,0.5384615384615385,0.5384615384615385,0.606060606060606,0.1739130434782609,0.4547126786257222,0.1531207839004028,20,MLPClassifier,,,,,,,logistic,"(100, 20)",adaptive,2000,adam
+0.0735023021697998,0.04297280081143553,0.003200244903564453,0.00010898346659552431,"{'activation': 'tanh', 'hidden_layer_sizes': (10,), 'learning_rate': 'constant', 'max_iter': 2000, 'solver': 'lbfgs'}",0.5625,0.3157894736842105,0.45,0.4375,0.21428571428571427,0.39601503759398493,0.11984515762755112,87,0.6,0.4,0.6,0.4666666666666667,0.2,0.4533333333333334,0.1484737163421339,16,0.5806451612903225,0.35294117647058826,0.5142857142857143,0.45161290322580644,0.20689655172413796,0.4212763013993139,0.1308417824463024,43,MLPClassifier,,,,,,,tanh,"(10,)",constant,2000,lbfgs
+0.23330154418945312,0.025409584035756685,0.002823209762573242,4.362906260465709e-05,"{'activation': 'tanh', 'hidden_layer_sizes': (10,), 'learning_rate': 'constant', 'max_iter': 2000, 'solver': 'sgd'}",0.625,0.42857142857142855,0.5454545454545454,0.625,0.5,0.5448051948051947,0.07534006784062329,44,0.3333333333333333,0.2,0.4,0.3333333333333333,0.26666666666666666,0.30666666666666664,0.06798692684790379,85,0.43478260869565216,0.27272727272727276,0.4615384615384615,0.43478260869565216,0.3478260869565218,0.3903314077227121,0.07022776930880646,80,MLPClassifier,,,,,,,tanh,"(10,)",constant,2000,sgd
+0.30579495429992676,0.026880923729556885,0.0028073787689208984,4.468246601668189e-05,"{'activation': 'tanh', 'hidden_layer_sizes': (10,), 'learning_rate': 'constant', 'max_iter': 2000, 'solver': 'adam'}",0.625,0.75,0.6666666666666666,0.6470588235294118,0.3333333333333333,0.6044117647058823,0.1419843442696606,25,0.3333333333333333,0.4,0.5333333333333333,0.7333333333333333,0.2,0.44000000000000006,0.18184242262647807,24,0.43478260869565216,0.5217391304347827,0.5925925925925926,0.6875,0.25,0.49732286634460543,0.148942741280385,4,MLPClassifier,,,,,,,tanh,"(10,)",constant,2000,adam
+0.05532646179199219,0.01984905783378854,0.002947568893432617,0.0001990153443400554,"{'activation': 'tanh', 'hidden_layer_sizes': (10,), 'learning_rate': 'invscaling', 'max_iter': 2000, 'solver': 'lbfgs'}",0.5333333333333333,0.5833333333333334,0.5555555555555556,0.38095238095238093,0.4444444444444444,0.49952380952380954,0.07536523983347677,55,0.5333333333333333,0.4666666666666667,0.6666666666666666,0.5333333333333333,0.5333333333333333,0.5466666666666666,0.06531972647421806,7,0.5333333333333333,0.5185185185185186,0.606060606060606,0.4444444444444444,0.4848484848484848,0.5174410774410775,0.053847805990810044,1,MLPClassifier,,,,,,,tanh,"(10,)",invscaling,2000,lbfgs
+0.054268169403076175,0.04850529363972415,0.0029219627380371095,0.00013884279700685358,"{'activation': 'tanh', 'hidden_layer_sizes': (10,), 'learning_rate': 'invscaling', 'max_iter': 2000, 'solver': 'sgd'}",0.14893617021276595,0.3055555555555556,0.10714285714285714,0.14285714285714285,0.20408163265306123,0.18171467168427652,0.06925624692440091,96,0.9333333333333333,0.7333333333333333,1.0,0.13333333333333333,0.6666666666666666,0.6933333333333332,0.3057958650981258,2,0.25688073394495414,0.43137254901960786,0.19354838709677416,0.1379310344827586,0.3125,0.2664465409088189,0.10123575468148033,88,MLPClassifier,,,,,,,tanh,"(10,)",invscaling,2000,sgd
+0.3439903259277344,0.029884819607975312,0.002832460403442383,3.7315181173268165e-05,"{'activation': 'tanh', 'hidden_layer_sizes': (10,), 'learning_rate': 'invscaling', 'max_iter': 2000, 'solver': 'adam'}",0.6666666666666666,0.7777777777777778,0.6363636363636364,0.6111111111111112,0.3,0.5983838383838384,0.1596759804503557,27,0.4,0.4666666666666667,0.4666666666666667,0.7333333333333333,0.2,0.4533333333333334,0.17074997966487596,16,0.5,0.5833333333333334,0.5384615384615385,0.6666666666666666,0.24,0.5056923076923077,0.1439718724812115,3,MLPClassifier,,,,,,,tanh,"(10,)",invscaling,2000,adam
+0.06870889663696289,0.03831671193962036,0.0028324127197265625,5.919410465086813e-05,"{'activation': 'tanh', 'hidden_layer_sizes': (10,), 'learning_rate': 'adaptive', 'max_iter': 2000, 'solver': 'lbfgs'}",0.47058823529411764,0.38461538461538464,0.5,0.46153846153846156,0.375,0.4383484162895928,0.04955474233893554,73,0.5333333333333333,0.3333333333333333,0.6666666666666666,0.4,0.4,0.4666666666666666,0.11925695879998877,15,0.5,0.3571428571428571,0.5714285714285715,0.42857142857142855,0.38709677419354843,0.44884792626728115,0.07781607810687231,23,MLPClassifier,,,,,,,tanh,"(10,)",adaptive,2000,lbfgs
+0.279741096496582,0.047131848681445,0.0028369903564453127,4.8859267808605723e-05,"{'activation': 'tanh', 'hidden_layer_sizes': (10,), 'learning_rate': 'adaptive', 'max_iter': 2000, 'solver': 'sgd'}",0.8,0.5,0.5833333333333334,0.6666666666666666,0.6,0.63,0.10022197585581939,16,0.26666666666666666,0.3333333333333333,0.4666666666666667,0.4,0.2,0.33333333333333337,0.09428090415820634,61,0.4,0.4,0.5185185185185186,0.5,0.3,0.42370370370370375,0.07904088990574379,41,MLPClassifier,,,,,,,tanh,"(10,)",adaptive,2000,sgd
+0.3252126216888428,0.04728604082254975,0.0028401851654052735,8.929501199730519e-05,"{'activation': 'tanh', 'hidden_layer_sizes': (10,), 'learning_rate': 'adaptive', 'max_iter': 2000, 'solver': 'adam'}",0.5,0.7777777777777778,0.6666666666666666,0.625,0.36363636363636365,0.5866161616161616,0.14258086523911065,39,0.3333333333333333,0.4666666666666667,0.5333333333333333,0.6666666666666666,0.26666666666666666,0.4533333333333333,0.14236104336041747,20,0.4,0.5833333333333334,0.5925925925925926,0.6451612903225806,0.30769230769230765,0.5057559047881628,0.12915589740823832,2,MLPClassifier,,,,,,,tanh,"(10,)",adaptive,2000,adam
+0.042188501358032225,0.004533299574197103,0.002889394760131836,3.5039582940702424e-05,"{'activation': 'tanh', 'hidden_layer_sizes': (50,), 'learning_rate': 'constant', 'max_iter': 2000, 'solver': 'lbfgs'}",0.5,0.6666666666666666,0.42857142857142855,0.3333333333333333,0.4444444444444444,0.4746031746031745,0.11001729175341877,59,0.4666666666666667,0.4,0.6,0.6,0.5333333333333333,0.52,0.07774602526460399,8,0.4827586206896552,0.5,0.5,0.42857142857142855,0.4848484848484848,0.4792357068219137,0.026355625708029434,7,MLPClassifier,,,,,,,tanh,"(50,)",constant,2000,lbfgs
+0.2983293056488037,0.031680255619424356,0.003059864044189453,4.750671915720951e-05,"{'activation': 'tanh', 'hidden_layer_sizes': (50,), 'learning_rate': 'constant', 'max_iter': 2000, 'solver': 'sgd'}",0.5,0.8333333333333334,0.7,0.6,0.5555555555555556,0.6377777777777778,0.11771486789465803,15,0.26666666666666666,0.3333333333333333,0.4666666666666667,0.4,0.3333333333333333,0.36,0.0679869268479038,56,0.3478260869565218,0.47619047619047616,0.56,0.48,0.4166666666666667,0.45613664596273296,0.07077890722870993,19,MLPClassifier,,,,,,,tanh,"(50,)",constant,2000,sgd
+1.8514402389526368,0.14303282559725186,0.0033551692962646485,0.0001438500507374196,"{'activation': 'tanh', 'hidden_layer_sizes': (50,), 'learning_rate': 'constant', 'max_iter': 2000, 'solver': 'adam'}",0.4,0.625,0.5294117647058824,0.6153846153846154,0.38461538461538464,0.5108823529411765,0.10249404159300556,52,0.26666666666666666,0.3333333333333333,0.6,0.5333333333333333,0.3333333333333333,0.4133333333333334,0.12927146286443544,39,0.32,0.43478260869565216,0.5625,0.5714285714285715,0.3571428571428571,0.4491708074534162,0.10310356747997484,22,MLPClassifier,,,,,,,tanh,"(50,)",constant,2000,adam
+0.040059995651245114,0.0015939162257844735,0.0033688068389892576,0.000186510537368315,"{'activation': 'tanh', 'hidden_layer_sizes': (50,), 'learning_rate': 'invscaling', 'max_iter': 2000, 'solver': 'lbfgs'}",0.3125,0.5454545454545454,0.5714285714285714,0.47058823529411764,0.2857142857142857,0.437137127578304,0.11777021875102951,74,0.3333333333333333,0.4,0.8,0.5333333333333333,0.26666666666666666,0.4666666666666667,0.1885618083164127,13,0.3225806451612903,0.4615384615384615,0.6666666666666666,0.5,0.2758620689655172,0.4453295684663871,0.13858443411729288,25,MLPClassifier,,,,,,,tanh,"(50,)",invscaling,2000,lbfgs
+0.116534423828125,0.06020136806794945,0.0034764289855957033,8.249329693611563e-05,"{'activation': 'tanh', 'hidden_layer_sizes': (50,), 'learning_rate': 'invscaling', 'max_iter': 2000, 'solver': 'sgd'}",0.06796116504854369,0.1276595744680851,0.0,0.2727272727272727,0.2727272727272727,0.14821505699423482,0.10939607112471537,99,0.4666666666666667,0.8,0.0,0.2,0.2,0.3333333333333333,0.2764858847110363,62,0.11864406779661017,0.2201834862385321,0.0,0.23076923076923075,0.23076923076923075,0.16007320311472076,0.09049741034758091,95,MLPClassifier,,,,,,,tanh,"(50,)",invscaling,2000,sgd
+1.9511982917785644,0.21572891140382686,0.0034070491790771486,0.0001838137295056284,"{'activation': 'tanh', 'hidden_layer_sizes': (50,), 'learning_rate': 'invscaling', 'max_iter': 2000, 'solver': 'adam'}",0.35714285714285715,0.5555555555555556,0.5263157894736842,0.6666666666666666,0.38461538461538464,0.4980592506908296,0.11424505109618006,56,0.3333333333333333,0.3333333333333333,0.6666666666666666,0.5333333333333333,0.3333333333333333,0.44000000000000006,0.13727506854649335,24,0.3448275862068965,0.4166666666666667,0.5882352941176471,0.5925925925925926,0.3571428571428571,0.45989299934533195,0.1093133875932921,14,MLPClassifier,,,,,,,tanh,"(50,)",invscaling,2000,adam
+0.045552396774291994,0.00258048129387556,0.0034361362457275392,4.4643368036889016e-05,"{'activation': 'tanh', 'hidden_layer_sizes': (50,), 'learning_rate': 'adaptive', 'max_iter': 2000, 'solver': 'lbfgs'}",0.3,0.6923076923076923,0.4090909090909091,0.42857142857142855,0.4,0.44599400599400607,0.13096985546611603,70,0.2,0.6,0.6,0.4,0.26666666666666666,0.4133333333333333,0.16546231527987806,42,0.24,0.6428571428571429,0.4864864864864865,0.4137931034482759,0.32,0.420627346558381,0.1389405029169313,45,MLPClassifier,,,,,,,tanh,"(50,)",adaptive,2000,lbfgs
+0.38925704956054685,0.01084902686677685,0.0034201622009277345,0.0002348708556576185,"{'activation': 'tanh', 'hidden_layer_sizes': (50,), 'learning_rate': 'adaptive', 'max_iter': 2000, 'solver': 'sgd'}",0.5,0.5555555555555556,0.6666666666666666,0.6153846153846154,0.5,0.5675213675213675,0.0653943906440635,41,0.26666666666666666,0.3333333333333333,0.5333333333333333,0.5333333333333333,0.3333333333333333,0.39999999999999997,0.1115546702045434,46,0.3478260869565218,0.4166666666666667,0.5925925925925926,0.5714285714285715,0.4,0.4657027835288705,0.09787264989404884,12,MLPClassifier,,,,,,,tanh,"(50,)",adaptive,2000,sgd
+1.8049395561218262,0.10115705318271331,0.003344440460205078,8.549340523387506e-05,"{'activation': 'tanh', 'hidden_layer_sizes': (50,), 'learning_rate': 'adaptive', 'max_iter': 2000, 'solver': 'adam'}",0.4166666666666667,0.625,0.5294117647058824,0.6153846153846154,0.35714285714285715,0.5087211807800044,0.1065607564812983,53,0.3333333333333333,0.3333333333333333,0.6,0.5333333333333333,0.3333333333333333,0.42666666666666664,0.11623730516108463,31,0.3703703703703704,0.43478260869565216,0.5625,0.5714285714285715,0.3448275862068965,0.4567818273402981,0.09466241660230847,18,MLPClassifier,,,,,,,tanh,"(50,)",adaptive,2000,adam
+0.10444097518920899,0.0104442568515255,0.003529262542724609,3.269427290173611e-05,"{'activation': 'tanh', 'hidden_layer_sizes': (100, 20), 'learning_rate': 'constant', 'max_iter': 2000, 'solver': 'lbfgs'}",0.3684210526315789,0.5454545454545454,0.5,0.42857142857142855,0.375,0.4434894053315106,0.06948228459176636,72,0.4666666666666667,0.4,0.6,0.4,0.2,0.4133333333333334,0.12927146286443542,39,0.4117647058823529,0.4615384615384615,0.5454545454545454,0.4137931034482759,0.26086956521739135,0.4186840763082055,0.09261152130754115,48,MLPClassifier,,,,,,,tanh,"(100, 20)",constant,2000,lbfgs
+0.4584303855895996,0.047589868422257646,0.0035296916961669923,9.227527175677256e-05,"{'activation': 'tanh', 'hidden_layer_sizes': (100, 20), 'learning_rate': 'constant', 'max_iter': 2000, 'solver': 'sgd'}",0.625,0.5,0.625,0.5454545454545454,0.5,0.5590909090909091,0.056315962242563036,43,0.3333333333333333,0.3333333333333333,0.3333333333333333,0.4,0.26666666666666666,0.3333333333333333,0.0421637021355784,62,0.43478260869565216,0.4,0.43478260869565216,0.4615384615384615,0.3478260869565218,0.41578595317725753,0.0391989224279597,50,MLPClassifier,,,,,,,tanh,"(100, 20)",constant,2000,sgd
+2.048261785507202,0.12304296298465582,0.0035881996154785156,8.7318115225509e-05,"{'activation': 'tanh', 'hidden_layer_sizes': (100, 20), 'learning_rate': 'constant', 'max_iter': 2000, 'solver': 'adam'}",0.42857142857142855,0.6363636363636364,0.5882352941176471,0.4666666666666667,0.4,0.5039674051438757,0.09221166830710863,54,0.4,0.4666666666666667,0.6666666666666666,0.4666666666666667,0.4,0.48,0.0979795897113271,12,0.4137931034482759,0.5384615384615385,0.625,0.4666666666666667,0.4000000000000001,0.48878426171529626,0.08368759303345308,5,MLPClassifier,,,,,,,tanh,"(100, 20)",constant,2000,adam
+0.11295123100280761,0.007178868221363568,0.003632211685180664,8.309174468394447e-05,"{'activation': 'tanh', 'hidden_layer_sizes': (100, 20), 'learning_rate': 'invscaling', 'max_iter': 2000, 'solver': 'lbfgs'}",0.3333333333333333,0.6666666666666666,0.5384615384615384,0.3333333333333333,0.4,0.4543589743589743,0.12992082755766138,66,0.2,0.5333333333333333,0.4666666666666667,0.4,0.4,0.4,0.1115546702045434,44,0.25,0.5925925925925926,0.5,0.3636363636363636,0.4000000000000001,0.42124579124579126,0.11718131467330822,44,MLPClassifier,,,,,,,tanh,"(100, 20)",invscaling,2000,lbfgs
+1.0702889919281007,0.48329102469891705,0.003557586669921875,3.6541596692378546e-05,"{'activation': 'tanh', 'hidden_layer_sizes': (100, 20), 'learning_rate': 'invscaling', 'max_iter': 2000, 'solver': 'sgd'}",0.0847457627118644,0.0625,0.06015037593984962,0.07894736842105263,0.15517241379310345,0.08830318417317402,0.034727698831627046,102,0.3333333333333333,0.06666666666666667,0.5333333333333333,0.2,0.6,0.3466666666666667,0.19955506062794354,59,0.13513513513513511,0.06451612903225808,0.10810810810810811,0.11320754716981134,0.24657534246575347,0.13350845238221323,0.06099838137664438,99,MLPClassifier,,,,,,,tanh,"(100, 20)",invscaling,2000,sgd
+2.093674898147583,0.20052294807354895,0.0038346767425537108,0.0002558114874855347,"{'activation': 'tanh', 'hidden_layer_sizes': (100, 20), 'learning_rate': 'invscaling', 'max_iter': 2000, 'solver': 'adam'}",0.38461538461538464,0.6363636363636364,0.5333333333333333,0.46153846153846156,0.3333333333333333,0.46983682983682995,0.1074502167964543,61,0.3333333333333333,0.4666666666666667,0.5333333333333333,0.4,0.3333333333333333,0.4133333333333334,0.07774602526460402,39,0.3571428571428571,0.5384615384615385,0.5333333333333333,0.42857142857142855,0.3333333333333333,0.43816849816849823,0.08574696474821124,30,MLPClassifier,,,,,,,tanh,"(100, 20)",invscaling,2000,adam
+0.12708439826965331,0.02499835821138367,0.003885030746459961,0.0004338437838139363,"{'activation': 'tanh', 'hidden_layer_sizes': (100, 20), 'learning_rate': 'adaptive', 'max_iter': 2000, 'solver': 'lbfgs'}",0.46153846153846156,0.5833333333333334,0.47058823529411764,0.34782608695652173,0.375,0.44765722342448677,0.0829118264196136,68,0.4,0.4666666666666667,0.5333333333333333,0.5333333333333333,0.2,0.42666666666666664,0.12364824660660936,31,0.42857142857142855,0.5185185185185186,0.5,0.4210526315789474,0.26086956521739135,0.42580242877725716,0.09092263776156755,37,MLPClassifier,,,,,,,tanh,"(100, 20)",adaptive,2000,lbfgs
+0.6415410041809082,0.05320448544756794,0.0036493778228759766,0.00015619387598895482,"{'activation': 'tanh', 'hidden_layer_sizes': (100, 20), 'learning_rate': 'adaptive', 'max_iter': 2000, 'solver': 'sgd'}",0.5555555555555556,0.8333333333333334,0.625,0.5833333333333334,0.4444444444444444,0.6083333333333334,0.12741494379088575,22,0.3333333333333333,0.3333333333333333,0.3333333333333333,0.4666666666666667,0.26666666666666666,0.3466666666666667,0.06531972647421809,59,0.4166666666666667,0.47619047619047616,0.43478260869565216,0.5185185185185186,0.33333333333333337,0.4358983206809294,0.062193621610738846,33,MLPClassifier,,,,,,,tanh,"(100, 20)",adaptive,2000,sgd
+2.061586618423462,0.06256793066074529,0.003636884689331055,7.811320390904681e-05,"{'activation': 'tanh', 'hidden_layer_sizes': (100, 20), 'learning_rate': 'adaptive', 'max_iter': 2000, 'solver': 'adam'}",0.3333333333333333,0.6,0.5294117647058824,0.46153846153846156,0.38461538461538464,0.4617797888386124,0.09605394982307672,64,0.26666666666666666,0.4,0.6,0.4,0.3333333333333333,0.39999999999999997,0.1115546702045434,46,0.2962962962962963,0.48,0.5625,0.42857142857142855,0.3571428571428571,0.4249021164021164,0.0928581632166312,39,MLPClassifier,,,,,,,tanh,"(100, 20)",adaptive,2000,adam
+0.10318431854248047,0.03479934907813475,0.0030486583709716797,0.00017016346716528863,"{'activation': 'relu', 'hidden_layer_sizes': (10,), 'learning_rate': 'constant', 'max_iter': 2000, 'solver': 'lbfgs'}",0.375,0.35294117647058826,0.42105263157894735,0.5,0.3157894736842105,0.39295665634674926,0.06344464589177647,88,0.4,0.4,0.5333333333333333,0.5333333333333333,0.4,0.4533333333333333,0.06531972647421808,20,0.38709677419354843,0.37500000000000006,0.47058823529411764,0.5161290322580646,0.35294117647058826,0.4203510436432638,0.06229583022396728,46,MLPClassifier,,,,,,,relu,"(10,)",constant,2000,lbfgs
+0.26506819725036623,0.03966394456860825,0.0028215885162353516,9.204162755871538e-05,"{'activation': 'relu', 'hidden_layer_sizes': (10,), 'learning_rate': 'constant', 'max_iter': 2000, 'solver': 'sgd'}",0.2857142857142857,1.0,0.4,0.0,0.3333333333333333,0.4038095238095238,0.3278515232772188,83,0.13333333333333333,0.13333333333333333,0.13333333333333333,0.0,0.2,0.12000000000000002,0.06531972647421809,99,0.18181818181818182,0.23529411764705882,0.2,0.0,0.25,0.17342245989304814,0.09004934765138274,94,MLPClassifier,,,,,,,relu,"(10,)",constant,2000,sgd
+0.34603633880615237,0.0649265436773322,0.00281219482421875,8.735279320805418e-05,"{'activation': 'relu', 'hidden_layer_sizes': (10,), 'learning_rate': 'constant', 'max_iter': 2000, 'solver': 'adam'}",0.38461538461538464,0.8,0.6153846153846154,0.5454545454545454,0.5,0.5690909090909091,0.13771388168398224,40,0.3333333333333333,0.26666666666666666,0.5333333333333333,0.4,0.3333333333333333,0.3733333333333333,0.09043106644167025,54,0.3571428571428571,0.4,0.5714285714285715,0.4615384615384615,0.4,0.43802197802197806,0.07454276354002475,31,MLPClassifier,,,,,,,relu,"(10,)",constant,2000,adam
+0.10606822967529297,0.053533481321843664,0.0027156352996826174,2.9628206893612443e-05,"{'activation': 'relu', 'hidden_layer_sizes': (10,), 'learning_rate': 'invscaling', 'max_iter': 2000, 'solver': 'lbfgs'}",0.3333333333333333,0.375,0.4,0.3125,0.2727272727272727,0.33871212121212124,0.045040029130305526,93,0.4666666666666667,0.4,0.5333333333333333,0.3333333333333333,0.2,0.3866666666666666,0.11469767022723501,50,0.3888888888888889,0.38709677419354843,0.4571428571428572,0.3225806451612903,0.23076923076923075,0.3572956792311631,0.07624928962977434,86,MLPClassifier,,,,,,,relu,"(10,)",invscaling,2000,lbfgs
+0.12356314659118653,0.12421942190111056,0.0026909828186035155,6.570781436380588e-05,"{'activation': 'relu', 'hidden_layer_sizes': (10,), 'learning_rate': 'invscaling', 'max_iter': 2000, 'solver': 'sgd'}",0.1,0.09243697478991597,0.3333333333333333,0.10067114093959731,0.2727272727272727,0.17983374435802385,0.10243990558139315,97,1.0,0.7333333333333333,0.06666666666666667,1.0,0.2,0.6,0.3955305859784354,4,0.18181818181818182,0.16417910447761194,0.1111111111111111,0.18292682926829268,0.23076923076923075,0.17416089148888564,0.03850945523905075,93,MLPClassifier,,,,,,,relu,"(10,)",invscaling,2000,sgd
+0.3445711612701416,0.0712838935691839,0.0027398586273193358,4.1858155276809265e-05,"{'activation': 'relu', 'hidden_layer_sizes': (10,), 'learning_rate': 'invscaling', 'max_iter': 2000, 'solver': 'adam'}",0.5,0.75,0.6666666666666666,0.5833333333333334,0.2222222222222222,0.5444444444444445,0.18138697463682787,45,0.3333333333333333,0.2,0.5333333333333333,0.4666666666666667,0.13333333333333333,0.3333333333333333,0.1520233900132184,62,0.4,0.31578947368421056,0.5925925925925926,0.5185185185185186,0.16666666666666669,0.3987134502923977,0.1501107105368813,77,MLPClassifier,,,,,,,relu,"(10,)",invscaling,2000,adam
+0.07240762710571289,0.03113599689325272,0.0027058601379394533,6.269518350269662e-05,"{'activation': 'relu', 'hidden_layer_sizes': (10,), 'learning_rate': 'adaptive', 'max_iter': 2000, 'solver': 'lbfgs'}",0.3181818181818182,0.6666666666666666,0.46153846153846156,0.38095238095238093,0.29411764705882354,0.4242913948796302,0.13432485500860764,76,0.4666666666666667,0.5333333333333333,0.4,0.5333333333333333,0.3333333333333333,0.4533333333333333,0.077746025264604,20,0.3783783783783784,0.5925925925925926,0.42857142857142855,0.4444444444444444,0.3125,0.4312973687973688,0.09285435902258966,35,MLPClassifier,,,,,,,relu,"(10,)",adaptive,2000,lbfgs
+0.2610966682434082,0.014935323166215772,0.0028059959411621095,5.1771517082965966e-05,"{'activation': 'relu', 'hidden_layer_sizes': (10,), 'learning_rate': 'adaptive', 'max_iter': 2000, 'solver': 'sgd'}",0.6666666666666666,0.5714285714285714,0.8,0.4444444444444444,0.6666666666666666,0.6298412698412699,0.11785604723984365,17,0.26666666666666666,0.26666666666666666,0.26666666666666666,0.26666666666666666,0.26666666666666666,0.26666666666666666,0.0,93,0.3809523809523809,0.36363636363636365,0.4,0.33333333333333337,0.3809523809523809,0.3717748917748918,0.022401332126470948,83,MLPClassifier,,,,,,,relu,"(10,)",adaptive,2000,sgd
+0.2892911911010742,0.030385564059187197,0.0028136253356933595,6.413617539858511e-05,"{'activation': 'relu', 'hidden_layer_sizes': (10,), 'learning_rate': 'adaptive', 'max_iter': 2000, 'solver': 'adam'}",0.625,1.0,0.5555555555555556,0.5454545454545454,0.2222222222222222,0.5896464646464646,0.24809497623477872,38,0.3333333333333333,0.3333333333333333,0.3333333333333333,0.4,0.13333333333333333,0.30666666666666664,0.09043106644167025,85,0.43478260869565216,0.5,0.4166666666666667,0.4615384615384615,0.16666666666666669,0.3959308807134894,0.11801376672200453,78,MLPClassifier,,,,,,,relu,"(10,)",adaptive,2000,adam
+0.04955000877380371,0.00372355649319816,0.002831697463989258,2.034983231097393e-05,"{'activation': 'relu', 'hidden_layer_sizes': (50,), 'learning_rate': 'constant', 'max_iter': 2000, 'solver': 'lbfgs'}",0.4375,0.5384615384615384,0.47058823529411764,0.35714285714285715,0.21428571428571427,0.4035956690368455,0.1111957130722278,84,0.4666666666666667,0.4666666666666667,0.5333333333333333,0.3333333333333333,0.2,0.4,0.11925695879998878,44,0.45161290322580644,0.5,0.5,0.3448275862068965,0.20689655172413796,0.40066740823136826,0.1122449122147155,67,MLPClassifier,,,,,,,relu,"(50,)",constant,2000,lbfgs
+0.3405783653259277,0.042148921484528666,0.003117942810058594,3.7836840712640754e-05,"{'activation': 'relu', 'hidden_layer_sizes': (50,), 'learning_rate': 'constant', 'max_iter': 2000, 'solver': 'sgd'}",0.6666666666666666,1.0,0.7142857142857143,0.4444444444444444,0.8333333333333334,0.7317460317460317,0.18404486400807296,3,0.26666666666666666,0.26666666666666666,0.3333333333333333,0.26666666666666666,0.3333333333333333,0.29333333333333333,0.03265986323710903,88,0.3809523809523809,0.4210526315789474,0.4545454545454545,0.33333333333333337,0.47619047619047616,0.41321485532011853,0.05129815357353517,54,MLPClassifier,,,,,,,relu,"(50,)",constant,2000,sgd
+1.406182336807251,0.12741938724800464,0.003235006332397461,0.00012487386517305267,"{'activation': 'relu', 'hidden_layer_sizes': (50,), 'learning_rate': 'constant', 'max_iter': 2000, 'solver': 'adam'}",0.38461538461538464,0.6666666666666666,0.5555555555555556,0.6363636363636364,0.36363636363636365,0.5213675213675214,0.12576568275645825,51,0.3333333333333333,0.5333333333333333,0.3333333333333333,0.4666666666666667,0.26666666666666666,0.3866666666666666,0.09797958971132713,50,0.3571428571428571,0.5925925925925926,0.4166666666666667,0.5384615384615385,0.30769230769230765,0.44251119251119253,0.10757575883244426,27,MLPClassifier,,,,,,,relu,"(50,)",constant,2000,adam
+0.052970504760742186,0.005452908684095196,0.0033062458038330077,2.9562211757353154e-05,"{'activation': 'relu', 'hidden_layer_sizes': (50,), 'learning_rate': 'invscaling', 'max_iter': 2000, 'solver': 'lbfgs'}",0.35294117647058826,0.4666666666666667,0.5789473684210527,0.3125,0.3,0.4022110423116615,0.10612534517325908,85,0.4,0.4666666666666667,0.7333333333333333,0.3333333333333333,0.2,0.42666666666666664,0.1768866554856213,31,0.37500000000000006,0.4666666666666667,0.6470588235294117,0.3225806451612903,0.24,0.4102612270714737,0.13940932488665292,59,MLPClassifier,,,,,,,relu,"(50,)",invscaling,2000,lbfgs
+0.1470343589782715,0.06399741224635598,0.0034874439239501952,0.0001061789693931547,"{'activation': 'relu', 'hidden_layer_sizes': (50,), 'learning_rate': 'invscaling', 'max_iter': 2000, 'solver': 'sgd'}",0.11392405063291139,0.12264150943396226,0.1095890410958904,0.0,0.10273972602739725,0.08977886543803226,0.0453511080011298,101,0.6,0.8666666666666667,0.5333333333333333,0.0,1.0,0.6,0.34512477614786097,4,0.19148936170212763,0.21487603305785125,0.18181818181818182,0.0,0.18633540372670807,0.15490379606097376,0.07828497809788673,97,MLPClassifier,,,,,,,relu,"(50,)",invscaling,2000,sgd
+1.6083802223205566,0.301994580639607,0.004079341888427734,0.001223078545507362,"{'activation': 'relu', 'hidden_layer_sizes': (50,), 'learning_rate': 'invscaling', 'max_iter': 2000, 'solver': 'adam'}",0.46153846153846156,0.45454545454545453,0.6666666666666666,0.42857142857142855,0.2222222222222222,0.4467088467088467,0.14098651209890012,69,0.4,0.3333333333333333,0.6666666666666666,0.4,0.13333333333333333,0.3866666666666666,0.17074997966487596,50,0.42857142857142855,0.3846153846153846,0.6666666666666666,0.4137931034482759,0.16666666666666669,0.41206264999368447,0.15878997484418497,56,MLPClassifier,,,,,,,relu,"(50,)",invscaling,2000,adam
+0.054295587539672854,0.006501607057676768,0.0033596038818359377,4.2841211503933764e-05,"{'activation': 'relu', 'hidden_layer_sizes': (50,), 'learning_rate': 'adaptive', 'max_iter': 2000, 'solver': 'lbfgs'}",0.38461538461538464,0.5,0.5,0.3888888888888889,0.25,0.4047008547008547,0.09246583477810955,82,0.3333333333333333,0.4,0.4666666666666667,0.4666666666666667,0.2,0.3733333333333334,0.09977753031397177,53,0.3571428571428571,0.4444444444444445,0.4827586206896552,0.42424242424242425,0.22222222222222224,0.3861621137483207,0.09153822213642739,82,MLPClassifier,,,,,,,relu,"(50,)",adaptive,2000,lbfgs
+0.42258143424987793,0.02879488291820619,0.0033105850219726563,6.506592852613603e-05,"{'activation': 'relu', 'hidden_layer_sizes': (50,), 'learning_rate': 'adaptive', 'max_iter': 2000, 'solver': 'sgd'}",0.5714285714285714,0.5,0.8,0.5,1.0,0.6742857142857143,0.1965830561345094,7,0.26666666666666666,0.2,0.26666666666666666,0.26666666666666666,0.2,0.24,0.03265986323710903,95,0.36363636363636365,0.28571428571428575,0.4,0.3478260869565218,0.33333333333333337,0.34610201392810086,0.03748369158197312,87,MLPClassifier,,,,,,,relu,"(50,)",adaptive,2000,sgd
+1.624642276763916,0.19872194693441425,0.0034543991088867186,0.0003602590534645065,"{'activation': 'relu', 'hidden_layer_sizes': (50,), 'learning_rate': 'adaptive', 'max_iter': 2000, 'solver': 'adam'}",0.38461538461538464,0.75,0.6428571428571429,0.46153846153846156,0.42857142857142855,0.5335164835164835,0.13941759584310953,48,0.3333333333333333,0.6,0.6,0.4,0.2,0.42666666666666664,0.155492050529208,31,0.3571428571428571,0.6666666666666665,0.6206896551724138,0.42857142857142855,0.27272727272727276,0.46915957605612774,0.15149337182982042,11,MLPClassifier,,,,,,,relu,"(50,)",adaptive,2000,adam
+0.12689056396484374,0.019380635646914092,0.0039041519165039064,0.0005076504639596825,"{'activation': 'relu', 'hidden_layer_sizes': (100, 20), 'learning_rate': 'constant', 'max_iter': 2000, 'solver': 'lbfgs'}",0.375,0.3888888888888889,0.5555555555555556,0.5,0.2777777777777778,0.41944444444444445,0.09797328938082443,79,0.4,0.4666666666666667,0.6666666666666666,0.4666666666666667,0.3333333333333333,0.4666666666666667,0.11155467020454339,13,0.38709677419354843,0.42424242424242425,0.606060606060606,0.4827586206896552,0.303030303030303,0.4406377456433074,0.10120290639445545,28,MLPClassifier,,,,,,,relu,"(100, 20)",constant,2000,lbfgs
+0.6938549995422363,0.0685513963084081,0.003431224822998047,7.781649329966713e-05,"{'activation': 'relu', 'hidden_layer_sizes': (100, 20), 'learning_rate': 'constant', 'max_iter': 2000, 'solver': 'sgd'}",0.5714285714285714,1.0,0.75,0.5555555555555556,0.6666666666666666,0.7087301587301587,0.1616322013273373,4,0.26666666666666666,0.2,0.4,0.3333333333333333,0.26666666666666666,0.29333333333333333,0.06798692684790379,88,0.36363636363636365,0.33333333333333337,0.5217391304347827,0.4166666666666667,0.3809523809523809,0.4032655750047055,0.06507602329837384,66,MLPClassifier,,,,,,,relu,"(100, 20)",constant,2000,sgd
+1.1876333236694336,0.08979259701925384,0.0034419536590576173,0.0001254144819289782,"{'activation': 'relu', 'hidden_layer_sizes': (100, 20), 'learning_rate': 'constant', 'max_iter': 2000, 'solver': 'adam'}",0.35714285714285715,0.7,0.6666666666666666,0.5454545454545454,0.38461538461538464,0.5307758907758908,0.14059152815711978,49,0.3333333333333333,0.4666666666666667,0.6666666666666666,0.4,0.3333333333333333,0.44000000000000006,0.12364824660660938,24,0.3448275862068965,0.56,0.6666666666666666,0.4615384615384615,0.3571428571428571,0.47803511431097645,0.12241848550740289,8,MLPClassifier,,,,,,,relu,"(100, 20)",constant,2000,adam
+0.10192203521728516,0.007974414784472122,0.0034714221954345705,0.00012783357681436726,"{'activation': 'relu', 'hidden_layer_sizes': (100, 20), 'learning_rate': 'invscaling', 'max_iter': 2000, 'solver': 'lbfgs'}",0.4666666666666667,0.4,0.5789473684210527,0.4,0.26666666666666666,0.4224561403508772,0.10170839856152558,77,0.4666666666666667,0.4,0.7333333333333333,0.4,0.26666666666666666,0.4533333333333333,0.154344492037203,20,0.4666666666666667,0.4000000000000001,0.6470588235294117,0.4000000000000001,0.26666666666666666,0.4360784313725491,0.12389674608676417,32,MLPClassifier,,,,,,,relu,"(100, 20)",invscaling,2000,lbfgs
+0.09777464866638183,0.027837176788495482,0.003357219696044922,0.0001439564509974002,"{'activation': 'relu', 'hidden_layer_sizes': (100, 20), 'learning_rate': 'invscaling', 'max_iter': 2000, 'solver': 'sgd'}",0.22580645161290322,0.0,0.0,0.5714285714285714,1.0,0.35944700460829493,0.3824940313849434,90,0.4666666666666667,0.0,0.0,0.26666666666666666,0.06666666666666667,0.16,0.18184242262647807,98,0.3043478260869565,0.0,0.0,0.36363636363636365,0.125,0.158596837944664,0.15147225433874276,96,MLPClassifier,,,,,,,relu,"(100, 20)",invscaling,2000,sgd
+1.173351764678955,0.03771937510383487,0.0034332275390625,5.926512339824602e-05,"{'activation': 'relu', 'hidden_layer_sizes': (100, 20), 'learning_rate': 'invscaling', 'max_iter': 2000, 'solver': 'adam'}",0.4,0.7,0.6666666666666666,0.7,0.25,0.5433333333333333,0.1848723283181606,46,0.4,0.4666666666666667,0.6666666666666666,0.4666666666666667,0.13333333333333333,0.42666666666666664,0.17178798302300166,31,0.4000000000000001,0.56,0.6666666666666666,0.56,0.1739130434782609,0.47211594202898555,0.171710732073877,10,MLPClassifier,,,,,,,relu,"(100, 20)",invscaling,2000,adam
+0.15829339027404785,0.07911208262281175,0.0034626007080078127,0.00010584059995727836,"{'activation': 'relu', 'hidden_layer_sizes': (100, 20), 'learning_rate': 'adaptive', 'max_iter': 2000, 'solver': 'lbfgs'}",0.4166666666666667,0.5454545454545454,0.5,0.5625,0.3333333333333333,0.47159090909090906,0.08562617087467662,60,0.3333333333333333,0.4,0.6,0.6,0.26666666666666666,0.44000000000000006,0.13727506854649332,24,0.3703703703703704,0.4615384615384615,0.5454545454545454,0.5806451612903225,0.2962962962962963,0.4508609669899992,0.10608945719305136,21,MLPClassifier,,,,,,,relu,"(100, 20)",adaptive,2000,lbfgs
+0.7347156524658203,0.07182041177507488,0.0034441471099853514,6.154994385330038e-05,"{'activation': 'relu', 'hidden_layer_sizes': (100, 20), 'learning_rate': 'adaptive', 'max_iter': 2000, 'solver': 'sgd'}",0.625,1.0,0.6666666666666666,0.4,0.8,0.6983333333333333,0.19835434062415785,5,0.3333333333333333,0.26666666666666666,0.26666666666666666,0.13333333333333333,0.26666666666666666,0.2533333333333333,0.06531972647421809,94,0.43478260869565216,0.4210526315789474,0.3809523809523809,0.2,0.4,0.367357524245396,0.08566002432699567,85,MLPClassifier,,,,,,,relu,"(100, 20)",adaptive,2000,sgd
+1.2371012687683105,0.07587795277792217,0.003482151031494141,7.824190178678591e-05,"{'activation': 'relu', 'hidden_layer_sizes': (100, 20), 'learning_rate': 'adaptive', 'max_iter': 2000, 'solver': 'adam'}",0.4666666666666667,0.5833333333333334,0.5625,0.5833333333333334,0.23076923076923078,0.4853205128205128,0.1343979318154324,58,0.4666666666666667,0.4666666666666667,0.6,0.4666666666666667,0.2,0.44000000000000006,0.13063945294843615,24,0.4666666666666667,0.5185185185185186,0.5806451612903225,0.5185185185185186,0.21428571428571427,0.45972691585594816,0.12792472062564747,15,MLPClassifier,,,,,,,relu,"(100, 20)",adaptive,2000,adam

bug-2022/requirements.txt

@@ -1,2 +1,3 @@
 javalang==0.13.0
 pandas==1.5.2
+scikit_learn==1.2.1

bug-2022/train_classifiers.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+import os
+import pandas as pd
+import glob
+
+from sklearn import preprocessing
+from sklearn.metrics import f1_score, precision_score, recall_score, make_scorer
+from sklearn.model_selection import GridSearchCV
+
+from sklearn.neural_network import MLPClassifier
+from sklearn.naive_bayes import GaussianNB
+from sklearn.svm import SVC
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.ensemble import RandomForestClassifier
+
+DIR: str = os.path.dirname(os.path.realpath(__file__))
+IN_DIR: str = DIR + '/metrics'
+OUT_DIR: str = DIR + '/models'
+RANDOM: int = 42
+
+
+def clean_output():
+    filelist = glob.glob(OUT_DIR + '/models.csv')
+    for f in filelist:
+        os.remove(f)
+
+
+def get_classifiers() -> list:
+    return [
+        (GaussianNB(), {}),
+        (SVC(), {
+            'kernel': ['linear', 'poly', 'rbf', 'sigmoid'],
+            'gamma': ['scale', 'auto']
+        }),
+        (DecisionTreeClassifier(), {
+            'criterion': ['gini', 'entropy'],
+            'splitter': ['best', 'random']
+        }),
+        (RandomForestClassifier(), {
+            'criterion': ['gini', 'entropy'],
+            'max_features': ['sqrt', 'log2'],
+            'class_weight': ['balanced', 'balanced_subsample']
+        }),
+        (MLPClassifier(), {
+            'max_iter': [2000],
+            'hidden_layer_sizes': [(10,), (50,), (100, 20,)],
+            'activation':  ['identity', 'logistic', 'tanh', 'relu'],
+            'solver': ['lbfgs', 'sgd', 'adam'],
+            'learning_rate': ['constant', 'invscaling', 'adaptive']
+        })
+    ]
+
+
+def main():
+    clean_output()
+
+    # load dataset
+    df = pd.read_csv(IN_DIR + '/feature_vectors_labeled.csv')
+    df = df.drop('class_name', axis=1)
+
+    X = df.iloc[:, :-1].to_numpy()
+    y = df.iloc[:, -1].to_numpy()
+
+    scaler = preprocessing.StandardScaler().fit(X)
+    X_scaled = scaler.transform(X)
+
+    df = None
+
+    for classifier, grid in get_classifiers():
+        # cross-validation splits are same across calls as data is not shuffled
+        # see: https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.GridSearchCV.html
+        clf = GridSearchCV(classifier, grid, cv=5, scoring={
+            'precision': make_scorer(precision_score, average='binary', zero_division=0),
+            'recall': make_scorer(recall_score, average='binary'),
+            'f1': make_scorer(f1_score, average='binary')
+        }, refit='f1')
+
+        clf.fit(X_scaled, y)
+
+        df_classifier = pd.DataFrame(clf.cv_results_)
+        df_classifier['classifier'] = type(classifier).__name__
+
+        df = df_classifier if df is None else pd.concat(
+            [df, df_classifier], ignore_index=True)
+
+    df.to_csv(OUT_DIR + '/models.csv', index=False)
+
+
+if __name__ == '__main__':
+    main()