muttest.py invokes mut.py and stores csv results

mutmuttest.py

@@ -1,61 +0,0 @@
-import argparse
-import os
-from mutmut.__main__ import do_run, run_mutation_tests
-from mutmut import mutations_by_type
-
-ROOT_DIR = os.path.dirname(__file__)
-IN_SOURCE_DIR = os.path.join(ROOT_DIR, "benchmark")
-IN_TEST_DIR = os.path.join(ROOT_DIR, "tests")
-OUT_DIR = os.path.join(ROOT_DIR, "tests")
-
-
-def run_mutmut(test_path: str, source_path: str):
-    run_mutation_tests(config=config, progress=progress, mutations_by_file=mutations_by_file)
-
-    do_run(None, source_path, None, None,
-           'python -m unittest ' + test_path, os.path.dirname(test_path), 0.0, 0.0,
-           False, False, '', None, None,
-           None, '', True, False, False, True)
-
-
-# def run_mutpy(test_path: str, source_path: str):
-#     # run_genetic([source_path], random.randint(0, 500))
-#
-#     argv = ['-t', source_path, '-u', test_path, '-m']
-#     argv = ['-t', source_path, '-u', test_path, '-m']
-#     cfg = commandline.build_parser().parse_args(args=argv)
-#
-#     mutation_controller = commandline.build_controller(cfg)
-#     mutation_controller.run()
-#
-#     # stream = os.popen(f'mut.py --target \'{source_path}\' --unit-test \'{test_path}\' -m | tee log.txt')
-#     # output = stream.read()
-#     # score = re.search('Mutation score \\[.*\\]: (\d+\.\d+)\%', output).group(1)
-#     # print(output, file=sys.stderr)
-#     # print(f"Score is: {score}")
-
-
-def main():
-    parser = argparse.ArgumentParser(prog='mutmuttest.py',
-                                     description='Runs MutPy over generated test suite.')
-    parser.add_argument('file', type=str, help="Source file to test",
-                        nargs="*")
-
-    files = parser.parse_args().file
-    if len(files) == 0:
-        files = [os.path.splitext(f) for f in os.listdir(IN_SOURCE_DIR)]
-        to_test = [file[0] for file in files if file[1] == ".py"]
-    else:
-        to_test = [os.path.splitext(os.path.basename(file))[0] for file in files]
-
-    for filename in to_test:
-        for i in range(10):
-            source_path = os.path.join(IN_SOURCE_DIR, f"{filename}.py")
-            test_path = os.path.join(IN_TEST_DIR, f"test_{filename}.py")
-            run_mutmut(test_path, source_path)
-            break
-        break
-
-
-if __name__ == "__main__":
-    main()

muttest.py

@@ -1,29 +1,49 @@
 import argparse
 import os
-from mutmut.__main__ import do_run, run_mutation_tests
+import re
-from mutmut import mutations_by_type
+from collections import defaultdict
+
+import pandas as pd
+from tqdm import tqdm
+
 from mutpy import commandline
+import sys
+from io import StringIO
+import contextlib
+import subprocess
+
+from typing import List, Dict, DefaultDict
 
 ROOT_DIR = os.path.dirname(__file__)
 IN_SOURCE_DIR = os.path.join(ROOT_DIR, "benchmark")
 IN_TEST_DIR = os.path.join(ROOT_DIR, "tests")
 OUT_DIR = os.path.join(ROOT_DIR, "tests")
+MUT_PY_PATH = os.path.join(ROOT_DIR, 'env37', 'bin', 'mut.py')
+REPS: int = 10
 
 
-def run_mutpy(test_path: str, source_path: str):
+class OutputCapture():
-    # run_genetic([source_path], random.randint(0, 500))
+    result: str
 
-    argv = ['-t', source_path, '-u', test_path, '-m']
-    cfg = commandline.build_parser().parse_args(args=argv)
 
-    mutation_controller = commandline.build_controller(cfg)
+@contextlib.contextmanager
-    mutation_controller.run()
+def capture_stdout():
+    old = sys.stdout
+    capturer = StringIO()
+    sys.stdout = capturer
 
-    # stream = os.popen(f'mut.py --target \'{source_path}\' --unit-test \'{test_path}\' -m | tee log.txt')
+    data = OutputCapture()
-    # output = stream.read()
+    yield data
-    # score = re.search('Mutation score \\[.*\\]: (\d+\.\d+)\%', output).group(1)
+
-    # print(output, file=sys.stderr)
+    sys.stdout = old
-    # print(f"Score is: {score}")
+    data.result = capturer.getvalue()
+
+
+def run_mutpy(test_path: str, source_path: str) -> float:
+    output = subprocess.check_output(
+        [sys.executable, MUT_PY_PATH, '-t', source_path, '-u', test_path]).decode('utf-8')
+    score = re.search('Mutation score \\[.*]: (\\d+\\.\\d+)%', output).group(1)
+    return float(score)
 
 
 def main():
@@ -39,11 +59,20 @@ def main():
     else:
         to_test = [os.path.splitext(os.path.basename(file))[0] for file in files]
 
-    for filename in to_test:
+    scores: List[Dict[str, any]] = []
-        for i in range(10):
+
-            source_path = os.path.join(IN_SOURCE_DIR, f"{filename}.py")
+    to_test = [e for t in to_test for e in ([t] * REPS)]
-            test_path = os.path.join(IN_TEST_DIR, f"test_{filename}.py")
+
-            run_mutpy(test_path, source_path)
+    for filename in tqdm(to_test, desc="Running mut.py over test suite"):
+        source_path = os.path.join(IN_SOURCE_DIR, f"{filename}.py")
+        test_path = os.path.join(IN_TEST_DIR, f"test_{filename}.py")
+        scores.append({
+            'file': filename,
+            'score': run_mutpy(test_path, source_path)
+        })
+
+    df = pd.DataFrame.from_records(scores)
+    df.to_csv(os.path.join(OUT_DIR, 'mutation_results.csv'))
 
 
 if __name__ == "__main__":

requirements.txt

@@ -3,4 +3,4 @@ deap==1.4.1
 astunparse==1.6.3
 frozendict==2.3.8
 tqdm==4.66.1
-mutmut==2.4.4
+pandas==1.3.5

setup.cfg

@@ -1,5 +0,0 @@
-[mutmut]
-paths_to_mutate=benchmark/
-backup=False
-runner=python -m unittest discover tests
-tests_dir=tests/

tests/mutation_results.csv

@@ -0,0 +1,101 @@
+,file,score
+0,anagram_check,38.5
+1,anagram_check,38.5
+2,anagram_check,38.5
+3,anagram_check,38.5
+4,anagram_check,38.5
+5,anagram_check,38.5
+6,anagram_check,38.5
+7,anagram_check,38.5
+8,anagram_check,38.5
+9,anagram_check,38.5
+10,caesar_cipher,64.7
+11,caesar_cipher,64.7
+12,caesar_cipher,64.7
+13,caesar_cipher,64.7
+14,caesar_cipher,64.7
+15,caesar_cipher,64.7
+16,caesar_cipher,64.7
+17,caesar_cipher,64.7
+18,caesar_cipher,64.7
+19,caesar_cipher,64.7
+20,check_armstrong,93.5
+21,check_armstrong,93.5
+22,check_armstrong,93.5
+23,check_armstrong,93.5
+24,check_armstrong,93.5
+25,check_armstrong,93.5
+26,check_armstrong,93.5
+27,check_armstrong,93.5
+28,check_armstrong,93.5
+29,check_armstrong,93.5
+30,common_divisor_count,80.9
+31,common_divisor_count,80.9
+32,common_divisor_count,80.9
+33,common_divisor_count,80.9
+34,common_divisor_count,80.9
+35,common_divisor_count,80.9
+36,common_divisor_count,80.9
+37,common_divisor_count,80.9
+38,common_divisor_count,80.9
+39,common_divisor_count,80.9
+40,exponentiation,71.4
+41,exponentiation,71.4
+42,exponentiation,71.4
+43,exponentiation,71.4
+44,exponentiation,71.4
+45,exponentiation,71.4
+46,exponentiation,71.4
+47,exponentiation,71.4
+48,exponentiation,71.4
+49,exponentiation,71.4
+50,gcd,60.9
+51,gcd,60.9
+52,gcd,60.9
+53,gcd,60.9
+54,gcd,60.9
+55,gcd,60.9
+56,gcd,60.9
+57,gcd,60.9
+58,gcd,60.9
+59,gcd,60.9
+60,longest_substring,69.6
+61,longest_substring,69.6
+62,longest_substring,69.6
+63,longest_substring,69.6
+64,longest_substring,69.6
+65,longest_substring,69.6
+66,longest_substring,69.6
+67,longest_substring,69.6
+68,longest_substring,69.6
+69,longest_substring,69.6
+70,rabin_karp,50.9
+71,rabin_karp,50.9
+72,rabin_karp,50.9
+73,rabin_karp,50.9
+74,rabin_karp,50.9
+75,rabin_karp,50.9
+76,rabin_karp,50.9
+77,rabin_karp,50.9
+78,rabin_karp,50.9
+79,rabin_karp,50.9
+80,railfence_cipher,86.2
+81,railfence_cipher,86.2
+82,railfence_cipher,86.2
+83,railfence_cipher,86.2
+84,railfence_cipher,86.2
+85,railfence_cipher,86.2
+86,railfence_cipher,86.2
+87,railfence_cipher,86.2
+88,railfence_cipher,86.2
+89,railfence_cipher,86.2
+90,zellers_birthday,65.0
+91,zellers_birthday,65.0
+92,zellers_birthday,65.0
+93,zellers_birthday,65.0
+94,zellers_birthday,65.0
+95,zellers_birthday,65.0
+96,zellers_birthday,65.0
+97,zellers_birthday,65.0
+98,zellers_birthday,65.0
+99,zellers_birthday,65.0