diff --git a/mutmuttest.py b/mutmuttest.py deleted file mode 100644 index 1615214..0000000 --- a/mutmuttest.py +++ /dev/null @@ -1,61 +0,0 @@ -import argparse -import os -from mutmut.__main__ import do_run, run_mutation_tests -from mutmut import mutations_by_type - -ROOT_DIR = os.path.dirname(__file__) -IN_SOURCE_DIR = os.path.join(ROOT_DIR, "benchmark") -IN_TEST_DIR = os.path.join(ROOT_DIR, "tests") -OUT_DIR = os.path.join(ROOT_DIR, "tests") - - -def run_mutmut(test_path: str, source_path: str): - run_mutation_tests(config=config, progress=progress, mutations_by_file=mutations_by_file) - - do_run(None, source_path, None, None, - 'python -m unittest ' + test_path, os.path.dirname(test_path), 0.0, 0.0, - False, False, '', None, None, - None, '', True, False, False, True) - - -# def run_mutpy(test_path: str, source_path: str): -# # run_genetic([source_path], random.randint(0, 500)) -# -# argv = ['-t', source_path, '-u', test_path, '-m'] -# argv = ['-t', source_path, '-u', test_path, '-m'] -# cfg = commandline.build_parser().parse_args(args=argv) -# -# mutation_controller = commandline.build_controller(cfg) -# mutation_controller.run() -# -# # stream = os.popen(f'mut.py --target \'{source_path}\' --unit-test \'{test_path}\' -m | tee log.txt') -# # output = stream.read() -# # score = re.search('Mutation score \\[.*\\]: (\d+\.\d+)\%', output).group(1) -# # print(output, file=sys.stderr) -# # print(f"Score is: {score}") - - -def main(): - parser = argparse.ArgumentParser(prog='mutmuttest.py', - description='Runs MutPy over generated test suite.') - parser.add_argument('file', type=str, help="Source file to test", - nargs="*") - - files = parser.parse_args().file - if len(files) == 0: - files = [os.path.splitext(f) for f in os.listdir(IN_SOURCE_DIR)] - to_test = [file[0] for file in files if file[1] == ".py"] - else: - to_test = [os.path.splitext(os.path.basename(file))[0] for file in files] - - for filename in to_test: - for i in range(10): - source_path = os.path.join(IN_SOURCE_DIR, f"{filename}.py") - test_path = os.path.join(IN_TEST_DIR, f"test_{filename}.py") - run_mutmut(test_path, source_path) - break - break - - -if __name__ == "__main__": - main() diff --git a/muttest.py b/muttest.py index 5ff5653..310ed0b 100644 --- a/muttest.py +++ b/muttest.py @@ -1,29 +1,49 @@ import argparse import os -from mutmut.__main__ import do_run, run_mutation_tests -from mutmut import mutations_by_type +import re +from collections import defaultdict + +import pandas as pd +from tqdm import tqdm + from mutpy import commandline +import sys +from io import StringIO +import contextlib +import subprocess + +from typing import List, Dict, DefaultDict ROOT_DIR = os.path.dirname(__file__) IN_SOURCE_DIR = os.path.join(ROOT_DIR, "benchmark") IN_TEST_DIR = os.path.join(ROOT_DIR, "tests") OUT_DIR = os.path.join(ROOT_DIR, "tests") +MUT_PY_PATH = os.path.join(ROOT_DIR, 'env37', 'bin', 'mut.py') +REPS: int = 10 -def run_mutpy(test_path: str, source_path: str): - # run_genetic([source_path], random.randint(0, 500)) +class OutputCapture(): + result: str - argv = ['-t', source_path, '-u', test_path, '-m'] - cfg = commandline.build_parser().parse_args(args=argv) - mutation_controller = commandline.build_controller(cfg) - mutation_controller.run() +@contextlib.contextmanager +def capture_stdout(): + old = sys.stdout + capturer = StringIO() + sys.stdout = capturer - # stream = os.popen(f'mut.py --target \'{source_path}\' --unit-test \'{test_path}\' -m | tee log.txt') - # output = stream.read() - # score = re.search('Mutation score \\[.*\\]: (\d+\.\d+)\%', output).group(1) - # print(output, file=sys.stderr) - # print(f"Score is: {score}") + data = OutputCapture() + yield data + + sys.stdout = old + data.result = capturer.getvalue() + + +def run_mutpy(test_path: str, source_path: str) -> float: + output = subprocess.check_output( + [sys.executable, MUT_PY_PATH, '-t', source_path, '-u', test_path]).decode('utf-8') + score = re.search('Mutation score \\[.*]: (\\d+\\.\\d+)%', output).group(1) + return float(score) def main(): @@ -39,11 +59,20 @@ def main(): else: to_test = [os.path.splitext(os.path.basename(file))[0] for file in files] - for filename in to_test: - for i in range(10): - source_path = os.path.join(IN_SOURCE_DIR, f"{filename}.py") - test_path = os.path.join(IN_TEST_DIR, f"test_{filename}.py") - run_mutpy(test_path, source_path) + scores: List[Dict[str, any]] = [] + + to_test = [e for t in to_test for e in ([t] * REPS)] + + for filename in tqdm(to_test, desc="Running mut.py over test suite"): + source_path = os.path.join(IN_SOURCE_DIR, f"{filename}.py") + test_path = os.path.join(IN_TEST_DIR, f"test_{filename}.py") + scores.append({ + 'file': filename, + 'score': run_mutpy(test_path, source_path) + }) + + df = pd.DataFrame.from_records(scores) + df.to_csv(os.path.join(OUT_DIR, 'mutation_results.csv')) if __name__ == "__main__": diff --git a/requirements.txt b/requirements.txt index bac24a2..3c1c5cb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,4 @@ deap==1.4.1 astunparse==1.6.3 frozendict==2.3.8 tqdm==4.66.1 -mutmut==2.4.4 \ No newline at end of file +pandas==1.3.5 \ No newline at end of file diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index be8f620..0000000 --- a/setup.cfg +++ /dev/null @@ -1,5 +0,0 @@ -[mutmut] -paths_to_mutate=benchmark/ -backup=False -runner=python -m unittest discover tests -tests_dir=tests/ \ No newline at end of file diff --git a/tests/mutation_results.csv b/tests/mutation_results.csv new file mode 100644 index 0000000..3385962 --- /dev/null +++ b/tests/mutation_results.csv @@ -0,0 +1,101 @@ +,file,score +0,anagram_check,38.5 +1,anagram_check,38.5 +2,anagram_check,38.5 +3,anagram_check,38.5 +4,anagram_check,38.5 +5,anagram_check,38.5 +6,anagram_check,38.5 +7,anagram_check,38.5 +8,anagram_check,38.5 +9,anagram_check,38.5 +10,caesar_cipher,64.7 +11,caesar_cipher,64.7 +12,caesar_cipher,64.7 +13,caesar_cipher,64.7 +14,caesar_cipher,64.7 +15,caesar_cipher,64.7 +16,caesar_cipher,64.7 +17,caesar_cipher,64.7 +18,caesar_cipher,64.7 +19,caesar_cipher,64.7 +20,check_armstrong,93.5 +21,check_armstrong,93.5 +22,check_armstrong,93.5 +23,check_armstrong,93.5 +24,check_armstrong,93.5 +25,check_armstrong,93.5 +26,check_armstrong,93.5 +27,check_armstrong,93.5 +28,check_armstrong,93.5 +29,check_armstrong,93.5 +30,common_divisor_count,80.9 +31,common_divisor_count,80.9 +32,common_divisor_count,80.9 +33,common_divisor_count,80.9 +34,common_divisor_count,80.9 +35,common_divisor_count,80.9 +36,common_divisor_count,80.9 +37,common_divisor_count,80.9 +38,common_divisor_count,80.9 +39,common_divisor_count,80.9 +40,exponentiation,71.4 +41,exponentiation,71.4 +42,exponentiation,71.4 +43,exponentiation,71.4 +44,exponentiation,71.4 +45,exponentiation,71.4 +46,exponentiation,71.4 +47,exponentiation,71.4 +48,exponentiation,71.4 +49,exponentiation,71.4 +50,gcd,60.9 +51,gcd,60.9 +52,gcd,60.9 +53,gcd,60.9 +54,gcd,60.9 +55,gcd,60.9 +56,gcd,60.9 +57,gcd,60.9 +58,gcd,60.9 +59,gcd,60.9 +60,longest_substring,69.6 +61,longest_substring,69.6 +62,longest_substring,69.6 +63,longest_substring,69.6 +64,longest_substring,69.6 +65,longest_substring,69.6 +66,longest_substring,69.6 +67,longest_substring,69.6 +68,longest_substring,69.6 +69,longest_substring,69.6 +70,rabin_karp,50.9 +71,rabin_karp,50.9 +72,rabin_karp,50.9 +73,rabin_karp,50.9 +74,rabin_karp,50.9 +75,rabin_karp,50.9 +76,rabin_karp,50.9 +77,rabin_karp,50.9 +78,rabin_karp,50.9 +79,rabin_karp,50.9 +80,railfence_cipher,86.2 +81,railfence_cipher,86.2 +82,railfence_cipher,86.2 +83,railfence_cipher,86.2 +84,railfence_cipher,86.2 +85,railfence_cipher,86.2 +86,railfence_cipher,86.2 +87,railfence_cipher,86.2 +88,railfence_cipher,86.2 +89,railfence_cipher,86.2 +90,zellers_birthday,65.0 +91,zellers_birthday,65.0 +92,zellers_birthday,65.0 +93,zellers_birthday,65.0 +94,zellers_birthday,65.0 +95,zellers_birthday,65.0 +96,zellers_birthday,65.0 +97,zellers_birthday,65.0 +98,zellers_birthday,65.0 +99,zellers_birthday,65.0