muttest.py invokes mut.py and stores csv results
This commit is contained in:
parent
c3c4095450
commit
5cec4b4680
5 changed files with 149 additions and 85 deletions
|
|
@ -1,61 +0,0 @@
|
|||
import argparse
|
||||
import os
|
||||
from mutmut.__main__ import do_run, run_mutation_tests
|
||||
from mutmut import mutations_by_type
|
||||
|
||||
ROOT_DIR = os.path.dirname(__file__)
|
||||
IN_SOURCE_DIR = os.path.join(ROOT_DIR, "benchmark")
|
||||
IN_TEST_DIR = os.path.join(ROOT_DIR, "tests")
|
||||
OUT_DIR = os.path.join(ROOT_DIR, "tests")
|
||||
|
||||
|
||||
def run_mutmut(test_path: str, source_path: str):
|
||||
run_mutation_tests(config=config, progress=progress, mutations_by_file=mutations_by_file)
|
||||
|
||||
do_run(None, source_path, None, None,
|
||||
'python -m unittest ' + test_path, os.path.dirname(test_path), 0.0, 0.0,
|
||||
False, False, '', None, None,
|
||||
None, '', True, False, False, True)
|
||||
|
||||
|
||||
# def run_mutpy(test_path: str, source_path: str):
|
||||
# # run_genetic([source_path], random.randint(0, 500))
|
||||
#
|
||||
# argv = ['-t', source_path, '-u', test_path, '-m']
|
||||
# argv = ['-t', source_path, '-u', test_path, '-m']
|
||||
# cfg = commandline.build_parser().parse_args(args=argv)
|
||||
#
|
||||
# mutation_controller = commandline.build_controller(cfg)
|
||||
# mutation_controller.run()
|
||||
#
|
||||
# # stream = os.popen(f'mut.py --target \'{source_path}\' --unit-test \'{test_path}\' -m | tee log.txt')
|
||||
# # output = stream.read()
|
||||
# # score = re.search('Mutation score \\[.*\\]: (\d+\.\d+)\%', output).group(1)
|
||||
# # print(output, file=sys.stderr)
|
||||
# # print(f"Score is: {score}")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(prog='mutmuttest.py',
|
||||
description='Runs MutPy over generated test suite.')
|
||||
parser.add_argument('file', type=str, help="Source file to test",
|
||||
nargs="*")
|
||||
|
||||
files = parser.parse_args().file
|
||||
if len(files) == 0:
|
||||
files = [os.path.splitext(f) for f in os.listdir(IN_SOURCE_DIR)]
|
||||
to_test = [file[0] for file in files if file[1] == ".py"]
|
||||
else:
|
||||
to_test = [os.path.splitext(os.path.basename(file))[0] for file in files]
|
||||
|
||||
for filename in to_test:
|
||||
for i in range(10):
|
||||
source_path = os.path.join(IN_SOURCE_DIR, f"{filename}.py")
|
||||
test_path = os.path.join(IN_TEST_DIR, f"test_{filename}.py")
|
||||
run_mutmut(test_path, source_path)
|
||||
break
|
||||
break
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
65
muttest.py
65
muttest.py
|
|
@ -1,29 +1,49 @@
|
|||
import argparse
|
||||
import os
|
||||
from mutmut.__main__ import do_run, run_mutation_tests
|
||||
from mutmut import mutations_by_type
|
||||
import re
|
||||
from collections import defaultdict
|
||||
|
||||
import pandas as pd
|
||||
from tqdm import tqdm
|
||||
|
||||
from mutpy import commandline
|
||||
import sys
|
||||
from io import StringIO
|
||||
import contextlib
|
||||
import subprocess
|
||||
|
||||
from typing import List, Dict, DefaultDict
|
||||
|
||||
ROOT_DIR = os.path.dirname(__file__)
|
||||
IN_SOURCE_DIR = os.path.join(ROOT_DIR, "benchmark")
|
||||
IN_TEST_DIR = os.path.join(ROOT_DIR, "tests")
|
||||
OUT_DIR = os.path.join(ROOT_DIR, "tests")
|
||||
MUT_PY_PATH = os.path.join(ROOT_DIR, 'env37', 'bin', 'mut.py')
|
||||
REPS: int = 10
|
||||
|
||||
|
||||
def run_mutpy(test_path: str, source_path: str):
|
||||
# run_genetic([source_path], random.randint(0, 500))
|
||||
class OutputCapture():
|
||||
result: str
|
||||
|
||||
argv = ['-t', source_path, '-u', test_path, '-m']
|
||||
cfg = commandline.build_parser().parse_args(args=argv)
|
||||
|
||||
mutation_controller = commandline.build_controller(cfg)
|
||||
mutation_controller.run()
|
||||
@contextlib.contextmanager
|
||||
def capture_stdout():
|
||||
old = sys.stdout
|
||||
capturer = StringIO()
|
||||
sys.stdout = capturer
|
||||
|
||||
# stream = os.popen(f'mut.py --target \'{source_path}\' --unit-test \'{test_path}\' -m | tee log.txt')
|
||||
# output = stream.read()
|
||||
# score = re.search('Mutation score \\[.*\\]: (\d+\.\d+)\%', output).group(1)
|
||||
# print(output, file=sys.stderr)
|
||||
# print(f"Score is: {score}")
|
||||
data = OutputCapture()
|
||||
yield data
|
||||
|
||||
sys.stdout = old
|
||||
data.result = capturer.getvalue()
|
||||
|
||||
|
||||
def run_mutpy(test_path: str, source_path: str) -> float:
|
||||
output = subprocess.check_output(
|
||||
[sys.executable, MUT_PY_PATH, '-t', source_path, '-u', test_path]).decode('utf-8')
|
||||
score = re.search('Mutation score \\[.*]: (\\d+\\.\\d+)%', output).group(1)
|
||||
return float(score)
|
||||
|
||||
|
||||
def main():
|
||||
|
|
@ -39,11 +59,20 @@ def main():
|
|||
else:
|
||||
to_test = [os.path.splitext(os.path.basename(file))[0] for file in files]
|
||||
|
||||
for filename in to_test:
|
||||
for i in range(10):
|
||||
source_path = os.path.join(IN_SOURCE_DIR, f"{filename}.py")
|
||||
test_path = os.path.join(IN_TEST_DIR, f"test_{filename}.py")
|
||||
run_mutpy(test_path, source_path)
|
||||
scores: List[Dict[str, any]] = []
|
||||
|
||||
to_test = [e for t in to_test for e in ([t] * REPS)]
|
||||
|
||||
for filename in tqdm(to_test, desc="Running mut.py over test suite"):
|
||||
source_path = os.path.join(IN_SOURCE_DIR, f"{filename}.py")
|
||||
test_path = os.path.join(IN_TEST_DIR, f"test_{filename}.py")
|
||||
scores.append({
|
||||
'file': filename,
|
||||
'score': run_mutpy(test_path, source_path)
|
||||
})
|
||||
|
||||
df = pd.DataFrame.from_records(scores)
|
||||
df.to_csv(os.path.join(OUT_DIR, 'mutation_results.csv'))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -3,4 +3,4 @@ deap==1.4.1
|
|||
astunparse==1.6.3
|
||||
frozendict==2.3.8
|
||||
tqdm==4.66.1
|
||||
mutmut==2.4.4
|
||||
pandas==1.3.5
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
[mutmut]
|
||||
paths_to_mutate=benchmark/
|
||||
backup=False
|
||||
runner=python -m unittest discover tests
|
||||
tests_dir=tests/
|
||||
101
tests/mutation_results.csv
Normal file
101
tests/mutation_results.csv
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
,file,score
|
||||
0,anagram_check,38.5
|
||||
1,anagram_check,38.5
|
||||
2,anagram_check,38.5
|
||||
3,anagram_check,38.5
|
||||
4,anagram_check,38.5
|
||||
5,anagram_check,38.5
|
||||
6,anagram_check,38.5
|
||||
7,anagram_check,38.5
|
||||
8,anagram_check,38.5
|
||||
9,anagram_check,38.5
|
||||
10,caesar_cipher,64.7
|
||||
11,caesar_cipher,64.7
|
||||
12,caesar_cipher,64.7
|
||||
13,caesar_cipher,64.7
|
||||
14,caesar_cipher,64.7
|
||||
15,caesar_cipher,64.7
|
||||
16,caesar_cipher,64.7
|
||||
17,caesar_cipher,64.7
|
||||
18,caesar_cipher,64.7
|
||||
19,caesar_cipher,64.7
|
||||
20,check_armstrong,93.5
|
||||
21,check_armstrong,93.5
|
||||
22,check_armstrong,93.5
|
||||
23,check_armstrong,93.5
|
||||
24,check_armstrong,93.5
|
||||
25,check_armstrong,93.5
|
||||
26,check_armstrong,93.5
|
||||
27,check_armstrong,93.5
|
||||
28,check_armstrong,93.5
|
||||
29,check_armstrong,93.5
|
||||
30,common_divisor_count,80.9
|
||||
31,common_divisor_count,80.9
|
||||
32,common_divisor_count,80.9
|
||||
33,common_divisor_count,80.9
|
||||
34,common_divisor_count,80.9
|
||||
35,common_divisor_count,80.9
|
||||
36,common_divisor_count,80.9
|
||||
37,common_divisor_count,80.9
|
||||
38,common_divisor_count,80.9
|
||||
39,common_divisor_count,80.9
|
||||
40,exponentiation,71.4
|
||||
41,exponentiation,71.4
|
||||
42,exponentiation,71.4
|
||||
43,exponentiation,71.4
|
||||
44,exponentiation,71.4
|
||||
45,exponentiation,71.4
|
||||
46,exponentiation,71.4
|
||||
47,exponentiation,71.4
|
||||
48,exponentiation,71.4
|
||||
49,exponentiation,71.4
|
||||
50,gcd,60.9
|
||||
51,gcd,60.9
|
||||
52,gcd,60.9
|
||||
53,gcd,60.9
|
||||
54,gcd,60.9
|
||||
55,gcd,60.9
|
||||
56,gcd,60.9
|
||||
57,gcd,60.9
|
||||
58,gcd,60.9
|
||||
59,gcd,60.9
|
||||
60,longest_substring,69.6
|
||||
61,longest_substring,69.6
|
||||
62,longest_substring,69.6
|
||||
63,longest_substring,69.6
|
||||
64,longest_substring,69.6
|
||||
65,longest_substring,69.6
|
||||
66,longest_substring,69.6
|
||||
67,longest_substring,69.6
|
||||
68,longest_substring,69.6
|
||||
69,longest_substring,69.6
|
||||
70,rabin_karp,50.9
|
||||
71,rabin_karp,50.9
|
||||
72,rabin_karp,50.9
|
||||
73,rabin_karp,50.9
|
||||
74,rabin_karp,50.9
|
||||
75,rabin_karp,50.9
|
||||
76,rabin_karp,50.9
|
||||
77,rabin_karp,50.9
|
||||
78,rabin_karp,50.9
|
||||
79,rabin_karp,50.9
|
||||
80,railfence_cipher,86.2
|
||||
81,railfence_cipher,86.2
|
||||
82,railfence_cipher,86.2
|
||||
83,railfence_cipher,86.2
|
||||
84,railfence_cipher,86.2
|
||||
85,railfence_cipher,86.2
|
||||
86,railfence_cipher,86.2
|
||||
87,railfence_cipher,86.2
|
||||
88,railfence_cipher,86.2
|
||||
89,railfence_cipher,86.2
|
||||
90,zellers_birthday,65.0
|
||||
91,zellers_birthday,65.0
|
||||
92,zellers_birthday,65.0
|
||||
93,zellers_birthday,65.0
|
||||
94,zellers_birthday,65.0
|
||||
95,zellers_birthday,65.0
|
||||
96,zellers_birthday,65.0
|
||||
97,zellers_birthday,65.0
|
||||
98,zellers_birthday,65.0
|
||||
99,zellers_birthday,65.0
|
||||
|
Reference in a new issue