import os import re import subprocess import sys from typing import List, Dict import pandas as pd from tqdm import tqdm ROOT_DIR = os.path.dirname(__file__) IN_SOURCE_DIR = os.path.join(ROOT_DIR, "benchmark") IN_TEST_DIR = os.path.join(ROOT_DIR, "tests") IN_FUZZER_TEST_DIR = os.path.join(ROOT_DIR, "fuzzer_tests") MUT_PY_PATH = os.path.join(ROOT_DIR, 'env37', 'bin', 'mut.py') REPS: int = 10 def run_mutpy(test_path: str, source_path: str) -> float: output = subprocess.check_output( [sys.executable, MUT_PY_PATH, '-t', source_path, '-u', test_path]).decode('utf-8') score = re.search('Mutation score \\[.*]: (\\d+\\.\\d+)%', output).group(1) return float(score) def mutate_suite(out_file: str, in_test_dir: str, to_test: List[str]): scores: List[Dict[str, any]] = [] if os.path.isfile(out_file): # do not re-generate if file exists return for filename in tqdm(to_test, desc=f"mut.py [{os.path.basename(out_file)}]"): source_path = os.path.join(IN_SOURCE_DIR, f"{filename}.py") test_path = os.path.join(in_test_dir, f"test_{filename}.py") scores.append({ 'file': filename, 'score': run_mutpy(test_path, source_path) }) df = pd.DataFrame.from_records(scores) df.to_csv(out_file) def main(): files = [os.path.splitext(f) for f in os.listdir(IN_SOURCE_DIR)] to_test = [file[0] for file in files if file[1] == ".py"] to_test = [e for t in to_test for e in ([t] * REPS)] mutate_suite(os.path.join(IN_TEST_DIR, 'mutation_results_genetic.csv'), IN_TEST_DIR, to_test) mutate_suite(os.path.join(IN_FUZZER_TEST_DIR, 'mutation_results_fuzzer.csv'), IN_FUZZER_TEST_DIR, to_test) if __name__ == "__main__": main()