diff --git a/fuzzer.py b/fuzzer.py index 5888e8a..fb99e38 100644 --- a/fuzzer.py +++ b/fuzzer.py @@ -166,10 +166,18 @@ def get_test_case_source(f_name: str, test_case: Params, i: int, indent: int): {space}{single_indent}assert {call_statement(f_name_orig, test_case)} == {repr(output)}""" -def get_test_class(f_name: str, cases: set[Params]) -> str: - f_name_orig = BranchTransformer.to_original_name(f_name) +def get_test_import_stmt(names: list[str]): + imports = ["from unittest import TestCase"] - test_class = (f"from unittest import TestCase\n\nfrom {module_of[f_name]} import {f_name_orig}\n\n\n" - f"class Test_{f_name_orig}(TestCase):\n") - test_class += "\n\n".join([get_test_case_source(f_name, case, i + 1, 1) for i, case in enumerate(cases)]) - return test_class + for orig_f_name in names: + f_name = BranchTransformer.to_instrumented_name(orig_f_name) + imports.append(f"from {'.'.join(module_of[f_name])} import {orig_f_name}") + + return "\n".join(imports) + "\n" + + +def get_test_class(orig_f_name: str, cases: set[Params]) -> str: + f_name = BranchTransformer.to_instrumented_name(orig_f_name) + return (f"class Test_{orig_f_name}(TestCase):\n" + + "\n\n".join([get_test_case_source(f_name, case, i + 1, 1) for i, case in enumerate(cases)]) + + "\n") diff --git a/genetic.py b/genetic.py index 886edc1..d2d2799 100644 --- a/genetic.py +++ b/genetic.py @@ -1,7 +1,6 @@ import argparse import os import random -import sys from functools import partial import frozendict @@ -56,8 +55,8 @@ def init_deap(): creator.create("Individual", list, fitness=creator.Fitness) -def generate(f_name: str): - orig_name = instrument.BranchTransformer.to_original_name(f_name) +def generate(orig_name: str) -> set[instrument.Params]: + f_name = instrument.BranchTransformer.to_instrumented_name(orig_name) args = instrument.functions[f_name] range_start, range_end = instrument.n_of_branches[f_name] @@ -106,6 +105,9 @@ def generate(f_name: str): top_result = archive.build_suite() top_coverage = cov + if tot_covered == total_branches: + break + return top_result @@ -150,11 +152,13 @@ def compute_fitness(f_name: str, archive: Archive, individual: list) -> tuple[fl return fitness, -def build_suite(f_name: str): - instr_name = instrument.BranchTransformer.to_instrumented_name(f_name) - cases = generate(instr_name) - with open(os.path.join(OUT_DIR, "test_" + f_name + ".py"), "w") as f: - f.write(get_test_class(instr_name, cases)) +def build_suite(filename: str, f_names: list[str]): + suite = [(name, generate(name)) for name in f_names] + + with open(os.path.join(OUT_DIR, f"test_{filename}.py"), "w") as f: + f.write(fuzzer.get_test_import_stmt(f_names)) + f.write("\n\n") + f.write("\n\n".join([get_test_class(name, cases) for name, cases in suite])) def main(): @@ -169,9 +173,8 @@ def main(): instrument.load_benchmark(save_instrumented=False, files=parser.parse_args().file) init_deap() - for instr_f in tqdm.tqdm(sorted(instrument.functions.keys()), desc="Generating tests"): - print("", file=sys.stderr) - build_suite(instrument.BranchTransformer.to_original_name(instr_f)) + for file_name, functions in tqdm.tqdm(instrument.get_benchmark().items(), desc="Generating tests"): + build_suite(file_name, functions) if __name__ == '__main__': diff --git a/instrument.py b/instrument.py index d62fad5..0b31d46 100644 --- a/instrument.py +++ b/instrument.py @@ -1,11 +1,13 @@ import ast import os.path import sys +from collections import defaultdict from typing import Optional import astunparse import tqdm from frozendict import frozendict + from operators import evaluate_condition ROOT_DIR: str = os.path.dirname(__file__) @@ -85,7 +87,7 @@ SignatureDict = dict[str, list[Arg]] n_of_branches: dict[str, tuple[int, int]] = {} functions: SignatureDict = {} -module_of: dict[str, str] = {} +module_of: dict[str, list[str]] = {} def instrument(source_path: str, target_path: str, save_instrumented=True): @@ -130,9 +132,7 @@ def instrument(source_path: str, target_path: str, save_instrumented=True): arg_types.append((arg.arg, arg_type)) functions[f.name] = arg_types - module_of[f.name] = os.path.normpath(os.path.relpath(source_path, ROOT_DIR)) \ - .replace(".py", "") \ - .replace("/", ".") + module_of[f.name] = os.path.splitext(os.path.normpath(os.path.relpath(source_path, ROOT_DIR)))[0].split("/") def invoke(f_name: str, f_args: Params) -> any: @@ -176,5 +176,19 @@ def call_statement(f_name: str, f_args: Params) -> str: return f"{f_name}({', '.join(arg_list)})" +def get_benchmark() -> dict[str, list[str]]: + """ + Returns a dictionary associated each source code file name loaded (without extension) with the list of + (non-instrumented) function names defined within it + """ + + names: defaultdict[str, list[str]] = defaultdict(list) + + for f in functions: + names[module_of[f][-1]].append(BranchTransformer.to_original_name(f)) + + return dict(names) + + if __name__ == '__main__': load_benchmark(save_instrumented=True) diff --git a/muttest.py b/muttest.py new file mode 100644 index 0000000..95ac21f --- /dev/null +++ b/muttest.py @@ -0,0 +1,41 @@ +import argparse +import os +import re +import sys + +import instrument + +ROOT_DIR = os.path.dirname(__file__) +IN_SOURCE_DIR = os.path.join(ROOT_DIR, "benchmark") +IN_TEST_DIR = os.path.join(ROOT_DIR, "tests") +OUT_DIR = os.path.join(ROOT_DIR, "tests") + + +def run_mutpy(test_path: str, source_path: str): + stream = os.popen(f'mut.py --target \'{source_path}\' --unit-test \'{test_path}\'') + output = stream.read() + score = re.search('Mutation score \[.*\]: (\d+\.\d+)\%', output).group(1) + print(output, file=sys.stderr) + print(f"Score is: {score}") + + +def main(): + parser = argparse.ArgumentParser(prog='muttest.py', + description='Runs MutPy over generated test suite.') + parser.add_argument('file', type=str, help="Source file to test", + nargs="*") + + files = parser.parse_args().file + if len(files) == 0: + to_test = instrument.get_benchmark().keys() + else: + to_test = [os.path.splitext(os.path.basename(file))[0] for file in files] + + for filename in to_test: + source_path = os.path.join(IN_SOURCE_DIR, f"{filename}.py") + test_path = os.path.join(IN_TEST_DIR, f"test_{filename}.py") + run_mutpy(test_path, source_path) + + +if __name__ == "__main__": + main() diff --git a/requirements.txt b/requirements.txt index b01f1ce..c184cdd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,5 @@ nltk==3.8.1 deap==1.4.1 astunparse==1.6.3 frozendict==2.3.8 -tqdm==4.66.1 \ No newline at end of file +tqdm==4.66.1 +MutPy==0.6.1 \ No newline at end of file diff --git a/tests/test_anagram_check.py b/tests/test_anagram_check.py index f3cc27b..e1d4108 100644 --- a/tests/test_anagram_check.py +++ b/tests/test_anagram_check.py @@ -1,20 +1,16 @@ from unittest import TestCase - from benchmark.anagram_check import anagram_check class Test_anagram_check(TestCase): def test_anagram_check_1(self): - assert anagram_check(s1='B', s2='o8') == False + assert anagram_check(s1='@', s2='{') == False def test_anagram_check_2(self): - assert anagram_check(s1=' ', s2='u') == False + assert anagram_check(s1='', s2='X~|') == False def test_anagram_check_3(self): - assert anagram_check(s1='', s2='') == True + assert anagram_check(s1='?kv|d', s2='43J!j') == False def test_anagram_check_4(self): - assert anagram_check(s1='?8H', s2='') == False - - def test_anagram_check_5(self): - assert anagram_check(s1='@m', s2='Pj') == False \ No newline at end of file + assert anagram_check(s1='U', s2='') == False diff --git a/tests/test_caesar_cipher.py b/tests/test_caesar_cipher.py new file mode 100644 index 0000000..6387ba4 --- /dev/null +++ b/tests/test_caesar_cipher.py @@ -0,0 +1,19 @@ +from unittest import TestCase +from benchmark.caesar_cipher import encrypt +from benchmark.caesar_cipher import decrypt + + +class Test_encrypt(TestCase): + def test_encrypt_1(self): + assert encrypt(strng='(B{6M K', key=90) == '#=v1HzF' + + def test_encrypt_2(self): + assert encrypt(strng='t3Cv', key=84) == 'i(8k' + + +class Test_decrypt(TestCase): + def test_decrypt_1(self): + assert decrypt(strng='4.J