diff --git a/genetic.py b/genetic.py index f77fff1..7a8a7da 100644 --- a/genetic.py +++ b/genetic.py @@ -18,7 +18,7 @@ CXPROB = 0.33 TOURNSIZE = 3 NPOP = 1000 NGEN = 200 -REPS = 1 +REPS = 10 OUT_DIR = os.path.join(os.path.dirname(__file__), "tests") @@ -93,7 +93,6 @@ def generate(orig_name: str) -> set[instrument.Params]: top_coverage = 0 for i in range(REPS): - archive.reset() population = toolbox.population(n=NPOP) # Create statistics object @@ -103,10 +102,20 @@ def generate(orig_name: str) -> set[instrument.Params]: population, logbook = algorithms.eaSimple(population, toolbox, CXPROB, MUPROB, NGEN, verbose=False, stats=stats) - for gen, record in enumerate(logbook): - print(f"Generation {gen}: min={record['min']} max={record['max']}") + print("population:\n" + "\n".join([str(p) for p in population]) + "\n") - print(population) + for member in population: + m = frozendict.frozendict(member) + for branch in range(range_start, range_end): + if (branch in operators.distances_true and + operators.distances_true[branch] == 0 and branch not in archive.true_branches): + archive.true_branches[branch] = m + if (branch in operators.distances_false and + operators.distances_false[branch] == 0 and branch not in archive.false_branches): + archive.false_branches[branch] = m + + # for gen, record in enumerate(logbook): + # print(f"Generation {gen}: min={record['min']} max={record['max']}") tot_covered = archive.branches_covered() @@ -114,6 +123,7 @@ def generate(orig_name: str) -> set[instrument.Params]: branches = archive.branches_str() print(f"{orig_name}: rep #{i:02d}: Cov: {cov:02.02f}% ({tot_covered}/{total_branches} branches): {branches}") + print(archive.build_suite()) if cov > top_coverage: top_result = archive.build_suite() @@ -132,8 +142,6 @@ def compute_fitness(f_name: str, archive: Archive, individual: list) -> tuple[fl # Reset any distance values from previous executions operators.distances_true = {} operators.distances_false = {} - # archive.true_branches = {} - # archive.false_branches = {} # the archive_true_branches and archive_false_branches are reset after # each generation. This is intentional as they are used to archive branches that @@ -144,38 +152,29 @@ def compute_fitness(f_name: str, archive: Archive, individual: list) -> tuple[fl try: out = instrument.invoke(f_name, x) except AssertionError: - print(f_name, x, "=", "[FAILS] fitness = 100.0") + # print(f_name, x, "=", "[FAILS] fitness = 100.0") return 100.0, fitness = 0.0 - branches = False - - # print(operators.distances_true, operators.distances_false) + #branches = False # Sum up branch distances for branch in range(range_start, range_end): if branch in operators.distances_true: - fitness += normalize(operators.distances_true[branch]) - branches = True - - if operators.distances_true[branch] == 0: # if test is true for this branch - if branch not in archive.false_score or archive.false_score[branch] > operators.distances_false[branch]: - archive.true_branches[branch] = x - archive.false_score[branch] = operators.distances_false[branch] + if branch not in archive.true_branches: + fitness += normalize(operators.distances_true[branch]) + #branches = True + for branch in range(range_start, range_end): if branch in operators.distances_false: - fitness += normalize(operators.distances_false[branch]) - branches = True + if branch not in archive.false_branches: + fitness += normalize(operators.distances_false[branch]) + #branches = True - if operators.distances_false[branch] == 0: # if test is true for this branch - if branch not in archive.true_score or archive.true_score[branch] > operators.distances_true[branch]: - archive.false_branches[branch] = x - archive.true_score[branch] = operators.distances_true[branch] + #if not branches: + # return 100.0, - if not branches: - return 100.0, - - print(f_name, x, "=", out, "fitness =", fitness) + # print(f_name, x, "=", out, "fitness =", fitness) return fitness, @@ -188,21 +187,27 @@ def build_suite(filename: str, f_names: list[str]): f.write("\n\n".join([get_test_class(name, cases) for name, cases in suite])) -def main(): - random.seed(0) # init random seed - - parser = argparse.ArgumentParser(prog='genetic.py', - description='Runs genetic algorithm for test case generation. Works on benchmark ' - 'files situated in the \'benchmark\' directory.') - parser.add_argument('file', type=str, help="File to test", - nargs="*") - - instrument.load_benchmark(save_instrumented=False, files=parser.parse_args().file) +def run_genetic(files: list[str], seed: int): + instrument.load_benchmark(save_instrumented=False, files=files) + random.seed(seed) # init random seed init_deap() for file_name, functions in tqdm.tqdm(instrument.get_benchmark().items(), desc="Generating tests"): build_suite(file_name, functions) +def main(): + parser = argparse.ArgumentParser(prog='genetic.py', + description='Runs genetic algorithm for test case generation. Works on benchmark ' + 'files situated in the \'benchmark\' directory.') + parser.add_argument('file', type=str, help="File to test", + nargs="*") + parser.add_argument('-s', '--seed', type=int, help="Random generator seed", + nargs="?", default=0) + args = parser.parse_args() + + run_genetic(args.file, args.seed) + + if __name__ == '__main__': main() diff --git a/instrument.py b/instrument.py index 0b31d46..571d860 100644 --- a/instrument.py +++ b/instrument.py @@ -119,9 +119,6 @@ def instrument(source_path: str, target_path: str, save_instrumented=True): top_level_f_ast: list[ast.FunctionDef] = [f for f in node.body if isinstance(f, ast.FunctionDef)] for f in top_level_f_ast: - if f.name in functions: - raise ValueError(f"Function '{f.name}' already loaded from another file") - arg_types: list[Arg] = [] for arg in f.args.args: diff --git a/muttest.py b/muttest.py index fa4575a..c585bdc 100644 --- a/muttest.py +++ b/muttest.py @@ -1,9 +1,11 @@ import argparse import os +import random import re import sys import instrument +from genetic import run_genetic ROOT_DIR = os.path.dirname(__file__) IN_SOURCE_DIR = os.path.join(ROOT_DIR, "benchmark") @@ -12,6 +14,8 @@ OUT_DIR = os.path.join(ROOT_DIR, "tests") def run_mutpy(test_path: str, source_path: str): + run_genetic([source_path], random.randint(0, 500)) + stream = os.popen(f'mut.py --target \'{source_path}\' --unit-test \'{test_path}\' -m') output = stream.read() score = re.search('Mutation score \\[.*\\]: (\d+\.\d+)\%', output).group(1) @@ -27,14 +31,16 @@ def main(): files = parser.parse_args().file if len(files) == 0: - to_test = instrument.get_benchmark().keys() + files = [os.path.splitext(f) for f in os.listdir(IN_SOURCE_DIR)] + to_test = [file[0] for file in files if file[1] == ".py"] else: to_test = [os.path.splitext(os.path.basename(file))[0] for file in files] for filename in to_test: - source_path = os.path.join(IN_SOURCE_DIR, f"{filename}.py") - test_path = os.path.join(IN_TEST_DIR, f"test_{filename}.py") - run_mutpy(test_path, source_path) + for i in range(10): + source_path = os.path.join(IN_SOURCE_DIR, f"{filename}.py") + test_path = os.path.join(IN_TEST_DIR, f"test_{filename}.py") + run_mutpy(test_path, source_path) if __name__ == "__main__": diff --git a/tests/test_anagram_check.py b/tests/test_anagram_check.py index e1d4108..6605991 100644 --- a/tests/test_anagram_check.py +++ b/tests/test_anagram_check.py @@ -3,14 +3,17 @@ from benchmark.anagram_check import anagram_check class Test_anagram_check(TestCase): + # distances_true = {1: [0], 2: [1], 3: [0]} + # distances_false = {1: [1], 2: [0], 3: [1]} def test_anagram_check_1(self): - assert anagram_check(s1='@', s2='{') == False + assert anagram_check(s1='Z', s2='') == False + # distances_true = {1: [0], 2: [0]} + # distances_false = {1: [1], 2: [1]} def test_anagram_check_2(self): - assert anagram_check(s1='', s2='X~|') == False + assert anagram_check(s1='w', s2='t') == False + # distances_true = {1: [1], 3: [0]} + # distances_false = {1: [0], 3: [1]} def test_anagram_check_3(self): - assert anagram_check(s1='?kv|d', s2='43J!j') == False - - def test_anagram_check_4(self): - assert anagram_check(s1='U', s2='') == False + assert anagram_check(s1='', s2='f') == False diff --git a/tests/test_caesar_cipher.py b/tests/test_caesar_cipher.py index a5ce417..8fd3440 100644 --- a/tests/test_caesar_cipher.py +++ b/tests/test_caesar_cipher.py @@ -4,24 +4,14 @@ from benchmark.caesar_cipher import decrypt class Test_encrypt(TestCase): - # distances_true = {1: [1]} - # distances_false = {1: [0]} + # distances_true = {} + # distances_false = {} def test_encrypt_1(self): - assert encrypt(strng='U', key=41) == '~' - - # distances_true = {1: [0]} - # distances_false = {1: [1]} - def test_encrypt_2(self): - assert encrypt(strng='h', key=23) == ' ' + assert encrypt(strng='', key=45) == '' class Test_decrypt(TestCase): - # distances_true = {2: [1]} - # distances_false = {2: [0]} + # distances_true = {2: [0, 211, 211, 0, 0, 196, 15, 221, 189]} + # distances_false = {2: [13, 0, 0, 9, 24, 0, 0, 0, 0]} def test_decrypt_1(self): - assert decrypt(strng='C', key=35) == ' ' - - # distances_true = {2: [0]} - # distances_false = {2: [1]} - def test_decrypt_2(self): - assert decrypt(strng='7', key=24) == '~' + assert decrypt(strng=']<