From f3106e28cdc99df041a1a24f65ac62353a699546 Mon Sep 17 00:00:00 2001 From: Claudio Maggioni Date: Sun, 24 Dec 2023 14:55:34 +0100 Subject: [PATCH] fuzzer tests added --- archive.py | 33 +++--- fuzzer.py | 86 +++++++++++++-- fuzzer_tests/mutation_results_fuzzer.csv | 101 ++++++++++++++++++ fuzzer_tests/test_anagram_check.py | 24 +++++ fuzzer_tests/test_caesar_cipher.py | 17 +++ fuzzer_tests/test_check_armstrong.py | 24 +++++ fuzzer_tests/test_common_divisor_count.py | 19 ++++ fuzzer_tests/test_exponentiation.py | 9 ++ fuzzer_tests/test_gcd.py | 14 +++ fuzzer_tests/test_longest_substring.py | 14 +++ fuzzer_tests/test_rabin_karp.py | 24 +++++ fuzzer_tests/test_railfence_cipher.py | 37 +++++++ fuzzer_tests/test_zellers_birthday.py | 24 +++++ genetic.py | 21 +--- muttest.py | 64 ++++------- ...sults.csv => mutation_results_genetic.csv} | 0 16 files changed, 422 insertions(+), 89 deletions(-) create mode 100644 fuzzer_tests/mutation_results_fuzzer.csv create mode 100644 fuzzer_tests/test_anagram_check.py create mode 100644 fuzzer_tests/test_caesar_cipher.py create mode 100644 fuzzer_tests/test_check_armstrong.py create mode 100644 fuzzer_tests/test_common_divisor_count.py create mode 100644 fuzzer_tests/test_exponentiation.py create mode 100644 fuzzer_tests/test_gcd.py create mode 100644 fuzzer_tests/test_longest_substring.py create mode 100644 fuzzer_tests/test_rabin_karp.py create mode 100644 fuzzer_tests/test_railfence_cipher.py create mode 100644 fuzzer_tests/test_zellers_birthday.py rename tests/{mutation_results.csv => mutation_results_genetic.csv} (100%) diff --git a/archive.py b/archive.py index 9851f7f..33e467c 100644 --- a/archive.py +++ b/archive.py @@ -1,4 +1,4 @@ -from typing import Optional, Dict, Set +from typing import Dict, Set, List, Tuple from frozendict import frozendict @@ -39,36 +39,35 @@ class Archive: return " ".join([",".join([f'{k}={repr(v)}' for k, v in test.items()]) for test in suite]) def consider_test(self, test_case: frozendict): - try: - instrument.invoke(self.f_name, test_case) - except AssertionError: - return + branch = self.satisfies_unseen_branches(test_case) - range_start, range_end = instrument.n_of_branches[self.f_name] - - for branch in range(range_start, range_end): - if (branch in operators.distances_true and - operators.distances_true[branch] == 0 and branch not in self.true_branches): + for branch, true_or_false in branch: + if true_or_false: self.true_branches[branch] = test_case - if (branch in operators.distances_false and - operators.distances_false[branch] == 0 and branch not in self.false_branches): + else: self.false_branches[branch] = test_case - def satisfies_unseen_branch(self, test_case: frozendict) -> Optional[str]: + def satisfies_unseen_branches(self, test_case: frozendict) -> List[Tuple[int, bool]]: try: instrument.invoke(self.f_name, test_case) except AssertionError: - return None + return [] + range_start, range_end = instrument.n_of_branches[self.f_name] + branches: List[Tuple[int, bool]] = [] + for branch in range(range_start, range_end): if (branch in operators.distances_true and operators.distances_true[branch] == 0 and branch not in self.true_branches): - return f"{branch}T" + branches.append((branch, True)) if (branch in operators.distances_false and operators.distances_false[branch] == 0 and branch not in self.false_branches): - return f"{branch}F" + branches.append((branch, False)) - return None + if len(branches) > 0: + print(list(test_case.items()), branches) + + return branches diff --git a/fuzzer.py b/fuzzer.py index 322540c..c72abd9 100644 --- a/fuzzer.py +++ b/fuzzer.py @@ -1,13 +1,16 @@ +import argparse import os -from random import randrange, choice, random, sample +from random import randrange, choice, random, sample, seed from frozendict import frozendict +from tqdm import tqdm +import instrument import operators from archive import Archive -from instrument import Arg, Params, invoke, call_statement, BranchTransformer, module_of - -from typing import Tuple, Dict, List, Set +from instrument import (Arg, Params, invoke, call_statement, BranchTransformer, + module_of, load_benchmark, get_benchmark, functions) +from typing import Tuple, Dict, List, Set, Callable Range = Tuple[int, int] @@ -15,8 +18,9 @@ INT_RANGE: Range = (-1000, 1000) STRING_LEN_RANGE: Range = (0, 10) STRING_CHAR_RANGE: Range = (32, 127) POOL_SIZE: int = 1000 +FUZZER_REPS: int = 1000 -OUT_DIR = os.path.join(os.path.dirname(__file__), "tests") +OUT_DIR = os.path.join(os.path.dirname(__file__), "fuzzer_tests") def random_int() -> int: @@ -88,6 +92,19 @@ def random_params(arguments: List[Arg]) -> Params: pools: Dict[tuple, Set[tuple]] = {} +def add_to_pool(arguments: List[Arg], params: Params): + arg_names = [arg_name for arg_name, _ in arguments] + arg_types = tuple([arg_type for _, arg_type in arguments]) + + if arg_types not in pools: + raise ValueError(f"{arguments} has no matching pool in pools") + + param_list: List[any] = [None] * len(arg_names) + for i, name in enumerate(arg_names): + param_list[i] = params[name] + pools[arg_types].add(tuple(param_list)) + + def get_pool(arguments: List[Arg]) -> List[Params]: arg_types = tuple([arg_type for _, arg_type in arguments]) arg_names = [arg_name for arg_name, _ in arguments] @@ -135,18 +152,18 @@ def crossover(chosen_test: Params, other_chosen_test: Params, arguments: List[Ar return t1, t2 -def generate_test_case(f_name: str, arguments: List[Arg], archive: Archive) -> Params: +def generate_test_case(f_name: str, arguments: List[Arg], archive: Archive, bias_unseen=True) -> Params: pool: List[Params] = get_pool(arguments) attempts = 20 # attempts to generate a random test that satisfies a new branch while True: test = sample(pool, 1)[0] - is_new = archive.satisfies_unseen_branch(test) + is_new = [] if not bias_unseen else archive.satisfies_unseen_branches(test) attempts -= 1 - if is_new is None and attempts > 0: + if bias_unseen and len(is_new) == 0 and attempts > 0: # print(f"Not new: {test}") continue @@ -199,3 +216,56 @@ def get_test_class(orig_f_name: str, cases: Set[Params]) -> str: return (f"class Test_{orig_f_name}(TestCase):\n" + "\n\n".join([get_test_case_source(f_name, case, i + 1, 1) for i, case in enumerate(cases)]) + "\n") + + +def generate_tests(files: List[str], seed_num: int, generation_fn: Callable[[str], Set[Params]]): + load_benchmark(save_instrumented=False, files=files) + seed(seed_num) # init random seed + + for file_name, f_names in tqdm(get_benchmark().items(), desc="Generating tests"): + suite = [(name, generation_fn(name)) for name in f_names] + with open(os.path.join(OUT_DIR, f"test_{file_name}.py"), "w") as f: + f.write(get_test_import_stmt(f_names)) + f.write("\n\n") + f.write("\n\n".join([get_test_class(name, cases) for name, cases in suite])) + + +def fuzzer_generate(f_name: str) -> Set[Params]: + instrumented = instrument.BranchTransformer.to_instrumented_name(f_name) + args = functions[instrumented] + + archive = Archive(instrumented) + + for _ in tqdm(range(FUZZER_REPS), desc=f"fuzzer [{f_name}]"): + test = generate_test_case(instrumented, args, archive, bias_unseen=False) + + alteration_choice = randrange(3) + if alteration_choice == 1: + test = mutate(test, args) + elif alteration_choice == 2: + test2 = generate_test_case(instrumented, args, archive, bias_unseen=False) + test, test2 = crossover(test, test2, args) + archive.consider_test(test2) + add_to_pool(args, test2) + + archive.consider_test(test) + add_to_pool(args, test) + + return archive.build_suite() + + +def main(): + parser = argparse.ArgumentParser(prog='fuzzer.py', + description='Runs fuzzer for test case generation. Works on benchmark ' + 'files situated in the \'benchmark\' directory.') + parser.add_argument('file', type=str, help="File to test", + nargs="*") + parser.add_argument('-s', '--seed', type=int, help="Random generator seed", + nargs="?", default=0) + args = parser.parse_args() + + generate_tests(args.file, args.seed, fuzzer_generate) + + +if __name__ == "__main__": + main() diff --git a/fuzzer_tests/mutation_results_fuzzer.csv b/fuzzer_tests/mutation_results_fuzzer.csv new file mode 100644 index 0000000..4d31830 --- /dev/null +++ b/fuzzer_tests/mutation_results_fuzzer.csv @@ -0,0 +1,101 @@ +,file,score +0,anagram_check,23.1 +1,anagram_check,23.1 +2,anagram_check,23.1 +3,anagram_check,23.1 +4,anagram_check,23.1 +5,anagram_check,23.1 +6,anagram_check,23.1 +7,anagram_check,23.1 +8,anagram_check,23.1 +9,anagram_check,23.1 +10,caesar_cipher,58.8 +11,caesar_cipher,58.8 +12,caesar_cipher,58.8 +13,caesar_cipher,58.8 +14,caesar_cipher,58.8 +15,caesar_cipher,58.8 +16,caesar_cipher,58.8 +17,caesar_cipher,58.8 +18,caesar_cipher,58.8 +19,caesar_cipher,58.8 +20,check_armstrong,90.3 +21,check_armstrong,90.3 +22,check_armstrong,90.3 +23,check_armstrong,90.3 +24,check_armstrong,90.3 +25,check_armstrong,90.3 +26,check_armstrong,90.3 +27,check_armstrong,90.3 +28,check_armstrong,90.3 +29,check_armstrong,90.3 +30,common_divisor_count,72.3 +31,common_divisor_count,72.3 +32,common_divisor_count,72.3 +33,common_divisor_count,72.3 +34,common_divisor_count,72.3 +35,common_divisor_count,72.3 +36,common_divisor_count,72.3 +37,common_divisor_count,72.3 +38,common_divisor_count,72.3 +39,common_divisor_count,72.3 +40,exponentiation,71.4 +41,exponentiation,71.4 +42,exponentiation,71.4 +43,exponentiation,71.4 +44,exponentiation,71.4 +45,exponentiation,71.4 +46,exponentiation,71.4 +47,exponentiation,71.4 +48,exponentiation,71.4 +49,exponentiation,71.4 +50,gcd,47.8 +51,gcd,47.8 +52,gcd,47.8 +53,gcd,47.8 +54,gcd,47.8 +55,gcd,47.8 +56,gcd,47.8 +57,gcd,47.8 +58,gcd,47.8 +59,gcd,47.8 +60,longest_substring,82.6 +61,longest_substring,82.6 +62,longest_substring,82.6 +63,longest_substring,82.6 +64,longest_substring,82.6 +65,longest_substring,82.6 +66,longest_substring,82.6 +67,longest_substring,82.6 +68,longest_substring,82.6 +69,longest_substring,82.6 +70,rabin_karp,64.9 +71,rabin_karp,64.9 +72,rabin_karp,64.9 +73,rabin_karp,64.9 +74,rabin_karp,64.9 +75,rabin_karp,64.9 +76,rabin_karp,64.9 +77,rabin_karp,64.9 +78,rabin_karp,64.9 +79,rabin_karp,64.9 +80,railfence_cipher,89.4 +81,railfence_cipher,89.4 +82,railfence_cipher,89.4 +83,railfence_cipher,89.4 +84,railfence_cipher,89.4 +85,railfence_cipher,89.4 +86,railfence_cipher,89.4 +87,railfence_cipher,89.4 +88,railfence_cipher,89.4 +89,railfence_cipher,89.4 +90,zellers_birthday,68.3 +91,zellers_birthday,68.3 +92,zellers_birthday,68.3 +93,zellers_birthday,68.3 +94,zellers_birthday,68.3 +95,zellers_birthday,68.3 +96,zellers_birthday,68.3 +97,zellers_birthday,68.3 +98,zellers_birthday,68.3 +99,zellers_birthday,68.3 diff --git a/fuzzer_tests/test_anagram_check.py b/fuzzer_tests/test_anagram_check.py new file mode 100644 index 0000000..b6731da --- /dev/null +++ b/fuzzer_tests/test_anagram_check.py @@ -0,0 +1,24 @@ +from unittest import TestCase +from benchmark.anagram_check import anagram_check + + +class Test_anagram_check(TestCase): + # distances_true = {1: [8], 3: [0]} + # distances_false = {1: [0], 3: [7]} + def test_anagram_check_1(self): + assert anagram_check(s1='gU(@sp?!<', s2='^$') == False + + # distances_true = {1: [2], 3: [1], 4: [3]} + # distances_false = {1: [0], 3: [0], 4: [0]} + def test_anagram_check_2(self): + assert anagram_check(s1='N9)', s2='%;r') == False + + # distances_true = {1: [0], 2: [0]} + # distances_false = {1: [1], 2: [1]} + def test_anagram_check_3(self): + assert anagram_check(s1='j', s2='7') == False + + # distances_true = {1: [0], 2: [2], 3: [0]} + # distances_false = {1: [1], 2: [0], 3: [2]} + def test_anagram_check_4(self): + assert anagram_check(s1='i', s2='E y') == False diff --git a/fuzzer_tests/test_caesar_cipher.py b/fuzzer_tests/test_caesar_cipher.py new file mode 100644 index 0000000..538cea5 --- /dev/null +++ b/fuzzer_tests/test_caesar_cipher.py @@ -0,0 +1,17 @@ +from unittest import TestCase +from benchmark.caesar_cipher import encrypt +from benchmark.caesar_cipher import decrypt + + +class Test_encrypt(TestCase): + # distances_true = {1: [0, 6, 0, 10, 0, 0, 0, 0]} + # distances_false = {1: [48, 0, 60, 0, 57, 30, 25, 47]} + def test_encrypt_1(self): + assert encrypt(strng='_*k&hMH^', key=79) == 'Oy[uX=8N' + + +class Test_decrypt(TestCase): + # distances_true = {2: [0, 0, 32, 0]} + # distances_false = {2: [1, 12, 0, 24]} + def test_decrypt_1(self): + assert decrypt(strng='\\Q|E', key=61) == '~s?g' diff --git a/fuzzer_tests/test_check_armstrong.py b/fuzzer_tests/test_check_armstrong.py new file mode 100644 index 0000000..cb064e8 --- /dev/null +++ b/fuzzer_tests/test_check_armstrong.py @@ -0,0 +1,24 @@ +from unittest import TestCase +from benchmark.check_armstrong import check_armstrong + + +class Test_check_armstrong(TestCase): + # distances_true = {1: [583], 2: [582], 3: [433], 4: [0, 0, 0, 1], 5: [81]} + # distances_false = {1: [0], 2: [0], 3: [0], 4: [583, 58, 5, 0], 5: [0]} + def test_check_armstrong_1(self): + assert check_armstrong(n=583) == False + + # distances_true = {1: [0]} + # distances_false = {1: [1]} + def test_check_armstrong_2(self): + assert check_armstrong(n=0) == True + + # distances_true = {1: [153], 2: [152], 3: [3], 4: [0, 0, 0, 1], 5: [0]} + # distances_false = {1: [0], 2: [0], 3: [0], 4: [153, 15, 1, 0], 5: [1]} + def test_check_armstrong_3(self): + assert check_armstrong(n=153) == True + + # distances_true = {1: [5], 2: [4], 3: [0]} + # distances_false = {1: [0], 2: [0], 3: [146]} + def test_check_armstrong_4(self): + assert check_armstrong(n=5) == False diff --git a/fuzzer_tests/test_common_divisor_count.py b/fuzzer_tests/test_common_divisor_count.py new file mode 100644 index 0000000..938fb70 --- /dev/null +++ b/fuzzer_tests/test_common_divisor_count.py @@ -0,0 +1,19 @@ +from unittest import TestCase +from benchmark.common_divisor_count import cd_count + + +class Test_cd_count(TestCase): + # distances_true = {1: [180], 2: [450], 3: [0], 4: [451], 5: [0, 0, 1], 6: [0, 0, 0, 2, 0, 0, 6, 2, 0], 7: [89, 43, 27, 13, 9, 1]} + # distances_false = {1: [0], 2: [0], 3: [180], 4: [0], 5: [180, 90, 0], 6: [1, 1, 1, 0, 1, 1, 0, 0, 1], 7: [0, 0, 0, 0, 0, 0]} + def test_cd_count_1(self): + assert cd_count(a=-180, b=450) == 12 + + # distances_true = {1: [524], 2: [858], 3: [525], 4: [0], 5: [0, 0, 0, 0, 0, 0, 0, 0, 1], 6: [0], 7: [1]} + # distances_false = {1: [0], 2: [0], 3: [0], 4: [858], 5: [524, 334, 190, 144, 46, 6, 4, 2, 0], 6: [1], 7: [0]} + def test_cd_count_2(self): + assert cd_count(a=524, b=-858) == 2 + + # distances_true = {1: [171], 2: [880], 3: [0], 4: [881], 5: [0, 0, 0, 0, 0, 1], 6: [0], 7: [0]} + # distances_false = {1: [0], 2: [0], 3: [171], 4: [0], 5: [171, 25, 21, 4, 1, 0], 6: [1], 7: [1]} + def test_cd_count_3(self): + assert cd_count(a=-171, b=880) == 1 diff --git a/fuzzer_tests/test_exponentiation.py b/fuzzer_tests/test_exponentiation.py new file mode 100644 index 0000000..e5ce5ca --- /dev/null +++ b/fuzzer_tests/test_exponentiation.py @@ -0,0 +1,9 @@ +from unittest import TestCase +from benchmark.exponentiation import exponentiation + + +class Test_exponentiation(TestCase): + # distances_true = {1: [0, 0, 0, 0, 0, 0, 0, 0, 0, 1], 3: [0], 2: [1, 1, 0, 0, 0, 1, 0, 0, 0]} + # distances_false = {1: [630, 314, 156, 77, 38, 18, 8, 3, 1, 0], 3: [1], 2: [0, 0, 1, 1, 1, 0, 1, 1, 1]} + def test_exponentiation_1(self): + assert exponentiation(baseNumber=229, power=631) == 11381234101445775753511392015341744682588570986102224039390720285077899169674568038197326342940064901244177560842841924371421802587044521315773772186187547659830685455568537528049212994980159679816838690878596437635556336390603217974843590523706585633093363478766247718669151124253597924271460074695331123030002477672470888683820892451691712794240150714666068396254837851712428304279437706402840278841022427798845642299620119122197582242489041676042030717852031723255821528213497142055551145575663634990077305256845727132894737670148720272258452398194448015830799903170149287684706972565823541486622411136605841318617521051177682587295876588617192583199512651351936677507861660939730278101046178558162552821556410557719089125378645170386340572632243077354907575845913647147461310394166725741203050654294205855227960718731811784697401084780988753863828019215023964034022075473428520603004040086437560944507735374937837026821029527152268670187574842115160681136501433596553090081291095916580753660680615454664168228698567169031280782756475821551196582762941080904807113833394318850338264484961526692224500757025151325286773628959521989541968478235125022809094655704670640668124923174654970259046198975073787375738402994461267901157313580710146722768111601235186132446805107849024669181014876768249128669364324993951100257739165644330717095953945583658926347402457474225454535689578418160695265405822296417373349554757877558059686302770208714142534097327460841575601330771335995265872402591629 diff --git a/fuzzer_tests/test_gcd.py b/fuzzer_tests/test_gcd.py new file mode 100644 index 0000000..6904080 --- /dev/null +++ b/fuzzer_tests/test_gcd.py @@ -0,0 +1,14 @@ +from unittest import TestCase +from benchmark.gcd import gcd + + +class Test_gcd(TestCase): + # distances_true = {1: [182], 2: [931], 3: [749], 4: [0], 5: [0, 0, 0, 0, 0, 1]} + # distances_false = {1: [0], 2: [0], 3: [0], 4: [749], 5: [183, 17, 13, 4, 1, 0]} + def test_gcd_1(self): + assert gcd(a=183, b=932) == 1 + + # distances_true = {1: [720], 2: [503], 3: [217], 4: [218], 5: [0, 0, 0, 0, 1]} + # distances_false = {1: [0], 2: [0], 3: [0], 4: [0], 5: [504, 217, 70, 7, 0]} + def test_gcd_2(self): + assert gcd(a=721, b=504) == 7 diff --git a/fuzzer_tests/test_longest_substring.py b/fuzzer_tests/test_longest_substring.py new file mode 100644 index 0000000..fd0ed20 --- /dev/null +++ b/fuzzer_tests/test_longest_substring.py @@ -0,0 +1,14 @@ +from unittest import TestCase +from benchmark.longest_substring import longest_sorted_substr + + +class Test_longest_sorted_substr(TestCase): + # distances_true = {1: [5, 0, 56, 0, 81, 0, 0], 2: [0, 1, 1, 0]} + # distances_false = {1: [0, 38, 0, 86, 0, 5, 57], 2: [1, 0, 0, 1]} + def test_longest_sorted_substr_1(self): + assert longest_sorted_substr(s='<7\\$y(,d') == '(,d' + + # distances_true = {1: [29, 2, 0, 50], 2: [0]} + # distances_false = {1: [0, 0, 35, 0], 2: [1]} + def test_longest_sorted_substr_2(self): + assert longest_sorted_substr(s='Q42T"') == '2T' diff --git a/fuzzer_tests/test_rabin_karp.py b/fuzzer_tests/test_rabin_karp.py new file mode 100644 index 0000000..5db2324 --- /dev/null +++ b/fuzzer_tests/test_rabin_karp.py @@ -0,0 +1,24 @@ +from unittest import TestCase +from benchmark.rabin_karp import rabin_karp_search + + +class Test_rabin_karp_search(TestCase): + # distances_true = {1: [0, 70, 54, 65, 28, 51, 66], 3: [1], 4: [0, 0, 0, 0, 0, 0, 1], 5: [71, 55, 66, 29, 52, 67]} + # distances_false = {1: [1, 0, 0, 0, 0, 0, 0], 3: [0], 4: [6, 5, 4, 3, 2, 1, 0], 5: [0, 0, 0, 0, 0, 0]} + def test_rabin_karp_search_1(self): + assert rabin_karp_search(pat='', txt=']vzK Tuple[fl return fitness, -def build_suite(filename: str, f_names: List[str]): - suite = [(name, generate(name)) for name in f_names] - - with open(os.path.join(OUT_DIR, f"test_{filename}.py"), "w") as f: - f.write(fuzzer.get_test_import_stmt(f_names)) - f.write("\n\n") - f.write("\n\n".join([get_test_class(name, cases) for name, cases in suite])) - - -def run_genetic(files: List[str], seed: int): - instrument.load_benchmark(save_instrumented=False, files=files) - random.seed(seed) # init random seed - init_deap() - - for file_name, functions in tqdm.tqdm(instrument.get_benchmark().items(), desc="Generating tests"): - build_suite(file_name, functions) - - def main(): parser = argparse.ArgumentParser(prog='genetic.py', description='Runs genetic algorithm for test case generation. Works on benchmark ' @@ -154,7 +136,8 @@ def main(): nargs="?", default=0) args = parser.parse_args() - run_genetic(args.file, args.seed) + init_deap() + fuzzer.generate_tests(args.file, args.seed, generate) if __name__ == '__main__': diff --git a/muttest.py b/muttest.py index 310ed0b..292cf05 100644 --- a/muttest.py +++ b/muttest.py @@ -1,44 +1,20 @@ -import argparse import os import re -from collections import defaultdict +import subprocess +import sys +from typing import List, Dict import pandas as pd from tqdm import tqdm -from mutpy import commandline -import sys -from io import StringIO -import contextlib -import subprocess - -from typing import List, Dict, DefaultDict - ROOT_DIR = os.path.dirname(__file__) IN_SOURCE_DIR = os.path.join(ROOT_DIR, "benchmark") IN_TEST_DIR = os.path.join(ROOT_DIR, "tests") -OUT_DIR = os.path.join(ROOT_DIR, "tests") +IN_FUZZER_TEST_DIR = os.path.join(ROOT_DIR, "fuzzer_tests") MUT_PY_PATH = os.path.join(ROOT_DIR, 'env37', 'bin', 'mut.py') REPS: int = 10 -class OutputCapture(): - result: str - - -@contextlib.contextmanager -def capture_stdout(): - old = sys.stdout - capturer = StringIO() - sys.stdout = capturer - - data = OutputCapture() - yield data - - sys.stdout = old - data.result = capturer.getvalue() - - def run_mutpy(test_path: str, source_path: str) -> float: output = subprocess.check_output( [sys.executable, MUT_PY_PATH, '-t', source_path, '-u', test_path]).decode('utf-8') @@ -46,33 +22,31 @@ def run_mutpy(test_path: str, source_path: str) -> float: return float(score) -def main(): - parser = argparse.ArgumentParser(prog='mutmuttest.py', - description='Runs MutPy over generated test suite.') - parser.add_argument('file', type=str, help="Source file to test", - nargs="*") - - files = parser.parse_args().file - if len(files) == 0: - files = [os.path.splitext(f) for f in os.listdir(IN_SOURCE_DIR)] - to_test = [file[0] for file in files if file[1] == ".py"] - else: - to_test = [os.path.splitext(os.path.basename(file))[0] for file in files] - +def mutate_suite(out_file: str, in_test_dir: str, to_test: List[str]): scores: List[Dict[str, any]] = [] - to_test = [e for t in to_test for e in ([t] * REPS)] + if os.path.isfile(out_file): # do not re-generate if file exists + return - for filename in tqdm(to_test, desc="Running mut.py over test suite"): + for filename in tqdm(to_test, desc=f"mut.py [{os.path.basename(out_file)}]"): source_path = os.path.join(IN_SOURCE_DIR, f"{filename}.py") - test_path = os.path.join(IN_TEST_DIR, f"test_{filename}.py") + test_path = os.path.join(in_test_dir, f"test_{filename}.py") scores.append({ 'file': filename, 'score': run_mutpy(test_path, source_path) }) df = pd.DataFrame.from_records(scores) - df.to_csv(os.path.join(OUT_DIR, 'mutation_results.csv')) + df.to_csv(out_file) + + +def main(): + files = [os.path.splitext(f) for f in os.listdir(IN_SOURCE_DIR)] + to_test = [file[0] for file in files if file[1] == ".py"] + to_test = [e for t in to_test for e in ([t] * REPS)] + + mutate_suite(os.path.join(IN_TEST_DIR, 'mutation_results_genetic.csv'), IN_TEST_DIR, to_test) + mutate_suite(os.path.join(IN_FUZZER_TEST_DIR, 'mutation_results_fuzzer.csv'), IN_FUZZER_TEST_DIR, to_test) if __name__ == "__main__": diff --git a/tests/mutation_results.csv b/tests/mutation_results_genetic.csv similarity index 100% rename from tests/mutation_results.csv rename to tests/mutation_results_genetic.csv