attempt 1

README.md

@@ -40,8 +40,8 @@ To generate test cases for all files in the benchmark run the command:
 python3 ./genetic.py
 ```
 
-The test suite is created in the directory `tests`. One test class is generated for each function defined in the 
-`benchmark` package. Run the command with the `-h` options for more details on partial generation.
+The test suite is created in the directory `tests`. One test file is generated for each file present in the 
+`benchmark` directory. Run the command with the `-h` options for more details on partial generation.
 
 The test suite can be then executed over the benchmark code with the command:
 

fuzzer.py

@@ -3,6 +3,7 @@ from random import randrange, choice, random, sample
 
 from frozendict import frozendict
 
+import operators
 from instrument import Arg, Params, invoke, call_statement, BranchTransformer, module_of
 
 Range = tuple[int, int]
@@ -160,9 +161,14 @@ def get_test_case_source(f_name: str, test_case: Params, i: int, indent: int):
     single_indent = " " * 4
     space = single_indent * indent
 
+    operators.distances_true_all = {}
+    operators.distances_false_all = {}
     output = invoke(f_name, test_case)
 
-    return f"""{space}def test_{f_name_orig}_{i}(self):
+    comment = (f"{space}#  distances_true = {repr(operators.distances_true_all)}\n"
+               f"{space}#  distances_false = {repr(operators.distances_false_all)}\n")
+
+    return f"""{comment}{space}def test_{f_name_orig}_{i}(self):
 {space}{single_indent}assert {call_statement(f_name_orig, test_case)} == {repr(output)}"""
 
 

genetic.py

@@ -18,7 +18,7 @@ CXPROB = 0.33
 TOURNSIZE = 3
 NPOP = 1000
 NGEN = 200
-REPS = 10
+REPS = 1
 
 OUT_DIR = os.path.join(os.path.dirname(__file__), "tests")
 
@@ -26,6 +26,8 @@ OUT_DIR = os.path.join(os.path.dirname(__file__), "tests")
 class Archive:
     true_branches: dict[int, any]
     false_branches: dict[int, any]
+    false_score: dict[int, any]
+    true_score: dict[int, any]
 
     def __init__(self):
         self.reset()
@@ -33,6 +35,8 @@ class Archive:
     def reset(self):
         self.true_branches = {}
         self.false_branches = {}
+        self.true_score = {}
+        self.false_score = {}
 
     def branches_covered(self) -> int:
         return len(self.true_branches.keys()) + len(self.false_branches.keys())
@@ -51,8 +55,8 @@ def normalize(x):
 
 
 def init_deap():
-    creator.create("Fitness", base.Fitness, weights=(-1.0,))
-    creator.create("Individual", list, fitness=creator.Fitness)
+    creator.create("FitnessMin", base.Fitness, weights=(-1.0,))
+    creator.create("Individual", list, fitness=creator.FitnessMin)
 
 
 def generate(orig_name: str) -> set[instrument.Params]:
@@ -92,7 +96,17 @@ def generate(orig_name: str) -> set[instrument.Params]:
         archive.reset()
         population = toolbox.population(n=NPOP)
 
-        algorithms.eaSimple(population, toolbox, CXPROB, MUPROB, NGEN, verbose=False)
+        # Create statistics object
+        stats = tools.Statistics(lambda ind: ind.fitness.values)
+        stats.register("min", min)
+        stats.register("max", max)
+
+        population, logbook = algorithms.eaSimple(population, toolbox, CXPROB, MUPROB, NGEN, verbose=False, stats=stats)
+
+        for gen, record in enumerate(logbook):
+            print(f"Generation {gen}: min={record['min']} max={record['max']}")
+
+        print(population)
 
         tot_covered = archive.branches_covered()
 
@@ -118,6 +132,8 @@ def compute_fitness(f_name: str, archive: Archive, individual: list) -> tuple[fl
     # Reset any distance values from previous executions
     operators.distances_true = {}
     operators.distances_false = {}
+    # archive.true_branches = {}
+    # archive.false_branches = {}
 
     # the archive_true_branches and archive_false_branches are reset after
     # each generation. This is intentional as they are used to archive branches that
@@ -126,29 +142,40 @@ def compute_fitness(f_name: str, archive: Archive, individual: list) -> tuple[fl
 
     # Run the function under test
     try:
-        instrument.invoke(f_name, x)
+        out = instrument.invoke(f_name, x)
     except AssertionError:
-        # print(to_test, x, "=", "[FAILS] fitness = 100.0")
+        print(f_name, x, "=", "[FAILS] fitness = 100.0")
         return 100.0,
 
     fitness = 0.0
+    branches = False
+
+    # print(operators.distances_true, operators.distances_false)
 
     # Sum up branch distances
     for branch in range(range_start, range_end):
         if branch in operators.distances_true:
-            if operators.distances_true[branch] == 0 and branch not in archive.true_branches:
-                archive.true_branches[branch] = x
-            if branch not in archive.true_branches:
-                fitness += normalize(operators.distances_true[branch])
+            fitness += normalize(operators.distances_true[branch])
+            branches = True
+
+            if operators.distances_true[branch] == 0:  # if test is true for this branch
+                if branch not in archive.false_score or archive.false_score[branch] > operators.distances_false[branch]:
+                    archive.true_branches[branch] = x
+                    archive.false_score[branch] = operators.distances_false[branch]
 
-    for branch in range(range_start, range_end):
         if branch in operators.distances_false:
-            if operators.distances_false[branch] == 0 and branch not in archive.false_branches:
-                archive.false_branches[branch] = x
-            if branch not in archive.false_branches:
-                fitness += normalize(operators.distances_false[branch])
+            fitness += normalize(operators.distances_false[branch])
+            branches = True
 
-    # print(to_test, x, "=", out, "fitness =", fitness)
+            if operators.distances_false[branch] == 0:  # if test is true for this branch
+                if branch not in archive.true_score or archive.true_score[branch] > operators.distances_true[branch]:
+                    archive.false_branches[branch] = x
+                    archive.true_score[branch] = operators.distances_true[branch]
+
+    if not branches:
+        return 100.0,
+
+    print(f_name, x, "=", out, "fitness =", fitness)
     return fitness,
 
 

muttest.py

@@ -12,9 +12,9 @@ OUT_DIR = os.path.join(ROOT_DIR, "tests")
 
 
 def run_mutpy(test_path: str, source_path: str):
-    stream = os.popen(f'mut.py --target \'{source_path}\' --unit-test \'{test_path}\'')
+    stream = os.popen(f'mut.py --target \'{source_path}\' --unit-test \'{test_path}\' -m')
     output = stream.read()
-    score = re.search('Mutation score \[.*\]: (\d+\.\d+)\%', output).group(1)
+    score = re.search('Mutation score \\[.*\\]: (\d+\.\d+)\%', output).group(1)
     print(output, file=sys.stderr)
     print(f"Score is: {score}")
 

operators.py

@@ -8,6 +8,9 @@ from nltk import edit_distance
 distances_true: dict[int, int] = {}
 distances_false: dict[int, int] = {}
 
+distances_true_all: dict[int, list[int]] = {}
+distances_false_all: dict[int, list[int]] = {}
+
 T = TypeVar('T')
 U = TypeVar('U')
 
@@ -58,7 +61,7 @@ int_str_ops: list[CmpOp[int | str]] = [
           false_dist=lambda lhs, rhs: 1 if lhs == rhs else 0),
     CmpOp(operator='!=',
           name='NotEq',
-          test=lambda lhs, rhs: lhs == rhs,
+          test=lambda lhs, rhs: lhs != rhs,
           true_dist=lambda lhs, rhs: 1 if lhs == rhs else 0,
           false_dist=lambda lhs, rhs: abs(lhs - rhs)),
 ]
@@ -92,7 +95,7 @@ str_ops: list[CmpOp[str]] = [
           false_dist=lambda lhs, rhs: 1 if lhs == rhs else 0),
     CmpOp(operator='!=',
           name='NotEq',
-          test=lambda lhs, rhs: lhs == rhs,
+          test=lambda lhs, rhs: lhs != rhs,
           true_dist=lambda lhs, rhs: 1 if lhs == rhs else 0,
           false_dist=lambda lhs, rhs: edit_distance(lhs, rhs)),
 ]
@@ -126,7 +129,7 @@ def compute_distances(name: str, lhs: any, rhs: any) -> tuple[int, int, bool]:
 
 
 def update_map(the_map: dict[int, int], condition_num: int, distance: int):
-    if condition_num in the_map.keys():
+    if condition_num in the_map:
         the_map[condition_num] = min(the_map[condition_num], distance)
     else:
         the_map[condition_num] = distance
@@ -134,8 +137,18 @@ def update_map(the_map: dict[int, int], condition_num: int, distance: int):
 
 def update_maps(condition_num, d_true, d_false):
     global distances_true, distances_false
+
     update_map(distances_true, condition_num, d_true)
+    if condition_num not in distances_true_all:
+        distances_true_all[condition_num] = [d_true]
+    else:
+        distances_true_all[condition_num].append(d_true)
+
     update_map(distances_false, condition_num, d_false)
+    if condition_num not in distances_false_all:
+        distances_false_all[condition_num] = [d_false]
+    else:
+        distances_false_all[condition_num].append(d_false)
 
 
 def in_op(num, lhs, rhs):

tests/test_caesar_cipher.py

@@ -4,16 +4,24 @@ from benchmark.caesar_cipher import decrypt
 
 
 class Test_encrypt(TestCase):
+    #  distances_true = {1: [1]}
+    #  distances_false = {1: [0]}
     def test_encrypt_1(self):
-        assert encrypt(strng='(B{6M K', key=90) == '#=v1HzF'
+        assert encrypt(strng='U', key=41) == '~'
 
+    #  distances_true = {1: [0]}
+    #  distances_false = {1: [1]}
     def test_encrypt_2(self):
-        assert encrypt(strng='t3Cv', key=84) == 'i(8k'
+        assert encrypt(strng='h', key=23) == ' '
 
 
 class Test_decrypt(TestCase):
+    #  distances_true = {2: [1]}
+    #  distances_false = {2: [0]}
     def test_decrypt_1(self):
-        assert decrypt(strng='4.J<IH{', key=11) == ')#?1>=p'
+        assert decrypt(strng='C', key=35) == ' '
 
+    #  distances_true = {2: [0]}
+    #  distances_false = {2: [1]}
     def test_decrypt_2(self):
-        assert decrypt(strng='5v8K', key=32) == 'tVw+'
+        assert decrypt(strng='7', key=24) == '~'

tests/test_check_armstrong.py

@@ -3,14 +3,27 @@ from benchmark.check_armstrong import check_armstrong
 
 
 class Test_check_armstrong(TestCase):
+    #  distances_true = {1: [0]}
+    #  distances_false = {1: [1]}
     def test_check_armstrong_1(self):
-        assert check_armstrong(n=220) == False
-
-    def test_check_armstrong_2(self):
         assert check_armstrong(n=0) == True
 
-    def test_check_armstrong_3(self):
-        assert check_armstrong(n=35) == False
+    #  distances_true = {1: [2], 2: [1], 3: [0]}
+    #  distances_false = {1: [0], 2: [0], 3: [149]}
+    def test_check_armstrong_2(self):
+        assert check_armstrong(n=2) == False
 
-    def test_check_armstrong_4(self):
+    #  distances_true = {1: [1], 2: [0]}
+    #  distances_false = {1: [0], 2: [1]}
+    def test_check_armstrong_3(self):
         assert check_armstrong(n=1) == True
+
+    #  distances_true = {1: [150], 2: [149], 3: [0]}
+    #  distances_false = {1: [0], 2: [0], 3: [1]}
+    def test_check_armstrong_4(self):
+        assert check_armstrong(n=150) == False
+
+    #  distances_true = {1: [151], 2: [150], 3: [1], 4: [0], 5: [151]}
+    #  distances_false = {1: [0], 2: [0], 3: [0], 4: [151], 5: [0]}
+    def test_check_armstrong_5(self):
+        assert check_armstrong(n=151) == False

tests/test_zellers_birthday.py

@@ -3,14 +3,47 @@ from benchmark.zellers_birthday import zeller
 
 
 class Test_zeller(TestCase):
+    #  distances_true = {1: [0], 2: [0], 3: [0], 4: [27], 5: [0], 6: [0], 7: [0], 8: [6, 5, 4, 3, 2, 1, 0]}
+    #  distances_false = {1: [607], 2: [301], 3: [51], 4: [0], 5: [51], 6: [0], 7: [1], 8: [0, 0, 0, 0, 0, 0, 1]}
     def test_zeller_1(self):
-        assert zeller(d=-466, m=3, y=76) == 'Tuesday'
+        assert zeller(d=-638, m=313, y=49) == 'Saturday'
 
+    #  distances_true = {1: [0], 2: [0], 3: [0], 4: [35], 5: [0], 6: [0], 7: [0], 8: [6, 5, 4, 3, 2, 1, 0]}
+    #  distances_false = {1: [125], 2: [1], 3: [43], 4: [0], 5: [43], 6: [0], 7: [1], 8: [0, 0, 0, 0, 0, 0, 1]}
     def test_zeller_2(self):
-        assert zeller(d=626, m=-928, y=27) == 'Saturday'
+        assert zeller(d=-156, m=13, y=-57) == 'Saturday'
 
+    #  distances_true = {1: [0], 2: [0], 3: [0], 4: [1], 5: [0], 6: [0], 7: [10], 8: [3, 2, 1, 0]}
+    #  distances_false = {1: [600], 2: [863], 3: [77], 4: [0], 5: [77], 6: [1], 7: [0], 8: [0, 0, 0, 1]}
     def test_zeller_3(self):
-        assert zeller(d=19, m=108, y=68) == 'Friday'
+        assert zeller(d=-631, m=-875, y=23) == 'Wednesday'
 
+    #  distances_true = {1: [0], 2: [0], 3: [0], 4: [37], 5: [0], 6: [0], 7: [1], 8: [6, 5, 4, 3, 2, 1, 0]}
+    #  distances_false = {1: [571], 2: [458], 3: [41], 4: [0], 5: [41], 6: [0], 7: [0], 8: [0, 0, 0, 0, 0, 0, 1]}
     def test_zeller_4(self):
-        assert zeller(d=284, m=255, y=-6) == 'Thursday'
+        assert zeller(d=602, m=-470, y=59) == 'Saturday'
+
+    #  distances_true = {1: [0], 2: [1], 3: [0], 4: [3], 5: [0], 6: [0], 7: [10], 8: [1, 0]}
+    #  distances_false = {1: [392], 2: [0], 3: [75], 4: [0], 5: [75], 6: [0], 7: [0], 8: [0, 1]}
+    def test_zeller_5(self):
+        assert zeller(d=423, m=12, y=-25) == 'Monday'
+
+    #  distances_true = {1: [0], 2: [0], 3: [0], 4: [0], 5: [1903], 7: [9], 8: [0]}
+    #  distances_false = {1: [9], 2: [898], 3: [98], 4: [21], 5: [0], 7: [0], 8: [1]}
+    def test_zeller_6(self):
+        assert zeller(d=40, m=910, y=2) == 'Sunday'
+
+    #  distances_true = {1: [0], 2: [0], 3: [0], 4: [77], 5: [0], 6: [0], 7: [3], 8: [4, 3, 2, 1, 0]}
+    #  distances_false = {1: [584], 2: [652], 3: [1], 4: [0], 5: [1], 6: [0], 7: [0], 8: [0, 0, 0, 0, 1]}
+    def test_zeller_7(self):
+        assert zeller(d=-615, m=-664, y=-99) == 'Thursday'
+
+    #  distances_true = {1: [0], 2: [0], 3: [0], 4: [0], 5: [1923], 7: [1], 8: [4, 3, 2, 1, 0]}
+    #  distances_false = {1: [40], 2: [758], 3: [78], 4: [1], 5: [0], 7: [0], 8: [0, 0, 0, 0, 1]}
+    def test_zeller_8(self):
+        assert zeller(d=-71, m=-770, y=-22) == 'Thursday'
+
+    #  distances_true = {1: [0], 2: [0], 3: [901], 5: [901], 7: [4], 8: [0]}
+    #  distances_false = {1: [338], 2: [905], 3: [0], 5: [0], 7: [0], 8: [1]}
+    def test_zeller_9(self):
+        assert zeller(d=369, m=-917, y=-1000) == 'Sunday'