implement replay analysis (#7)
Co-authored-by: Donovan <donovan.a.kelly@pm.me> Reviewed-on: https://git.infra.nkode.tech/dkelly/evilnkode/pulls/7
This commit is contained in:
116
src/benchmark.py
116
src/benchmark.py
@@ -8,9 +8,8 @@ from src.utils import ShuffleTypes, observations, passcode_generator
|
||||
|
||||
@dataclass
|
||||
class Benchmark:
|
||||
mean: int
|
||||
variance: int
|
||||
runs: list[int]
|
||||
iterations_to_break: list[int]
|
||||
iterations_to_replay: list[int]
|
||||
|
||||
|
||||
def shuffle_benchmark(
|
||||
@@ -25,20 +24,21 @@ def shuffle_benchmark(
|
||||
file_path: str = '../output',
|
||||
overwrite: bool = False
|
||||
) -> Benchmark:
|
||||
file_name = f"{shuffle_type.name.lower()}-{number_of_keys}-{properties_per_key}-{passcode_len}-{max_tries_before_lockout}-{complexity}-{disparity}-{run_count}.txt"
|
||||
full_path = Path(file_path) / file_name
|
||||
if not overwrite and full_path.exists():
|
||||
print(f"file exists {file_path}")
|
||||
with open(full_path, "r") as fp:
|
||||
runs = fp.readline()
|
||||
runs = runs.split(',')
|
||||
runs = [int(i) for i in runs]
|
||||
return Benchmark(
|
||||
mean=mean(runs),
|
||||
variance=variance(runs),
|
||||
runs=runs
|
||||
)
|
||||
runs = []
|
||||
# file_name_break = f"{shuffle_type.name.lower()}-{number_of_keys}-{properties_per_key}-{passcode_len}-{max_tries_before_lockout}-{complexity}-{disparity}-{run_count}.txt"
|
||||
# full_path_iter_break = Path(file_path) / "iterations_to_break" /file_name_break
|
||||
# if not overwrite and full_path_iter_break.exists():
|
||||
# print(f"file exists {file_path}")
|
||||
# with open(full_path_iter_break, "r") as fp:
|
||||
# iterations_to_break = fp.readline()
|
||||
# iterations_to_break = iterations_to_break.split(',')
|
||||
# iterations_to_break = [int(i) for i in iterations_to_break]
|
||||
# return Benchmark(
|
||||
# mean=mean(iterations_to_break),
|
||||
# variance=variance(iterations_to_break),
|
||||
# iterations_to_break=iterations_to_break
|
||||
# )
|
||||
iterations_to_break = []
|
||||
iterations_to_replay = []
|
||||
for _ in range(run_count):
|
||||
passcode = passcode_generator(number_of_keys, properties_per_key, passcode_len, complexity, disparity)
|
||||
evilkode = Evilkode(
|
||||
@@ -56,50 +56,50 @@ def shuffle_benchmark(
|
||||
max_tries_before_lockout=max_tries_before_lockout,
|
||||
)
|
||||
evilout = evilkode.run()
|
||||
runs.append(evilout.iterations)
|
||||
iterations_to_break.append(evilout.iterations_to_break)
|
||||
iterations_to_replay.append(evilout.iterations_to_replay)
|
||||
|
||||
full_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(full_path, "w") as fp:
|
||||
fp.write(",".join([str(i) for i in runs])),
|
||||
# full_path_iter_break.parent.mkdir(parents=True, exist_ok=True)
|
||||
# with open(full_path_iter_break, "w") as fp:
|
||||
# fp.write(",".join([str(i) for i in iterations_to_break])),
|
||||
|
||||
return Benchmark(
|
||||
mean=mean(runs),
|
||||
variance=variance(runs),
|
||||
runs=runs
|
||||
iterations_to_break=iterations_to_break,
|
||||
iterations_to_replay=iterations_to_replay
|
||||
)
|
||||
|
||||
|
||||
def full_shuffle_benchmark(
|
||||
number_of_keys: int,
|
||||
properties_per_key: int,
|
||||
passcode_len: int,
|
||||
max_tries_before_lockout: int,
|
||||
run_count: int,
|
||||
complexity: int,
|
||||
disparity: int,
|
||||
) -> Benchmark:
|
||||
runs = []
|
||||
for _ in range(run_count):
|
||||
passcode = passcode_generator(number_of_keys, properties_per_key, passcode_len, complexity, disparity)
|
||||
evilkode = Evilkode(
|
||||
observations=observations(
|
||||
target_passcode=passcode,
|
||||
number_of_keys=number_of_keys,
|
||||
properties_per_key=properties_per_key,
|
||||
min_complexity=complexity,
|
||||
min_disparity=disparity,
|
||||
shuffle_type=ShuffleTypes.FULL_SHUFFLE,
|
||||
),
|
||||
number_of_keys=number_of_keys,
|
||||
properties_per_key=properties_per_key,
|
||||
passcode_len=passcode_len,
|
||||
max_tries_before_lockout=max_tries_before_lockout,
|
||||
)
|
||||
evilout = evilkode.run()
|
||||
runs.append(evilout.iterations)
|
||||
|
||||
return Benchmark(
|
||||
mean=mean(runs),
|
||||
variance=variance(runs),
|
||||
runs=runs
|
||||
)
|
||||
# def full_shuffle_benchmark(
|
||||
# number_of_keys: int,
|
||||
# properties_per_key: int,
|
||||
# passcode_len: int,
|
||||
# max_tries_before_lockout: int,
|
||||
# run_count: int,
|
||||
# complexity: int,
|
||||
# disparity: int,
|
||||
# ) -> Benchmark:
|
||||
# runs = []
|
||||
# for _ in range(run_count):
|
||||
# passcode = passcode_generator(number_of_keys, properties_per_key, passcode_len, complexity, disparity)
|
||||
# evilkode = Evilkode(
|
||||
# observations=observations(
|
||||
# target_passcode=passcode,
|
||||
# number_of_keys=number_of_keys,
|
||||
# properties_per_key=properties_per_key,
|
||||
# min_complexity=complexity,
|
||||
# min_disparity=disparity,
|
||||
# shuffle_type=ShuffleTypes.FULL_SHUFFLE,
|
||||
# ),
|
||||
# number_of_keys=number_of_keys,
|
||||
# properties_per_key=properties_per_key,
|
||||
# passcode_len=passcode_len,
|
||||
# max_tries_before_lockout=max_tries_before_lockout,
|
||||
# )
|
||||
# evilout = evilkode.run()
|
||||
# runs.append(evilout.iterations_to_break)
|
||||
#
|
||||
# return Benchmark(
|
||||
# mean=mean(runs),
|
||||
# variance=variance(runs),
|
||||
# iterations_to_break=runs
|
||||
# )
|
||||
|
||||
@@ -12,15 +12,20 @@ class Observation:
|
||||
def property_list(self) -> list[set[int]]:
|
||||
return [set(self.keypad[idx]) for idx in self.key_selection]
|
||||
|
||||
@property
|
||||
def flat_keypad(self) -> list[int]:
|
||||
return [num for row in self.keypad for num in row]
|
||||
|
||||
|
||||
@dataclass
|
||||
class EvilOutput:
|
||||
possible_nkodes: list[list[int]]
|
||||
iterations: int
|
||||
# possible_nkodes: list[list[int]]
|
||||
iterations_to_break: int
|
||||
iterations_to_replay: int
|
||||
|
||||
@property
|
||||
def number_of_possible_nkode(self):
|
||||
return math.prod([len(el) for el in self.possible_nkodes])
|
||||
# @property
|
||||
# def number_of_possible_nkode(self):
|
||||
# return math.prod([len(el) for el in self.possible_nkodes])
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -39,9 +44,25 @@ class Evilkode:
|
||||
|
||||
def run(self) -> EvilOutput:
|
||||
self.initialize()
|
||||
iterations_to_replay = None
|
||||
for idx, obs in enumerate(self.observations):
|
||||
if iterations_to_replay is None:
|
||||
replay_possibilities = self.replay_attack(obs)
|
||||
if replay_possibilities <= self.max_tries_before_lockout:
|
||||
iterations_to_replay = idx + 1
|
||||
if math.prod([len(el) for el in self.possible_nkode]) <= self.max_tries_before_lockout:
|
||||
return EvilOutput(possible_nkodes=[list(el) for el in self.possible_nkode], iterations=idx+1)
|
||||
assert iterations_to_replay <= idx +1
|
||||
return EvilOutput(
|
||||
# possible_nkodes=[list(el) for el in self.possible_nkode],
|
||||
iterations_to_break=idx + 1,
|
||||
iterations_to_replay=iterations_to_replay
|
||||
)
|
||||
for jdx, props in enumerate(obs.property_list):
|
||||
self.possible_nkode[jdx] = props.intersection(self.possible_nkode[jdx])
|
||||
raise Exception("error in Evilkode, observations stopped yielding")
|
||||
|
||||
def replay_attack(self, obs: Observation) -> int:
|
||||
possible_combos = 1
|
||||
for el in self.possible_nkode:
|
||||
possible_combos *= len({obs.flat_keypad.index(el2) // self.properties_per_key for el2 in el})
|
||||
return possible_combos
|
||||
@@ -59,7 +59,7 @@ class TowerShuffle:
|
||||
|
||||
@classmethod
|
||||
def new(cls, total_pos:int):
|
||||
assert total_pos >= 4
|
||||
assert total_pos >= 3
|
||||
rand_pos = np.random.permutation(total_pos)
|
||||
return TowerShuffle(
|
||||
total_positions=total_pos,
|
||||
|
||||
Reference in New Issue
Block a user