Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Allow multiple input test cases #85

Merged
merged 1 commit into from
Oct 16, 2023
Merged

Allow multiple input test cases #85

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 14 additions & 5 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,20 @@ You will get output like so:
```bash
$ invoke test python any

> 🟢 script "./src/python/sort_builtin.py" succeeded
> 🟢 script "./src/python/sort_bubble_sort.py" succeeded
> 🟢 script "./src/python/sort_selection_sort.py" succeeded
> 🟢 script "./src/python/sort_insertion_sort.py" succeeded
> ✨ script run success ✨
docker run ...
⏱ ./src/python/sort_builtin.py on ./data/sort_input_0.txt ran for 0.5 seconds
🟢 ./src/python/sort_builtin.py on ./data/sort_input_0.txt succeeded
docker run ...
⏱ ./src/python/sort_bubble_sort.py on ./data/sort_input_0.txt ran for 0.51 seconds
🟢 ./src/python/sort_bubble_sort.py on ./data/sort_input_0.txt succeeded
docker run ...
⏱ ./src/python/sort_selection_sort.py on ./data/sort_input_0.txt ran for 0.58 seconds
🟢 ./src/python/sort_selection_sort.py on ./data/sort_input_0.txt succeeded
docker run ...
⏱ ./src/python/sort_insertion_sort.py on ./data/sort_input_0.txt ran for 0.52 seconds
🟢 ./src/python/sort_insertion_sort.py on ./data/sort_input_0.txt succeeded

✨ script run success ✨
```

_(note: I will likely not bother to update the above example when the output changes in minor ways)_
Expand Down
0 data/sort_input.txt → data/sort_input_0.txt
View file
Open in desktop
File renamed without changes.
0 data/sort_output.txt → data/sort_output_0.txt
View file
Open in desktop
File renamed without changes.
3 changes: 0 additions & 3 deletions src/readme.md
View file
Open in desktop

This file was deleted.

54 changes: 39 additions & 15 deletions tasks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,10 +42,12 @@ class TestRunnerContext:
script_name: str
docker_pull: str
docker_run_test: str
script_type: str
script_relative_path: str
script_name_with_file_type: str
script_output_file_path: str
script_output_file_name: str
input_file_path: str
prepared_file_path: str
snippet_start_line: int
snippet_end_line: int
Expand Down Expand Up @@ -73,23 +75,31 @@ def __init__(self, language) -> None:
# generate the contexts
self.ctxs = []
for script_path in glob.glob(f"{self.base_directory}/src/{language}/*"):
# given "src/python/sort_builtin.py" => return "sort"
script_type = script_path.split("/")[-1].split("_")[0]

# ignore helpers, metadata files, etc
if config.get("ignoreFiles") and script_path.split("/")[-1] in config.get(
"ignoreFiles"
):
continue

# ignore directories, generally compiled code
if not os.path.isfile(script_path):
continue
# generate a context for this particular script
if ctx := self.generate(language, config, script_path):
self.ctxs.append(ctx)

for input_file_path in glob.glob(
f"{self.data_folder_path}/{script_type}_input_*.txt"
):
# generate a context for this particular script
if ctx := self.generate(language, config, script_path, input_file_path):
self.ctxs.append(ctx)

@property
def data(self):
return [ctx.data for ctx in self.ctxs]

def generate(self, language, config, script_path):
def generate(self, language, config, script_path, input_file_path):
# given "src/python/sort_builtin.py" => split on "/" and return "sort_builtin.py"
script_path_split_on_slash = script_path.split("/")
script_name_with_file_type = script_path_split_on_slash[-1]
Expand All @@ -106,11 +116,14 @@ def generate(self, language, config, script_path):

# get the path of the file that's been prepared in advance
# and has the output we would be expecting from out script
prepared_file_path = f"{self.data_folder_path}/{script_type}_output.txt"
prepared_file_path = input_file_path.replace("input", "output")

# given "data/sort_input_1.txt" => return "1"
prepared_file_index = prepared_file_path.split("_")[-1].split(".")[0]

# our scripts write their output files to this path
script_output_file_name = (
f"output_{script_type}_via_{language}_{script_name}.txt"
f"output_{language}_{script_name}_{prepared_file_index}.txt"
)
script_output_file_path = f"{self.data_folder_path}/{script_output_file_name}"

Expand Down Expand Up @@ -142,7 +155,7 @@ def generate(self, language, config, script_path):

# construct ending call args
docker_run_test_list += [
f"-e=INPUT_PATH={self.data_folder_path}/{script_type}_input.txt",
f"-e=INPUT_PATH={input_file_path}",
f"-e=OUTPUT_PATH={script_output_file_path}",
config["dockerImage"],
*script_invoker,
Expand Down Expand Up @@ -170,14 +183,14 @@ def generate(self, language, config, script_path):
if snippet_start_line != 0:
raise Exception(
f'Found multiple "{self.snippet_start_text}" lines in {script_relative_path}.\n'
f"The text with found on lines {snippet_start_line - snippet_start_line_offset + 1} and {idx + 1}."
f"The lines were {snippet_start_line - snippet_start_line_offset + 1} and {idx + 1}."
)
snippet_start_line = idx + 3
if self.snippet_end_text in line:
if snippet_end_line != 0:
raise Exception(
f'Found multiple "{self.snippet_end_text}" lines in {script_relative_path}.\n'
f"The text with found on lines {snippet_end_line + snippet_end_line_offset + 1} and {idx + 1}."
f"The lines were {snippet_end_line + snippet_end_line_offset + 1} and {idx + 1}."
)
snippet_end_line = idx - snippet_end_line_offset
if snippet_start_line == 0:
Expand All @@ -195,10 +208,12 @@ def generate(self, language, config, script_path):
script_name=script_name,
docker_pull=docker_pull,
docker_run_test=docker_run_test,
script_type=script_type,
script_relative_path=script_relative_path,
script_name_with_file_type=script_name_with_file_type,
script_output_file_path=script_output_file_path,
script_output_file_name=script_output_file_name,
input_file_path=input_file_path,
prepared_file_path=prepared_file_path,
snippet_start_line=snippet_start_line,
snippet_end_line=snippet_end_line,
Expand Down Expand Up @@ -244,19 +259,24 @@ def run_tests(self, input_script):
# report timing
# we round the number so humans dont over-index on small differences
print(
f'\t⏱ script "{ctx.script_relative_path}" run for {round(end_time - start_time, 2)} seconds'
f"\t⏱ {ctx.script_relative_path} on {ctx.input_file_path} "
f"ran for {round(end_time - start_time, 2)} seconds"
)

# check if the script invoke failed
if output.exited != 0:
self.set_success_status(False)
print(f'\t🔴 script "{ctx.script_relative_path}" failed, reason:')
print(
f"\t🔴 {ctx.script_relative_path} on {ctx.input_file_path} failed, reason:"
)
print(f'\t\t the exit code "{output.exited}" was not 0')

# check if the output file was created
if not os.path.exists(ctx.script_output_file_path):
self.set_success_status(False)
print(f'\t🔴 script "{ctx.script_relative_path}" failed, reason:')
print(
f"\t🔴 {ctx.script_relative_path} on {ctx.input_file_path} failed, reason:"
)
print(
f"\t\t the output {ctx.script_output_file_name} file was not created"
)
Expand All @@ -266,14 +286,18 @@ def run_tests(self, input_script):
ctx.prepared_file_path, ctx.script_output_file_path
):
self.set_success_status(True)
print(f'\t🟢 script "{ctx.script_relative_path}" succeeded')
print(
f"\t🟢 {ctx.script_relative_path} on {ctx.input_file_path} succeeded"
)

# check if the output file does not match the prepared file
if os.path.exists(ctx.script_output_file_path) and not filecmp.cmp(
ctx.prepared_file_path, ctx.script_output_file_path
):
self.set_success_status(False)
print(f'\t🔴 script "{ctx.script_relative_path}" failed, reason:')
print(
f"\t🔴 {ctx.script_relative_path} on {ctx.input_file_path} failed, reason:"
)
print(
f"\t\t output file {ctx.script_output_file_name} has does not match the prepared file"
)
Expand Down Expand Up @@ -370,4 +394,4 @@ def test(ctx: invoke.Context, language, input_script):

@invoke.task
def clean(ctx: invoke.Context):
ctx.run("git clean -fdx ./data/*")
ctx.run("git clean -fdx ./data/output_*")