-
Notifications
You must be signed in to change notification settings - Fork 5
/
Copy pathrun-tests.sh
executable file
·152 lines (130 loc) · 8.84 KB
/
run-tests.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
#!/bin/bash
RED='\033[0;31m'
GREEN='\033[0;32m'
NC='\033[0m'
print_test_parameters() {
local test_filename="$1"
local test_command_extra_parameters="$2"
local output_contains_test_result="$3"
local output_contains_text="$4"
local output_contains_text2="$5"
local output_not_contains_text="$6"
echo
echo "******************************************"
echo "Testing:"
echo
echo "filename: $test_filename"
echo "wacat parameters: $test_command_extra_parameters"
echo "contains result: $output_contains_test_result"
echo "contains: $output_contains_text"
if [[ -n "$output_contains_text2" ]]; then
echo "contains: $output_contains_text2"
fi
if [[ -n "$output_not_contains_text" ]]; then
echo "not contains: $output_not_contains_text"
fi
}
run_playwright_tests() {
local test_filename="$1"
local test_command_extra_parameters="$2"
local output_contains_test_result="$3"
local output_contains_text="$4"
local output_contains_text2="$5"
local output_not_contains_text="$6"
print_test_parameters "$test_filename" "$test_command_extra_parameters" "$output_contains_test_result" \
"$output_contains_text" "$output_contains_text2" "$output_not_contains_text"
echo "******************************************"
echo
cp "test-app/test-app/pages/$test_filename" test-app/test-app/pages/index.tsx
sleep 5
test_output=$(wacat test --headless $test_command_extra_parameters http://localhost:3000 2>&1)
echo "$test_output"
if (
[[ $test_output == *$output_contains_test_result* ]] && \
[[ $test_output == *$output_contains_text* ]] && \
([[ -z "$output_contains_text2" ]] || [[ $test_output == *$output_contains_text2* ]]) && \
([[ -z "$output_not_contains_text" ]] || [[ ! $test_output == *$output_not_contains_text* ]]) \
); then \
echo -e "${GREEN}"
print_test_parameters "$test_filename" "$test_command_extra_parameters" "$output_contains_test_result" \
"$output_contains_text" "$output_contains_text2" "$output_not_contains_text"
echo
echo "successful"
echo "******************************************"
echo
echo -e "${NC}"
else
echo -e "${RED}"
print_test_parameters "$test_filename" "$test_command_extra_parameters" "$output_contains_test_result" \
"$output_contains_text" "$output_contains_text2" "$output_not_contains_text"
echo
echo "failed!"
echo "******************************************"
echo "******************************************"
echo "******************************************"
echo "Testing failed!"
echo "******************************************"
echo "******************************************"
echo -e "${NC}"
pkill -f "next"
exit 1
fi
}
echo
echo "******************************************"
echo "******************************************"
echo "Start testing"
echo "******************************************"
echo "******************************************"
echo
(cd test-app/test-app && npm run dev &)
sleep 10
run_playwright_tests "index-simple.tsx" "--wait 20000 --timeout 500 --broken-input-values --broken-input-values-percentage 100 --conf example-files/configuration-error-texts-have-nice-day.json" "1 failed" "(the broken input value used)" "Check that the page doesn't contain the Have a nice day! text"
run_playwright_tests "index-simple.tsx" "--wait 20000 --timeout 500 --conf example-files/configuration-error-texts-have-nice-day.json" "1 failed" "Filling the #1 input field with the AI" "Check that the page doesn't contain the Have a nice day! text"
run_playwright_tests "index-working-page3.tsx" "--wait 20000 --timeout 500 --ignore-ai-generated-input-texts-in-test" "1 passed" "Check with the AI that the page doesn't contain errors." "In the page: http://localhost:3000/working-page"
run_playwright_tests "index-errors-in-page-and-console.tsx" "--wait 20000 --bypass-browser-console-errors --timeout 500 --ignore-ai-generated-input-texts-in-test" \
"1 failed" "The AI detected that current page contains error" "Test page An unexpected error occurred!"
run_playwright_tests "index-errors-in-page-and-console.tsx" "--wait 20000 --bypass-browser-console-errors --timeout 500 --bypass-ai-errors --ignore-ai-generated-input-texts-in-test" \
"1 passed" "The AI detected that current page contains error" "Test page An unexpected error occurred!"
run_playwright_tests "index-errors-in-page-and-console.tsx" "--conf example-files/configuration-error-texts.json --wait 3000 --bypass-browser-console-errors --ignore-ai-in-test" \
"1 failed" "expect(content).not.toContain"
run_playwright_tests "index-different-types-inputs-and-button-push-causes-error.tsx" "--conf example-files/configuration-error-texts.json --wait 3000 --ignore-ai-in-test" \
"1 failed" "expect(content).not.toContain"
run_playwright_tests "index-different-types-inputs-and-button-push-causes-error.tsx" "--conf example-files/configuration-error-texts.json --random-input-texts-min-length 5 --random-input-texts-max-length 5 --random-input-texts-charset ¿ --wait 3000 --ignore-ai-in-test" \
"1 failed" "expect(content).not.toContain"
run_playwright_tests "index-different-types-inputs-and-button-push-causes-error.tsx" "--conf example-files/configuration-error-texts.json --input-texts example-files/input-texts.txt --wait 3000 --ignore-ai-in-test" \
"1 failed" "expect(content).not.toContain"
run_playwright_tests "index-working-page3.tsx" "--input-texts example-files/input-texts.txt --wait 3000 --ignore-ai-in-test" "1 passed" "ybyb"
run_playwright_tests "index-working-page3.tsx" "--input-texts https://raw.githubusercontent.com/mikesmallhelp/wacat/main/example-files/input-texts.txt --wait 3000 --ignore-ai-in-test" "1 passed" "ybyb"
run_playwright_tests "index-api-returns-http-500.tsx" "--wait 3000 --ignore-ai-in-test" "1 failed" \
"http://localhost:3000/api-returns-http-500: Request to" \
"http://localhost:3000/api/http-500 resulted in status code 500"
run_playwright_tests "index-api-returns-http-500.tsx" "--bypass-http-errors --wait 3000 --ignore-ai-in-test" "1 passed" \
"In the page: http://localhost:3000/api-returns-http-500: Request to http://localhost:3000/api/http-500 resulted in status code 500"
run_playwright_tests "index-errors-in-page-and-console.tsx" "--ignore-ai-in-test" "1 failed" "Hello! Something wrong!"
run_playwright_tests "index-working-page3.tsx" "--conf example-files/configuration-error-texts.json --wait 3000 --ignore-ai-in-test" "1 passed" \
"Check that the page doesn't contain the An unexpected error occurred! Please try again after some time. text"
run_playwright_tests "index-working-page3.tsx" "--ignore-ai-in-test" "1 passed" "Push the button"
run_playwright_tests "index-working-page3.tsx" "--debug --wait 3000 --ignore-ai-in-test" "1 passed" "Went outside of the tested application to the page https://mikesmallhelp-test-application.vercel.app/, returning back to the test application"
run_playwright_tests "index-working-page3.tsx" "--wait 3000 --random-input-texts-min-length 5 --random-input-texts-max-length 5 --random-input-texts-charset ¿ --ignore-ai-in-test" \
"1 passed" "¿¿¿¿¿"
run_playwright_tests "index-working-page3.tsx" "--only-links --wait 3000 --ignore-ai-in-test" "1 passed" \
"In the page: http://localhost:3000/working-page2" \
"In the page: http://localhost:3000/working-page3" "Push the button"
run_playwright_tests "index-auth.tsx" "--conf example-files/configuration-authentication.json --wait 3000 --ignore-ai-in-test" "1 passed" \
"In the page: http://localhost:3000/working-page2"
run_playwright_tests "index-auth-complicated.tsx" "--conf example-files/configuration-complicated-authentication.json --wait 3000 --ignore-ai-in-test" \
"1 passed" "In the page: http://localhost:3000/working-page3" "In the page: http://localhost:3000/logout"
run_playwright_tests "index-auth-complicated.tsx" \
"--conf example-files/configuration-complicated-authentication-with-not-visit-link-urls.json --wait 3000 --ignore-ai-in-test" \
"1 passed" "In the page: http://localhost:3000/working-page2" "In the page: http://localhost:3000/working-page3" \
"In the page: http://localhost:3000/logout"
run_playwright_tests "index-working-page3.tsx" "--timeout 1 --ignore-ai-in-test" "1 failed" "Test timeout of 1000ms exceeded."
pkill -f "next"
echo -e "${GREEN}"
echo "******************************************"
echo "******************************************"
echo "Testing successful"
echo "******************************************"
echo "******************************************"
echo -e "${NC}"