Index: openacs-4/packages/acs-automated-testing/tcl/aa-test-procs.tcl
===================================================================
RCS file: /usr/local/cvsroot/openacs-4/packages/acs-automated-testing/tcl/aa-test-procs.tcl,v
diff -u -r1.27 -r1.28
--- openacs-4/packages/acs-automated-testing/tcl/aa-test-procs.tcl 15 Feb 2004 12:25:01 -0000 1.27
+++ openacs-4/packages/acs-automated-testing/tcl/aa-test-procs.tcl 16 Feb 2004 16:47:25 -0000 1.28
@@ -25,6 +25,7 @@
nsv_set aa_test components {}
nsv_set aa_test init_classes {}
nsv_set aa_test categories { config db api web smoke stress security_risk populator production_safe }
+ nsv_set aa_test exclusion_categories { stress security_risk }
}
ad_proc -public aa_stub {
@@ -293,15 +294,15 @@
@param libraries A list of keywords of additional code modules to load. The entire test case will fail if any package is missing. Currently includes <b>tclwebtest</b>.
@param cats Properties of the test case. Must be zero or more of the following:
-<ul>
-<li><b>db</b>: Tests the database directly
-<li><b>api</b>: tests the TCL API
-<li><b>web</b>: tests HTTP interface
-<li><b>smoke</b>: Minimal test to assure functionality and catch basic errors.
-<li><b>stress</b>: Puts heavy load on server or creates large numbers of records. Intended to simulate maximal production load.
-<li><b>security_risk</b>: May introduce a security risk.
-<li><b>populator</b>: Creates and leaves sample data.
-<li><b>production_safe</b>: Can be run without the risk of adding or deleting data.
+ <ul>
+ <li><b>db</b>: Tests the database directly
+ <li><b>api</b>: tests the TCL API
+ <li><b>web</b>: tests HTTP interface
+ <li><b>smoke</b>: Minimal test to assure functionality and catch basic errors.
+ <li><b>stress</b>: Puts heavy load on server or creates large numbers of records. Intended to simulate maximal production load.
+ <li><b>security_risk</b>: May introduce a security risk.
+ <li><b>populator</b>: Creates sample data for future use.
+ <li><b>production_safe</b>: Can be used on a live production site, ie for sanity checking or keepalive purposes. Implies: no risk of adding or deleting data; no risk of crashing; minimal cpu/db/net load.
</ul>
@param error_level Force all test failures to this error level. One of
<ul>
@@ -490,6 +491,8 @@
}
ad_proc -public aa_runseries {
+ {-stress 0}
+ {-security_risk 0}
-quiet:boolean
{-testcase_id ""}
by_package_key
@@ -530,13 +533,34 @@
set package_key [lindex $testcase 3]
set categories [lindex $testcase 4]
set init_classes [lindex $testcase 5]
- if {($by_package_key == "" || ($by_package_key == $package_key)) && \
- ($by_category == "" || ([lsearch $categories $by_category] != -1))} {
- lappend testcase_ids $testcase_id
- foreach init_class $init_classes {
- set classes([list $package_key $init_class]) 1
- }
+
+ # try to disqualify the test case
+
+ # if category is specified,
+ if { [exists_and_not_null by_package_key] && $by_package_key != $package_key } {
+ continue
}
+
+ # is it the wrong category?
+ if { [exists_and_not_null by_category] && [lsearch $categories $by_category] < 0 } {
+ continue
+ }
+
+ # if we don't want stress, then the test must not be stress
+ if { ! $stress && [lsearch $categories "stress"] >= 0 } {
+ continue
+ }
+
+ # if we don't want security risks, then the test must not be stress
+ if { ! $security_risk && [lsearch $categories "security_risk"] >= 0 } {
+ continue
+ }
+
+ # we made it through the filters, so add the test case
+ lappend testcase_ids $testcase_id
+ foreach init_class $init_classes {
+ set classes([list $package_key $init_class]) 1
+ }
}
}
#
@@ -1044,12 +1068,3 @@
set service(rebuild_cmd) "sh [file join $service(script_path) recreate.sh]"
}
-
-
-
-
-
-
-
-
-�
\ No newline at end of file
Index: openacs-4/packages/acs-automated-testing/tcl/test/example-test-init.tcl
===================================================================
RCS file: /usr/local/cvsroot/openacs-4/packages/acs-automated-testing/tcl/test/Attic/example-test-init.tcl,v
diff -u -r1.1 -r1.2
--- openacs-4/packages/acs-automated-testing/tcl/test/example-test-init.tcl 4 Apr 2003 09:46:47 -0000 1.1
+++ openacs-4/packages/acs-automated-testing/tcl/test/example-test-init.tcl 16 Feb 2004 16:47:27 -0000 1.2
@@ -45,7 +45,7 @@
}
aa_register_case -cats {
- script
+ tcl
} -init_classes {
my_init
} "aa_example-000" {
@@ -84,7 +84,7 @@
}
aa_register_case -cats {
- script
+ tcl
} -init_classes {
my_init my_init2
} "aa-example-001" {
@@ -116,7 +116,7 @@
}
aa_register_case -cats {
- script
+ tcl
} "aa_example-002" {
Tests un-successful audit writing.
First call fails.
@@ -137,6 +137,14 @@
set result [aa_example_write_audit_entries $entries]
aa_false "return value false" $result
- aa_equals "entries parameter not currupted" $entries $entries_ex
+ aa_equals "entries parameter not corrupted" $entries $entries_ex
}
+aa_register_case -cats {
+ security_risk
+} "aa_example-exclusion-security-risk" {
+ If security-risk is not checked, this test shouldn't run
+} {
+ aa_log "Unless security-risk is was checked, you shouldn't see this test."
+}
+
Index: openacs-4/packages/acs-automated-testing/www/admin/index.adp
===================================================================
RCS file: /usr/local/cvsroot/openacs-4/packages/acs-automated-testing/www/admin/index.adp,v
diff -u -r1.7 -r1.8
--- openacs-4/packages/acs-automated-testing/www/admin/index.adp 12 Sep 2003 12:03:34 -0000 1.7
+++ openacs-4/packages/acs-automated-testing/www/admin/index.adp 16 Feb 2004 16:47:28 -0000 1.8
@@ -10,48 +10,88 @@
<th bgcolor="#ECECEC">View by</th>
</tr>
<tr>
- <td align=center> [
+
+ <td> [
<if @by_category@ eq "">
<strong> all </strong>
</if><else>
- <a href="index?by_package_key=@by_package_key@&view_by=@view_by@&quiet=@quiet@">all</a>
+ <a href="index?stress=@stress@&security_risk=@security_risk@&by_package_key=@by_package_key@&view_by=@view_by@&quiet=@quiet@">all</a>
</else>
- <multiple name="all_categories">
+ <multiple name="main_categories">
|
- <if @by_category@ eq @all_categories.name@>
- <strong> @all_categories.name@ </strong>
+ <if @by_category@ eq @main_categories.name@>
+ <strong> @main_categories.name@ </strong>
</if><else>
- <a href="index?by_package_key=@by_package_key@&view_by=@view_by@&by_category=@all_categories.name@&quiet=@quiet@">@all_categories.name@</a>
+ <a href="index?stress=@stress@&security_risk=@security_risk@&by_package_key=@by_package_key@&view_by=@view_by@&by_category=@main_categories.name@&quiet=@quiet@">@main_categories.name@</a>
</else>
</multiple> ]
+ <div class="form-help-text">
+ <img src="/shared/images/info.gif" width="12" height="9" alt="[i]" title="Help text" border="0">
+ View and run only tests in this category (<a
+ href="/api-doc/proc-view?proc=aa%5fregister%5fcase">more info</a>)
+ </div>
+
+<p><if @stress@ eq 1>
+ <a href="index?stress=0&security_risk=@security_risk@&by_package_key=@by_package_key@&view_by=@view_by@&by_category=@by_category@&quiet=0" style="padding-top: 2px; padding-bottom: -2px;">
+ <img src="/resources/acs-subsite/checkboxchecked.gif" border="0" height="13" width="13"/>
+ </a>
+ </if>
+ <else>
+ <a href="index?stress=1&security_risk=@security_risk@&by_package_key=@by_package_key@&view_by=@view_by@&by_category=@by_category@&quiet=0" style="padding: 0px;">
+ <img src="/resources/acs-subsite/checkbox.gif" border="0" height="13" width="13"/>
+ </a>
+ </button>
+ </else>
+ Include Stress tests
+</p>
+<p><if @security_risk@ eq 1>
+ <a href="index?stress=@stress@&security_risk=0&by_package_key=@by_package_key@&view_by=@view_by@&by_category=@by_category@&quiet=0" style="padding-top: 2px; padding-bottom: -2px;">
+ <img src="/resources/acs-subsite/checkboxchecked.gif" border="0" height="13" width="13"/>
+ </a>
+ </if>
+ <else>
+ <a href="index?stress=@stress@&security_risk=1&by_package_key=@by_package_key@&view_by=@view_by@&by_category=@by_category@&quiet=0" style="padding: 0px;">
+ <img src="/resources/acs-subsite/checkbox.gif" border="0" height="13" width="13"/>
+ </a>
+ </button>
+ </else>
+ Include tests that may compromise security
+</p>
</td>
- <td align=center> [
+
+ <td align=center valign="top"> [
<if @quiet@ eq "1">
<strong> quiet </strong> |
- <a href="index?by_package_key=@by_package_key@&view_by=@view_by@&by_category=@by_category@&quiet=0">verbose</a>
+ <a href="index?stress=@stress@&security_risk=@security_risk@&by_package_key=@by_package_key@&view_by=@view_by@&by_category=@by_category@&quiet=0">verbose</a>
</if><else>
- <a href="index?by_package_key=@by_package_key@&view_by=@view_by@&by_category=@by_category@&quiet=1">quiet</a>
+ <a href="index?stress=@stress@&security_risk=@security_risk@&by_package_key=@by_package_key@&view_by=@view_by@&by_category=@by_category@&quiet=1">quiet</a>
| <strong> verbose </strong>
</else> ]
+ <div class="form-help-text">
+ <img src="/shared/images/info.gif" width="12" height="9" alt="[i]" title="Help text" border="0">
+ Quiet mode shows only test failures.
+ </div>
</td>
- <td align=center> [
+
+ <td align=center valign="top"> [
<if @view_by@ eq "" or @view_by@ eq "package">
<strong> package </strong> |
- <a href="index?by_package_key=@by_package_key@&view_by=testcase&by_category=@by_category@&quiet=@quiet@">testcase</a>
+ <a href="index?stress=@stress@&security_risk=@security_risk@&by_package_key=@by_package_key@&view_by=testcase&by_category=@by_category@&quiet=@quiet@">testcase</a>
</if><else>
- <a href="index?view_by=package&by_category=@by_category@&quiet=@quiet@">package</a>
+ <a href="index?stress=@stress@&security_risk=@security_risk@&view_by=package&by_category=@by_category@&quiet=@quiet@">package</a>
| <strong> testcase </strong>
</else> ]
</td>
+
</tr>
</table>
<blockquote>
<b>»</b>
- <a href="rerun?package_key=@by_package_key@&category=@by_category@&view_by=@view_by@&quiet=@quiet@"> Rerun displayed test cases</a>
+ <a href="rerun?package_key=@by_package_key@&category=@by_category@&view_by=@view_by@&quiet=@quiet@&stress=@stress@&security_risk=@security_risk@"> Rerun displayed test cases</a>
<br>
<b>»</b>
- <a href="clear?package_key=@by_package_key@&category=@by_category@&view_by=@view_by@&quiet=@quiet@"> Clear test result data</a> </li>
+ <a href="clear?package_key=@by_package_key@&category=@by_category@&view_by=@view_by@&quiet=@quiet@&stress=@stress@&security_risk=@security_risk@"> Clear test result data</a> </li>
</blockquote>
<if @view_by@ eq "package">
@@ -65,7 +105,7 @@
</tr>
<multiple name="packageinfo">
<tr>
- <td> <a href="index?by_package_key=@packageinfo.key@&view_by=testcase&quiet=@quiet@">@packageinfo.key@</a></td>
+ <td> <a href="index?stress=@stress@&security_risk=@security_risk@&by_package_key=@packageinfo.key@&view_by=testcase&quiet=@quiet@">@packageinfo.key@</a></td>
<if @packageinfo.total@ eq "0">
<td> No Data </td><td>-</td><td>-</td>
<td>
@@ -131,10 +171,10 @@
<blockquote>
<b>»</b>
- <a href="rerun?package_key=@by_package_key@&category=@by_category@&view_by=@view_by@&quiet=@quiet@"> Rerun displayed test cases</a>
+ <a href="rerun?package_key=@by_package_key@&category=@by_category@&view_by=@view_by@&quiet=@quiet@&stress=@stress@&security_risk=@security_risk@"> Rerun displayed test cases</a>
<br>
<b>»</b>
- <a href="clear?package_key=@by_package_key@&category=@by_category@&view_by=@view_by@&quiet=@quiet@"> Clear test result data</a> </li>
+ <a href="clear?package_key=@by_package_key@&category=@by_category@&view_by=@view_by@&quiet=@quiet@&stress=@stress@&security_risk=@security_risk@"> Clear test result data</a> </li>
</blockquote>
</body>
Index: openacs-4/packages/acs-automated-testing/www/admin/index.tcl
===================================================================
RCS file: /usr/local/cvsroot/openacs-4/packages/acs-automated-testing/www/admin/index.tcl,v
diff -u -r1.4 -r1.5
--- openacs-4/packages/acs-automated-testing/www/admin/index.tcl 14 Feb 2004 19:19:38 -0000 1.4
+++ openacs-4/packages/acs-automated-testing/www/admin/index.tcl 16 Feb 2004 16:47:28 -0000 1.5
@@ -1,143 +1,151 @@
ad_page_contract {
- @cvs-id $Id$
+ @cvs-id $Id$
} {
- {quiet 0}
- {by_package_key ""}
- {by_category:aa_test_category ""}
- {view_by:aa_test_view_by "package"}
+ {quiet 0}
+ {by_package_key ""}
+ {by_category:aa_test_category ""}
+ {view_by:aa_test_view_by "package"}
+ {stress 0}
+ {security_risk 0}
+ {populator 0}
} -properties {
- context_bar:onevalue
- title:onevalue
- server_name:onevalue
- tests:multirow
- packages:multirow
- categories:multirow
- by_package_key:onevalue
- by_category:onevalue
- view_by:onevalue
- quiet:onevalue
+ context_bar:onevalue
+ title:onevalue
+ server_name:onevalue
+ tests:multirow
+ packages:multirow
+ categories:multirow
+ by_package_key:onevalue
+ by_category:onevalue
+ view_by:onevalue
+ quiet:onevalue
}
set title "System test cases"
if {$by_package_key != ""} {
- append title " for package $by_package_key"
+ append title " for package $by_package_key"
}
if {$by_category != ""} {
- append title ", category $by_category"
+ append title ", category $by_category"
} else {
- append title ", all categories"
+ append title ", all categories"
}
-
foreach testcase [nsv_get aa_test cases] {
- set testcase_id [lindex $testcase 0]
- set testcase_desc [lindex $testcase 1]
- set package_key [lindex $testcase 3]
- set categories [lindex $testcase 4]
- set results("$testcase_id,$package_key") \
- [list $testcase_desc $package_key $categories]
- set packages($package_key) [list 0 0 0]
+ set testcase_id [lindex $testcase 0]
+ set testcase_desc [lindex $testcase 1]
+ set package_key [lindex $testcase 3]
+ set categories [lindex $testcase 4]
+ set results("$testcase_id,$package_key") \
+ [list $testcase_desc $package_key $categories]
+ set packages($package_key) [list 0 0 0]
}
db_foreach acs-automated-testing.results_query {
- select testcase_id, package_key,
- to_char(timestamp,'DD-MM-YYYY HH24:MI:SS') timestamp, passes, fails
- from aa_test_final_results
+ select testcase_id, package_key,
+ to_char(timestamp,'DD-MM-YYYY HH24:MI:SS') timestamp, passes, fails
+ from aa_test_final_results
} {
- if {[info exists results("$testcase_id,$package_key")]} {
- # Append results to individual testcase
- lappend results("$testcase_id,$package_key") $timestamp $passes $fails
+ if {[info exists results("$testcase_id,$package_key")]} {
+ # Append results to individual testcase
+ lappend results("$testcase_id,$package_key") $timestamp $passes $fails
- #
- # If viewing by package, update the by-package results, taking into
- # account whether a specific category has been specified.
- #
- if {$view_by == "package"} {
- set package_total [lindex $packages($package_key) 0]
- set package_pass [lindex $packages($package_key) 1]
- set package_fail [lindex $packages($package_key) 2]
- if {$by_category != ""} {
- # Category specific, only add results if this testcase is of the
- # specified category.
- set categories [lindex $results("$testcase_id,$package_key") 2]
- if {[lsearch $categories $by_category] != -1} {
- incr package_total
- incr package_pass $passes
- incr package_fail $fails
- set packages($package_key) [list $package_total \
- $package_pass $package_fail]
+ #
+ # If viewing by package, update the by-package results, taking into
+ # account whether a specific category has been specified.
+ #
+ if {$view_by == "package"} {
+ set package_total [lindex $packages($package_key) 0]
+ set package_pass [lindex $packages($package_key) 1]
+ set package_fail [lindex $packages($package_key) 2]
+ if {$by_category != ""} {
+ # Category specific, only add results if this testcase is of the
+ # specified category.
+ set categories [lindex $results("$testcase_id,$package_key") 2]
+ if {[lsearch $categories $by_category] != -1} {
+ incr package_total
+ incr package_pass $passes
+ incr package_fail $fails
+ set packages($package_key) [list $package_total \
+ $package_pass $package_fail]
+ }
+ } else {
+ # No category specified, add results.
+ incr package_total
+ incr package_pass $passes
+ incr package_fail $fails
+ set packages($package_key) [list $package_total \
+ $package_pass $package_fail]
+ }
}
- } else {
- # No category specified, add results.
- incr package_total
- incr package_pass $passes
- incr package_fail $fails
- set packages($package_key) [list $package_total \
- $package_pass $package_fail]
- }
}
- }
}
if {$view_by == "package"} {
- #
- # Prepare the template data for a view_by "package"
- #
- template::multirow create packageinfo key total passes fails
- foreach package_key [lsort [array names packages]] {
- set total [lindex $packages($package_key) 0]
- set passes [lindex $packages($package_key) 1]
- set fails [lindex $packages($package_key) 2]
- template::multirow append packageinfo $package_key $total $passes $fails
- }
+ #
+ # Prepare the template data for a view_by "package"
+ #
+ template::multirow create packageinfo key total passes fails
+ foreach package_key [lsort [array names packages]] {
+ set total [lindex $packages($package_key) 0]
+ set passes [lindex $packages($package_key) 1]
+ set fails [lindex $packages($package_key) 2]
+ template::multirow append packageinfo $package_key $total $passes $fails
+ }
} else {
- #
- # Prepare the template data for a view_by "testcase"
- #
- template::multirow create tests id description package_key categories \
- timestamp passes fails marker
- set old_package_key ""
- foreach testcase [nsv_get aa_test cases] {
- set testcase_id [lindex $testcase 0]
- set package_key [lindex $testcase 3]
-
- set testcase_desc [lindex $results("$testcase_id,$package_key") 0]
- regexp {^(.+?\.)\s} $testcase_desc "" testcase_desc
- set categories [lindex $results("$testcase_id,$package_key") 2]
- set categories_str [join $categories ", "]
- set testcase_timestamp [lindex $results("$testcase_id,$package_key") 3]
- set testcase_passes [lindex $results("$testcase_id,$package_key") 4]
- set testcase_fails [lindex $results("$testcase_id,$package_key") 5]
#
- # Only add the testcase to the template multirow if either
- # - The package key is blank or it matches the specified.
- # - The category is blank or it matches the specified.
+ # Prepare the template data for a view_by "testcase"
#
- if {($by_package_key == "" || ($by_package_key == $package_key)) && \
- ($by_category == "" || ([lsearch $categories $by_category] != -1))} {
- # Swap the highlight flag between packages.
- if {$old_package_key != $package_key} {
- set marker 1
- set old_package_key $package_key
- } else {
- set marker 0
- }
- template::multirow append tests $testcase_id $testcase_desc \
- $package_key \
- $categories_str \
- $testcase_timestamp \
- $testcase_passes $testcase_fails \
- $marker
+ template::multirow create tests id description package_key categories \
+ timestamp passes fails marker
+ set old_package_key ""
+ foreach testcase [nsv_get aa_test cases] {
+ set testcase_id [lindex $testcase 0]
+ set package_key [lindex $testcase 3]
+
+ set testcase_desc [lindex $results("$testcase_id,$package_key") 0]
+ regexp {^(.+?\.)\s} $testcase_desc "" testcase_desc
+ set categories [lindex $results("$testcase_id,$package_key") 2]
+ set categories_str [join $categories ", "]
+ set testcase_timestamp [lindex $results("$testcase_id,$package_key") 3]
+ set testcase_passes [lindex $results("$testcase_id,$package_key") 4]
+ set testcase_fails [lindex $results("$testcase_id,$package_key") 5]
+ #
+ # Only add the testcase to the template multirow if either
+ # - The package key is blank or it matches the specified.
+ # - The category is blank or it matches the specified.
+ #
+ if {($by_package_key == "" || ($by_package_key == $package_key)) && \
+ ($by_category == "" || ([lsearch $categories $by_category] != -1))} {
+ # Swap the highlight flag between packages.
+ if {$old_package_key != $package_key} {
+ set marker 1
+ set old_package_key $package_key
+ } else {
+ set marker 0
+ }
+ template::multirow append tests $testcase_id $testcase_desc \
+ $package_key \
+ $categories_str \
+ $testcase_timestamp \
+ $testcase_passes $testcase_fails \
+ $marker
+ }
}
- }
}
#
# Create the category multirow
#
-template::multirow create all_categories name
+template::multirow create main_categories name
+template::multirow create exclusion_categories name
foreach category [nsv_get aa_test categories] {
- template::multirow append all_categories $category
+ # joel@aufrecht.org: putting in special cases for exclusionary categories
+ if { [lsearch [nsv_get aa_test exclusion_categories] $category ] < 0 } {
+ template::multirow append main_categories $category
+ } else {
+ template::multirow append exclusion_categories $category
+ }
}
ad_return_template
Index: openacs-4/packages/acs-automated-testing/www/admin/rerun.tcl
===================================================================
RCS file: /usr/local/cvsroot/openacs-4/packages/acs-automated-testing/www/admin/rerun.tcl,v
diff -u -r1.2 -r1.3
--- openacs-4/packages/acs-automated-testing/www/admin/rerun.tcl 10 Sep 2002 22:22:04 -0000 1.2
+++ openacs-4/packages/acs-automated-testing/www/admin/rerun.tcl 16 Feb 2004 16:47:28 -0000 1.3
@@ -1,21 +1,23 @@
ad_page_contract {
- @cvs-id $Id$
+ @cvs-id $Id$
} {
- {package_key ""}
- {category:aa_test_category ""}
- {view_by:aa_test_view_by "package"}
- {testcase_id:nohtml ""}
- {quiet "0"}
+ {package_key ""}
+ {category:aa_test_category ""}
+ {view_by:aa_test_view_by "package"}
+ {testcase_id:nohtml ""}
+ {quiet "0"}
+ {stress "0"}
+ {security_risk "0"}
} -properties {
}
if {$testcase_id == ""} {
if {$quiet} {
- aa_runseries -quiet $package_key $category
+ aa_runseries -stress $stress -security_risk $security_risk -quiet $package_key $category
} else {
- aa_runseries $package_key $category
+ aa_runseries -stress $stress -security_risk $security_risk $package_key $category
}
- ad_returnredirect "index?by_package_key=$package_key&by_category=$category&view_by=$view_by&quiet=$quiet"
+ ad_returnredirect "index?by_package_key=$package_key&by_category=$category&view_by=$view_by&quiet=$quiet&stress=$stress&security_risk=$security_risk"
} else {
if {$quiet} {
aa_runseries -quiet -testcase_id $testcase_id "" ""
Index: openacs-4/packages/acs-automated-testing/www/doc/xml/requirements.xml
===================================================================
RCS file: /usr/local/cvsroot/openacs-4/packages/acs-automated-testing/www/doc/xml/requirements.xml,v
diff -u -r1.4 -r1.5
--- openacs-4/packages/acs-automated-testing/www/doc/xml/requirements.xml 29 Jan 2004 12:57:18 -0000 1.4
+++ openacs-4/packages/acs-automated-testing/www/doc/xml/requirements.xml 16 Feb 2004 16:47:28 -0000 1.5
@@ -36,13 +36,13 @@
<row>
<entry>1.1</entry>
<entry>partial</entry>
- <entry>A</entry>
+ <entry>Done</entry>
<entry><emphasis role="strong">Execute HTTP tests</emphasis>. Execute tests that can interact with a the webserver via the external, HTTP interface, including retrieving pages, following links, and submitting forms. (This is partially done in the sense that we can make http calls from tcl api, but there is no framework for doing anything complicated.)</entry>
</row>
<row>
<entry>1.1.1</entry>
<entry></entry>
- <entry>A</entry>
+ <entry>Done</entry>
<entry><emphasis role="strong">Execute tclwebtest scripts</emphasis>. A test can contain tclwebtest commands. If tclwebtest is not installed, those commands fail gracefully.</entry>
</row>
<row>
@@ -54,34 +54,11 @@
<row>
<entry>2</entry>
- <entry></entry>
- <entry>A</entry>
- <entry><emphasis role="strong">Tests have categories</emphasis>. Individual tests can be marked as belonging to zero, one, or many of these categories. The UI provides for running only tests in selected categories, and for viewing only results of tests in selected categories.
-<programlisting>Testing Mode
- Regression
- Smoke
- Stress
- Default-Only (for tests, such as front page UI tests, that will break
- once the default site is modified and can be ignored on
- non-default sites)
-Layer
- Web Page
- TCL Page Contract
- TCL API
- SQL
-Severity (derives from ns_log values)
- Notice (use this for informational items that do not imply a problem)
- Warning (use this for submitted tests w/o fixes; hygiene tests such as deprecated function sweeps)
- Error (default severity)
- Test Validity Bug (use this for problems that suggest the test itself in invalid or broken)
-Test Data
- Self-contained Requires no test data, leaves no test data.
- Populate Generates and leaves test data, for other tests or for end users.
-Package Version
- 5.0.0
- etc
-</programlisting></entry>
- </row>
+ <entry>Done</entry>
+ <entry>Done</entry>
+ <entry><emphasis role="strong">Tests have categories</emphasis>. Individual tests can be marked as belonging to zero, one, or many of these categories. The UI provides for running only tests in selected categories, and for viewing only results of tests in selected categories.</entry>
+ </row>
+
<row>
<entry>2.1</entry>
@@ -129,7 +106,38 @@
</tbody>
</tgroup>
</informaltable>
- </sect2>
+
+ <para>Because the current test package uses in-memory variables instead of database objects to track its tests, it is incompatible with the standard category package. It uses an internal, single-dimension category field. Should this eventually get extended, a more complete list of categories to implement could be:</para>
+<programlisting>Testing Mode
+ Regression
+ Smoke
+ Stress
+ Default-Only (for tests, such as front page UI tests, that will break
+ once the default site is modified and can be ignored on
+ non-default sites)
+ production-safe
+ security_risk
+Layer
+ Web Page
+ TCL Page Contract
+ TCL API
+ SQL
+Severity (derives from ns_log values)
+ Notice (use this for informational items that do not imply a problem)
+ Warning (use this for submitted tests w/o fixes; hygiene tests such as deprecated function sweeps)
+ Error (default severity)
+ Test Validity Bug (use this for problems that suggest the test itself in invalid or broken)
+Test Data
+ Self-contained Requires no test data, leaves no test data.
+ Populate Generates and leaves test data, for other tests or for end users.
+Package Version
+ 5.0.0
+ etc
+</programlisting>
+
+ </sect2>
+
+
<sect2>
<title>References</title>
<itemizedlist>