[ŧests] more tests, improve handling

tests/base.py

@@ -50,6 +50,7 @@ import sys
 import traceback
 import re
 import subprocess
+import inspect
 
 from service import *
 
@@ -90,17 +91,22 @@ class TestBase(object):
 	runnable = True
 	todo = False
 	subdomains = False # set to true to match all subdomains too
+	inherit_docroot = False
 
-	def __init__(self):
+	def __init__(self, parent = None):
 		self._test_cleanup_files = []
 		self._test_cleanup_dirs = []
 		self._test_failed = False # "not run" is "successful"
+		self._parent = parent
 
 	# internal methods, do not override
 	def _register(self, tests):
 		self.tests = tests
 		if not self.vhost: self.vhost = vhostname(self.name)
-		self.vhostdir = os.path.join(Env.dir, 'www', 'vhosts', self.vhost)
+		if self.inherit_docroot:
+			self.vhostdir = self._parent.vhostdir
+		else:
+			self.vhostdir = os.path.join(Env.dir, 'www', 'vhosts', self.vhost)
 		if self.FeatureCheck():
 			tests.add_test(self)
 			return True
@@ -116,7 +122,7 @@ class TestBase(object):
 			errorconfig = Env.debug and " " or """log [ default => "file:%s" ];""" % (errorlog)
 			accesslog = self.PrepareFile("log/access.log-%s" % self.vhost, "")
 			if None != self.vhostdir:
-				docroot = 'docroot "%s";' % self.vhostdir
+				docroot = 'local var.docroot = "%s"; docroot var.docroot;' % self.vhostdir
 			else:
 				docroot = ''
 			if self.subdomains:
@@ -228,9 +234,14 @@ class GroupTest(TestBase):
 
 	def __init__(self):
 		super(GroupTest, self).__init__()
+		test_module = self.__class__.__module__
+		module_classes = inspect.getmembers(sys.modules[test_module], lambda member: inspect.isclass(member) and member.__module__ == test_module)
+		for name, obj in module_classes:
+			if name.startswith("Test") and not obj in self.group and obj != self.__class__:
+				print >> sys.stderr, "Test class", name, "not listed in group test list in", test_module, ".", self.__class__.__name__
 		self.subtests = []
 		for c in self.group:
-			t = c()
+			t = c(self)
 			self.subtests.append(t)
 
 	def _register(self, tests):
@@ -331,35 +342,53 @@ class Tests(object):
 		errorconfig = Env.debug and " " or """log [ default => "file:%s" ];""" % (errorlog)
 		accesslog = self.PrepareFile("log/access.log", "")
 		self.config = """
+global var.docdir = "{Env.docdir}";
+global var.default_docroot = "{Env.defaultwww}";
+
 setup {{
 	workers 2;
 
-	module_load (
+	module_load [
 		"mod_accesslog",
 		"mod_cache_disk_etag",
 		"mod_deflate",
 		"mod_dirlist",
 		"mod_lua",
 		"mod_vhost"
-	);
+	];
 
 	listen "127.0.0.2:{Env.port}";
 	log [ default => "stderr" ];
 
-	lua.plugin "{Env.luadir}/core.lua";
-	lua.plugin "{Env.luadir}/secdownload.lua";
+	lua.plugin var.docdir + "/core.lua";
+	lua.plugin var.docdir + "/secdownload.lua";
 
 	accesslog.format "%h %V %u %t \\"%r\\" %>s %b \\"%{{Referer}}i\\" \\"%{{User-Agent}}i\\"";
 	accesslog "{accesslog}";
+
+	debug.log_request_handling true;
+
+	# default values, just check whether they parse
+	static.range_requests true;
+	keepalive.timeout 5;
+	keepalive.requests 0;
+	etag.use ["inode", "mtime", "size"];
+	stat.async true;
+	buffer_request_body true;
+
+	io.timeout 300;
+	stat_cache.ttl 10;
 }}
 
 {errorconfig}
 
-defaultaction = {{
-	docroot "{Env.defaultwww}";
+include var.docdir + "/mimetypes.conf";
+
+global defaultaction = {{
+	docroot var.default_docroot;
 }};
 
-do_deflate = {{
+global do_deflate = {{
 	if request.is_handled {{
 		deflate;
 	}}

tests/requests.py

@@ -9,6 +9,13 @@ import bz2
 
 from base import *
 
+TEST_TXT="""Hi!
+0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
+0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
+0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
+0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
+"""
+
 class CurlRequestException(Exception):
 	def __init__(self, value): self.value = value
 	def __str__(self): return repr(self.value)
@@ -26,8 +33,8 @@ class CurlRequest(TestBase):
 	EXPECT_RESPONSE_CODE = None
 	EXPECT_RESPONSE_HEADERS = []
 
-	def __init__(self):
-		super(CurlRequest, self).__init__()
+	def __init__(self, parent):
+		super(CurlRequest, self).__init__(parent)
 		self.resp_header_list = []
 		self.resp_headers = { }
 		self.resp_first_line = None

tests/runtests.py

@@ -72,7 +72,7 @@ Env.force_cleanup = options.force_cleanup
 Env.port = find_port(options.port)
 Env.tests = options.tests
 Env.sourcedir = os.path.abspath(os.path.dirname(__file__))
-Env.luadir = os.path.join(os.path.dirname(Env.sourcedir), "doc")
+Env.docdir = os.path.join(os.path.dirname(Env.sourcedir), "doc")
 Env.debugRequests = options.debug_requests
 Env.strace = options.strace
 Env.truss = options.truss
@@ -119,7 +119,8 @@ try:
 	try:
 		tests.Prepare()
 	except:
-		raise
+		import traceback
+		print >> sys.stderr, traceback.format_exc()
 	else:
 		if tests.Run():
 			failed = False
@@ -127,7 +128,10 @@ try:
 		tests.Cleanup()
 		if not Env.no_cleanup and not failed:
 			os.remove(os.path.join(Env.dir, "tests.log"))
-
+except:
+	import traceback
+	print >> sys.stderr, traceback.format_exc()
+	failed = True
 finally:
 	try:
 		if Env.force_cleanup:

tests/t-alias.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+
+from base import *
+from requests import *
+
+class TestAlias1(CurlRequest):
+	URL = "/alias1"
+	EXPECT_RESPONSE_BODY = TEST_TXT
+	EXPECT_RESPONSE_CODE = 200
+	EXPECT_RESPONSE_HEADERS = [("Content-Type", "text/plain; charset=utf-8")]
+	config = """
+alias "/alias1" => var.default_docroot + "/test.txt";
+"""
+
+class TestAlias2(CurlRequest):
+	URL = "/alias2"
+	EXPECT_RESPONSE_BODY = TEST_TXT
+	EXPECT_RESPONSE_CODE = 200
+	EXPECT_RESPONSE_HEADERS = [("Content-Type", "text/plain; charset=utf-8")]
+	config = """
+alias "/alias1" => "/nothing", "/alias2" => var.default_docroot + "/test.txt";
+"""
+
+class TestAlias3(CurlRequest):
+	URL = "/alias3/test.txt"
+	EXPECT_RESPONSE_BODY = TEST_TXT
+	EXPECT_RESPONSE_CODE = 200
+	EXPECT_RESPONSE_HEADERS = [("Content-Type", "text/plain; charset=utf-8")]
+	config = """
+alias "/alias3" => var.default_docroot + "/";
+"""
+
+class Test(GroupTest):
+	group = [
+		TestAlias1,
+		TestAlias2,
+		TestAlias3
+	]

tests/t-basic-gets.py

@@ -3,13 +3,6 @@
 from base import *
 from requests import *
 
-TEST_TXT="""Hi!
-0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
-0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
-0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
-0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
-"""
-
 LUA_SHOW_ENV_INFO="""
 
 function show_env_info(vr)
@@ -28,6 +21,41 @@ class TestSimpleRequest(CurlRequest):
 	URL = "/test.txt"
 	EXPECT_RESPONSE_BODY = TEST_TXT
 	EXPECT_RESPONSE_CODE = 200
+	EXPECT_RESPONSE_HEADERS = [("Content-Type", "text/plain; charset=utf-8")]
+
+class TestSimpleRequestStatus(CurlRequest):
+	URL = "/test.txt"
+	EXPECT_RESPONSE_BODY = TEST_TXT
+	EXPECT_RESPONSE_CODE = 403
+	config = """
+defaultaction;
+static_no_fail;
+set_status 403;
+"""
+
+class TestSimpleRespond(CurlRequest):
+	URL = "/test.txt"
+	EXPECT_RESPONSE_BODY = "hello"
+	EXPECT_RESPONSE_CODE = 200
+	config = 'respond "hello";'
+
+class TestIndex1(CurlRequest):
+	URL = "/"
+	EXPECT_RESPONSE_BODY = TEST_TXT
+	EXPECT_RESPONSE_CODE = 200
+	config = """
+defaultaction;
+index "test.txt";
+"""
+
+class TestIndex2(CurlRequest):
+	URL = "/"
+	EXPECT_RESPONSE_BODY = TEST_TXT
+	EXPECT_RESPONSE_CODE = 200
+	config = """
+defaultaction;
+index "index.html", "test.txt";
+"""
 
 class TestSimpleInfo(CurlRequest):
 	URL = "/?a_simple_query"
@@ -44,6 +72,64 @@ class TestBadRequest1(CurlRequest):
 	URL = "/?complicated?query= $"
 	EXPECT_RESPONSE_CODE = 400
 
+class TestStaticExcludeExtensions1(CurlRequest):
+	URL = "/test.php"
+	EXPECT_RESPONSE_CODE = 403
+	config = """
+defaultaction;
+static.exclude_extensions ".php";
+"""
+
+class TestStaticExcludeExtensions2(CurlRequest):
+	URL = "/test.php"
+	EXPECT_RESPONSE_CODE = 403
+	config = """
+defaultaction;
+static.exclude_extensions (".php", ".py");
+"""
+
+class TestServerTag(CurlRequest):
+	URL = "/test.txt"
+	EXPECT_RESPONSE_BODY = TEST_TXT
+	EXPECT_RESPONSE_CODE = 200
+	EXPECT_RESPONSE_HEADERS = [("Server", "apache - no really!")]
+	config = """
+defaultaction;
+server.tag "apache - no really!";
+"""
+
+class TestConditionalHeader1(CurlRequest):
+	URL = "/"
+	EXPECT_RESPONSE_BODY = "a"
+	REQUEST_HEADERS = ["X-Select: a"]
+	config = """
+if req.header["X-Select"] == "a" {
+	respond "a";
+} else {
+	respond "b";
+}
+"""
+
+class TestConditionalHeader2(CurlRequest):
+	URL = "/"
+	EXPECT_RESPONSE_BODY = "b"
+	config = """
+if req.header["X-Select"] == "a" {
+	respond "a";
+} else {
+	respond "b";
+}
+"""
+
+class TestSimplePattern1(CurlRequest):
+	URL = "/"
+	EXPECT_RESPONSE_CODE = 403
+	EXPECT_RESPONSE_BODY = "hello"
+	REQUEST_HEADERS = ["X-Select: hello"]
+	config = """
+respond 403 => "%{req.header[X-Select]}";
+"""
+
 class ProvideStatus(TestBase):
 	runnable = False
 	vhost = "status"
@@ -53,10 +139,26 @@ status.info;
 """
 
 class Test(GroupTest):
-	group = [TestSimpleRequest,TestSimpleInfo,TestBadRequest1,ProvideStatus]
+	group = [
+		TestSimpleRequest,
+		TestSimpleRequestStatus,
+		TestSimpleRespond,
+		TestIndex1,
+		TestIndex2,
+		TestSimpleInfo,
+		TestBadRequest1,
+		TestStaticExcludeExtensions1,
+		TestStaticExcludeExtensions2,
+		TestServerTag,
+		TestConditionalHeader1,
+		TestConditionalHeader2,
+		TestSimplePattern1,
+		ProvideStatus
+	]
 
 	def Prepare(self):
 		self.PrepareFile("www/default/test.txt", TEST_TXT)
+		self.PrepareFile("www/default/test.php", "")
 		show_env_info_lua = self.PrepareFile("lua/show_env_info.lua", LUA_SHOW_ENV_INFO)
 		self.plain_config = """
 show_env_info = {{

tests/t-deflate.py

@@ -3,13 +3,6 @@
 from base import *
 from requests import *
 
-TEST_TXT="""Hi!
-0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
-0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
-0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
-0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
-"""
-
 class DeflateRequest(CurlRequest):
 	URL = "/test.txt"
 	EXPECT_RESPONSE_BODY = TEST_TXT
@@ -51,6 +44,8 @@ class Test(GroupTest):
 	group = [TestGzip, TestXGzip, TestDeflate, TestBzip2, TestXBzip2, TestDisableDeflate]
 
 	def Prepare(self):
-		self.PrepareVHostFile("test.txt", TEST_TXT)
 		# deflate is enabled global too; force it here anyway
-		self.config = """if req.query == "nodeflate" { req_header.remove "Accept-Encoding"; } static; do_deflate;"""
+		self.config = """
+defaultaction;
+if req.query == "nodeflate" { req_header.remove "Accept-Encoding"; } static; do_deflate;
+"""

tests/t-etag.py

@@ -3,13 +3,6 @@
 from base import *
 from requests import *
 
-TEST_TXT="""Hi!
-0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
-0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
-0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
-0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
-"""
-
 retrieved_etag1 = None
 
 class TestGetEtag1(CurlRequest):

tests/t-redirect.py

@@ -3,32 +3,31 @@
 from base import *
 from requests import *
 
-class TestForceHttps(CurlRequest):
+class TestRewrite1(CurlRequest):
+	URL = "/somefile"
+	EXPECT_RESPONSE_BODY = TEST_TXT
+	EXPECT_RESPONSE_CODE = 200
+	EXPECT_RESPONSE_HEADERS = [("Content-Type", "text/plain; charset=utf-8")]
 	config = """
-redirect "https://%{request.host}%{enc:request.path}?%{request.query}";
+rewrite "^/somefile$" => "/test.txt";
+defaultaction;
 """
-	URL = "/?a_simple_query"
-	EXPECT_RESPONSE_CODE = 301
-	EXPECT_RESPONSE_HEADERS = [("Location", "https://forcehttps.redirect/?a_simple_query")]
 
-class TestRemoveWWW(CurlRequest):
+class TestRewrite2(CurlRequest):
+	URL = "/somefile"
+	EXPECT_RESPONSE_BODY = TEST_TXT
+	EXPECT_RESPONSE_CODE = 200
+	EXPECT_RESPONSE_HEADERS = [("Content-Type", "text/plain; charset=utf-8")]
 	config = """
-if req.host =~ "^www\.(.+)$" {
-	redirect "http://%1%{enc:request.path}?%{request.query}";
-}
+rewrite "/somethingelse" => "/nothing", "^/somefile$" => "/test.txt";
+defaultaction;
 """
-
-	vhost = "www.test"
-	URL = "/foo?bar"
-	EXPECT_RESPONSE_CODE = 301
-	EXPECT_RESPONSE_HEADERS = [("Location", "http://test/foo?bar")]
-
 class Test(GroupTest):
 	plain_config = """
-setup { module_load "mod_redirect"; }
+setup { module_load "mod_rewrite"; }
 """
 
 	group = [
-		TestForceHttps,
-		TestRemoveWWW
+		TestRewrite1,
+		TestRewrite2
 	]

tests/t-rewrite.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+
+from base import *
+from requests import *
+
+class TestRewrite1(CurlRequest):
+	URL = "/somefile"
+	EXPECT_RESPONSE_BODY = TEST_TXT
+	EXPECT_RESPONSE_CODE = 200
+	EXPECT_RESPONSE_HEADERS = [("Content-Type", "text/plain; charset=utf-8")]
+	config = """
+rewrite "^/somefile$" => "/test.txt";
+defaultaction;
+"""
+
+class TestRewrite2(CurlRequest):
+	URL = "/somefile"
+	EXPECT_RESPONSE_BODY = TEST_TXT
+	EXPECT_RESPONSE_CODE = 200
+	EXPECT_RESPONSE_HEADERS = [("Content-Type", "text/plain; charset=utf-8")]
+	config = """
+rewrite "/somethingelse" => "/nothing", "^/somefile$" => "/test.txt";
+defaultaction;
+"""
+class Test(GroupTest):
+	plain_config = """
+setup { module_load "mod_rewrite"; }
+"""
+
+	group = [
+		TestRewrite1,
+		TestRewrite2
+	]

tests/t-secdownload.py

@@ -5,13 +5,6 @@ from requests import *
 import time
 from md5 import md5
 
-TEST_TXT="""Hi!
-0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
-0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
-0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
-0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
-"""
-
 def securl(prefix, path, secret, tstamp = None):
 	if tstamp == None: tstamp = time.time()
 	tstamp = '%x' % int(tstamp)
@@ -40,10 +33,9 @@ class SecdownloadGone(CurlRequest):
 
 class Test(GroupTest):
 	group = [SecdownloadFail, SecdownloadSuccess, SecdownloadGone]
+	config = """
+secdownload ( "prefix" => "/", "document-root" => var.default_docroot, "secret" => "abc", "timeout" => 600 );
+"""
 
 	def Prepare(self):
-		self.PrepareVHostFile("test.txt", TEST_TXT)
-		self.config = """
-			secdownload ( "prefix" => "/", "document-root" => "{docroot}", "secret" => "abc", "timeout" => 600 );
-		""".format(docroot = self.vhostdir)
 		self.vhostdir = None