[core] server.core-files support for solaris (fixes #3135)

server.core-files support for solaris based systems.

- using setpflags and disable process tracing protection for the current process.

(edited: gstrauss)

x-ref:
  "server.core-files support on Solaris based system"
  https://redmine.lighttpd.net/issues/3135

SConstruct

@@ -463,6 +463,9 @@ if 1:
 	])
 	autoconf.haveFunc('getentropy', 'sys/random.h')
 	autoconf.haveFunc('getrandom', 'linux/random.h')
+	if re.compile("sunos|solaris").search(env['PLATFORM']):
+		autoconf.haveCHeader('priv.h')
+		autoconf.haveFunc('setpflags', 'priv.h')
 
 	autoconf.haveTypes(Split('pid_t size_t off_t'))
 

configure.ac

@@ -1472,6 +1472,10 @@ AC_CHECK_FUNCS([\
 ])
 AC_CHECK_HEADERS([sys/random.h], [AC_CHECK_FUNCS([getentropy])])
 AC_CHECK_HEADERS([linux/random.h], [AC_CHECK_FUNCS([getrandom])])
+case "$host_os" in
+*sunos*|*solaris* ) AC_CHECK_HEADERS([priv.h], [AC_CHECK_FUNCS([setpflags])]) ;;
+* ) ;;
+esac
 
 AC_MSG_NOTICE([----------------------------------------])
 AC_MSG_CHECKING([if weak symbols are supported])

src/CMakeLists.txt

@@ -155,6 +155,11 @@ set(CMAKE_EXTRA_INCLUDE_FILES time.h)
 check_function_exists(timegm HAVE_TIMEGM)
 set(CMAKE_EXTRA_INCLUDE_FILES)
 
+if(CMAKE_SYSTEM_NAME MATCHES "SunOS")
+check_include_files(priv.h HAVE_PRIV_H)
+check_function_exists(setpflags HAVE_SETPFLAGS)
+endif()
+
 check_type_size(long SIZEOF_LONG)
 check_type_size(off_t SIZEOF_OFF_T)
 

src/config.h.cmake

@@ -20,6 +20,7 @@
 #cmakedefine  HAVE_MALLOC_H
 #cmakedefine  HAVE_POLL_H
 #cmakedefine  HAVE_PORT_H
+#cmakedefine  HAVE_PRIV_H
 #cmakedefine  HAVE_PWD_H
 #cmakedefine  HAVE_STDINT_H
 #cmakedefine  HAVE_STDLIB_H
@@ -186,6 +187,7 @@
 #cmakedefine  HAVE_SEND_FILE
 #cmakedefine  HAVE_SENDFILE64
 #cmakedefine  HAVE_SENDFILEV
+#cmakedefine  HAVE_SETPFLAGS
 #cmakedefine  HAVE_SIGACTION
 #cmakedefine  HAVE_SIGNAL
 #cmakedefine  HAVE_SIGTIMEDWAIT

src/meson.build

@@ -118,6 +118,11 @@ if conf_data.get('HAVE_LINUX_RANDOM_H')
 	))
 endif
 
+if target_machine.system() == 'sunos'
+conf_data.set('HAVE_PRIV_H', compiler.has_header('priv.h'))
+conf_data.set('HAVE_SETPFLAGS', compiler.has_function('setpflags', args: defs))
+endif
+
 conf_data.set('SIZEOF_LONG', compiler.sizeof('long', args: defs))
 conf_data.set('SIZEOF_OFF_T', compiler.sizeof('off_t', args: defs))
 

src/server.c

@@ -68,6 +68,9 @@ static const buffer default_server_tag = { CONST_STR_LEN(PACKAGE_DESC)+1, 0 };
 #ifdef HAVE_SYS_PROCCTL_H
 # include <sys/procctl.h>
 #endif
+#ifdef HAVE_PRIV_H
+# include <priv.h>
+#endif
 
 #ifdef HAVE_MALLOC_H
 #ifndef LIGHTTPD_STATIC
@@ -1474,6 +1477,14 @@ static int server_main_setup (server * const srv, int argc, char **argv) {
 			int dumpable = PROC_TRACE_CTL_ENABLE;
 			procctl(P_PID, 0, PROC_TRACE_CTL, &dumpable);
 		}
+#elif defined(HAVE_SETPFLAGS) && defined(__PROC_PROTECT)
+		/**
+		 * setpflags seems uniquely a solaris/illumos feature
+		 * but just taking extra precautions clearing __PROC_PROTECT option
+		 */
+		if (srv->srvconf.enable_cores) {
+			setpflags(__PROC_PROTECT, 0);
+		}
 #endif
 	}