diff --git a/src/network.c b/src/network.c
index 1ee43bf0..382f570f 100644
--- a/src/network.c
+++ b/src/network.c
@@ -271,11 +271,6 @@ static int network_server_init(server *srv, buffer *host_token, size_t sidx) {
 		goto error_free_socket;
 	}
 
-	if (-1 == listen(srv_socket->fd, s->listen_backlog)) {
-		log_error_write(srv, __FILE__, __LINE__, "ss", "listen failed: ", strerror(errno));
-		goto error_free_socket;
-	}
-
 	if (srv_socket->addr.plain.sa_family == AF_UNIX && !buffer_string_is_empty(s->socket_perms)) {
 		mode_t m = 0;
 		for (char *str = s->socket_perms->ptr; *str; ++str) {
@@ -287,6 +282,11 @@ static int network_server_init(server *srv, buffer *host_token, size_t sidx) {
 		}
 	}
 
+	if (-1 == listen(srv_socket->fd, s->listen_backlog)) {
+		log_error_write(srv, __FILE__, __LINE__, "ss", "listen failed: ", strerror(errno));
+		goto error_free_socket;
+	}
+
 	if (s->ssl_enabled) {
 #ifdef TCP_DEFER_ACCEPT
 	} else if (s->defer_accept) {