Author: psmith
Date: Tue Oct 10 01:55:24 2006
New Revision: 10
Modified:
branches/home/psmith/stress-mods/event-notification.asd
branches/home/psmith/stress-mods/src/async-fd.lisp
branches/home/psmith/stress-mods/src/async-socket.lisp
branches/home/psmith/stress-mods/src/epoll.lisp
branches/home/psmith/stress-mods/src/event-notification.lisp
branches/home/psmith/stress-mods/src/nio-httpd.lisp
branches/home/psmith/stress-mods/src/nio-server.lisp
Log:
Moved to event driven SM:
Server socket in level triggered mode - (TODO: why does this improve performance?)
Put accepted connections fd's in nonblocking, RW notification and left as ET (as suggested in 'man epoll')
Increased accept backlog to 1k (TODO: put on config)
Included code review suggestions from Risto
Modified: branches/home/psmith/stress-mods/event-notification.asd
==============================================================================
--- branches/home/psmith/stress-mods/event-notification.asd (original)
+++ branches/home/psmith/stress-mods/event-notification.asd Tue Oct 10 01:55:24 2006
@@ -8,7 +8,8 @@
(:file "src/kqueue-cffi" :depends-on ("src/event-notification"))
(:file "src/epoll-cffi" :depends-on ("src/event-notification"))
(:file "src/kqueue" :depends-on ("src/event-notification" "src/kqueue-cffi"))
- (:file "src/epoll" :depends-on ("src/event-notification" "src/epoll-cffi")))
+ (:file "src/errno")
+ (:file "src/epoll" :depends-on ("src/event-notification" "src/epoll-cffi" "src/errno")))
:depends-on (:cffi))
Modified: branches/home/psmith/stress-mods/src/async-fd.lisp
==============================================================================
--- branches/home/psmith/stress-mods/src/async-fd.lisp (original)
+++ branches/home/psmith/stress-mods/src/async-fd.lisp Tue Oct 10 01:55:24 2006
@@ -105,6 +105,7 @@
(defun close-fd (unix-fd)
"Close UNIX-FD."
+#+nio-debug (format t "close-fd ~A~%" unix-fd)
(%close unix-fd))
@@ -145,15 +146,20 @@
(define-condition read-error (error) ())
(defun read-more (async-fd)
+#+nio-debug (format t "read-more called with ~A~%" async-fd)
"Read more data from ASYNC-FD."
(with-slots (foreign-read-buffer foreign-read-buffer-size) async-fd
(with-slots (read-fd lisp-read-buffer lisp-read-buffer-write-ptr) async-fd
-
+#+nio-debug (format t "read-more - calling read()~%")
+#+nio-debug (force-output t)
(let ((new-bytes (%read read-fd foreign-read-buffer foreign-read-buffer-size)))
-
+#+nio-debug (format t "read-more : Read ~A bytes~%" new-bytes)
+#+nio-debug (force-output t)
(cond
- ((< new-bytes 0)
- (error 'read-error))
+ ((< new-bytes 0)
+ (progn
+ (format t "read-error - Errno: ~A~%" (sb-alien:get-errno))
+ (error 'read-error)))
((= new-bytes 0)
nil);;(throw 'end-of-file nil))
@@ -168,6 +174,8 @@
(mem-aref foreign-read-buffer :uint8 i)))
(incf lisp-read-buffer-write-ptr new-bytes)
+#+nio-debug (format t "read-more prior to callback")
+#+nio-debug (force-output t)
;; call callback
(with-slots (accept-filter read-callback) async-fd
(if accept-filter
@@ -182,8 +190,8 @@
(defun close-async-fd (async-fd)
"Close ASYNC-FD's fd after everything has been written from write-queue."
(with-slots (write-queue read-fd write-fd foreign-read-buffer) async-fd
+#+nio-debug (format t "close-asyn-fd called with :read-fd ~A :write-fd ~A~%" read-fd write-fd)
(cond
-
;; if write-queue is emtpy, close now
((null write-queue)
(close-fd read-fd)
@@ -197,6 +205,7 @@
(defun write-more (async-fd)
"Write data from ASYNC-FD's write-queue."
+#+nio-debug (format t "write-more called with ~A~%" async-fd)
(with-slots (write-fd write-queue) async-fd
;; loop for packets in queue
@@ -234,7 +243,7 @@
(defun async-write-seq (async-fd seq &optional (start 0) (end (length seq)))
"Queue from SEQ between START and END to write-queue."
-
+#+nio-debug (format t "async-write-seq - called ~A~%" async-fd)
(assert (and (numberp start) (not (null seq))))
;; enqueue sequence
@@ -243,7 +252,8 @@
(setf write-queue (append write-queue (list entry)))))
;; start writing
- (write-more async-fd))
+; (write-more async-fd)
+)
@@ -258,13 +268,15 @@
(defun add-async-fd (event-queue async-fd mode)
(ecase mode
(:read (add-fd event-queue (slot-value async-fd 'read-fd) :read))
- (:write (add-fd event-queue (slot-value async-fd 'write-fd) :write))))
+ (:write (add-fd event-queue (slot-value async-fd 'write-fd) :write))
+ (:read-write (add-fd event-queue (slot-value async-fd 'write-fd) :read-write))))
(defun remove-async-fd (event-queue async-fd mode)
(ecase mode
(:read (remove-fd event-queue (slot-value async-fd 'read-fd) :read))
- (:write (remove-fd event-queue (slot-value async-fd 'write-fd) :write))))
+ (:write (remove-fd event-queue (slot-value async-fd 'write-fd) :write))
+ (:read-write (remove-fd event-queue (slot-value async-fd 'write-fd) :read-write))))
(defun async-fd-read-fd (async-fd)
Modified: branches/home/psmith/stress-mods/src/async-socket.lisp
==============================================================================
--- branches/home/psmith/stress-mods/src/async-socket.lisp (original)
+++ branches/home/psmith/stress-mods/src/async-socket.lisp Tue Oct 10 01:55:24 2006
@@ -82,8 +82,8 @@
(sockaddr :pointer)
(socklen :pointer))
-
-(defun start-listen (socket-fd &optional (backlog 7))
+;;TODO put backlog on config
+(defun start-listen (socket-fd &optional (backlog 1000))
(%listen socket-fd backlog))
Modified: branches/home/psmith/stress-mods/src/epoll.lisp
==============================================================================
--- branches/home/psmith/stress-mods/src/epoll.lisp (original)
+++ branches/home/psmith/stress-mods/src/epoll.lisp Tue Oct 10 01:55:24 2006
@@ -29,23 +29,38 @@
#+linux
(progn
- (defun make-event-queue ()
- (%epoll-create 10))
+ (defconstant +epoll-size+ 1000)
+ (defun make-event-queue ()
+ (%epoll-create +epoll-size+))
- (defun add-fd (event-queue fd mode &key (trigger :edge))
- (with-foreign-object (ev 'epoll-event)
- (memzero ev +epoll-event-size+)
+ (defun read-event-p (event)
+ (not (eql (logand event +epoll-in+) 0)))
- (setf (foreign-slot-value ev 'epoll-event 'fd) fd
- (foreign-slot-value ev 'epoll-event 'events)
- (logior (if (eql :read mode) +epoll-in+ 0)
- (if (eql :write mode) +epoll-out+ 0)
- (if (eql trigger :edge) +epoll-et+)))
+ (defun write-event-p (event)
+ (not (eql (logand event +epoll-out+) 0)))
- (%epoll-ctl event-queue +epoll-ctl-add+ fd ev)))
+ (defun add-fd (event-queue fd mode &key (trigger :edge))
+ (with-foreign-object (ev 'epoll-event)
+ (memzero ev +epoll-event-size+)
+ (let ((actual-mode (logior (if (eql :read mode) +epoll-in+ 0)
+ (if (eql :write mode) +epoll-out+ 0)
+ (if (eql :read-write mode) (logior +epoll-in+ +epoll-out+) 0)
+ (if (eql trigger :edge) +epoll-et+ 0))))
+ #+nio-debug (format t "Add-fd called with :fd ~A :event-queue ~A :mode ~A :trigger ~A :actual-mode ~A~%" fd event-queue mode trigger actual-mode)
+
+ (setf (foreign-slot-value ev 'epoll-event 'fd) fd
+ (foreign-slot-value ev 'epoll-event 'events)
+ actual-mode))
+
+ (if (eql (%epoll-ctl event-queue +epoll-ctl-add+ fd ev) -1)
+ (progn
+ (format t "add-fd (epoll_ctl) error occurred: ~A~%" (get-errno))
+ ;; (error 'poll-error)
+ ))))
+
(defun remove-fd (event-queue fd mode)
(with-foreign-object (ev 'epoll-event)
(memzero ev +epoll-event-size+)
@@ -58,21 +73,26 @@
(define-condition poll-error (error) ())
(defun poll-events (event-queue)
- (with-foreign-object (events 'epoll-event 10)
- (memzero events (* +epoll-event-size+ 10))
- (loop for res = (%epoll-wait event-queue events 10 -1)
+#+nio-debug (format t "poll-events called with :event-queue ~A~%" event-queue)
+ (with-foreign-object (events 'epoll-event +epoll-size+)
+ (memzero events (* +epoll-event-size+ +epoll-size+))
+ (loop for res = (%epoll-wait event-queue events +epoll-size+ -1)
+
do
- (case res
- (-1 (error 'poll-error))
- (0 nil)
- (t
- (let ((idents nil))
- (loop for i from 0 below res do
- (push (foreign-slot-value
- (mem-aref events 'epoll-event i)
- 'epoll-event 'fd)
- idents))
- (return idents)))))))
-
-
- )
+ (progn
+#+nio-debug (format t "poll-events - dealing with ~A~%" res)
+ (case res
+ (-1 (error 'poll-error))
+ (0 nil)
+ (t
+ (let ((idents nil))
+ (loop for i from 0 below res do
+ (push (cons (foreign-slot-value
+ (mem-aref events 'epoll-event i)
+ 'epoll-event 'fd)
+ (foreign-slot-value
+ (mem-aref events 'epoll-event i)
+ 'epoll-event 'events))
+ idents))
+ (return idents))))))))
+)
Modified: branches/home/psmith/stress-mods/src/event-notification.lisp
==============================================================================
--- branches/home/psmith/stress-mods/src/event-notification.lisp (original)
+++ branches/home/psmith/stress-mods/src/event-notification.lisp Tue Oct 10 01:55:24 2006
@@ -26,4 +26,4 @@
|#
(defpackage :event-notification (:use :cl :cffi)
(:export
- make-event-queue add-fd remove-fd poll-events poll-error))
\ No newline at end of file
+ make-event-queue add-fd remove-fd poll-events poll-error read-event-p write-event-p))
\ No newline at end of file
Modified: branches/home/psmith/stress-mods/src/nio-httpd.lisp
==============================================================================
--- branches/home/psmith/stress-mods/src/nio-httpd.lisp (original)
+++ branches/home/psmith/stress-mods/src/nio-httpd.lisp Tue Oct 10 01:55:24 2006
@@ -94,6 +94,7 @@
(defun serve-content (client status content-type content)
+#+nio-debug (format t "serve-content :client ~A~%" client)
(let ((status-line (make-status-line status))
(content-type
(concatenate 'vector
Modified: branches/home/psmith/stress-mods/src/nio-server.lisp
==============================================================================
--- branches/home/psmith/stress-mods/src/nio-server.lisp (original)
+++ branches/home/psmith/stress-mods/src/nio-server.lisp Tue Oct 10 01:55:24 2006
@@ -59,7 +59,8 @@
(format t "~&Starting server on ~S port ~S.. (socket fd is ~D)~%" host port sock)
(start-listen sock)
- (add-fd event-queue sock :read)
+
+ (add-fd event-queue sock :read :trigger :level)
(format t "waiting for events..~%") (force-output)
@@ -69,38 +70,39 @@
(format t "Poll-error, exiting..~%")
(throw 'poll-error-exit nil))))
- (loop for unix-fds = (poll-events event-queue) do
+ (loop for unix-epoll-events = (poll-events event-queue) do
- (loop for fd in unix-fds do
-
+ (loop for (fd . event) in unix-epoll-events do
(cond
;; new connection
((= fd sock)
(let ((async-fd (socket-accept fd)))
-
+#+nio-debug (format t "start-server - New conn: ~A~%" async-fd)
(cond
((null async-fd)
(format t "Accept failed.~%"))
;; accept connection ?
- ((funcall accept-connection async-fd)
+ ((set-fd-nonblocking (async-fd-read-fd async-fd))
+ (funcall accept-connection async-fd)
(setf (gethash (async-fd-read-fd async-fd) client-hash) async-fd)
(set-accept-filter async-fd accept-filter)
(set-read-callback async-fd connection-handler)
- (add-async-fd event-queue async-fd :read)
- (add-async-fd event-queue async-fd :write)
+ (add-async-fd event-queue async-fd :read-write)
+; (add-async-fd event-queue async-fd :write)
)
;; no accept, close
(t
+ (format t "start-server - accept-connection closed~%")
(close-async-fd async-fd)))))
;; socket i/o available
(t
(let ((async-fd (gethash fd client-hash)))
-
+#+nio-debug (format t "IO event ~A on ~A~%" event async-fd)
(unless (null async-fd)
(catch 'error-exit
(handler-bind ((read-error #'(lambda (x)
@@ -112,9 +114,9 @@
(force-close-async-fd async-fd)
(throw 'error-exit nil))))
- (read-more async-fd))))
+ (when (read-event-p event) (read-more async-fd))
+ (when (write-event-p event) (write-more async-fd)))))
))
- )))))
-
+ )))))
(ignore-errors
(close-fd sock))))
