- nio-cvs@common-lisp.net - mailman3.common-lisp.net

[nio-cvs] r75 - branches/home/psmith/restructure/src/statemachine
by psmith＠common-lisp.net 07 Feb '07

07 Feb '07

Author: psmith Date: Tue Feb 6 20:17:03 2007 New Revision: 75 Modified: branches/home/psmith/restructure/src/statemachine/state-machine.lisp Log: Removed close con in sm if process-packet returns null (is allowed) Modified: branches/home/psmith/restructure/src/statemachine/state-machine.lisp ============================================================================== --- branches/home/psmith/restructure/src/statemachine/state-machine.lisp (original) +++ branches/home/psmith/restructure/src/statemachine/state-machine.lisp Tue Feb 6 20:17:03 2007 @@ -65,8 +65,7 @@ (error 'not-implemented-yet-read-resize-failure)))))) (format-log t "state-machine::process-read - incoming packet: ~A~%" incoming-packet) (when incoming-packet - (when (not (process-incoming-packet sm incoming-packet)) - (close-sm sm)))))) + (process-incoming-packet sm incoming-packet))))) ;The connection is write ready. ;See if theres anything ready to be written in the SM

1 0

[nio-cvs] r74 - in branches/home/psmith/restructure/src: io nio-logger protocol/yarpc
by psmith＠common-lisp.net 06 Feb '07

06 Feb '07

Author: psmith Date: Mon Feb 5 22:43:57 2007 New Revision: 74 Modified: branches/home/psmith/restructure/src/io/async-fd.lisp branches/home/psmith/restructure/src/io/nio-package.lisp branches/home/psmith/restructure/src/io/nio-server.lisp branches/home/psmith/restructure/src/nio-logger/nio-logger.asd branches/home/psmith/restructure/src/nio-logger/nio-logger.lisp branches/home/psmith/restructure/src/protocol/yarpc/yarpc-state-machine.lisp Log: Changes for server close Modified: branches/home/psmith/restructure/src/io/async-fd.lisp ============================================================================== --- branches/home/psmith/restructure/src/io/async-fd.lisp (original) +++ branches/home/psmith/restructure/src/io/async-fd.lisp Mon Feb 5 22:43:57 2007 @@ -43,7 +43,8 @@ (write-ready :initform nil :accessor write-ready :documentation "Have we been notified as write ready and not received EAGAIN from %write?") - (close-pending :initform nil) + (close-pending :initform nil + :accessor close-pending) (socket :initarg :socket :accessor socket))) @@ -151,7 +152,7 @@ (error 'not-implemented-yet)))) #+nio-debug (format t "write buffer after write :~A~%" foreign-write-buffer) - (when (eql (remaining foreign-write-buffer) 0) + (when (eql (buffer-position foreign-write-buffer) 0) (when close-pending (close-async-fd async-fd))))) Modified: branches/home/psmith/restructure/src/io/nio-package.lisp ============================================================================== --- branches/home/psmith/restructure/src/io/nio-package.lisp (original) +++ branches/home/psmith/restructure/src/io/nio-package.lisp Mon Feb 5 22:43:57 2007 @@ -30,7 +30,7 @@ ;; async-fd.lisp async-fd process-read process-write foreign-read-buffer foreign-write-buffer close-sm - recommend-buffer-size + recommend-buffer-size close-pending ;; async-socket.lisp Modified: branches/home/psmith/restructure/src/io/nio-server.lisp ============================================================================== --- branches/home/psmith/restructure/src/io/nio-server.lisp (original) +++ branches/home/psmith/restructure/src/io/nio-server.lisp Mon Feb 5 22:43:57 2007 @@ -39,7 +39,8 @@ ;loop over hashtable (defun process-async-fds (client-hash) - (maphash #'(lambda (k async-fd) + (let ((removals nil)) + (maphash #'(lambda (k async-fd) (format-log t "Dealing with ~a => ~a~%" k async-fd) ;process reads @@ -50,9 +51,16 @@ ;process-writes (process-write async-fd) (when (and (write-ready async-fd) - (> (buffer-position (foreign-write-buffer async-fd)) 0)) - (write-more async-fd))) - client-hash)) + (> (buffer-position (foreign-write-buffer async-fd)) 0)) + (write-more async-fd)) + + (when (close-pending async-fd) + (write-more async-fd) + (push async-fd removals))) + client-hash) + (dolist (async-fd removals) + (remhash (async-fd-read-fd async-fd) client-hash)) + (format t "client-hash list ~A~%"client-hash ))) Modified: branches/home/psmith/restructure/src/nio-logger/nio-logger.asd ============================================================================== --- branches/home/psmith/restructure/src/nio-logger/nio-logger.asd (original) +++ branches/home/psmith/restructure/src/nio-logger/nio-logger.asd Mon Feb 5 22:43:57 2007 @@ -8,5 +8,5 @@ (:file "nio-logger" :depends-on ("nio-logger-package")) ) - :depends-on (:nio-yarpc :nio-utils)) + :depends-on (:nio-yarpc :nio-utils :cl-base64)) Modified: branches/home/psmith/restructure/src/nio-logger/nio-logger.lisp ============================================================================== --- branches/home/psmith/restructure/src/nio-logger/nio-logger.lisp (original) +++ branches/home/psmith/restructure/src/nio-logger/nio-logger.lisp Mon Feb 5 22:43:57 2007 @@ -50,7 +50,7 @@ (let ((sm (nio:add-connection ip-address 16323 'nio-yarpc:yarpc-client-state-machine))) (nio-utils:format-log t "toplevel adding conn ~A to ~A~%" sm ip-address) (with-line-from-tailed-file (text filename 1) - (let ((rpc (format nil "(nio-logger:remote-log \"~A\")" text))) + (let ((rpc (format nil "(nio-logger:remote-log \"~A\")" (cl-base64:string-to-base64-string text)))) (nio-utils:format-log t "Toplevel Submitting job~A~%" rpc) (nio-utils:format-log t "Result of remote-log ~A~%" (nio-yarpc:remote-execute sm rpc)))))) @@ -67,8 +67,8 @@ ;;block waiting for jobs (nio-yarpc:run-job))) -(nio-yarpc:defremote remote-log(str) +(nio-yarpc:defremote remote-log(base64-str) (with-open-file (out +log-file-name+ :direction :output :if-exists :append) - (nio-utils:format-log out "~A~%" str)) + (nio-utils:format-log out "~A~%" (cl-base64:base64-string-to-string base64-str))) t) Modified: branches/home/psmith/restructure/src/protocol/yarpc/yarpc-state-machine.lisp ============================================================================== --- branches/home/psmith/restructure/src/protocol/yarpc/yarpc-state-machine.lisp (original) +++ branches/home/psmith/restructure/src/protocol/yarpc/yarpc-state-machine.lisp Mon Feb 5 22:43:57 2007 @@ -102,12 +102,14 @@ (define-condition authorization-error (error) ()) (defun execute-call (call-string) + (handler-case (let* ((rpc-call-list (read-from-string call-string )) (fn (member (symbol-function (first rpc-call-list)) *remote-fns* ))) (format-log t "yarpc-state-machine:execute-call - fn ~A authorised? : ~A~%" (symbol-function (first rpc-call-list)) fn) (if fn (apply (first rpc-call-list) (rest rpc-call-list)) - (error 'authorization-error)))) + (error 'authorization-error))) + (reader-error (re) (format-log t "yarpc-state-machine:execute-call - reader error on call-string ~A ~%" re)))) ;;end move TODO

1 0

[nio-cvs] r73 - branches/home/psmith/restructure/src/io
by psmith＠common-lisp.net 06 Feb '07

06 Feb '07

Author: psmith Date: Mon Feb 5 20:30:09 2007 New Revision: 73 Modified: branches/home/psmith/restructure/src/io/fd-helper.lisp Log: Corrected typo Modified: branches/home/psmith/restructure/src/io/fd-helper.lisp ============================================================================== --- branches/home/psmith/restructure/src/io/fd-helper.lisp (original) +++ branches/home/psmith/restructure/src/io/fd-helper.lisp Mon Feb 5 20:30:09 2007 @@ -46,7 +46,7 @@ (fd :int) (cmd :int) #+(or darwin macosx freebsd) - (arg :int)) + (arg :int) #+linux (arg :long))

1 0

[nio-cvs] r72 - branches/home/psmith/restructure/src/nio-logger
by psmith＠common-lisp.net 06 Feb '07

06 Feb '07

Author: psmith Date: Mon Feb 5 20:29:41 2007 New Revision: 72 Modified: branches/home/psmith/restructure/src/nio-logger/nio-logger.lisp branches/home/psmith/restructure/src/nio-logger/run-logging-server.lisp branches/home/psmith/restructure/src/nio-logger/run.sh Log: Updates to logging server for cmd line usage Modified: branches/home/psmith/restructure/src/nio-logger/nio-logger.lisp ============================================================================== --- branches/home/psmith/restructure/src/nio-logger/nio-logger.lisp (original) +++ branches/home/psmith/restructure/src/nio-logger/nio-logger.lisp Mon Feb 5 20:29:41 2007 @@ -58,10 +58,11 @@ (defparameter +log-file-name+ "./out") -(defun run-logging-server() +(defun run-logging-server(listen-ip out-file &optional (allowed-ips "ips.txt")) (setf nio-yarpc:+process-jobs-inline+ nil) - (nio:load-ips "ips.txt") - (sb-thread:make-thread #'(lambda()(nio:start-server 'identity 'identity 'nio-yarpc:yarpc-state-machine :host "127.0.0.1" :accept-connection 'nio:check-ip)) :name "nio-server") + (setf +log-file-name+ out-file) + (nio:load-ips allowed-ips) + (sb-thread:make-thread #'(lambda()(nio:start-server 'identity 'identity 'nio-yarpc:yarpc-state-machine :host listen-ip :accept-connection 'nio:check-ip)) :name "nio-server") (loop ;;block waiting for jobs (nio-yarpc:run-job))) Modified: branches/home/psmith/restructure/src/nio-logger/run-logging-server.lisp ============================================================================== --- branches/home/psmith/restructure/src/nio-logger/run-logging-server.lisp (original) +++ branches/home/psmith/restructure/src/nio-logger/run-logging-server.lisp Mon Feb 5 20:29:41 2007 @@ -28,4 +28,9 @@ (push :nio-debug *features*) (require :asdf) (require :nio-logger) -(nio-logger:run-logging-server) + +(let ((listen-ip (second sb-ext:*posix-argv*)) + (out-file (third sb-ext:*posix-argv*)) + (allowed-ips-filename (fourth sb-ext:*posix-argv*))) + (format t "Starting logging Server with ~A ~A ~A~%" listen-ip out-file allowed-ips-filename) + (nio-logger:run-logging-server listen-ip out-file allowed-ips-filename)) Modified: branches/home/psmith/restructure/src/nio-logger/run.sh ============================================================================== --- branches/home/psmith/restructure/src/nio-logger/run.sh (original) +++ branches/home/psmith/restructure/src/nio-logger/run.sh Mon Feb 5 20:29:41 2007 @@ -1,8 +1,8 @@ #!/bin/bash # # run.sh <logging-client in-file dest-ip> -# run.sh <logging-server out-file> +# run.sh <logging-server listen-ip out-file allowed-ips-filename> # export LANG=en_US.UTF-8 -sbcl --load run-$1.lisp --end-toplevel-options $2 $3 +sbcl --load run-$1.lisp --end-toplevel-options $2 $3 $4

1 0

[nio-cvs] r71 - branches/home/psmith/restructure/src/nio-logger
by psmith＠common-lisp.net 05 Feb '07

05 Feb '07

Author: psmith Date: Mon Feb 5 11:57:21 2007 New Revision: 71 Modified: branches/home/psmith/restructure/src/nio-logger/run-logging-client.lisp branches/home/psmith/restructure/src/nio-logger/run.sh Log: Made startup more firendly Modified: branches/home/psmith/restructure/src/nio-logger/run-logging-client.lisp ============================================================================== --- branches/home/psmith/restructure/src/nio-logger/run-logging-client.lisp (original) +++ branches/home/psmith/restructure/src/nio-logger/run-logging-client.lisp Mon Feb 5 11:57:21 2007 @@ -28,4 +28,9 @@ (push :nio-debug *features*) (require :asdf) (require :nio-logger) -(nio-logger:tail-log "./test" "127.0.0.1") + +;Use run.sh logging-client <file> <ip> +(let ((log-file (second sb-ext:*posix-argv*)) + (ip (third sb-ext:*posix-argv*))) + (format t "Starting logging client with ~A ~A~%" log-file ip) + (nio-logger:tail-log log-file ip)) Modified: branches/home/psmith/restructure/src/nio-logger/run.sh ============================================================================== --- branches/home/psmith/restructure/src/nio-logger/run.sh (original) +++ branches/home/psmith/restructure/src/nio-logger/run.sh Mon Feb 5 11:57:21 2007 @@ -1,7 +1,8 @@ #!/bin/bash # -# run.sh <http|yarpc> +# run.sh <logging-client in-file dest-ip> +# run.sh <logging-server out-file> # export LANG=en_US.UTF-8 -sbcl --load run-$1.lisp +sbcl --load run-$1.lisp --end-toplevel-options $2 $3

1 0

[nio-cvs] r70 - branches/home/psmith/restructure/src/io
by psmith＠common-lisp.net 05 Feb '07

05 Feb '07

Author: psmith Date: Sun Feb 4 23:51:51 2007 New Revision: 70 Modified: branches/home/psmith/restructure/src/io/async-fd.lisp branches/home/psmith/restructure/src/io/nio-server.lisp Log: Large packet support working Modified: branches/home/psmith/restructure/src/io/async-fd.lisp ============================================================================== --- branches/home/psmith/restructure/src/io/async-fd.lisp (original) +++ branches/home/psmith/restructure/src/io/async-fd.lisp Sun Feb 4 23:51:51 2007 @@ -77,7 +77,9 @@ (with-slots (close-pending) async-fd (setf close-pending t))) -(define-condition read-error (error) ()) +(define-condition read-error (error) + ((errno :initform :errno + :accessor read-error-errno))) ;; "Read more data from STATE-MACHINE." (defun read-more (state-machine) @@ -88,16 +90,19 @@ #+nio-debug (format t "read-more : Read ~A bytes into ~A~%" new-bytes foreign-read-buffer) (cond ((< new-bytes 0) - (progn -;;TODO if ret is -1 and errno is EAGAIN save state and wait for notification - (format t "read-error - Errno: ~A~%" (get-errno)) - (error 'read-error))) + (let ((errno (get-errno))) + (format t "read-error - Errno: ~A~%" errno) + (cond ((eql errno +ERRNO_EAGAIN+) + (setf (read-ready state-machine) nil)) + (t + (error 'read-error :errno errno))))) ((= new-bytes 0) nil);;(throw 'end-of-file nil) (t ;;Update buffer position (inc-position foreign-read-buffer new-bytes) - (setf (read-ready state-machine) nil)))))) + (when (> (remaining foreign-read-buffer) 0) + (setf (read-ready state-machine) nil))))))) (defun close-async-fd (async-fd) "Close ASYNC-FD's fd after everything has been written from write-queue." Modified: branches/home/psmith/restructure/src/io/nio-server.lisp ============================================================================== --- branches/home/psmith/restructure/src/io/nio-server.lisp (original) +++ branches/home/psmith/restructure/src/io/nio-server.lisp Sun Feb 4 23:51:51 2007 @@ -109,7 +109,12 @@ ;; accept connection ? ((funcall accept-connection async-fd) - (set-fd-nonblocking (async-fd-read-fd async-fd)) + (sleep 0.1) + (let ((nb-ret (set-fd-nonblocking (async-fd-read-fd async-fd)))) + (format t "set bb ret: ~A :flags ~A~%" nb-ret (get-fd-flags (async-fd-read-fd async-fd))) + (when (< nb-ret 0) + (format t "Error setting socket non-blocking: ") + (perror))) (setf (gethash (async-fd-read-fd async-fd) client-hash) async-fd) (add-async-fd event-queue async-fd :read-write) ; (add-async-fd event-queue async-fd :write)

1 0

[nio-cvs] r69 - branches/home/psmith/restructure/src/io
by psmith＠common-lisp.net 05 Feb '07

05 Feb '07

Author: psmith Date: Sun Feb 4 23:51:27 2007 New Revision: 69 Modified: branches/home/psmith/restructure/src/io/fd-helper.lisp Log: Corrections to fcntl and O_NONBLOCK definitions for linux Modified: branches/home/psmith/restructure/src/io/fd-helper.lisp ============================================================================== --- branches/home/psmith/restructure/src/io/fd-helper.lisp (original) +++ branches/home/psmith/restructure/src/io/fd-helper.lisp Sun Feb 4 23:51:27 2007 @@ -32,7 +32,12 @@ (defconstant +cmd-get-flags+ 3) (defconstant +cmd-set-flags+ 4) -(defconstant +arg-nonblock+ #x0004) + +#+(or darwin macosx freebsd) +(defconstant +arg-nonblock+ #x4) + +#+linux +(defconstant +arg-nonblock+ #x800) (defcfun ("close" %close) :int (fd :int)) @@ -40,7 +45,10 @@ (defcfun ("fcntl" %fcntl) :int (fd :int) (cmd :int) +#+(or darwin macosx freebsd) (arg :int)) +#+linux + (arg :long)) (defcfun ("read" %read) :long (fd :int) @@ -60,10 +68,15 @@ (defun memzero (ptr size) (%memset ptr 0 size)) +(defun get-fd-flags(unix-fd) + (%fcntl unix-fd +cmd-get-flags+ 0)) (defun set-fd-nonblocking (unix-fd) "Set UNIX-FD to non-blocking mode (O_NONBLOCK)." - (%fcntl unix-fd +cmd-set-flags+ +arg-nonblock+)) + (let* ((flags (get-fd-flags unix-fd)) + (new-flags (logior flags +arg-nonblock+))) + (format t "set-fd-nonblocking :fd ~A :old-flags 0x~X 0x~X~%" unix-fd flags new-flags) + (%fcntl unix-fd +cmd-set-flags+ new-flags))) (defun close-fd (unix-fd)

1 0

[nio-cvs] r68 - in branches/home/psmith/restructure/src: buffer io statemachine
by psmith＠common-lisp.net 04 Feb '07

04 Feb '07

Author: psmith Date: Sun Feb 4 18:04:14 2007 New Revision: 68 Modified: branches/home/psmith/restructure/src/buffer/buffer.lisp branches/home/psmith/restructure/src/buffer/nio-buffer-package.lisp branches/home/psmith/restructure/src/io/async-fd.lisp branches/home/psmith/restructure/src/statemachine/state-machine.lisp Log: large packet server side OK. Modified: branches/home/psmith/restructure/src/buffer/buffer.lisp ============================================================================== --- branches/home/psmith/restructure/src/buffer/buffer.lisp (original) +++ branches/home/psmith/restructure/src/buffer/buffer.lisp Sun Feb 4 18:04:14 2007 @@ -107,6 +107,10 @@ (defun byte-buffer (capacity) (make-instance 'byte-buffer :capacity capacity :limit capacity :position 0 :buf (cffi:foreign-alloc :uint8 :count capacity))) +;Gets a pointer to the address in the native memory of the position index +(defmethod buffer-pointer ((bb byte-buffer)) + (cffi:make-pointer (+ (cffi:pointer-address (buffer-buf bb)) (buffer-position bb)))) + (defmethod print-object ((byte-buffer byte-buffer) stream) (with-slots (capacity position limit buf) byte-buffer (format stream "<byte-buffer :capacity ~A :position ~A :limit ~A :buf ~%~A>~%" capacity position limit (if buf (pretty-hex-dump (cffi:pointer-address buf) limit) nil)))) @@ -176,7 +180,8 @@ ;Used to signal either an attempt has been made to write data to a buffer that is too small using a write (overflow) ; or an incomming packet doesn't have enough room to fit (define-condition buffer-too-small-error (error) - ((recommended-size :initarg :recommended-size))) + ((recommended-size :initarg :recommended-size + :accessor recommended-size))) (defun buffer-too-small-error(recommended-size) (make-instance 'buffer-too-small-error :recommended-size recommended-size)) Modified: branches/home/psmith/restructure/src/buffer/nio-buffer-package.lisp ============================================================================== --- branches/home/psmith/restructure/src/buffer/nio-buffer-package.lisp (original) +++ branches/home/psmith/restructure/src/buffer/nio-buffer-package.lisp Sun Feb 4 18:04:14 2007 @@ -32,5 +32,5 @@ bytebuffer-read-vector bytebuffer-read-string bytebuffer-read-8 bytebuffer-read-32 bytebuffer-write-8 bytebuffer-write-32 bytebuffer-insert-8 bytebuffer-insert-32 flip unflip clear buffer-position copy-buffer buffer-capacity compact mark reset - buffer-too-small-error + buffer-too-small-error recommended-size buffer-pointer )) Modified: branches/home/psmith/restructure/src/io/async-fd.lisp ============================================================================== --- branches/home/psmith/restructure/src/io/async-fd.lisp (original) +++ branches/home/psmith/restructure/src/io/async-fd.lisp Sun Feb 4 18:04:14 2007 @@ -84,7 +84,7 @@ (with-slots (foreign-read-buffer read-fd) state-machine #+nio-debug (format t "read-more called with ~A~%" state-machine) #+nio-debug (format t "read-more - calling read() into ~A~%" foreign-read-buffer) - (let ((new-bytes (%read read-fd (buffer-buf foreign-read-buffer) (remaining foreign-read-buffer)))) + (let ((new-bytes (%read read-fd (buffer-pointer foreign-read-buffer) (remaining foreign-read-buffer)))) #+nio-debug (format t "read-more : Read ~A bytes into ~A~%" new-bytes foreign-read-buffer) (cond ((< new-bytes 0) Modified: branches/home/psmith/restructure/src/statemachine/state-machine.lisp ============================================================================== --- branches/home/psmith/restructure/src/statemachine/state-machine.lisp (original) +++ branches/home/psmith/restructure/src/statemachine/state-machine.lisp Sun Feb 4 18:04:14 2007 @@ -56,7 +56,13 @@ ;Use the packet factory to obtain any valid packet and pass it through (defmethod process-read((sm state-machine)) (with-slots (foreign-read-buffer) sm - (let ((incoming-packet (get-packet (get-packet-factory sm) foreign-read-buffer))) + (let ((incoming-packet + (handler-case + (get-packet (get-packet-factory sm) foreign-read-buffer) + (buffer-too-small-error (read-err) + (if (recommend-buffer-size sm :read (recommended-size read-err)) + (format-log t "resized incomming buffer ~A~%"foreign-read-buffer) + (error 'not-implemented-yet-read-resize-failure)))))) (format-log t "state-machine::process-read - incoming packet: ~A~%" incoming-packet) (when incoming-packet (when (not (process-incoming-packet sm incoming-packet))

1 0

[nio-cvs] r67 - in branches/home/psmith/restructure/src: buffer io protocol/yarpc statemachine
by psmith＠common-lisp.net 04 Feb '07

04 Feb '07

Author: psmith Date: Sun Feb 4 15:53:04 2007 New Revision: 67 Modified: branches/home/psmith/restructure/src/buffer/buffer.lisp branches/home/psmith/restructure/src/buffer/nio-buffer-package.lisp branches/home/psmith/restructure/src/io/async-fd.lisp branches/home/psmith/restructure/src/protocol/yarpc/yarpc-packet-factory.lisp branches/home/psmith/restructure/src/statemachine/state-machine.lisp Log: client side large packet OK Modified: branches/home/psmith/restructure/src/buffer/buffer.lisp ============================================================================== --- branches/home/psmith/restructure/src/buffer/buffer.lisp (original) +++ branches/home/psmith/restructure/src/buffer/buffer.lisp Sun Feb 4 15:53:04 2007 @@ -48,6 +48,9 @@ :initform 0 :accessor buffer-position :documentation "Index of next element to be read/written 0<=position<=limit") + (mark :initarg :position + :initform 0 + :documentation "A marked position") (buf :initarg :buf :accessor buffer-buf))) @@ -159,6 +162,15 @@ (setf position remaining) (setf limit capacity)))) +(defmethod mark((bb byte-buffer)) + :documentation "mark a position in the buffer for subsequent use with reset" + (with-slots (position mark) bb + (setf mark position))) + +(defmethod reset((bb byte-buffer)) + (with-slots (position mark) bb + (setf position mark))) + ;Used to signal either an attempt has been made to write data to a buffer that is too small using a write (overflow) @@ -231,9 +243,7 @@ ;TODO move string-to-octets into nio-compat (defmethod bytebuffer-write-string((bb byte-buffer) str &optional (external-format :ascii)) :documentation "Returns number of bytes written to bytebuffer" - (let ((vec (sb-ext:string-to-octets str :external-format external-format))) - (when (< (remaining bb) (length vec)) (error 'buffer-too-small-error)) - (bytebuffer-write-vector bb vec))) + (bytebuffer-write-vector bb (sb-ext:string-to-octets str :external-format external-format))) @@ -241,7 +251,7 @@ (assert (<= (buffer-capacity old) (buffer-capacity new))) (%memcpy (buffer-buf new) (buffer-buf old) (buffer-capacity old)) (setf (buffer-position new) (buffer-position old)) - (setf (buffer-limit new) (buffer-limit old))) + (setf (buffer-limit new) (buffer-capacity new))) ;void *memcpy(void *dest, const void *src, size_t n); (cffi:defcfun ("memcpy" %memcpy) :pointer Modified: branches/home/psmith/restructure/src/buffer/nio-buffer-package.lisp ============================================================================== --- branches/home/psmith/restructure/src/buffer/nio-buffer-package.lisp (original) +++ branches/home/psmith/restructure/src/buffer/nio-buffer-package.lisp Sun Feb 4 15:53:04 2007 @@ -31,6 +31,6 @@ bytebuffer-write-vector bytebuffer-write-string bytebuffer-read-vector bytebuffer-read-string bytebuffer-read-8 bytebuffer-read-32 bytebuffer-write-8 bytebuffer-write-32 bytebuffer-insert-8 bytebuffer-insert-32 - flip unflip clear buffer-position copy-buffer buffer-capacity compact + flip unflip clear buffer-position copy-buffer buffer-capacity compact mark reset buffer-too-small-error )) Modified: branches/home/psmith/restructure/src/io/async-fd.lisp ============================================================================== --- branches/home/psmith/restructure/src/io/async-fd.lisp (original) +++ branches/home/psmith/restructure/src/io/async-fd.lisp Sun Feb 4 15:53:04 2007 @@ -152,21 +152,34 @@ (defconstant +MAX-BUFFER-SIZE-BYTES+ (* 1024 1024)) -(defun realloc-buffer (async-fd buffer size) - (if (>= (length buffer) size) - t - (let ((new-buffer (byte-buffer size))) - (copy-buffer buffer new-buffer) - (free-buffer buffer) - (setf (foreign-read-buffer async-fd) new-buffer)))) + + +;(let ((buffer (foreign-read-buffer async-fd))) +; (if (>= (length buffer) size) +; t +; (let ((new-buffer (byte-buffer size))) +; (copy-buffer buffer new-buffer) +; (free-buffer buffer) +; (setf (foreign-read-buffer async-fd) new-buffer))))) + + +(defmacro realloc-buffer(async-fd accessor size) + `(let ((buffer (,accessor ,async-fd))) + (if (>= (buffer-capacity buffer) size) + t + (let ((new-buffer (byte-buffer ,size))) + (copy-buffer buffer new-buffer) + (free-buffer buffer) + (setf (,accessor ,async-fd) new-buffer))))) + ;TODO actually deal with buffer allocation failure (defmethod recommend-buffer-size((async-fd async-fd) mode size) (if (> size +MAX-BUFFER-SIZE-BYTES+) nil (ecase mode - (:read (realloc-buffer (foreign-read-buffer async-fd) size)) - (:write (realloc-buffer (foreign-write-buffer async-fd) size))))) + (:read (realloc-buffer async-fd foreign-read-buffer size)) + (:write (realloc-buffer async-fd foreign-write-buffer size))))) Modified: branches/home/psmith/restructure/src/protocol/yarpc/yarpc-packet-factory.lisp ============================================================================== --- branches/home/psmith/restructure/src/protocol/yarpc/yarpc-packet-factory.lisp (original) +++ branches/home/psmith/restructure/src/protocol/yarpc/yarpc-packet-factory.lisp Sun Feb 4 15:53:04 2007 @@ -74,12 +74,19 @@ (defmethod write-bytes((packet call-method-packet) buf) #+nio-debug (format-log t "yarpc-packet-factory:write-bytes(call-method-packet) - writing ~%~A to ~%~A~%" packet buf) - (nio-buffer:bytebuffer-write-8 buf +CALL-METHOD-PACKET-ID+) - (nio-buffer:bytebuffer-write-32 buf 0) ; come back and write length later - (nio-buffer:bytebuffer-write-string buf (call-string packet)) - (nio-buffer:bytebuffer-insert-32 buf (buffer-position buf) 1) + (nio-buffer:mark buf) + (handler-case + (progn + (nio-buffer:bytebuffer-write-8 buf +CALL-METHOD-PACKET-ID+) + (nio-buffer:bytebuffer-write-32 buf 0) ; come back and write length later + (nio-buffer:bytebuffer-write-string buf (call-string packet)) + (nio-buffer:bytebuffer-insert-32 buf (buffer-position buf) 1) #+nio-debug (format-log t "yarpc-packet-factory:write-bytes(call-method-packet) - written ~%~A ~%" buf) - ) + ) + (buffer-too-small-error (err) + (nio-buffer:reset buf) + (error err)))) + (defmethod get-packet-size ((packet call-method-packet)) (+ +yarpc-packet-header-size+ @@ -97,12 +104,18 @@ (defmethod write-bytes((packet method-response-packet) buf) #+nio-debug (format-log t "yarpc-packet-factory:write-bytes - writing ~A to ~A~%" packet buf) - (nio-buffer:bytebuffer-write-8 buf +METHOD-RESPONSE-PACKET-ID+) - (nio-buffer:bytebuffer-write-32 buf 0) ; come back and write length later - (nio-buffer:bytebuffer-write-string buf (write-to-string (response packet))) - (nio-buffer:bytebuffer-insert-32 buf (buffer-position buf) 1) + (nio-buffer:mark buf) + (handler-case + (progn + (nio-buffer:bytebuffer-write-8 buf +METHOD-RESPONSE-PACKET-ID+) + (nio-buffer:bytebuffer-write-32 buf 0) ; come back and write length later + (nio-buffer:bytebuffer-write-string buf (write-to-string (response packet))) + (nio-buffer:bytebuffer-insert-32 buf (buffer-position buf) 1) #+nio-debug (format-log t "yarpc-packet-factory:write-bytes - written ~A~%" buf) -) + ) + (buffer-too-small-error (err) + (nio-buffer:reset buf) + (error err)))) (defmethod get-packet-size ((packet method-response-packet)) (+ +yarpc-packet-header-size+ Modified: branches/home/psmith/restructure/src/statemachine/state-machine.lisp ============================================================================== --- branches/home/psmith/restructure/src/statemachine/state-machine.lisp (original) +++ branches/home/psmith/restructure/src/statemachine/state-machine.lisp Sun Feb 4 15:53:04 2007 @@ -71,9 +71,11 @@ (when outgoing-packet (handler-case (write-bytes outgoing-packet foreign-write-buffer) - (buffer-too-small-error (e) + (buffer-too-small-error (write-error1) (if (recommend-buffer-size sm :write (get-packet-size outgoing-packet)) - (write-bytes outgoing-packet foreign-write-buffer) + (handler-case + (write-bytes outgoing-packet foreign-write-buffer) + (buffer-too-small-error (write-error1) (format t "Failed to write packet after resize (something already in write buffer?, dropping packet ~A~% out buffer:~%~A~%" outgoing-packet foreign-write-buffer))) (format t "Failed to resize io buffer, dropping packet: ~A~%" outgoing-packet))))))))

1 0

[nio-cvs] r66 - in branches/home/psmith/restructure/src: buffer io nio-logger protocol/yarpc statemachine
by psmith＠common-lisp.net 04 Feb '07

04 Feb '07

Author: psmith Date: Sun Feb 4 14:44:58 2007 New Revision: 66 Modified: branches/home/psmith/restructure/src/buffer/buffer.lisp branches/home/psmith/restructure/src/buffer/nio-buffer-package.lisp branches/home/psmith/restructure/src/io/async-fd.lisp branches/home/psmith/restructure/src/io/nio-package.lisp branches/home/psmith/restructure/src/io/packet.lisp branches/home/psmith/restructure/src/nio-logger/nio-logger.lisp branches/home/psmith/restructure/src/nio-logger/run-logging-client.lisp branches/home/psmith/restructure/src/protocol/yarpc/yarpc-packet-factory.lisp branches/home/psmith/restructure/src/statemachine/state-machine.lisp Log: plumb in large packet support Modified: branches/home/psmith/restructure/src/buffer/buffer.lisp ============================================================================== --- branches/home/psmith/restructure/src/buffer/buffer.lisp (original) +++ branches/home/psmith/restructure/src/buffer/buffer.lisp Sun Feb 4 14:44:58 2007 @@ -159,6 +159,18 @@ (setf position remaining) (setf limit capacity)))) + + +;Used to signal either an attempt has been made to write data to a buffer that is too small using a write (overflow) +; or an incomming packet doesn't have enough room to fit +(define-condition buffer-too-small-error (error) + ((recommended-size :initarg :recommended-size))) + +(defun buffer-too-small-error(recommended-size) + (make-instance 'buffer-too-small-error :recommended-size recommended-size)) + + + ;reads bytes from byte-buffer and returns a vector (unsigned-byte 8) (defmethod bytebuffer-read-vector((bb byte-buffer) &optional (num-bytes-to-read (remaining bb))) (let ((vec (make-uint8-seq num-bytes-to-read))) @@ -185,11 +197,13 @@ ;write an 8 bit value and up date position in buffer (defmethod bytebuffer-write-8 ((bb byte-buffer) value) + (when (< (remaining bb) 1) (error 'buffer-too-small-error)) (setf (cffi:mem-ref (buffer-buf bb) :unsigned-char (buffer-position bb)) value) (inc-position bb 1)) ;write a 32 bit value and up date position in buffer (defmethod bytebuffer-write-32 ((bb byte-buffer) value) + (when (< (remaining bb) 4) (error 'buffer-too-small-error)) (setf (cffi:mem-ref (buffer-buf bb) :unsigned-int (buffer-position bb)) value) (inc-position bb 4)) @@ -202,27 +216,24 @@ (setf (cffi:mem-ref (buffer-buf bb) :unsigned-int byte-position) value)) - - ;; Write bytes from vector vec to bytebuffer (defmethod bytebuffer-write-vector((bb byte-buffer) vec) :documentation "Returns number of bytes written to bytebuffer" #+nio-debug (format t "bytebuffer-write-vector - called with ~A ~A"bb vec) -; (if (> (remaining bb) 0) -; 0 - (progn -; (clear bb) - (let ((bytes-written (cffi:mem-write-vector vec (buffer-buf bb) :unsigned-char (length vec) (buffer-position bb)))) + (when (< (remaining bb) (length vec)) (error 'buffer-too-small-error)) + (let ((bytes-written (cffi:mem-write-vector vec (buffer-buf bb) :unsigned-char (length vec) (buffer-position bb)))) #+nio-debug (format t "bytebuffer-write-vector - byteswritten: ~A~%" bytes-written) - (inc-position bb bytes-written) - bytes-written))) -;) + (inc-position bb bytes-written) + bytes-written)) + ;; Writes data from string str to bytebuffer using specified encoding ;TODO move string-to-octets into nio-compat (defmethod bytebuffer-write-string((bb byte-buffer) str &optional (external-format :ascii)) :documentation "Returns number of bytes written to bytebuffer" - (bytebuffer-write-vector bb (sb-ext:string-to-octets str :external-format external-format))) + (let ((vec (sb-ext:string-to-octets str :external-format external-format))) + (when (< (remaining bb) (length vec)) (error 'buffer-too-small-error)) + (bytebuffer-write-vector bb vec))) Modified: branches/home/psmith/restructure/src/buffer/nio-buffer-package.lisp ============================================================================== --- branches/home/psmith/restructure/src/buffer/nio-buffer-package.lisp (original) +++ branches/home/psmith/restructure/src/buffer/nio-buffer-package.lisp Sun Feb 4 14:44:58 2007 @@ -32,4 +32,5 @@ bytebuffer-read-vector bytebuffer-read-string bytebuffer-read-8 bytebuffer-read-32 bytebuffer-write-8 bytebuffer-write-32 bytebuffer-insert-8 bytebuffer-insert-32 flip unflip clear buffer-position copy-buffer buffer-capacity compact + buffer-too-small-error )) Modified: branches/home/psmith/restructure/src/io/async-fd.lisp ============================================================================== --- branches/home/psmith/restructure/src/io/async-fd.lisp (original) +++ branches/home/psmith/restructure/src/io/async-fd.lisp Sun Feb 4 14:44:58 2007 @@ -152,11 +152,8 @@ (defconstant +MAX-BUFFER-SIZE-BYTES+ (* 1024 1024)) -(defmacro check-buffer-size (buffer size) - `(>= (length ,buffer) ,size)) - (defun realloc-buffer (async-fd buffer size) - (if (check-buffer-size buffer size) + (if (>= (length buffer) size) t (let ((new-buffer (byte-buffer size))) (copy-buffer buffer new-buffer) @@ -164,7 +161,7 @@ (setf (foreign-read-buffer async-fd) new-buffer)))) -;TODO actually deal with cuffer allocation failure +;TODO actually deal with buffer allocation failure (defmethod recommend-buffer-size((async-fd async-fd) mode size) (if (> size +MAX-BUFFER-SIZE-BYTES+) nil (ecase mode Modified: branches/home/psmith/restructure/src/io/nio-package.lisp ============================================================================== --- branches/home/psmith/restructure/src/io/nio-package.lisp (original) +++ branches/home/psmith/restructure/src/io/nio-package.lisp Sun Feb 4 14:44:58 2007 @@ -38,7 +38,7 @@ start-server add-connection ;;packet - packet write-bytes + packet write-bytes get-packet-size ;;ip-authorisation check-ip load-ips Modified: branches/home/psmith/restructure/src/io/packet.lisp ============================================================================== --- branches/home/psmith/restructure/src/io/packet.lisp (original) +++ branches/home/psmith/restructure/src/io/packet.lisp Sun Feb 4 14:44:58 2007 @@ -38,3 +38,6 @@ ;Implement in concrete (defgeneric write-bytes(packet nio-buffer)) + +(defgeneric get-packet-size(packet)) + Modified: branches/home/psmith/restructure/src/nio-logger/nio-logger.lisp ============================================================================== --- branches/home/psmith/restructure/src/nio-logger/nio-logger.lisp (original) +++ branches/home/psmith/restructure/src/nio-logger/nio-logger.lisp Sun Feb 4 14:44:58 2007 @@ -56,7 +56,7 @@ ;Runs a multithreaded system with an IO thread dealing with IO only and a 'job' thread taking and executing jobs -(defparameter +log-file-name+ "/tmp/out") +(defparameter +log-file-name+ "./out") (defun run-logging-server() (setf nio-yarpc:+process-jobs-inline+ nil) Modified: branches/home/psmith/restructure/src/nio-logger/run-logging-client.lisp ============================================================================== --- branches/home/psmith/restructure/src/nio-logger/run-logging-client.lisp (original) +++ branches/home/psmith/restructure/src/nio-logger/run-logging-client.lisp Sun Feb 4 14:44:58 2007 @@ -28,4 +28,4 @@ (push :nio-debug *features*) (require :asdf) (require :nio-logger) -(nio-logger:tail-log "/tmp/test" "127.0.0.1") +(nio-logger:tail-log "./test" "127.0.0.1") Modified: branches/home/psmith/restructure/src/protocol/yarpc/yarpc-packet-factory.lisp ============================================================================== --- branches/home/psmith/restructure/src/protocol/yarpc/yarpc-packet-factory.lisp (original) +++ branches/home/psmith/restructure/src/protocol/yarpc/yarpc-packet-factory.lisp Sun Feb 4 14:44:58 2007 @@ -80,7 +80,10 @@ (nio-buffer:bytebuffer-insert-32 buf (buffer-position buf) 1) #+nio-debug (format-log t "yarpc-packet-factory:write-bytes(call-method-packet) - written ~%~A ~%" buf) ) - + +(defmethod get-packet-size ((packet call-method-packet)) + (+ +yarpc-packet-header-size+ + (length (sb-ext:string-to-octets (write-to-string (call-string packet)))))) (defclass method-response-packet (packet) ((response :initarg :response @@ -100,3 +103,7 @@ (nio-buffer:bytebuffer-insert-32 buf (buffer-position buf) 1) #+nio-debug (format-log t "yarpc-packet-factory:write-bytes - written ~A~%" buf) ) + +(defmethod get-packet-size ((packet method-response-packet)) + (+ +yarpc-packet-header-size+ + (length (sb-ext:string-to-octets (write-to-string (response packet)))))) Modified: branches/home/psmith/restructure/src/statemachine/state-machine.lisp ============================================================================== --- branches/home/psmith/restructure/src/statemachine/state-machine.lisp (original) +++ branches/home/psmith/restructure/src/statemachine/state-machine.lisp Sun Feb 4 14:44:58 2007 @@ -68,7 +68,13 @@ (with-slots (foreign-write-buffer) sm (let ((outgoing-packet (process-outgoing-packet sm))) (format-log t "state-machine::process-write - outgoing packet: ~A~%" outgoing-packet) - (when outgoing-packet (write-bytes outgoing-packet foreign-write-buffer))))) + (when outgoing-packet + (handler-case + (write-bytes outgoing-packet foreign-write-buffer) + (buffer-too-small-error (e) + (if (recommend-buffer-size sm :write (get-packet-size outgoing-packet)) + (write-bytes outgoing-packet foreign-write-buffer) + (format t "Failed to resize io buffer, dropping packet: ~A~%" outgoing-packet)))))))) @@ -79,11 +85,7 @@ ; Get the packet in buf using the packet factory (defgeneric get-packet (packet-factory buf)) -;Used to signal that the packet wants a larger buffer to complete this packet -(define-condition buffer-too-small-error (error) - ((recommended-size :initarg :recommended-size))) - -(defun buffer-too-small-error(recommended-size) - (make-instance 'buffer-too-small-error :recommended-size recommended-size)) +; Get size of packet +(defgeneric get-packet (packet-factory buf))

1 0