nio-cvs February 2007

nio-cvs@common-lisp.net

1 participants
41 discussions

[nio-cvs] r72 - branches/home/psmith/restructure/src/nio-logger
by psmith＠common-lisp.net 06 Feb '07

06 Feb '07

Author: psmith Date: Mon Feb 5 20:29:41 2007 New Revision: 72 Modified: branches/home/psmith/restructure/src/nio-logger/nio-logger.lisp branches/home/psmith/restructure/src/nio-logger/run-logging-server.lisp branches/home/psmith/restructure/src/nio-logger/run.sh Log: Updates to logging server for cmd line usage Modified: branches/home/psmith/restructure/src/nio-logger/nio-logger.lisp ============================================================================== --- branches/home/psmith/restructure/src/nio-logger/nio-logger.lisp (original) +++ branches/home/psmith/restructure/src/nio-logger/nio-logger.lisp Mon Feb 5 20:29:41 2007 @@ -58,10 +58,11 @@ (defparameter +log-file-name+ "./out") -(defun run-logging-server() +(defun run-logging-server(listen-ip out-file &optional (allowed-ips "ips.txt")) (setf nio-yarpc:+process-jobs-inline+ nil) - (nio:load-ips "ips.txt") - (sb-thread:make-thread #'(lambda()(nio:start-server 'identity 'identity 'nio-yarpc:yarpc-state-machine :host "127.0.0.1" :accept-connection 'nio:check-ip)) :name "nio-server") + (setf +log-file-name+ out-file) + (nio:load-ips allowed-ips) + (sb-thread:make-thread #'(lambda()(nio:start-server 'identity 'identity 'nio-yarpc:yarpc-state-machine :host listen-ip :accept-connection 'nio:check-ip)) :name "nio-server") (loop ;;block waiting for jobs (nio-yarpc:run-job))) Modified: branches/home/psmith/restructure/src/nio-logger/run-logging-server.lisp ============================================================================== --- branches/home/psmith/restructure/src/nio-logger/run-logging-server.lisp (original) +++ branches/home/psmith/restructure/src/nio-logger/run-logging-server.lisp Mon Feb 5 20:29:41 2007 @@ -28,4 +28,9 @@ (push :nio-debug *features*) (require :asdf) (require :nio-logger) -(nio-logger:run-logging-server) + +(let ((listen-ip (second sb-ext:*posix-argv*)) + (out-file (third sb-ext:*posix-argv*)) + (allowed-ips-filename (fourth sb-ext:*posix-argv*))) + (format t "Starting logging Server with ~A ~A ~A~%" listen-ip out-file allowed-ips-filename) + (nio-logger:run-logging-server listen-ip out-file allowed-ips-filename)) Modified: branches/home/psmith/restructure/src/nio-logger/run.sh ============================================================================== --- branches/home/psmith/restructure/src/nio-logger/run.sh (original) +++ branches/home/psmith/restructure/src/nio-logger/run.sh Mon Feb 5 20:29:41 2007 @@ -1,8 +1,8 @@ #!/bin/bash # # run.sh <logging-client in-file dest-ip> -# run.sh <logging-server out-file> +# run.sh <logging-server listen-ip out-file allowed-ips-filename> # export LANG=en_US.UTF-8 -sbcl --load run-$1.lisp --end-toplevel-options $2 $3 +sbcl --load run-$1.lisp --end-toplevel-options $2 $3 $4

1 0

[nio-cvs] r71 - branches/home/psmith/restructure/src/nio-logger
by psmith＠common-lisp.net 05 Feb '07

05 Feb '07

Author: psmith Date: Mon Feb 5 11:57:21 2007 New Revision: 71 Modified: branches/home/psmith/restructure/src/nio-logger/run-logging-client.lisp branches/home/psmith/restructure/src/nio-logger/run.sh Log: Made startup more firendly Modified: branches/home/psmith/restructure/src/nio-logger/run-logging-client.lisp ============================================================================== --- branches/home/psmith/restructure/src/nio-logger/run-logging-client.lisp (original) +++ branches/home/psmith/restructure/src/nio-logger/run-logging-client.lisp Mon Feb 5 11:57:21 2007 @@ -28,4 +28,9 @@ (push :nio-debug *features*) (require :asdf) (require :nio-logger) -(nio-logger:tail-log "./test" "127.0.0.1") + +;Use run.sh logging-client <file> <ip> +(let ((log-file (second sb-ext:*posix-argv*)) + (ip (third sb-ext:*posix-argv*))) + (format t "Starting logging client with ~A ~A~%" log-file ip) + (nio-logger:tail-log log-file ip)) Modified: branches/home/psmith/restructure/src/nio-logger/run.sh ============================================================================== --- branches/home/psmith/restructure/src/nio-logger/run.sh (original) +++ branches/home/psmith/restructure/src/nio-logger/run.sh Mon Feb 5 11:57:21 2007 @@ -1,7 +1,8 @@ #!/bin/bash # -# run.sh <http|yarpc> +# run.sh <logging-client in-file dest-ip> +# run.sh <logging-server out-file> # export LANG=en_US.UTF-8 -sbcl --load run-$1.lisp +sbcl --load run-$1.lisp --end-toplevel-options $2 $3

1 0

[nio-cvs] r70 - branches/home/psmith/restructure/src/io
by psmith＠common-lisp.net 05 Feb '07

05 Feb '07

Author: psmith Date: Sun Feb 4 23:51:51 2007 New Revision: 70 Modified: branches/home/psmith/restructure/src/io/async-fd.lisp branches/home/psmith/restructure/src/io/nio-server.lisp Log: Large packet support working Modified: branches/home/psmith/restructure/src/io/async-fd.lisp ============================================================================== --- branches/home/psmith/restructure/src/io/async-fd.lisp (original) +++ branches/home/psmith/restructure/src/io/async-fd.lisp Sun Feb 4 23:51:51 2007 @@ -77,7 +77,9 @@ (with-slots (close-pending) async-fd (setf close-pending t))) -(define-condition read-error (error) ()) +(define-condition read-error (error) + ((errno :initform :errno + :accessor read-error-errno))) ;; "Read more data from STATE-MACHINE." (defun read-more (state-machine) @@ -88,16 +90,19 @@ #+nio-debug (format t "read-more : Read ~A bytes into ~A~%" new-bytes foreign-read-buffer) (cond ((< new-bytes 0) - (progn -;;TODO if ret is -1 and errno is EAGAIN save state and wait for notification - (format t "read-error - Errno: ~A~%" (get-errno)) - (error 'read-error))) + (let ((errno (get-errno))) + (format t "read-error - Errno: ~A~%" errno) + (cond ((eql errno +ERRNO_EAGAIN+) + (setf (read-ready state-machine) nil)) + (t + (error 'read-error :errno errno))))) ((= new-bytes 0) nil);;(throw 'end-of-file nil) (t ;;Update buffer position (inc-position foreign-read-buffer new-bytes) - (setf (read-ready state-machine) nil)))))) + (when (> (remaining foreign-read-buffer) 0) + (setf (read-ready state-machine) nil))))))) (defun close-async-fd (async-fd) "Close ASYNC-FD's fd after everything has been written from write-queue." Modified: branches/home/psmith/restructure/src/io/nio-server.lisp ============================================================================== --- branches/home/psmith/restructure/src/io/nio-server.lisp (original) +++ branches/home/psmith/restructure/src/io/nio-server.lisp Sun Feb 4 23:51:51 2007 @@ -109,7 +109,12 @@ ;; accept connection ? ((funcall accept-connection async-fd) - (set-fd-nonblocking (async-fd-read-fd async-fd)) + (sleep 0.1) + (let ((nb-ret (set-fd-nonblocking (async-fd-read-fd async-fd)))) + (format t "set bb ret: ~A :flags ~A~%" nb-ret (get-fd-flags (async-fd-read-fd async-fd))) + (when (< nb-ret 0) + (format t "Error setting socket non-blocking: ") + (perror))) (setf (gethash (async-fd-read-fd async-fd) client-hash) async-fd) (add-async-fd event-queue async-fd :read-write) ; (add-async-fd event-queue async-fd :write)

1 0

[nio-cvs] r69 - branches/home/psmith/restructure/src/io
by psmith＠common-lisp.net 05 Feb '07

05 Feb '07

Author: psmith Date: Sun Feb 4 23:51:27 2007 New Revision: 69 Modified: branches/home/psmith/restructure/src/io/fd-helper.lisp Log: Corrections to fcntl and O_NONBLOCK definitions for linux Modified: branches/home/psmith/restructure/src/io/fd-helper.lisp ============================================================================== --- branches/home/psmith/restructure/src/io/fd-helper.lisp (original) +++ branches/home/psmith/restructure/src/io/fd-helper.lisp Sun Feb 4 23:51:27 2007 @@ -32,7 +32,12 @@ (defconstant +cmd-get-flags+ 3) (defconstant +cmd-set-flags+ 4) -(defconstant +arg-nonblock+ #x0004) + +#+(or darwin macosx freebsd) +(defconstant +arg-nonblock+ #x4) + +#+linux +(defconstant +arg-nonblock+ #x800) (defcfun ("close" %close) :int (fd :int)) @@ -40,7 +45,10 @@ (defcfun ("fcntl" %fcntl) :int (fd :int) (cmd :int) +#+(or darwin macosx freebsd) (arg :int)) +#+linux + (arg :long)) (defcfun ("read" %read) :long (fd :int) @@ -60,10 +68,15 @@ (defun memzero (ptr size) (%memset ptr 0 size)) +(defun get-fd-flags(unix-fd) + (%fcntl unix-fd +cmd-get-flags+ 0)) (defun set-fd-nonblocking (unix-fd) "Set UNIX-FD to non-blocking mode (O_NONBLOCK)." - (%fcntl unix-fd +cmd-set-flags+ +arg-nonblock+)) + (let* ((flags (get-fd-flags unix-fd)) + (new-flags (logior flags +arg-nonblock+))) + (format t "set-fd-nonblocking :fd ~A :old-flags 0x~X 0x~X~%" unix-fd flags new-flags) + (%fcntl unix-fd +cmd-set-flags+ new-flags))) (defun close-fd (unix-fd)

1 0

[nio-cvs] r68 - in branches/home/psmith/restructure/src: buffer io statemachine
by psmith＠common-lisp.net 05 Feb '07

05 Feb '07

Author: psmith Date: Sun Feb 4 18:04:14 2007 New Revision: 68 Modified: branches/home/psmith/restructure/src/buffer/buffer.lisp branches/home/psmith/restructure/src/buffer/nio-buffer-package.lisp branches/home/psmith/restructure/src/io/async-fd.lisp branches/home/psmith/restructure/src/statemachine/state-machine.lisp Log: large packet server side OK. Modified: branches/home/psmith/restructure/src/buffer/buffer.lisp ============================================================================== --- branches/home/psmith/restructure/src/buffer/buffer.lisp (original) +++ branches/home/psmith/restructure/src/buffer/buffer.lisp Sun Feb 4 18:04:14 2007 @@ -107,6 +107,10 @@ (defun byte-buffer (capacity) (make-instance 'byte-buffer :capacity capacity :limit capacity :position 0 :buf (cffi:foreign-alloc :uint8 :count capacity))) +;Gets a pointer to the address in the native memory of the position index +(defmethod buffer-pointer ((bb byte-buffer)) + (cffi:make-pointer (+ (cffi:pointer-address (buffer-buf bb)) (buffer-position bb)))) + (defmethod print-object ((byte-buffer byte-buffer) stream) (with-slots (capacity position limit buf) byte-buffer (format stream "<byte-buffer :capacity ~A :position ~A :limit ~A :buf ~%~A>~%" capacity position limit (if buf (pretty-hex-dump (cffi:pointer-address buf) limit) nil)))) @@ -176,7 +180,8 @@ ;Used to signal either an attempt has been made to write data to a buffer that is too small using a write (overflow) ; or an incomming packet doesn't have enough room to fit (define-condition buffer-too-small-error (error) - ((recommended-size :initarg :recommended-size))) + ((recommended-size :initarg :recommended-size + :accessor recommended-size))) (defun buffer-too-small-error(recommended-size) (make-instance 'buffer-too-small-error :recommended-size recommended-size)) Modified: branches/home/psmith/restructure/src/buffer/nio-buffer-package.lisp ============================================================================== --- branches/home/psmith/restructure/src/buffer/nio-buffer-package.lisp (original) +++ branches/home/psmith/restructure/src/buffer/nio-buffer-package.lisp Sun Feb 4 18:04:14 2007 @@ -32,5 +32,5 @@ bytebuffer-read-vector bytebuffer-read-string bytebuffer-read-8 bytebuffer-read-32 bytebuffer-write-8 bytebuffer-write-32 bytebuffer-insert-8 bytebuffer-insert-32 flip unflip clear buffer-position copy-buffer buffer-capacity compact mark reset - buffer-too-small-error + buffer-too-small-error recommended-size buffer-pointer )) Modified: branches/home/psmith/restructure/src/io/async-fd.lisp ============================================================================== --- branches/home/psmith/restructure/src/io/async-fd.lisp (original) +++ branches/home/psmith/restructure/src/io/async-fd.lisp Sun Feb 4 18:04:14 2007 @@ -84,7 +84,7 @@ (with-slots (foreign-read-buffer read-fd) state-machine #+nio-debug (format t "read-more called with ~A~%" state-machine) #+nio-debug (format t "read-more - calling read() into ~A~%" foreign-read-buffer) - (let ((new-bytes (%read read-fd (buffer-buf foreign-read-buffer) (remaining foreign-read-buffer)))) + (let ((new-bytes (%read read-fd (buffer-pointer foreign-read-buffer) (remaining foreign-read-buffer)))) #+nio-debug (format t "read-more : Read ~A bytes into ~A~%" new-bytes foreign-read-buffer) (cond ((< new-bytes 0) Modified: branches/home/psmith/restructure/src/statemachine/state-machine.lisp ============================================================================== --- branches/home/psmith/restructure/src/statemachine/state-machine.lisp (original) +++ branches/home/psmith/restructure/src/statemachine/state-machine.lisp Sun Feb 4 18:04:14 2007 @@ -56,7 +56,13 @@ ;Use the packet factory to obtain any valid packet and pass it through (defmethod process-read((sm state-machine)) (with-slots (foreign-read-buffer) sm - (let ((incoming-packet (get-packet (get-packet-factory sm) foreign-read-buffer))) + (let ((incoming-packet + (handler-case + (get-packet (get-packet-factory sm) foreign-read-buffer) + (buffer-too-small-error (read-err) + (if (recommend-buffer-size sm :read (recommended-size read-err)) + (format-log t "resized incomming buffer ~A~%"foreign-read-buffer) + (error 'not-implemented-yet-read-resize-failure)))))) (format-log t "state-machine::process-read - incoming packet: ~A~%" incoming-packet) (when incoming-packet (when (not (process-incoming-packet sm incoming-packet))

1 0

[nio-cvs] r67 - in branches/home/psmith/restructure/src: buffer io protocol/yarpc statemachine
by psmith＠common-lisp.net 04 Feb '07

04 Feb '07

Author: psmith Date: Sun Feb 4 15:53:04 2007 New Revision: 67 Modified: branches/home/psmith/restructure/src/buffer/buffer.lisp branches/home/psmith/restructure/src/buffer/nio-buffer-package.lisp branches/home/psmith/restructure/src/io/async-fd.lisp branches/home/psmith/restructure/src/protocol/yarpc/yarpc-packet-factory.lisp branches/home/psmith/restructure/src/statemachine/state-machine.lisp Log: client side large packet OK Modified: branches/home/psmith/restructure/src/buffer/buffer.lisp ============================================================================== --- branches/home/psmith/restructure/src/buffer/buffer.lisp (original) +++ branches/home/psmith/restructure/src/buffer/buffer.lisp Sun Feb 4 15:53:04 2007 @@ -48,6 +48,9 @@ :initform 0 :accessor buffer-position :documentation "Index of next element to be read/written 0<=position<=limit") + (mark :initarg :position + :initform 0 + :documentation "A marked position") (buf :initarg :buf :accessor buffer-buf))) @@ -159,6 +162,15 @@ (setf position remaining) (setf limit capacity)))) +(defmethod mark((bb byte-buffer)) + :documentation "mark a position in the buffer for subsequent use with reset" + (with-slots (position mark) bb + (setf mark position))) + +(defmethod reset((bb byte-buffer)) + (with-slots (position mark) bb + (setf position mark))) + ;Used to signal either an attempt has been made to write data to a buffer that is too small using a write (overflow) @@ -231,9 +243,7 @@ ;TODO move string-to-octets into nio-compat (defmethod bytebuffer-write-string((bb byte-buffer) str &optional (external-format :ascii)) :documentation "Returns number of bytes written to bytebuffer" - (let ((vec (sb-ext:string-to-octets str :external-format external-format))) - (when (< (remaining bb) (length vec)) (error 'buffer-too-small-error)) - (bytebuffer-write-vector bb vec))) + (bytebuffer-write-vector bb (sb-ext:string-to-octets str :external-format external-format))) @@ -241,7 +251,7 @@ (assert (<= (buffer-capacity old) (buffer-capacity new))) (%memcpy (buffer-buf new) (buffer-buf old) (buffer-capacity old)) (setf (buffer-position new) (buffer-position old)) - (setf (buffer-limit new) (buffer-limit old))) + (setf (buffer-limit new) (buffer-capacity new))) ;void *memcpy(void *dest, const void *src, size_t n); (cffi:defcfun ("memcpy" %memcpy) :pointer Modified: branches/home/psmith/restructure/src/buffer/nio-buffer-package.lisp ============================================================================== --- branches/home/psmith/restructure/src/buffer/nio-buffer-package.lisp (original) +++ branches/home/psmith/restructure/src/buffer/nio-buffer-package.lisp Sun Feb 4 15:53:04 2007 @@ -31,6 +31,6 @@ bytebuffer-write-vector bytebuffer-write-string bytebuffer-read-vector bytebuffer-read-string bytebuffer-read-8 bytebuffer-read-32 bytebuffer-write-8 bytebuffer-write-32 bytebuffer-insert-8 bytebuffer-insert-32 - flip unflip clear buffer-position copy-buffer buffer-capacity compact + flip unflip clear buffer-position copy-buffer buffer-capacity compact mark reset buffer-too-small-error )) Modified: branches/home/psmith/restructure/src/io/async-fd.lisp ============================================================================== --- branches/home/psmith/restructure/src/io/async-fd.lisp (original) +++ branches/home/psmith/restructure/src/io/async-fd.lisp Sun Feb 4 15:53:04 2007 @@ -152,21 +152,34 @@ (defconstant +MAX-BUFFER-SIZE-BYTES+ (* 1024 1024)) -(defun realloc-buffer (async-fd buffer size) - (if (>= (length buffer) size) - t - (let ((new-buffer (byte-buffer size))) - (copy-buffer buffer new-buffer) - (free-buffer buffer) - (setf (foreign-read-buffer async-fd) new-buffer)))) + + +;(let ((buffer (foreign-read-buffer async-fd))) +; (if (>= (length buffer) size) +; t +; (let ((new-buffer (byte-buffer size))) +; (copy-buffer buffer new-buffer) +; (free-buffer buffer) +; (setf (foreign-read-buffer async-fd) new-buffer))))) + + +(defmacro realloc-buffer(async-fd accessor size) + `(let ((buffer (,accessor ,async-fd))) + (if (>= (buffer-capacity buffer) size) + t + (let ((new-buffer (byte-buffer ,size))) + (copy-buffer buffer new-buffer) + (free-buffer buffer) + (setf (,accessor ,async-fd) new-buffer))))) + ;TODO actually deal with buffer allocation failure (defmethod recommend-buffer-size((async-fd async-fd) mode size) (if (> size +MAX-BUFFER-SIZE-BYTES+) nil (ecase mode - (:read (realloc-buffer (foreign-read-buffer async-fd) size)) - (:write (realloc-buffer (foreign-write-buffer async-fd) size))))) + (:read (realloc-buffer async-fd foreign-read-buffer size)) + (:write (realloc-buffer async-fd foreign-write-buffer size))))) Modified: branches/home/psmith/restructure/src/protocol/yarpc/yarpc-packet-factory.lisp ============================================================================== --- branches/home/psmith/restructure/src/protocol/yarpc/yarpc-packet-factory.lisp (original) +++ branches/home/psmith/restructure/src/protocol/yarpc/yarpc-packet-factory.lisp Sun Feb 4 15:53:04 2007 @@ -74,12 +74,19 @@ (defmethod write-bytes((packet call-method-packet) buf) #+nio-debug (format-log t "yarpc-packet-factory:write-bytes(call-method-packet) - writing ~%~A to ~%~A~%" packet buf) - (nio-buffer:bytebuffer-write-8 buf +CALL-METHOD-PACKET-ID+) - (nio-buffer:bytebuffer-write-32 buf 0) ; come back and write length later - (nio-buffer:bytebuffer-write-string buf (call-string packet)) - (nio-buffer:bytebuffer-insert-32 buf (buffer-position buf) 1) + (nio-buffer:mark buf) + (handler-case + (progn + (nio-buffer:bytebuffer-write-8 buf +CALL-METHOD-PACKET-ID+) + (nio-buffer:bytebuffer-write-32 buf 0) ; come back and write length later + (nio-buffer:bytebuffer-write-string buf (call-string packet)) + (nio-buffer:bytebuffer-insert-32 buf (buffer-position buf) 1) #+nio-debug (format-log t "yarpc-packet-factory:write-bytes(call-method-packet) - written ~%~A ~%" buf) - ) + ) + (buffer-too-small-error (err) + (nio-buffer:reset buf) + (error err)))) + (defmethod get-packet-size ((packet call-method-packet)) (+ +yarpc-packet-header-size+ @@ -97,12 +104,18 @@ (defmethod write-bytes((packet method-response-packet) buf) #+nio-debug (format-log t "yarpc-packet-factory:write-bytes - writing ~A to ~A~%" packet buf) - (nio-buffer:bytebuffer-write-8 buf +METHOD-RESPONSE-PACKET-ID+) - (nio-buffer:bytebuffer-write-32 buf 0) ; come back and write length later - (nio-buffer:bytebuffer-write-string buf (write-to-string (response packet))) - (nio-buffer:bytebuffer-insert-32 buf (buffer-position buf) 1) + (nio-buffer:mark buf) + (handler-case + (progn + (nio-buffer:bytebuffer-write-8 buf +METHOD-RESPONSE-PACKET-ID+) + (nio-buffer:bytebuffer-write-32 buf 0) ; come back and write length later + (nio-buffer:bytebuffer-write-string buf (write-to-string (response packet))) + (nio-buffer:bytebuffer-insert-32 buf (buffer-position buf) 1) #+nio-debug (format-log t "yarpc-packet-factory:write-bytes - written ~A~%" buf) -) + ) + (buffer-too-small-error (err) + (nio-buffer:reset buf) + (error err)))) (defmethod get-packet-size ((packet method-response-packet)) (+ +yarpc-packet-header-size+ Modified: branches/home/psmith/restructure/src/statemachine/state-machine.lisp ============================================================================== --- branches/home/psmith/restructure/src/statemachine/state-machine.lisp (original) +++ branches/home/psmith/restructure/src/statemachine/state-machine.lisp Sun Feb 4 15:53:04 2007 @@ -71,9 +71,11 @@ (when outgoing-packet (handler-case (write-bytes outgoing-packet foreign-write-buffer) - (buffer-too-small-error (e) + (buffer-too-small-error (write-error1) (if (recommend-buffer-size sm :write (get-packet-size outgoing-packet)) - (write-bytes outgoing-packet foreign-write-buffer) + (handler-case + (write-bytes outgoing-packet foreign-write-buffer) + (buffer-too-small-error (write-error1) (format t "Failed to write packet after resize (something already in write buffer?, dropping packet ~A~% out buffer:~%~A~%" outgoing-packet foreign-write-buffer))) (format t "Failed to resize io buffer, dropping packet: ~A~%" outgoing-packet))))))))

1 0

[nio-cvs] r66 - in branches/home/psmith/restructure/src: buffer io nio-logger protocol/yarpc statemachine
by psmith＠common-lisp.net 04 Feb '07

04 Feb '07

Author: psmith Date: Sun Feb 4 14:44:58 2007 New Revision: 66 Modified: branches/home/psmith/restructure/src/buffer/buffer.lisp branches/home/psmith/restructure/src/buffer/nio-buffer-package.lisp branches/home/psmith/restructure/src/io/async-fd.lisp branches/home/psmith/restructure/src/io/nio-package.lisp branches/home/psmith/restructure/src/io/packet.lisp branches/home/psmith/restructure/src/nio-logger/nio-logger.lisp branches/home/psmith/restructure/src/nio-logger/run-logging-client.lisp branches/home/psmith/restructure/src/protocol/yarpc/yarpc-packet-factory.lisp branches/home/psmith/restructure/src/statemachine/state-machine.lisp Log: plumb in large packet support Modified: branches/home/psmith/restructure/src/buffer/buffer.lisp ============================================================================== --- branches/home/psmith/restructure/src/buffer/buffer.lisp (original) +++ branches/home/psmith/restructure/src/buffer/buffer.lisp Sun Feb 4 14:44:58 2007 @@ -159,6 +159,18 @@ (setf position remaining) (setf limit capacity)))) + + +;Used to signal either an attempt has been made to write data to a buffer that is too small using a write (overflow) +; or an incomming packet doesn't have enough room to fit +(define-condition buffer-too-small-error (error) + ((recommended-size :initarg :recommended-size))) + +(defun buffer-too-small-error(recommended-size) + (make-instance 'buffer-too-small-error :recommended-size recommended-size)) + + + ;reads bytes from byte-buffer and returns a vector (unsigned-byte 8) (defmethod bytebuffer-read-vector((bb byte-buffer) &optional (num-bytes-to-read (remaining bb))) (let ((vec (make-uint8-seq num-bytes-to-read))) @@ -185,11 +197,13 @@ ;write an 8 bit value and up date position in buffer (defmethod bytebuffer-write-8 ((bb byte-buffer) value) + (when (< (remaining bb) 1) (error 'buffer-too-small-error)) (setf (cffi:mem-ref (buffer-buf bb) :unsigned-char (buffer-position bb)) value) (inc-position bb 1)) ;write a 32 bit value and up date position in buffer (defmethod bytebuffer-write-32 ((bb byte-buffer) value) + (when (< (remaining bb) 4) (error 'buffer-too-small-error)) (setf (cffi:mem-ref (buffer-buf bb) :unsigned-int (buffer-position bb)) value) (inc-position bb 4)) @@ -202,27 +216,24 @@ (setf (cffi:mem-ref (buffer-buf bb) :unsigned-int byte-position) value)) - - ;; Write bytes from vector vec to bytebuffer (defmethod bytebuffer-write-vector((bb byte-buffer) vec) :documentation "Returns number of bytes written to bytebuffer" #+nio-debug (format t "bytebuffer-write-vector - called with ~A ~A"bb vec) -; (if (> (remaining bb) 0) -; 0 - (progn -; (clear bb) - (let ((bytes-written (cffi:mem-write-vector vec (buffer-buf bb) :unsigned-char (length vec) (buffer-position bb)))) + (when (< (remaining bb) (length vec)) (error 'buffer-too-small-error)) + (let ((bytes-written (cffi:mem-write-vector vec (buffer-buf bb) :unsigned-char (length vec) (buffer-position bb)))) #+nio-debug (format t "bytebuffer-write-vector - byteswritten: ~A~%" bytes-written) - (inc-position bb bytes-written) - bytes-written))) -;) + (inc-position bb bytes-written) + bytes-written)) + ;; Writes data from string str to bytebuffer using specified encoding ;TODO move string-to-octets into nio-compat (defmethod bytebuffer-write-string((bb byte-buffer) str &optional (external-format :ascii)) :documentation "Returns number of bytes written to bytebuffer" - (bytebuffer-write-vector bb (sb-ext:string-to-octets str :external-format external-format))) + (let ((vec (sb-ext:string-to-octets str :external-format external-format))) + (when (< (remaining bb) (length vec)) (error 'buffer-too-small-error)) + (bytebuffer-write-vector bb vec))) Modified: branches/home/psmith/restructure/src/buffer/nio-buffer-package.lisp ============================================================================== --- branches/home/psmith/restructure/src/buffer/nio-buffer-package.lisp (original) +++ branches/home/psmith/restructure/src/buffer/nio-buffer-package.lisp Sun Feb 4 14:44:58 2007 @@ -32,4 +32,5 @@ bytebuffer-read-vector bytebuffer-read-string bytebuffer-read-8 bytebuffer-read-32 bytebuffer-write-8 bytebuffer-write-32 bytebuffer-insert-8 bytebuffer-insert-32 flip unflip clear buffer-position copy-buffer buffer-capacity compact + buffer-too-small-error )) Modified: branches/home/psmith/restructure/src/io/async-fd.lisp ============================================================================== --- branches/home/psmith/restructure/src/io/async-fd.lisp (original) +++ branches/home/psmith/restructure/src/io/async-fd.lisp Sun Feb 4 14:44:58 2007 @@ -152,11 +152,8 @@ (defconstant +MAX-BUFFER-SIZE-BYTES+ (* 1024 1024)) -(defmacro check-buffer-size (buffer size) - `(>= (length ,buffer) ,size)) - (defun realloc-buffer (async-fd buffer size) - (if (check-buffer-size buffer size) + (if (>= (length buffer) size) t (let ((new-buffer (byte-buffer size))) (copy-buffer buffer new-buffer) @@ -164,7 +161,7 @@ (setf (foreign-read-buffer async-fd) new-buffer)))) -;TODO actually deal with cuffer allocation failure +;TODO actually deal with buffer allocation failure (defmethod recommend-buffer-size((async-fd async-fd) mode size) (if (> size +MAX-BUFFER-SIZE-BYTES+) nil (ecase mode Modified: branches/home/psmith/restructure/src/io/nio-package.lisp ============================================================================== --- branches/home/psmith/restructure/src/io/nio-package.lisp (original) +++ branches/home/psmith/restructure/src/io/nio-package.lisp Sun Feb 4 14:44:58 2007 @@ -38,7 +38,7 @@ start-server add-connection ;;packet - packet write-bytes + packet write-bytes get-packet-size ;;ip-authorisation check-ip load-ips Modified: branches/home/psmith/restructure/src/io/packet.lisp ============================================================================== --- branches/home/psmith/restructure/src/io/packet.lisp (original) +++ branches/home/psmith/restructure/src/io/packet.lisp Sun Feb 4 14:44:58 2007 @@ -38,3 +38,6 @@ ;Implement in concrete (defgeneric write-bytes(packet nio-buffer)) + +(defgeneric get-packet-size(packet)) + Modified: branches/home/psmith/restructure/src/nio-logger/nio-logger.lisp ============================================================================== --- branches/home/psmith/restructure/src/nio-logger/nio-logger.lisp (original) +++ branches/home/psmith/restructure/src/nio-logger/nio-logger.lisp Sun Feb 4 14:44:58 2007 @@ -56,7 +56,7 @@ ;Runs a multithreaded system with an IO thread dealing with IO only and a 'job' thread taking and executing jobs -(defparameter +log-file-name+ "/tmp/out") +(defparameter +log-file-name+ "./out") (defun run-logging-server() (setf nio-yarpc:+process-jobs-inline+ nil) Modified: branches/home/psmith/restructure/src/nio-logger/run-logging-client.lisp ============================================================================== --- branches/home/psmith/restructure/src/nio-logger/run-logging-client.lisp (original) +++ branches/home/psmith/restructure/src/nio-logger/run-logging-client.lisp Sun Feb 4 14:44:58 2007 @@ -28,4 +28,4 @@ (push :nio-debug *features*) (require :asdf) (require :nio-logger) -(nio-logger:tail-log "/tmp/test" "127.0.0.1") +(nio-logger:tail-log "./test" "127.0.0.1") Modified: branches/home/psmith/restructure/src/protocol/yarpc/yarpc-packet-factory.lisp ============================================================================== --- branches/home/psmith/restructure/src/protocol/yarpc/yarpc-packet-factory.lisp (original) +++ branches/home/psmith/restructure/src/protocol/yarpc/yarpc-packet-factory.lisp Sun Feb 4 14:44:58 2007 @@ -80,7 +80,10 @@ (nio-buffer:bytebuffer-insert-32 buf (buffer-position buf) 1) #+nio-debug (format-log t "yarpc-packet-factory:write-bytes(call-method-packet) - written ~%~A ~%" buf) ) - + +(defmethod get-packet-size ((packet call-method-packet)) + (+ +yarpc-packet-header-size+ + (length (sb-ext:string-to-octets (write-to-string (call-string packet)))))) (defclass method-response-packet (packet) ((response :initarg :response @@ -100,3 +103,7 @@ (nio-buffer:bytebuffer-insert-32 buf (buffer-position buf) 1) #+nio-debug (format-log t "yarpc-packet-factory:write-bytes - written ~A~%" buf) ) + +(defmethod get-packet-size ((packet method-response-packet)) + (+ +yarpc-packet-header-size+ + (length (sb-ext:string-to-octets (write-to-string (response packet)))))) Modified: branches/home/psmith/restructure/src/statemachine/state-machine.lisp ============================================================================== --- branches/home/psmith/restructure/src/statemachine/state-machine.lisp (original) +++ branches/home/psmith/restructure/src/statemachine/state-machine.lisp Sun Feb 4 14:44:58 2007 @@ -68,7 +68,13 @@ (with-slots (foreign-write-buffer) sm (let ((outgoing-packet (process-outgoing-packet sm))) (format-log t "state-machine::process-write - outgoing packet: ~A~%" outgoing-packet) - (when outgoing-packet (write-bytes outgoing-packet foreign-write-buffer))))) + (when outgoing-packet + (handler-case + (write-bytes outgoing-packet foreign-write-buffer) + (buffer-too-small-error (e) + (if (recommend-buffer-size sm :write (get-packet-size outgoing-packet)) + (write-bytes outgoing-packet foreign-write-buffer) + (format t "Failed to resize io buffer, dropping packet: ~A~%" outgoing-packet)))))))) @@ -79,11 +85,7 @@ ; Get the packet in buf using the packet factory (defgeneric get-packet (packet-factory buf)) -;Used to signal that the packet wants a larger buffer to complete this packet -(define-condition buffer-too-small-error (error) - ((recommended-size :initarg :recommended-size))) - -(defun buffer-too-small-error(recommended-size) - (make-instance 'buffer-too-small-error :recommended-size recommended-size)) +; Get size of packet +(defgeneric get-packet (packet-factory buf))

1 0

[nio-cvs] r65 - branches/home/psmith/restructure/src/io
by psmith＠common-lisp.net 03 Feb '07

03 Feb '07

Author: psmith Date: Fri Feb 2 23:45:48 2007 New Revision: 65 Modified: branches/home/psmith/restructure/src/io/nio-server.lisp Log: Only attempt write when theres something to be written Modified: branches/home/psmith/restructure/src/io/nio-server.lisp ============================================================================== --- branches/home/psmith/restructure/src/io/nio-server.lisp (original) +++ branches/home/psmith/restructure/src/io/nio-server.lisp Fri Feb 2 23:45:48 2007 @@ -49,7 +49,9 @@ ;process-writes (process-write async-fd) - (when (write-ready async-fd) (write-more async-fd))) + (when (and (write-ready async-fd) + (> (buffer-position (foreign-write-buffer async-fd)) 0)) + (write-more async-fd))) client-hash))

1 0

[nio-cvs] r64 - in branches/home/psmith/restructure/src: io nio-logger
by psmith＠common-lisp.net 03 Feb '07

03 Feb '07

Author: psmith Date: Fri Feb 2 22:20:01 2007 New Revision: 64 Modified: branches/home/psmith/restructure/src/io/async-fd.lisp branches/home/psmith/restructure/src/nio-logger/nio-logger.lisp Log: logger updates Modified: branches/home/psmith/restructure/src/io/async-fd.lisp ============================================================================== --- branches/home/psmith/restructure/src/io/async-fd.lisp (original) +++ branches/home/psmith/restructure/src/io/async-fd.lisp Fri Feb 2 22:20:01 2007 @@ -136,6 +136,7 @@ (let ((err (get-errno))) (format t "write-more - write returned -1 :errno ~A~%" err) (unless (eql err 11) ;; eagain - failed to write whole buffer need to wait for next notify + (perror) (let ((err-cond (make-instance 'write-error :error err))) (close err-cond) (error err-cond)))) Modified: branches/home/psmith/restructure/src/nio-logger/nio-logger.lisp ============================================================================== --- branches/home/psmith/restructure/src/nio-logger/nio-logger.lisp (original) +++ branches/home/psmith/restructure/src/nio-logger/nio-logger.lisp Fri Feb 2 22:20:01 2007 @@ -29,6 +29,18 @@ (declaim (optimize (debug 3) (speed 3) (space 0))) + + +(defmacro with-line-from-tailed-file ((line filename delay) &rest body) + `(with-open-file (in ,filename :direction :input) + (loop for ,line = (read-line in nil nil) do + (if ,line + (progn ,@body) + (progn + (format t "read nil~%") + (sleep ,delay)))))) + + ;;Tail the given log and write to remote logger ;;e.g. (tail-log "/var/log/httpd/access_log" "192.168.1.1") (defun tail-log(filename ip-address) @@ -37,11 +49,10 @@ (sleep 4) (let ((sm (nio:add-connection ip-address 16323 'nio-yarpc:yarpc-client-state-machine))) (nio-utils:format-log t "toplevel adding conn ~A to ~A~%" sm ip-address) - (with-open-file (in filename :direction :input) - (loop for text = (read-line in nil nil) do - (let ((rpc (format nil "(nio-logger:remote-log \"~A\")" text))) - (nio-utils:format-log t "Toplevel Submitting job~A~%" rpc) - (nio-utils:format-log t "Result of remote-log ~A~%" (nio-yarpc:remote-execute sm rpc))))))) + (with-line-from-tailed-file (text filename 1) + (let ((rpc (format nil "(nio-logger:remote-log \"~A\")" text))) + (nio-utils:format-log t "Toplevel Submitting job~A~%" rpc) + (nio-utils:format-log t "Result of remote-log ~A~%" (nio-yarpc:remote-execute sm rpc)))))) ;Runs a multithreaded system with an IO thread dealing with IO only and a 'job' thread taking and executing jobs @@ -57,4 +68,6 @@ (nio-yarpc:defremote remote-log(str) (with-open-file (out +log-file-name+ :direction :output :if-exists :append) - (nio-utils:format-log out "~A~%" str))) + (nio-utils:format-log out "~A~%" str)) + t) +

1 0

[nio-cvs] r63 - branches/home/psmith/restructure/src/buffer
by psmith＠common-lisp.net 03 Feb '07

03 Feb '07

Author: psmith Date: Fri Feb 2 22:16:10 2007 New Revision: 63 Modified: branches/home/psmith/restructure/src/buffer/buffer.lisp Log: Corrected compact Modified: branches/home/psmith/restructure/src/buffer/buffer.lisp ============================================================================== --- branches/home/psmith/restructure/src/buffer/buffer.lisp (original) +++ branches/home/psmith/restructure/src/buffer/buffer.lisp Fri Feb 2 22:16:10 2007 @@ -88,8 +88,8 @@ ; (format t "Current address : ~A~%" address) (if (>= address (+ start-address length)) (progn - (format str "--") - (format readable "--")) + (format str (if (eql column-index 7) " " ".. ")) + (format readable ".")) (progn (format str (if (eql column-index 7) "~A " "~A ") (hex-dump-byte address)) (format readable "~A" (get-readable-char (byte-value address))))))))))))) @@ -153,10 +153,11 @@ (defmethod compact((byte-buffer byte-buffer)) :documentation "copy remaining bytes to the beginning of this buffer and set position to number of bytes copied (ready for a new put" - (with-slots (buf position limit) byte-buffer + (with-slots (buf position limit capacity) byte-buffer (let ((remaining (remaining byte-buffer))) (%memcpy buf (cffi:make-pointer (+ (cffi:pointer-address buf) position)) remaining) - (setf position remaining)))) + (setf position remaining) + (setf limit capacity)))) ;reads bytes from byte-buffer and returns a vector (unsigned-byte 8) (defmethod bytebuffer-read-vector((bb byte-buffer) &optional (num-bytes-to-read (remaining bb)))

1 0

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

2008

2007

2006

nio-cvs February 2007