Author: psmith Date: Sun Feb 4 14:44:58 2007 New Revision: 66 Modified: branches/home/psmith/restructure/src/buffer/buffer.lisp branches/home/psmith/restructure/src/buffer/nio-buffer-package.lisp branches/home/psmith/restructure/src/io/async-fd.lisp branches/home/psmith/restructure/src/io/nio-package.lisp branches/home/psmith/restructure/src/io/packet.lisp branches/home/psmith/restructure/src/nio-logger/nio-logger.lisp branches/home/psmith/restructure/src/nio-logger/run-logging-client.lisp branches/home/psmith/restructure/src/protocol/yarpc/yarpc-packet-factory.lisp branches/home/psmith/restructure/src/statemachine/state-machine.lisp Log: plumb in large packet support Modified: branches/home/psmith/restructure/src/buffer/buffer.lisp ============================================================================== --- branches/home/psmith/restructure/src/buffer/buffer.lisp (original) +++ branches/home/psmith/restructure/src/buffer/buffer.lisp Sun Feb 4 14:44:58 2007 @@ -159,6 +159,18 @@ (setf position remaining) (setf limit capacity)))) + + +;Used to signal either an attempt has been made to write data to a buffer that is too small using a write (overflow) +; or an incomming packet doesn't have enough room to fit +(define-condition buffer-too-small-error (error) + ((recommended-size :initarg :recommended-size))) + +(defun buffer-too-small-error(recommended-size) + (make-instance 'buffer-too-small-error :recommended-size recommended-size)) + + + ;reads bytes from byte-buffer and returns a vector (unsigned-byte 8) (defmethod bytebuffer-read-vector((bb byte-buffer) &optional (num-bytes-to-read (remaining bb))) (let ((vec (make-uint8-seq num-bytes-to-read))) @@ -185,11 +197,13 @@ ;write an 8 bit value and up date position in buffer (defmethod bytebuffer-write-8 ((bb byte-buffer) value) + (when (< (remaining bb) 1) (error 'buffer-too-small-error)) (setf (cffi:mem-ref (buffer-buf bb) :unsigned-char (buffer-position bb)) value) (inc-position bb 1)) ;write a 32 bit value and up date position in buffer (defmethod bytebuffer-write-32 ((bb byte-buffer) value) + (when (< (remaining bb) 4) (error 'buffer-too-small-error)) (setf (cffi:mem-ref (buffer-buf bb) :unsigned-int (buffer-position bb)) value) (inc-position bb 4)) @@ -202,27 +216,24 @@ (setf (cffi:mem-ref (buffer-buf bb) :unsigned-int byte-position) value)) - - ;; Write bytes from vector vec to bytebuffer (defmethod bytebuffer-write-vector((bb byte-buffer) vec) :documentation "Returns number of bytes written to bytebuffer" #+nio-debug (format t "bytebuffer-write-vector - called with ~A ~A"bb vec) -; (if (> (remaining bb) 0) -; 0 - (progn -; (clear bb) - (let ((bytes-written (cffi:mem-write-vector vec (buffer-buf bb) :unsigned-char (length vec) (buffer-position bb)))) + (when (< (remaining bb) (length vec)) (error 'buffer-too-small-error)) + (let ((bytes-written (cffi:mem-write-vector vec (buffer-buf bb) :unsigned-char (length vec) (buffer-position bb)))) #+nio-debug (format t "bytebuffer-write-vector - byteswritten: ~A~%" bytes-written) - (inc-position bb bytes-written) - bytes-written))) -;) + (inc-position bb bytes-written) + bytes-written)) + ;; Writes data from string str to bytebuffer using specified encoding ;TODO move string-to-octets into nio-compat (defmethod bytebuffer-write-string((bb byte-buffer) str &optional (external-format :ascii)) :documentation "Returns number of bytes written to bytebuffer" - (bytebuffer-write-vector bb (sb-ext:string-to-octets str :external-format external-format))) + (let ((vec (sb-ext:string-to-octets str :external-format external-format))) + (when (< (remaining bb) (length vec)) (error 'buffer-too-small-error)) + (bytebuffer-write-vector bb vec))) Modified: branches/home/psmith/restructure/src/buffer/nio-buffer-package.lisp ============================================================================== --- branches/home/psmith/restructure/src/buffer/nio-buffer-package.lisp (original) +++ branches/home/psmith/restructure/src/buffer/nio-buffer-package.lisp Sun Feb 4 14:44:58 2007 @@ -32,4 +32,5 @@ bytebuffer-read-vector bytebuffer-read-string bytebuffer-read-8 bytebuffer-read-32 bytebuffer-write-8 bytebuffer-write-32 bytebuffer-insert-8 bytebuffer-insert-32 flip unflip clear buffer-position copy-buffer buffer-capacity compact + buffer-too-small-error )) Modified: branches/home/psmith/restructure/src/io/async-fd.lisp ============================================================================== --- branches/home/psmith/restructure/src/io/async-fd.lisp (original) +++ branches/home/psmith/restructure/src/io/async-fd.lisp Sun Feb 4 14:44:58 2007 @@ -152,11 +152,8 @@ (defconstant +MAX-BUFFER-SIZE-BYTES+ (* 1024 1024)) -(defmacro check-buffer-size (buffer size) - `(>= (length ,buffer) ,size)) - (defun realloc-buffer (async-fd buffer size) - (if (check-buffer-size buffer size) + (if (>= (length buffer) size) t (let ((new-buffer (byte-buffer size))) (copy-buffer buffer new-buffer) @@ -164,7 +161,7 @@ (setf (foreign-read-buffer async-fd) new-buffer)))) -;TODO actually deal with cuffer allocation failure +;TODO actually deal with buffer allocation failure (defmethod recommend-buffer-size((async-fd async-fd) mode size) (if (> size +MAX-BUFFER-SIZE-BYTES+) nil (ecase mode Modified: branches/home/psmith/restructure/src/io/nio-package.lisp ============================================================================== --- branches/home/psmith/restructure/src/io/nio-package.lisp (original) +++ branches/home/psmith/restructure/src/io/nio-package.lisp Sun Feb 4 14:44:58 2007 @@ -38,7 +38,7 @@ start-server add-connection ;;packet - packet write-bytes + packet write-bytes get-packet-size ;;ip-authorisation check-ip load-ips Modified: branches/home/psmith/restructure/src/io/packet.lisp ============================================================================== --- branches/home/psmith/restructure/src/io/packet.lisp (original) +++ branches/home/psmith/restructure/src/io/packet.lisp Sun Feb 4 14:44:58 2007 @@ -38,3 +38,6 @@ ;Implement in concrete (defgeneric write-bytes(packet nio-buffer)) + +(defgeneric get-packet-size(packet)) + Modified: branches/home/psmith/restructure/src/nio-logger/nio-logger.lisp ============================================================================== --- branches/home/psmith/restructure/src/nio-logger/nio-logger.lisp (original) +++ branches/home/psmith/restructure/src/nio-logger/nio-logger.lisp Sun Feb 4 14:44:58 2007 @@ -56,7 +56,7 @@ ;Runs a multithreaded system with an IO thread dealing with IO only and a 'job' thread taking and executing jobs -(defparameter +log-file-name+ "/tmp/out") +(defparameter +log-file-name+ "./out") (defun run-logging-server() (setf nio-yarpc:+process-jobs-inline+ nil) Modified: branches/home/psmith/restructure/src/nio-logger/run-logging-client.lisp ============================================================================== --- branches/home/psmith/restructure/src/nio-logger/run-logging-client.lisp (original) +++ branches/home/psmith/restructure/src/nio-logger/run-logging-client.lisp Sun Feb 4 14:44:58 2007 @@ -28,4 +28,4 @@ (push :nio-debug *features*) (require :asdf) (require :nio-logger) -(nio-logger:tail-log "/tmp/test" "127.0.0.1") +(nio-logger:tail-log "./test" "127.0.0.1") Modified: branches/home/psmith/restructure/src/protocol/yarpc/yarpc-packet-factory.lisp ============================================================================== --- branches/home/psmith/restructure/src/protocol/yarpc/yarpc-packet-factory.lisp (original) +++ branches/home/psmith/restructure/src/protocol/yarpc/yarpc-packet-factory.lisp Sun Feb 4 14:44:58 2007 @@ -80,7 +80,10 @@ (nio-buffer:bytebuffer-insert-32 buf (buffer-position buf) 1) #+nio-debug (format-log t "yarpc-packet-factory:write-bytes(call-method-packet) - written ~%~A ~%" buf) ) - + +(defmethod get-packet-size ((packet call-method-packet)) + (+ +yarpc-packet-header-size+ + (length (sb-ext:string-to-octets (write-to-string (call-string packet)))))) (defclass method-response-packet (packet) ((response :initarg :response @@ -100,3 +103,7 @@ (nio-buffer:bytebuffer-insert-32 buf (buffer-position buf) 1) #+nio-debug (format-log t "yarpc-packet-factory:write-bytes - written ~A~%" buf) ) + +(defmethod get-packet-size ((packet method-response-packet)) + (+ +yarpc-packet-header-size+ + (length (sb-ext:string-to-octets (write-to-string (response packet)))))) Modified: branches/home/psmith/restructure/src/statemachine/state-machine.lisp ============================================================================== --- branches/home/psmith/restructure/src/statemachine/state-machine.lisp (original) +++ branches/home/psmith/restructure/src/statemachine/state-machine.lisp Sun Feb 4 14:44:58 2007 @@ -68,7 +68,13 @@ (with-slots (foreign-write-buffer) sm (let ((outgoing-packet (process-outgoing-packet sm))) (format-log t "state-machine::process-write - outgoing packet: ~A~%" outgoing-packet) - (when outgoing-packet (write-bytes outgoing-packet foreign-write-buffer))))) + (when outgoing-packet + (handler-case + (write-bytes outgoing-packet foreign-write-buffer) + (buffer-too-small-error (e) + (if (recommend-buffer-size sm :write (get-packet-size outgoing-packet)) + (write-bytes outgoing-packet foreign-write-buffer) + (format t "Failed to resize io buffer, dropping packet: ~A~%" outgoing-packet)))))))) @@ -79,11 +85,7 @@ ; Get the packet in buf using the packet factory (defgeneric get-packet (packet-factory buf)) -;Used to signal that the packet wants a larger buffer to complete this packet -(define-condition buffer-too-small-error (error) - ((recommended-size :initarg :recommended-size))) - -(defun buffer-too-small-error(recommended-size) - (make-instance 'buffer-too-small-error :recommended-size recommended-size)) +; Get size of packet +(defgeneric get-packet (packet-factory buf))