This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "CMU Common Lisp". The branch, master has been updated via a60274829b62c938292920f8f19b1c269240e8c2 (commit) via 99d39125b83f6173912c2a689e41db4c40096061 (commit) from 9f6b065b0ea4b758a00fa9b3b790b5ef2bb8853d (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit a60274829b62c938292920f8f19b1c269240e8c2 Author: Raymond Toy <toy.raymond@gmail.com> Date: Fri Apr 20 09:35:43 2012 -0700 Oops. Debugging print accidentally left in. diff --git a/src/code/stream.lisp b/src/code/stream.lisp index a91d4d7..b5d34ad 100644 --- a/src/code/stream.lisp +++ b/src/code/stream.lisp @@ -829,6 +829,7 @@ (flet ((convert-buffer () (let ((old-state (fd-stream-oc-state stream))) + #+(or debug-frc-sr) (format t "old-state = ~S~%" old-state) (multiple-value-bind (s char-count octet-count new-state) (stream::octets-to-string-counted commit 99d39125b83f6173912c2a689e41db4c40096061 Author: Raymond Toy <toy.raymond@gmail.com> Date: Fri Apr 20 09:34:48 2012 -0700 UTF-16-BE and UTF-16-LE external formats were returning the incorrect number of octets for surrogate pairs. Two was returned instead of four. * src/pcl/simple-streams/external-formats/utf-16-be.lisp: * src/pcl/simple-streams/external-formats/utf-16-le.lisp: * Return correct number of octets * src/general-info/release-20d.txt * Update. diff --git a/src/general-info/release-20d.txt b/src/general-info/release-20d.txt index 6bd52d5..fe9baaf 100644 --- a/src/general-info/release-20d.txt +++ b/src/general-info/release-20d.txt @@ -61,6 +61,9 @@ New in this release: double-float multiplier from being used when sse3 is available. * External format for UTF-32 was generating an error when converting octets to a string. + * The UTF-16-BE and UTF-16-LE external formats were returning the + incorrect number of octets when surrogates pairs were decoded. + This confuses the stream buffering code. * Trac Tickets: * #50: Print/read error with make-pathname. diff --git a/src/pcl/simple-streams/external-formats/utf-16-be.lisp b/src/pcl/simple-streams/external-formats/utf-16-be.lisp index e2780ea..948a24e 100644 --- a/src/pcl/simple-streams/external-formats/utf-16-be.lisp +++ b/src/pcl/simple-streams/external-formats/utf-16-be.lisp @@ -21,10 +21,11 @@ By default, illegal inputs and illegal outputs are replaced by the Unicode replacement character.") () - (octets-to-code (state input unput error c1 c2 code next) + (octets-to-code (state input unput error c1 c2 code wd next) `(let* ((,c1 ,input) (,c2 ,input) - (,code (+ (* 256 ,c1) ,c2))) + (,code (+ (* 256 ,c1) ,c2)) + (, wd 2)) (declare (type lisp:codepoint ,code)) (cond ((lisp::surrogatep ,code :low) ;; Got low surrogate. Combine with the state (high @@ -55,7 +56,8 @@ Unicode replacement character.") ;; unput 2 so it'll be read as another character ;; next time around? (if (lisp::surrogatep ,next :low) - (setq ,code (+ (ash (- ,code #xD800) 10) ,next #x2400)) + (setq ,code (+ (ash (- ,code #xD800) 10) ,next #x2400) + ,wd 4) (setf ,code (if ,error (locally @@ -74,7 +76,7 @@ Unicode replacement character.") (funcall ,error "BOM is not valid within a UTF-16 stream" ,code 2)) +replacement-character-code+))) (t (setf ,state nil))) - (values ,code 2))) + (values ,code ,wd))) (code-to-octets (code state output error c c1 c2) `(flet ((output (code) (,output (ldb (byte 8 8) code)) diff --git a/src/pcl/simple-streams/external-formats/utf-16-le.lisp b/src/pcl/simple-streams/external-formats/utf-16-le.lisp index b7faa28..6fb4b65 100644 --- a/src/pcl/simple-streams/external-formats/utf-16-le.lisp +++ b/src/pcl/simple-streams/external-formats/utf-16-le.lisp @@ -22,10 +22,11 @@ By default, illegal inputs and illegal outputs are replaced by the Unicode replacement character.") () - (octets-to-code (state input unput error c1 c2 code next) + (octets-to-code (state input unput error c1 c2 code wd next) `(let* ((,c1 ,input) (,c2 ,input) - (,code (+ (* 256 ,c2) ,c1))) + (,code (+ (* 256 ,c2) ,c1)) + (,wd 2)) (declare (type lisp:codepoint ,code)) (cond ((lisp::surrogatep ,code :low) ;; If possible combine this low surrogate with the @@ -56,14 +57,15 @@ Unicode replacement character.") ;; unput 2 so it'll be read as another character ;; next time around? (if (lisp::surrogatep ,next :low) - (setq ,code (+ (ash (- ,code #xD800) 10) ,next #x2400)) + (setq ,code (+ (ash (- ,code #xD800) 10) ,next #x2400) + ,wd 4) (setq ,code (if ,error (locally ;; No warnings about fdefinition (declare (optimize (ext:inhibit-warnings 3))) (funcall ,error "High surrogate followed by #x~4,'0X ~ - instead of low surrogate" ,next 2)) + instead of low surrogate" ,next ,wd)) +replacement-character-code+))))) ((= ,code #xFFFE) ;; replace with REPLACEMENT CHARACTER. @@ -75,7 +77,7 @@ Unicode replacement character.") (funcall ,error "BOM is not valid within a UTF-16 stream" ,code 2)) +replacement-character-code+))) (t (setf ,state nil))) - (values ,code 2))) + (values ,code ,wd))) (code-to-octets (code state output error c c1 c2) `(flet ((output (code) (,output (ldb (byte 8 0) code)) ----------------------------------------------------------------------- Summary of changes: src/code/stream.lisp | 1 + src/general-info/release-20d.txt | 3 +++ .../simple-streams/external-formats/utf-16-be.lisp | 10 ++++++---- .../simple-streams/external-formats/utf-16-le.lisp | 12 +++++++----- 4 files changed, 17 insertions(+), 9 deletions(-) hooks/post-receive -- CMU Common Lisp