- cmucl-cvs - mailman3.common-lisp.net

[Git][cmucl/cmucl][issue-158-darwin-pathnames] 8 commits: Address #139: Set filename encoding to :utf-8
by Raymond Toy (＠rtoy) 15 Feb '23

15 Feb '23

Raymond Toy pushed to branch issue-158-darwin-pathnames at cmucl / cmucl Commits: dbdec3a5 by Raymond Toy at 2023-01-13T23:33:47+00:00 Address #139: Set filename encoding to :utf-8 - - - - - d004986e by Raymond Toy at 2023-01-13T23:33:49+00:00 Merge branch 'issue-139-set-filename-encoding-to-utf8' into 'master' Address #139: Set filename encoding to :utf-8 See merge request cmucl/cmucl!109 - - - - - d01f2cf9 by Raymond Toy at 2023-01-18T08:00:32-08:00 Fix #162: Change *filename-encoding* to use :no-encoding Instead of using `NIL` to indicate that `*filename-encoding*` is not to be done, use `:no-encoding` to indicate that. This makes it a bit clearer what `*filename-encoding*` means. - - - - - 7c44d848 by Raymond Toy at 2023-01-19T15:30:06-08:00 Use :null instead of :no-encoding for no filename encoding The advantage of using `:null` is that it's a recognized external format (that aliases to `:void`). So if we inadvertently use `:null` as a filename encoding somewhere unexpected, it will cause an error (because the `:void` encoding does). - - - - - ce202074 by Raymond Toy at 2023-02-10T08:29:32-08:00 Fix stupid typos Really stupid typos/thinkos: * Forgot to change initial value if `*filename-encoding*` from `:no-encoding` to `:null`. (Stupid!) * Fix typo: `:nul` instead of `:null` in `%name->file`. Update cmucl-unix.pot too for the change in the docstring for `*filename-encoding*`. - - - - - 9eb801f6 by Raymond Toy at 2023-02-15T13:01:43-08:00 Disable issue.41.1 when running CI This test was previously disabled only for Linux when running the CI. However, it's now also failing when running the CI for Darwin. Thus disable it whenever we're running the CI. I just manually tested this on my Linux and Mac boxes. This test passes without any problem. Not sure what's going on. - - - - - 4be1d90c by Raymond Toy at 2023-02-15T21:23:15+00:00 Merge branch 'issue-162-filename-encoding-no-encoding' into 'master' Fix #162: Change *filename-encoding* to use :null Closes #162 See merge request cmucl/cmucl!111 - - - - - e5a4b66d by Raymond Toy at 2023-02-15T13:51:37-08:00 Merge branch 'master' into issue-158-darwin-pathnames - - - - - 6 changed files: - src/code/extfmts.lisp - src/code/lispinit.lisp - src/code/save.lisp - src/code/unix.lisp - src/i18n/locale/cmucl-unix.pot - tests/issues.lisp Changes: ===================================== src/code/extfmts.lisp ===================================== @@ -370,8 +370,10 @@ #() '()))))) (defun load-external-format-aliases () + ;; Set filename encoding to NIL to bypass any encoding; it's not + ;; needed to open the aliases file. NIL means the pathname string is passed as is where only the low 8 bits of the (let ((*package* (find-package "KEYWORD")) - (unix::*filename-encoding* :iso8859-1)) + (unix::*filename-encoding* :null)) (with-open-file (stm "ext-formats:aliases" :if-does-not-exist nil :external-format :iso8859-1) (when stm @@ -486,11 +488,16 @@ (and (consp name) (find-external-format name)) (and (with-standard-io-syntax ;; Use standard IO syntax so that changes by the user - ;; don't mess up compiling the external format. - (let ((*package* (find-package "STREAM")) - (lisp::*enable-package-locked-errors* nil) - (s (open (format nil "ext-formats:~(~A~).lisp" name) - :if-does-not-exist nil :external-format :iso8859-1))) + ;; don't mess up compiling the external format, but we + ;; don't need to print readably. Also, set filename + ;; encoding to NIL because we don't need any special + ;; encoding to open the format files. + (let* ((*print-readably* nil) + (unix::*filename-encoding* :null) + (*package* (find-package "STREAM")) + (lisp::*enable-package-locked-errors* nil) + (s (open (format nil "ext-formats:~(~A~).lisp" name) + :if-does-not-exist nil :external-format :iso8859-1))) (when s (null (nth-value 1 (ext:compile-from-stream s)))))) (gethash name *external-formats*)))) @@ -1150,7 +1157,7 @@ character and illegal outputs are replaced by a question mark.") (unless (find-external-format filenames) (error (intl:gettext "Can't find external-format ~S.") filenames)) (setq filenames (ef-name (find-external-format filenames))) - (when (and unix::*filename-encoding* + (when (and (not (eq unix::*filename-encoding* :null)) (not (eq unix::*filename-encoding* filenames))) (cerror (intl:gettext "Change it anyway.") (intl:gettext "The external-format for encoding filenames is already set."))) ===================================== src/code/lispinit.lisp ===================================== @@ -344,7 +344,7 @@ #-gengc (setf unix::*interrupt-pending* nil) (setf *type-system-initialized* nil) (setf *break-on-signals* nil) - (setf unix::*filename-encoding* nil) + (setf unix::*filename-encoding* :null) (setf *enable-darwin-path-normalization* nil) #+gengc (setf conditions::*handler-clusters* nil) (setq intl::*default-domain* "cmucl") ===================================== src/code/save.lisp ===================================== @@ -164,7 +164,35 @@ *default-external-format*)))) (values)) - +(defun decode-runtime-strings (locale file-locale) + ;; The C runtime can initialize the following strings from the + ;; command line or the environment. We need to decode these into + ;; the utf-16 strings that Lisp uses. + (setf lisp-command-line-list + (mapcar #'(lambda (s) + (stream:string-decode s locale)) + lisp-command-line-list)) + (setf lisp-environment-list + (mapcar #'(lambda (s) + (stream:string-decode s locale)) + lisp-environment-list)) + ;; This needs more work.. *cmucl-lib* could be set from the the envvar + ;; "CMUCLLIB" or from the "-lib" command-line option, and thus + ;; should use the LOCALE to decode the string. + (when *cmucl-lib* + (setf *cmucl-lib* + (stream:string-decode *cmucl-lib* file-locale))) + ;; This also needs more work since the core path could come from the + ;; "-core" command-line option and should thus use LOCALE to decode + ;; the string. It could also come from the "CMUCLCORE" envvar. + (setf *cmucl-core-path* + (stream:string-decode *cmucl-core-path* file-locale)) + ;; *unidata-path* defaults to a pathname object, but the user can + ;; specify a path, so we need to decode the string path if given. + (when (and *unidata-path* (stringp *unidata-path*)) + (setf *unidata-path* + (stream:string-decode *unidata-path* file-locale)))) + (defun save-lisp (core-file-name &key (purify t) (root-structures ()) @@ -278,10 +306,9 @@ ;; Load external format aliases now so we can aliases to ;; specify the external format. (stream::load-external-format-aliases) - ;; Set the locale for lisp - (intl::setlocale) ;; Set up :locale format (set-up-locale-external-format) +<<<<<<< HEAD ;; Set terminal encodings to :locale (set-system-external-format :locale) #+darwin @@ -295,6 +322,18 @@ (lisp::load-decomp) (lisp::load-combining) (setf *enable-darwin-path-normalization* t)) +======= + ;; Set terminal encodings to :locale and filename encoding to :utf-8. + ;; (This needs more work on Darwin.) + (set-system-external-format :locale :utf-8) + (decode-runtime-strings :locale :utf-8) + ;; Need to reinitialize the environment again because + ;; we've possibly changed the environment variables and + ;; pathnames. + (environment-init) + ;; Set the locale for lisp + (intl::setlocale) +>>>>>>> master (ext::process-command-strings process-command-line) (setf *editor-lisp-p* nil) (macrolet ((find-switch (name) ===================================== src/code/unix.lisp ===================================== @@ -25,17 +25,22 @@ ;; it must be set to :iso8859-1 (or left as NIL), making files with ;; non-Latin-1 characters "mojibake", but otherwise they'll be inaccessible. ;; Must be set to NIL initially to enable building Lisp! -(defvar *filename-encoding* nil) +(defvar *filename-encoding* :null + "The encoding to use for converting a namestring to a string that can + be used by the operations system. It must be a valid + external-format name or :NULL. :NULL means the string + is passed as is to the operating system. The operating system will + get the low 8 bits of each UTF-16 code unit of the string.") (eval-when (:compile-toplevel :load-toplevel :execute) (defmacro %name->file (string) - `(if *filename-encoding* - (string-encode ,string *filename-encoding*) - ,string)) + `(if (eql *filename-encoding* :null) + ,string + (string-encode ,string *filename-encoding*))) (defmacro %file->name (string) - `(if *filename-encoding* - (string-decode ,string *filename-encoding*) - ,string))) + `(if (eql *filename-encoding* :null) + ,string + (string-decode ,string *filename-encoding*)))) ;;;; Common machine independent structures. ===================================== src/i18n/locale/cmucl-unix.pot ===================================== @@ -15,6 +15,15 @@ msgstr "" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" +#: src/code/unix.lisp +msgid "" +"The encoding to use for converting a namestring to a string that can\n" +" be used by the operations system. It must be a valid\n" +" external-format name or :NULL. :NULL means the string\n" +" is passed as is to the operating system. The operating system will\n" +" get the low 8 bits of each UTF-16 code unit of the string." +msgstr "" + #: src/code/unix.lisp msgid "Syscall ~A failed: ~A" msgstr "" ===================================== tests/issues.lisp ===================================== @@ -258,6 +258,13 @@ (assert-equal (map 'list #'char-code out-string) (map 'list #'char-code expected)))))) +(define-test issue.25c-setup + (:tag :issues) + ;; Get the external format before running the test issue.25c. See + ;; issue #161 + ;; (https://gitlab.common-lisp.net/cmucl/cmucl/-/issues/161) + (assert-true (stream::find-external-format :utf16-be))) + (define-test issue.25c (:tag :issues) ;; Modified test to verify that each octet read from run-program is @@ -409,9 +416,12 @@ ;; running a pipeline with linux, but otherwise enable it. The ;; pipeline defines the envvar GITLAB_CI so check for that. ;; +;; This also fails on Darwin CI now. Let's just disable the test if +;; running on CI. +;; ;; It would be better if lisp-unit had a way of marking tests as known ;; failures, but it doesn't. -#+#.(cl:if (cl:and (ext:featurep :linux) (unix:unix-getenv "GITLAB_CI")) '(or) '(and)) +#+#.(cl:if (cl:and (unix:unix-getenv "GITLAB_CI")) '(or) '(and)) (define-test issue.41.1 (:tag :issues) (issue-41-tester unix:sigstop)) @@ -682,10 +692,7 @@ ;; work and not return NIL. (assert-true (file-author ".")) (assert-true (file-author "bin/build.sh")) - (let ((unix::*filename-encoding* :utf-8)) - ;; Set filename encoding to utf-8 so that we can encode the - ;; filename properly. - (assert-true + (assert-true (file-author (merge-pathnames (concatenate 'string @@ -696,7 +703,7 @@ '(#\Hangul_Syllable_An #\Hangul_Syllable_Nyeong #\Hangul_Syllable_Ha #\Hangul_Syllable_Sib #\Hangul_Syllable_Ni #\Hangul_Syllable_Gga) ".txt") - *test-path*))))) + *test-path*)))) (define-test issue.139-default-external-format (:tag :issues) View it on GitLab: https://gitlab.common-lisp.net/cmucl/cmucl/-/compare/08496138d12cbf3f2b839d… -- View it on GitLab: https://gitlab.common-lisp.net/cmucl/cmucl/-/compare/08496138d12cbf3f2b839d… You're receiving this email because of your account on gitlab.common-lisp.net.

1 0

[Git][cmucl/cmucl][issue-158-darwin-pathnames-utf8] 6 commits: Fix #162: Change *filename-encoding* to use :no-encoding
by Raymond Toy (＠rtoy) 15 Feb '23

15 Feb '23

Raymond Toy pushed to branch issue-158-darwin-pathnames-utf8 at cmucl / cmucl Commits: d01f2cf9 by Raymond Toy at 2023-01-18T08:00:32-08:00 Fix #162: Change *filename-encoding* to use :no-encoding Instead of using `NIL` to indicate that `*filename-encoding*` is not to be done, use `:no-encoding` to indicate that. This makes it a bit clearer what `*filename-encoding*` means. - - - - - 7c44d848 by Raymond Toy at 2023-01-19T15:30:06-08:00 Use :null instead of :no-encoding for no filename encoding The advantage of using `:null` is that it's a recognized external format (that aliases to `:void`). So if we inadvertently use `:null` as a filename encoding somewhere unexpected, it will cause an error (because the `:void` encoding does). - - - - - ce202074 by Raymond Toy at 2023-02-10T08:29:32-08:00 Fix stupid typos Really stupid typos/thinkos: * Forgot to change initial value if `*filename-encoding*` from `:no-encoding` to `:null`. (Stupid!) * Fix typo: `:nul` instead of `:null` in `%name->file`. Update cmucl-unix.pot too for the change in the docstring for `*filename-encoding*`. - - - - - 9eb801f6 by Raymond Toy at 2023-02-15T13:01:43-08:00 Disable issue.41.1 when running CI This test was previously disabled only for Linux when running the CI. However, it's now also failing when running the CI for Darwin. Thus disable it whenever we're running the CI. I just manually tested this on my Linux and Mac boxes. This test passes without any problem. Not sure what's going on. - - - - - 4be1d90c by Raymond Toy at 2023-02-15T21:23:15+00:00 Merge branch 'issue-162-filename-encoding-no-encoding' into 'master' Fix #162: Change *filename-encoding* to use :null Closes #162 See merge request cmucl/cmucl!111 - - - - - aca1967b by Raymond Toy at 2023-02-15T13:43:20-08:00 Merge branch 'master' into issue-158-darwin-pathnames-utf8 - - - - - 5 changed files: - src/code/extfmts.lisp - src/code/lispinit.lisp - src/code/unix.lisp - src/i18n/locale/cmucl-unix.pot - tests/issues.lisp Changes: ===================================== src/code/extfmts.lisp ===================================== @@ -373,7 +373,7 @@ ;; Set filename encoding to NIL to bypass any encoding; it's not ;; needed to open the aliases file. NIL means the pathname string is passed as is where only the low 8 bits of the (let ((*package* (find-package "KEYWORD")) - (unix::*filename-encoding* nil)) + (unix::*filename-encoding* :null)) (with-open-file (stm "ext-formats:aliases" :if-does-not-exist nil :external-format :iso8859-1) (when stm @@ -493,7 +493,7 @@ ;; encoding to NIL because we don't need any special ;; encoding to open the format files. (let* ((*print-readably* nil) - (unix::*filename-encoding* nil) + (unix::*filename-encoding* :null) (*package* (find-package "STREAM")) (lisp::*enable-package-locked-errors* nil) (s (open (format nil "ext-formats:~(~A~).lisp" name) @@ -1157,7 +1157,7 @@ character and illegal outputs are replaced by a question mark.") (unless (find-external-format filenames) (error (intl:gettext "Can't find external-format ~S.") filenames)) (setq filenames (ef-name (find-external-format filenames))) - (when (and unix::*filename-encoding* + (when (and (not (eq unix::*filename-encoding* :null)) (not (eq unix::*filename-encoding* filenames))) (cerror (intl:gettext "Change it anyway.") (intl:gettext "The external-format for encoding filenames is already set."))) ===================================== src/code/lispinit.lisp ===================================== @@ -344,7 +344,7 @@ #-gengc (setf unix::*interrupt-pending* nil) (setf *type-system-initialized* nil) (setf *break-on-signals* nil) - (setf unix::*filename-encoding* nil) + (setf unix::*filename-encoding* :null) (setf *enable-darwin-path-normalization* nil) #+gengc (setf conditions::*handler-clusters* nil) (setq intl::*default-domain* "cmucl") ===================================== src/code/unix.lisp ===================================== @@ -25,22 +25,22 @@ ;; it must be set to :iso8859-1 (or left as NIL), making files with ;; non-Latin-1 characters "mojibake", but otherwise they'll be inaccessible. ;; Must be set to NIL initially to enable building Lisp! -(defvar *filename-encoding* nil +(defvar *filename-encoding* :null "The encoding to use for converting a namestring to a string that can be used by the operations system. It must be a valid - external-format name or NIL. NIL means the string is passed as is - to the operating system. The operating system will get the low 8 - bits of each UTF-16 code unit of the string.") + external-format name or :NULL. :NULL means the string + is passed as is to the operating system. The operating system will + get the low 8 bits of each UTF-16 code unit of the string.") (eval-when (:compile-toplevel :load-toplevel :execute) (defmacro %name->file (string) - `(if *filename-encoding* - (string-encode ,string *filename-encoding*) - ,string)) + `(if (eql *filename-encoding* :null) + ,string + (string-encode ,string *filename-encoding*))) (defmacro %file->name (string) - `(if *filename-encoding* - (string-decode ,string *filename-encoding*) - ,string))) + `(if (eql *filename-encoding* :null) + ,string + (string-decode ,string *filename-encoding*)))) ;;;; Common machine independent structures. ===================================== src/i18n/locale/cmucl-unix.pot ===================================== @@ -15,6 +15,15 @@ msgstr "" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" +#: src/code/unix.lisp +msgid "" +"The encoding to use for converting a namestring to a string that can\n" +" be used by the operations system. It must be a valid\n" +" external-format name or :NULL. :NULL means the string\n" +" is passed as is to the operating system. The operating system will\n" +" get the low 8 bits of each UTF-16 code unit of the string." +msgstr "" + #: src/code/unix.lisp msgid "Syscall ~A failed: ~A" msgstr "" ===================================== tests/issues.lisp ===================================== @@ -416,9 +416,12 @@ ;; running a pipeline with linux, but otherwise enable it. The ;; pipeline defines the envvar GITLAB_CI so check for that. ;; +;; This also fails on Darwin CI now. Let's just disable the test if +;; running on CI. +;; ;; It would be better if lisp-unit had a way of marking tests as known ;; failures, but it doesn't. -#+#.(cl:if (cl:and (ext:featurep :linux) (unix:unix-getenv "GITLAB_CI")) '(or) '(and)) +#+#.(cl:if (cl:and (unix:unix-getenv "GITLAB_CI")) '(or) '(and)) (define-test issue.41.1 (:tag :issues) (issue-41-tester unix:sigstop)) View it on GitLab: https://gitlab.common-lisp.net/cmucl/cmucl/-/compare/34d66158f3f06e2f5cabc9… -- View it on GitLab: https://gitlab.common-lisp.net/cmucl/cmucl/-/compare/34d66158f3f06e2f5cabc9… You're receiving this email because of your account on gitlab.common-lisp.net.

1 0

[Git][cmucl/cmucl][master] 5 commits: Fix #162: Change *filename-encoding* to use :no-encoding
by Raymond Toy (＠rtoy) 15 Feb '23

15 Feb '23

Raymond Toy pushed to branch master at cmucl / cmucl Commits: d01f2cf9 by Raymond Toy at 2023-01-18T08:00:32-08:00 Fix #162: Change *filename-encoding* to use :no-encoding Instead of using `NIL` to indicate that `*filename-encoding*` is not to be done, use `:no-encoding` to indicate that. This makes it a bit clearer what `*filename-encoding*` means. - - - - - 7c44d848 by Raymond Toy at 2023-01-19T15:30:06-08:00 Use :null instead of :no-encoding for no filename encoding The advantage of using `:null` is that it's a recognized external format (that aliases to `:void`). So if we inadvertently use `:null` as a filename encoding somewhere unexpected, it will cause an error (because the `:void` encoding does). - - - - - ce202074 by Raymond Toy at 2023-02-10T08:29:32-08:00 Fix stupid typos Really stupid typos/thinkos: * Forgot to change initial value if `*filename-encoding*` from `:no-encoding` to `:null`. (Stupid!) * Fix typo: `:nul` instead of `:null` in `%name->file`. Update cmucl-unix.pot too for the change in the docstring for `*filename-encoding*`. - - - - - 9eb801f6 by Raymond Toy at 2023-02-15T13:01:43-08:00 Disable issue.41.1 when running CI This test was previously disabled only for Linux when running the CI. However, it's now also failing when running the CI for Darwin. Thus disable it whenever we're running the CI. I just manually tested this on my Linux and Mac boxes. This test passes without any problem. Not sure what's going on. - - - - - 4be1d90c by Raymond Toy at 2023-02-15T21:23:15+00:00 Merge branch 'issue-162-filename-encoding-no-encoding' into 'master' Fix #162: Change *filename-encoding* to use :null Closes #162 See merge request cmucl/cmucl!111 - - - - - 5 changed files: - src/code/extfmts.lisp - src/code/lispinit.lisp - src/code/unix.lisp - src/i18n/locale/cmucl-unix.pot - tests/issues.lisp Changes: ===================================== src/code/extfmts.lisp ===================================== @@ -373,7 +373,7 @@ ;; Set filename encoding to NIL to bypass any encoding; it's not ;; needed to open the aliases file. NIL means the pathname string is passed as is where only the low 8 bits of the (let ((*package* (find-package "KEYWORD")) - (unix::*filename-encoding* nil)) + (unix::*filename-encoding* :null)) (with-open-file (stm "ext-formats:aliases" :if-does-not-exist nil :external-format :iso8859-1) (when stm @@ -493,7 +493,7 @@ ;; encoding to NIL because we don't need any special ;; encoding to open the format files. (let* ((*print-readably* nil) - (unix::*filename-encoding* nil) + (unix::*filename-encoding* :null) (*package* (find-package "STREAM")) (lisp::*enable-package-locked-errors* nil) (s (open (format nil "ext-formats:~(~A~).lisp" name) @@ -1157,7 +1157,7 @@ character and illegal outputs are replaced by a question mark.") (unless (find-external-format filenames) (error (intl:gettext "Can't find external-format ~S.") filenames)) (setq filenames (ef-name (find-external-format filenames))) - (when (and unix::*filename-encoding* + (when (and (not (eq unix::*filename-encoding* :null)) (not (eq unix::*filename-encoding* filenames))) (cerror (intl:gettext "Change it anyway.") (intl:gettext "The external-format for encoding filenames is already set."))) ===================================== src/code/lispinit.lisp ===================================== @@ -343,7 +343,7 @@ #-gengc (setf unix::*interrupt-pending* nil) (setf *type-system-initialized* nil) (setf *break-on-signals* nil) - (setf unix::*filename-encoding* nil) + (setf unix::*filename-encoding* :null) #+gengc (setf conditions::*handler-clusters* nil) (setq intl::*default-domain* "cmucl") (setq intl::*locale* "C") ===================================== src/code/unix.lisp ===================================== @@ -25,22 +25,22 @@ ;; it must be set to :iso8859-1 (or left as NIL), making files with ;; non-Latin-1 characters "mojibake", but otherwise they'll be inaccessible. ;; Must be set to NIL initially to enable building Lisp! -(defvar *filename-encoding* nil +(defvar *filename-encoding* :null "The encoding to use for converting a namestring to a string that can be used by the operations system. It must be a valid - external-format name or NIL. NIL means the string is passed as is - to the operating system. The operating system will get the low 8 - bits of each UTF-16 code unit of the string.") + external-format name or :NULL. :NULL means the string + is passed as is to the operating system. The operating system will + get the low 8 bits of each UTF-16 code unit of the string.") (eval-when (:compile-toplevel :load-toplevel :execute) (defmacro %name->file (string) - `(if *filename-encoding* - (string-encode ,string *filename-encoding*) - ,string)) + `(if (eql *filename-encoding* :null) + ,string + (string-encode ,string *filename-encoding*))) (defmacro %file->name (string) - `(if *filename-encoding* - (string-decode ,string *filename-encoding*) - ,string))) + `(if (eql *filename-encoding* :null) + ,string + (string-decode ,string *filename-encoding*)))) ;;;; Common machine independent structures. ===================================== src/i18n/locale/cmucl-unix.pot ===================================== @@ -15,6 +15,15 @@ msgstr "" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" +#: src/code/unix.lisp +msgid "" +"The encoding to use for converting a namestring to a string that can\n" +" be used by the operations system. It must be a valid\n" +" external-format name or :NULL. :NULL means the string\n" +" is passed as is to the operating system. The operating system will\n" +" get the low 8 bits of each UTF-16 code unit of the string." +msgstr "" + #: src/code/unix.lisp msgid "Syscall ~A failed: ~A" msgstr "" ===================================== tests/issues.lisp ===================================== @@ -416,9 +416,12 @@ ;; running a pipeline with linux, but otherwise enable it. The ;; pipeline defines the envvar GITLAB_CI so check for that. ;; +;; This also fails on Darwin CI now. Let's just disable the test if +;; running on CI. +;; ;; It would be better if lisp-unit had a way of marking tests as known ;; failures, but it doesn't. -#+#.(cl:if (cl:and (ext:featurep :linux) (unix:unix-getenv "GITLAB_CI")) '(or) '(and)) +#+#.(cl:if (cl:and (unix:unix-getenv "GITLAB_CI")) '(or) '(and)) (define-test issue.41.1 (:tag :issues) (issue-41-tester unix:sigstop)) View it on GitLab: https://gitlab.common-lisp.net/cmucl/cmucl/-/compare/d004986e80238a6ddc6dc8… -- View it on GitLab: https://gitlab.common-lisp.net/cmucl/cmucl/-/compare/d004986e80238a6ddc6dc8… You're receiving this email because of your account on gitlab.common-lisp.net.

1 0

[Git][cmucl/cmucl][issue-162-filename-encoding-no-encoding] Disable issue.41.1 when running CI
by Raymond Toy (＠rtoy) 15 Feb '23

15 Feb '23

Raymond Toy pushed to branch issue-162-filename-encoding-no-encoding at cmucl / cmucl Commits: 9eb801f6 by Raymond Toy at 2023-02-15T13:01:43-08:00 Disable issue.41.1 when running CI This test was previously disabled only for Linux when running the CI. However, it's now also failing when running the CI for Darwin. Thus disable it whenever we're running the CI. I just manually tested this on my Linux and Mac boxes. This test passes without any problem. Not sure what's going on. - - - - - 1 changed file: - tests/issues.lisp Changes: ===================================== tests/issues.lisp ===================================== @@ -416,9 +416,12 @@ ;; running a pipeline with linux, but otherwise enable it. The ;; pipeline defines the envvar GITLAB_CI so check for that. ;; +;; This also fails on Darwin CI now. Let's just disable the test if +;; running on CI. +;; ;; It would be better if lisp-unit had a way of marking tests as known ;; failures, but it doesn't. -#+#.(cl:if (cl:and (ext:featurep :linux) (unix:unix-getenv "GITLAB_CI")) '(or) '(and)) +#+#.(cl:if (cl:and (unix:unix-getenv "GITLAB_CI")) '(or) '(and)) (define-test issue.41.1 (:tag :issues) (issue-41-tester unix:sigstop)) View it on GitLab: https://gitlab.common-lisp.net/cmucl/cmucl/-/commit/9eb801f6a320f02c222624f… -- View it on GitLab: https://gitlab.common-lisp.net/cmucl/cmucl/-/commit/9eb801f6a320f02c222624f… You're receiving this email because of your account on gitlab.common-lisp.net.

1 0

[Git][cmucl/cmucl][issue-162-filename-encoding-no-encoding] Fix stupid typos
by Raymond Toy (＠rtoy) 10 Feb '23

10 Feb '23

Raymond Toy pushed to branch issue-162-filename-encoding-no-encoding at cmucl / cmucl Commits: ce202074 by Raymond Toy at 2023-02-10T08:29:32-08:00 Fix stupid typos Really stupid typos/thinkos: * Forgot to change initial value if `*filename-encoding*` from `:no-encoding` to `:null`. (Stupid!) * Fix typo: `:nul` instead of `:null` in `%name->file`. Update cmucl-unix.pot too for the change in the docstring for `*filename-encoding*`. - - - - - 2 changed files: - src/code/unix.lisp - src/i18n/locale/cmucl-unix.pot Changes: ===================================== src/code/unix.lisp ===================================== @@ -25,7 +25,7 @@ ;; it must be set to :iso8859-1 (or left as NIL), making files with ;; non-Latin-1 characters "mojibake", but otherwise they'll be inaccessible. ;; Must be set to NIL initially to enable building Lisp! -(defvar *filename-encoding* :no-encoding +(defvar *filename-encoding* :null "The encoding to use for converting a namestring to a string that can be used by the operations system. It must be a valid external-format name or :NULL. :NULL means the string @@ -34,7 +34,7 @@ (eval-when (:compile-toplevel :load-toplevel :execute) (defmacro %name->file (string) - `(if (eql *filename-encoding* :nul) + `(if (eql *filename-encoding* :null) ,string (string-encode ,string *filename-encoding*))) (defmacro %file->name (string) ===================================== src/i18n/locale/cmucl-unix.pot ===================================== @@ -19,7 +19,7 @@ msgstr "" msgid "" "The encoding to use for converting a namestring to a string that can\n" " be used by the operations system. It must be a valid\n" -" external-format name or :NO-ENCODING. :NO-ENCODING means the string\n" +" external-format name or :NULL. :NULL means the string\n" " is passed as is to the operating system. The operating system will\n" " get the low 8 bits of each UTF-16 code unit of the string." msgstr "" View it on GitLab: https://gitlab.common-lisp.net/cmucl/cmucl/-/commit/ce202074d2ffa1b1626f7bd… -- View it on GitLab: https://gitlab.common-lisp.net/cmucl/cmucl/-/commit/ce202074d2ffa1b1626f7bd… You're receiving this email because of your account on gitlab.common-lisp.net.

1 0

[Git][cmucl/cmucl] Pushed new branch issue-163-add-command-line-version
by Raymond Toy (＠rtoy) 20 Jan '23

20 Jan '23

Raymond Toy pushed new branch issue-163-add-command-line-version at cmucl / cmucl -- View it on GitLab: https://gitlab.common-lisp.net/cmucl/cmucl/-/tree/issue-163-add-command-lin… You're receiving this email because of your account on gitlab.common-lisp.net.

1 0

[Git][cmucl/cmucl][issue-162-filename-encoding-no-encoding] Use :null instead of :no-encoding for no filename encoding
by Raymond Toy (＠rtoy) 19 Jan '23

19 Jan '23

Raymond Toy pushed to branch issue-162-filename-encoding-no-encoding at cmucl / cmucl Commits: 7c44d848 by Raymond Toy at 2023-01-19T15:30:06-08:00 Use :null instead of :no-encoding for no filename encoding The advantage of using `:null` is that it's a recognized external format (that aliases to `:void`). So if we inadvertently use `:null` as a filename encoding somewhere unexpected, it will cause an error (because the `:void` encoding does). - - - - - 3 changed files: - src/code/extfmts.lisp - src/code/lispinit.lisp - src/code/unix.lisp Changes: ===================================== src/code/extfmts.lisp ===================================== @@ -373,7 +373,7 @@ ;; Set filename encoding to NIL to bypass any encoding; it's not ;; needed to open the aliases file. NIL means the pathname string is passed as is where only the low 8 bits of the (let ((*package* (find-package "KEYWORD")) - (unix::*filename-encoding* :no-encoding)) + (unix::*filename-encoding* :null)) (with-open-file (stm "ext-formats:aliases" :if-does-not-exist nil :external-format :iso8859-1) (when stm @@ -493,7 +493,7 @@ ;; encoding to NIL because we don't need any special ;; encoding to open the format files. (let* ((*print-readably* nil) - (unix::*filename-encoding* :no-encoding) + (unix::*filename-encoding* :null) (*package* (find-package "STREAM")) (lisp::*enable-package-locked-errors* nil) (s (open (format nil "ext-formats:~(~A~).lisp" name) @@ -1157,7 +1157,7 @@ character and illegal outputs are replaced by a question mark.") (unless (find-external-format filenames) (error (intl:gettext "Can't find external-format ~S.") filenames)) (setq filenames (ef-name (find-external-format filenames))) - (when (and (not (eq unix::*filename-encoding* :no-encoding)) + (when (and (not (eq unix::*filename-encoding* :null)) (not (eq unix::*filename-encoding* filenames))) (cerror (intl:gettext "Change it anyway.") (intl:gettext "The external-format for encoding filenames is already set."))) ===================================== src/code/lispinit.lisp ===================================== @@ -343,7 +343,7 @@ #-gengc (setf unix::*interrupt-pending* nil) (setf *type-system-initialized* nil) (setf *break-on-signals* nil) - (setf unix::*filename-encoding* :no-encoding) + (setf unix::*filename-encoding* :null) #+gengc (setf conditions::*handler-clusters* nil) (setq intl::*default-domain* "cmucl") (setq intl::*locale* "C") ===================================== src/code/unix.lisp ===================================== @@ -28,17 +28,17 @@ (defvar *filename-encoding* :no-encoding "The encoding to use for converting a namestring to a string that can be used by the operations system. It must be a valid - external-format name or :NO-ENCODING. :NO-ENCODING means the string + external-format name or :NULL. :NULL means the string is passed as is to the operating system. The operating system will get the low 8 bits of each UTF-16 code unit of the string.") (eval-when (:compile-toplevel :load-toplevel :execute) (defmacro %name->file (string) - `(if (eql *filename-encoding* :no-encoding) + `(if (eql *filename-encoding* :nul) ,string (string-encode ,string *filename-encoding*))) (defmacro %file->name (string) - `(if (eql *filename-encoding* :no-encoding) + `(if (eql *filename-encoding* :null) ,string (string-decode ,string *filename-encoding*)))) View it on GitLab: https://gitlab.common-lisp.net/cmucl/cmucl/-/commit/7c44d848ce1724bf5cfed2e… -- View it on GitLab: https://gitlab.common-lisp.net/cmucl/cmucl/-/commit/7c44d848ce1724bf5cfed2e… You're receiving this email because of your account on gitlab.common-lisp.net.

1 0

[Git][cmucl/cmucl] Pushed new branch issue-162-filename-encoding-no-encoding
by Raymond Toy (＠rtoy) 18 Jan '23

18 Jan '23

Raymond Toy pushed new branch issue-162-filename-encoding-no-encoding at cmucl / cmucl -- View it on GitLab: https://gitlab.common-lisp.net/cmucl/cmucl/-/tree/issue-162-filename-encodi… You're receiving this email because of your account on gitlab.common-lisp.net.

1 0

[Git][cmucl/cmucl][issue-158-darwin-pathnames-utf8] 3 commits: Address #139: Set filename encoding to :utf-8
by Raymond Toy (＠rtoy) 14 Jan '23

14 Jan '23

Raymond Toy pushed to branch issue-158-darwin-pathnames-utf8 at cmucl / cmucl Commits: dbdec3a5 by Raymond Toy at 2023-01-13T23:33:47+00:00 Address #139: Set filename encoding to :utf-8 - - - - - d004986e by Raymond Toy at 2023-01-13T23:33:49+00:00 Merge branch 'issue-139-set-filename-encoding-to-utf8' into 'master' Address #139: Set filename encoding to :utf-8 See merge request cmucl/cmucl!109 - - - - - 34d66158 by Raymond Toy at 2023-01-14T11:52:03-08:00 Merge branch 'master' into issue-158-darwin-pathnames-utf8 - - - - - 1 changed file: - src/code/extfmts.lisp Changes: ===================================== src/code/extfmts.lisp ===================================== @@ -493,7 +493,7 @@ ;; encoding to NIL because we don't need any special ;; encoding to open the format files. (let* ((*print-readably* nil) - ;;(unix::*filename-encoding* nil) + (unix::*filename-encoding* nil) (*package* (find-package "STREAM")) (lisp::*enable-package-locked-errors* nil) (s (open (format nil "ext-formats:~(~A~).lisp" name) View it on GitLab: https://gitlab.common-lisp.net/cmucl/cmucl/-/compare/abd552b0ebeba75f3f8af8… -- View it on GitLab: https://gitlab.common-lisp.net/cmucl/cmucl/-/compare/abd552b0ebeba75f3f8af8… You're receiving this email because of your account on gitlab.common-lisp.net.

1 0

[Git][cmucl/cmucl][master] 2 commits: Address #139: Set filename encoding to :utf-8
by Raymond Toy (＠rtoy) 13 Jan '23

13 Jan '23

Raymond Toy pushed to branch master at cmucl / cmucl Commits: dbdec3a5 by Raymond Toy at 2023-01-13T23:33:47+00:00 Address #139: Set filename encoding to :utf-8 - - - - - d004986e by Raymond Toy at 2023-01-13T23:33:49+00:00 Merge branch 'issue-139-set-filename-encoding-to-utf8' into 'master' Address #139: Set filename encoding to :utf-8 See merge request cmucl/cmucl!109 - - - - - 4 changed files: - src/code/extfmts.lisp - src/code/save.lisp - src/code/unix.lisp - tests/issues.lisp Changes: ===================================== src/code/extfmts.lisp ===================================== @@ -370,8 +370,10 @@ #() '()))))) (defun load-external-format-aliases () + ;; Set filename encoding to NIL to bypass any encoding; it's not + ;; needed to open the aliases file. NIL means the pathname string is passed as is where only the low 8 bits of the (let ((*package* (find-package "KEYWORD")) - (unix::*filename-encoding* :iso8859-1)) + (unix::*filename-encoding* nil)) (with-open-file (stm "ext-formats:aliases" :if-does-not-exist nil :external-format :iso8859-1) (when stm @@ -486,11 +488,16 @@ (and (consp name) (find-external-format name)) (and (with-standard-io-syntax ;; Use standard IO syntax so that changes by the user - ;; don't mess up compiling the external format. - (let ((*package* (find-package "STREAM")) - (lisp::*enable-package-locked-errors* nil) - (s (open (format nil "ext-formats:~(~A~).lisp" name) - :if-does-not-exist nil :external-format :iso8859-1))) + ;; don't mess up compiling the external format, but we + ;; don't need to print readably. Also, set filename + ;; encoding to NIL because we don't need any special + ;; encoding to open the format files. + (let* ((*print-readably* nil) + (unix::*filename-encoding* nil) + (*package* (find-package "STREAM")) + (lisp::*enable-package-locked-errors* nil) + (s (open (format nil "ext-formats:~(~A~).lisp" name) + :if-does-not-exist nil :external-format :iso8859-1))) (when s (null (nth-value 1 (ext:compile-from-stream s)))))) (gethash name *external-formats*)))) ===================================== src/code/save.lisp ===================================== @@ -164,7 +164,35 @@ *default-external-format*)))) (values)) - +(defun decode-runtime-strings (locale file-locale) + ;; The C runtime can initialize the following strings from the + ;; command line or the environment. We need to decode these into + ;; the utf-16 strings that Lisp uses. + (setf lisp-command-line-list + (mapcar #'(lambda (s) + (stream:string-decode s locale)) + lisp-command-line-list)) + (setf lisp-environment-list + (mapcar #'(lambda (s) + (stream:string-decode s locale)) + lisp-environment-list)) + ;; This needs more work.. *cmucl-lib* could be set from the the envvar + ;; "CMUCLLIB" or from the "-lib" command-line option, and thus + ;; should use the LOCALE to decode the string. + (when *cmucl-lib* + (setf *cmucl-lib* + (stream:string-decode *cmucl-lib* file-locale))) + ;; This also needs more work since the core path could come from the + ;; "-core" command-line option and should thus use LOCALE to decode + ;; the string. It could also come from the "CMUCLCORE" envvar. + (setf *cmucl-core-path* + (stream:string-decode *cmucl-core-path* file-locale)) + ;; *unidata-path* defaults to a pathname object, but the user can + ;; specify a path, so we need to decode the string path if given. + (when (and *unidata-path* (stringp *unidata-path*)) + (setf *unidata-path* + (stream:string-decode *unidata-path* file-locale)))) + (defun save-lisp (core-file-name &key (purify t) (root-structures ()) @@ -278,12 +306,18 @@ ;; Load external format aliases now so we can aliases to ;; specify the external format. (stream::load-external-format-aliases) - ;; Set the locale for lisp - (intl::setlocale) ;; Set up :locale format (set-up-locale-external-format) - ;; Set terminal encodings to :locale - (set-system-external-format :locale) + ;; Set terminal encodings to :locale and filename encoding to :utf-8. + ;; (This needs more work on Darwin.) + (set-system-external-format :locale :utf-8) + (decode-runtime-strings :locale :utf-8) + ;; Need to reinitialize the environment again because + ;; we've possibly changed the environment variables and + ;; pathnames. + (environment-init) + ;; Set the locale for lisp + (intl::setlocale) (ext::process-command-strings process-command-line) (setf *editor-lisp-p* nil) (macrolet ((find-switch (name) ===================================== src/code/unix.lisp ===================================== @@ -25,7 +25,12 @@ ;; it must be set to :iso8859-1 (or left as NIL), making files with ;; non-Latin-1 characters "mojibake", but otherwise they'll be inaccessible. ;; Must be set to NIL initially to enable building Lisp! -(defvar *filename-encoding* nil) +(defvar *filename-encoding* nil + "The encoding to use for converting a namestring to a string that can + be used by the operations system. It must be a valid + external-format name or NIL. NIL means the string is passed as is + to the operating system. The operating system will get the low 8 + bits of each UTF-16 code unit of the string.") (eval-when (:compile-toplevel :load-toplevel :execute) (defmacro %name->file (string) ===================================== tests/issues.lisp ===================================== @@ -258,6 +258,13 @@ (assert-equal (map 'list #'char-code out-string) (map 'list #'char-code expected)))))) +(define-test issue.25c-setup + (:tag :issues) + ;; Get the external format before running the test issue.25c. See + ;; issue #161 + ;; (https://gitlab.common-lisp.net/cmucl/cmucl/-/issues/161) + (assert-true (stream::find-external-format :utf16-be))) + (define-test issue.25c (:tag :issues) ;; Modified test to verify that each octet read from run-program is @@ -682,10 +689,7 @@ ;; work and not return NIL. (assert-true (file-author ".")) (assert-true (file-author "bin/build.sh")) - (let ((unix::*filename-encoding* :utf-8)) - ;; Set filename encoding to utf-8 so that we can encode the - ;; filename properly. - (assert-true + (assert-true (file-author (merge-pathnames (concatenate 'string @@ -696,7 +700,7 @@ '(#\Hangul_Syllable_An #\Hangul_Syllable_Nyeong #\Hangul_Syllable_Ha #\Hangul_Syllable_Sib #\Hangul_Syllable_Ni #\Hangul_Syllable_Gga) ".txt") - *test-path*))))) + *test-path*)))) (define-test issue.139-default-external-format (:tag :issues) View it on GitLab: https://gitlab.common-lisp.net/cmucl/cmucl/-/compare/6fc2e38e925ab9f3fcfb7e… -- View it on GitLab: https://gitlab.common-lisp.net/cmucl/cmucl/-/compare/6fc2e38e925ab9f3fcfb7e… You're receiving this email because of your account on gitlab.common-lisp.net.

1 0