Raymond Toy pushed to branch issue-139-set-filename-encoding-to-utf8 at cmucl / cmucl
Commits: 1af83384 by Raymond Toy at 2022-12-08T14:57:43+00:00 Address #139: Set terminal format to :locale
- - - - - 6fc2e38e by Raymond Toy at 2022-12-08T14:57:45+00:00 Merge branch 'issue-139-set-terminal-to-utf8' into 'master'
Address #139: Set terminal format to :locale
See merge request cmucl/cmucl!108 - - - - - 7478e387 by Raymond Toy at 2022-12-08T07:04:12-08:00 Merge branch 'master' into issue-139-set-filename-encoding-to-utf8
- - - - - 8367cd33 by Raymond Toy at 2022-12-08T07:26:45-08:00 Fix merge mistake. LANG should not be set.
When merging master, we added LANG again when it should have been deleted.
- - - - - fde3634e by Raymond Toy at 2022-12-08T07:31:53-08:00 Set filename encoding to utf-8.
- - - - - 354f94f5 by Raymond Toy at 2022-12-08T07:32:26-08:00 Fix up tests for filename encoding
In test issue.130, we set `*filename-encoding*` to `:utf-8`. This shouldn't be needed anymore, so remove it.
We also note here the in tests issue.139-default-external-format-read-file and issue.139-default-external-format-write-file, we were creating a pathname consisting of Korean letters. We didn't set the filename encoding to utf-8 here when opening thes file so I don't know how that ever worked. But since the encoding is now :utf-8 it should be ok now.
- - - - -
3 changed files:
- bin/run-tests.sh - src/code/save.lisp - tests/issues.lisp
Changes:
===================================== bin/run-tests.sh ===================================== @@ -47,13 +47,9 @@ function cleanup {
trap cleanup EXIT
-# Set a known value for LANG here so that we can tests consistently. -# The most important part is that the codeset is UTF-8. -LANG=en_US.UTF-8 - if [ $# -eq 0 ]; then # No args so run all the tests - $LISP -noinit -load tests/run-tests.lisp -eval '(cmucl-test-runner:run-all-tests)' + $LISP -noinit -nositeinit -load tests/run-tests.lisp -eval '(cmucl-test-runner:run-all-tests)' else # Run selected files. Convert each file name to uppercase and append "-TESTS" result="" @@ -62,6 +58,6 @@ else new=`echo $f | tr '[a-z]' '[A-Z]'` result="$result ""$new-TESTS" done - $LISP -noinit -load tests/run-tests.lisp -eval "(progn (cmucl-test-runner:load-test-files) (cmucl-test-runner:run-test $result))" + $LISP -noinit -nositeinit -load tests/run-tests.lisp -eval "(progn (cmucl-test-runner:load-test-files) (cmucl-test-runner:run-test $result))" fi
===================================== src/code/save.lisp ===================================== @@ -145,26 +145,23 @@ (defun set-up-locale-external-format () "Add external format alias for :locale to the format specified by the locale as set by setlocale(3C)." - (let ((codeset (unix::unix-get-locale-codeset))) + (let ((codeset (unix::unix-get-locale-codeset)) + (external-format nil)) (cond ((zerop (length codeset)) - ;; Codeset was the empty string, so just set :locale to - ;; alias to the default external format. + (setq external-format *default-external-format*)) + (t + (let ((name (intern codeset "KEYWORD"))) + (setq external-format + (stream::ef-name (stream::find-external-format name nil)))))) + (cond (external-format (setf (gethash :locale stream::*external-format-aliases*) - *default-external-format*)) + external-format)) (t - ;; If we know the format. This could be an alias to - ;; another format and so on, so use FIND-EXTERNAL-FORMAT - ;; to determine the final format and use that as the - ;; alias. - (let* ((codeset-format (intern codeset "KEYWORD")) - (final-format (stream::find-external-format codeset-format))) - (setf (gethash :locale stream::*external-format-aliases*) - (if final-format - (stream::ef-name final-format) - (progn - (warn "Unsupported external format; using :iso8859-1 instead: ~S" - codeset-format) - :iso8859-1))))))) + (warn "No external format found for codeset "~S"; using ~S instead" + codeset + *default-external-format*) + (setf (gethash :locale stream::*external-format-aliases*) + *default-external-format*)))) (values))
@@ -285,8 +282,9 @@ (intl::setlocale) ;; Set up :locale format (set-up-locale-external-format) - ;; Set terminal encodings to :locale - (set-system-external-format :locale) + ;; Set terminal encodings to :locale and filename encoding to :utf-8. + ;; (This needs more work on Darwin.) + (set-system-external-format :locale :utf-8) (ext::process-command-strings process-command-line) (setf *editor-lisp-p* nil) (macrolet ((find-switch (name)
===================================== tests/issues.lisp ===================================== @@ -682,10 +682,7 @@ ;; work and not return NIL. (assert-true (file-author ".")) (assert-true (file-author "bin/build.sh")) - (let ((unix::*filename-encoding* :utf-8)) - ;; Set filename encoding to utf-8 so that we can encode the - ;; filename properly. - (assert-true + (assert-true (file-author (merge-pathnames (concatenate 'string @@ -696,7 +693,7 @@ '(#\Hangul_Syllable_An #\Hangul_Syllable_Nyeong #\Hangul_Syllable_Ha #\Hangul_Syllable_Sib #\Hangul_Syllable_Ni #\Hangul_Syllable_Gga) ".txt") - *test-path*))))) + *test-path*))))
(define-test issue.139-default-external-format (:tag :issues) @@ -709,9 +706,14 @@ (assert-eq locale-format (stream-external-format sys:*stdout*)) (assert-eq locale-format (stream-external-format sys:*stderr*)) ;; sys:*tty* can either be an fd-stream or a two-way-stream. - ;; stream-external-format doesn't work for a two-way-stream. - (unless (typep sys:*tty* 'two-way-stream) - (assert-eq locale-format (stream-external-format sys:*tty*))))) + (etypecase sys:*tty* + (system:fd-stream + (assert-eq locale-format (stream-external-format sys:*tty*))) + (two-way-stream + (assert-eq locale-format + (stream-external-format (two-way-stream-input-stream sys:*tty*))) + (assert-eq locale-format + (stream-external-format (two-way-stream-output-stream sys:*tty*)))))))
(define-test issue.139-default-external-format-read-file (:tag :issues)
View it on GitLab: https://gitlab.common-lisp.net/cmucl/cmucl/-/compare/11e9eadcc18a3e0edd412c4...