GitLab

at 2023-01-13T23:33:47+00:00

Address #139: Set filename encoding to :utf-8

Merge branch 'issue-139-set-filename-encoding-to-utf8' into 'master'

Address #139: Set filename encoding to :utf-8

See merge request cmucl/cmucl!109
 		    #() '())))))

 (defun load-external-format-aliases ()

+  ;; Set filename encoding to NIL to bypass any encoding; it's not

+  ;; needed to open the aliases file.  NIL means the pathname string is passed as is where only the low 8 bits of the 

   (let ((*package* (find-package "KEYWORD"))

-	(unix::*filename-encoding* :iso8859-1))

+	(unix::*filename-encoding* nil))

     (with-open-file (stm "ext-formats:aliases" :if-does-not-exist nil

 			 :external-format :iso8859-1)

       (when stm

       (and (consp name) (find-external-format name))

       (and (with-standard-io-syntax

 	     ;; Use standard IO syntax so that changes by the user

-	     ;; don't mess up compiling the external format.

-	     (let ((*package* (find-package "STREAM"))

-		   (lisp::*enable-package-locked-errors* nil)

-		   (s (open (format nil "ext-formats:~(~A~).lisp" name)

-			    :if-does-not-exist nil :external-format :iso8859-1)))

+	     ;; don't mess up compiling the external format, but we

+	     ;; don't need to print readably.  Also, set filename

+	     ;; encoding to NIL because we don't need any special

+	     ;; encoding to open the format files.

+	     (let* ((*print-readably* nil)

+		    (unix::*filename-encoding* nil)

+		    (*package* (find-package "STREAM"))

+		    (lisp::*enable-package-locked-errors* nil)

+		    (s (open (format nil "ext-formats:~(~A~).lisp" name)

+			     :if-does-not-exist nil :external-format :iso8859-1)))

 	       (when s

 		 (null (nth-value 1 (ext:compile-from-stream s))))))

            (gethash name *external-formats*))))

 		 *default-external-format*))))

   (values))

- 

+(defun decode-runtime-strings (locale file-locale)

+  ;; The C runtime can initialize the following strings from the

+  ;; command line or the environment.  We need to decode these into

+  ;; the utf-16 strings that Lisp uses.

+  (setf lisp-command-line-list

+	(mapcar #'(lambda (s)

+		    (stream:string-decode s locale))

+		lisp-command-line-list))

+  (setf lisp-environment-list

+	(mapcar #'(lambda (s)

+		    (stream:string-decode s locale))

+		lisp-environment-list))

+  ;; This needs more work..  *cmucl-lib* could be set from the the envvar

+  ;; "CMUCLLIB" or from the "-lib" command-line option, and thus

+  ;; should use the LOCALE to decode the string.

+  (when *cmucl-lib*

+    (setf *cmucl-lib*

+	  (stream:string-decode *cmucl-lib* file-locale)))

+  ;; This also needs more work since the core path could come from the

+  ;; "-core" command-line option and should thus use LOCALE to decode

+  ;; the string.  It could also come from the "CMUCLCORE" envvar.

+  (setf *cmucl-core-path*

+	(stream:string-decode *cmucl-core-path* file-locale))

+  ;; *unidata-path* defaults to a pathname object, but the user can

+  ;; specify a path, so we need to decode the string path if given.

+  (when (and *unidata-path* (stringp *unidata-path*))

+    (setf *unidata-path*

+	  (stream:string-decode *unidata-path* file-locale))))

+

 (defun save-lisp (core-file-name &key

 				 (purify t)

 				 (root-structures ())

 	     ;; Load external format aliases now so we can aliases to

 	     ;; specify the external format.

 	     (stream::load-external-format-aliases)

-	     ;; Set the locale for lisp

-	     (intl::setlocale)

 	     ;; Set up :locale format

 	     (set-up-locale-external-format)

-	     ;; Set terminal encodings to :locale

-	     (set-system-external-format :locale)

+	     ;; Set terminal encodings to :locale and filename encoding to :utf-8.

+	     ;; (This needs more work on Darwin.)

+	     (set-system-external-format :locale :utf-8)

+	     (decode-runtime-strings :locale :utf-8)

+	     ;; Need to reinitialize the environment again because

+	     ;; we've possibly changed the environment variables and

+	     ;; pathnames.

+	     (environment-init)

+	     ;; Set the locale for lisp

+	     (intl::setlocale)

 	     (ext::process-command-strings process-command-line)

 	     (setf *editor-lisp-p* nil)

 	     (macrolet ((find-switch (name)

 ;; it must be set to :iso8859-1 (or left as NIL), making files with

 ;; non-Latin-1 characters "mojibake", but otherwise they'll be inaccessible.

 ;; Must be set to NIL initially to enable building Lisp!

-(defvar *filename-encoding* nil)

+(defvar *filename-encoding* nil

+  "The encoding to use for converting a namestring to a string that can

+  be used by the operations system.  It must be a valid

+  external-format name or NIL.  NIL means the string is passed as is

+  to the operating system.  The operating system will get the low 8

+  bits of each UTF-16 code unit of the string.")

 (eval-when (:compile-toplevel :load-toplevel :execute)

   (defmacro %name->file (string)

 	(assert-equal (map 'list #'char-code out-string)

 		      (map 'list #'char-code expected))))))

+(define-test issue.25c-setup

+    (:tag :issues)

+  ;; Get the external format before running the test issue.25c.  See

+  ;; issue #161

+  ;; (https://gitlab.common-lisp.net/cmucl/cmucl/-/issues/161).

+  (assert-true (stream::find-external-format :utf16-be)))

+

 (define-test issue.25c

     (:tag :issues)

   ;; Modified test to verify that each octet read from run-program is

   ;; work and not return NIL.

   (assert-true (file-author "."))

   (assert-true (file-author "bin/build.sh"))

-  (let ((unix::*filename-encoding* :utf-8))

-    ;; Set filename encoding to utf-8 so that we can encode the

-    ;; filename properly.

-    (assert-true

+  (assert-true

    (file-author

     (merge-pathnames 

      (concatenate 'string

 		  '(#\Hangul_Syllable_An #\Hangul_Syllable_Nyeong #\Hangul_Syllable_Ha

 		    #\Hangul_Syllable_Sib #\Hangul_Syllable_Ni #\Hangul_Syllable_Gga)

 		  ".txt")

-     *test-path*)))))

+     *test-path*))))

 (define-test issue.139-default-external-format

     (:tag :issues)

Raymond Toy pushed to branch master at cmucl / cmucl

Commits:

4 changed files:

Changes: