[BUG] ${#var} returns length in bytes, not characters

[BUG] ${#var} returns length in bytes, not characters

[Martijn Dekker]

POSIX:
http://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_06_02
> ${#parameter}
> String Length. The length in characters of the value of parameter
> shall be substituted. [...]

dash does not expand the length in characters; it expands the length in
bytes instead. That is invalid for locales that include multi-byte
characters, such as the now ubiquitous UTF-8 set.

Test case:

$ locale
LANG="nl_NL.UTF-8"
LC_COLLATE="nl_NL.UTF-8"
LC_CTYPE="nl_NL.UTF-8"
LC_MESSAGES="nl_NL.UTF-8"
LC_MONETARY="nl_NL.UTF-8"
LC_NUMERIC="nl_NL.UTF-8"
LC_TIME="nl_NL.UTF-8"
LC_ALL=
$ word='bètatest'	# length: 8
$ echo ${#word}
9

Expected output: 8
Got output: 9

(bash, ksh93, mksh, and zsh all do this correctly.)

- Martijn

Re: [BUG] ${#var} returns length in bytes, not characters

[Stephane Chazelas]

[-- Attachment #1: Type: text/plain, Size: 765 bytes --]

2015-06-03 13:29:33 +0200, Martijn Dekker:
> POSIX:
> http://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_06_02
> > ${#parameter}
> > String Length. The length in characters of the value of parameter
> > shall be substituted. [...]
> 
> dash does not expand the length in characters; it expands the length in
> bytes instead. That is invalid for locales that include multi-byte
> characters, such as the now ubiquitous UTF-8 set.
[...]

See also:

http://thread.gmane.org/gmane.comp.standards.posix.austin.general/9972/focus=10040

For a few UTF-8 related variations in behaviour between shells
(including this one), though many of them are about "unspecified
behaviour".

The script mentioned there is also attached here.

-- 
Stephane

[-- Attachment #2: u8-tests --]
[-- Type: text/plain, Size: 4137 bytes --]

euro=$(printf '\342\202\254')
o342=$(printf '\342')
o202=$(printf '\202')
o254=$(printf '\254')

test_shell=$1

n=0
case $test_shell in
  zsh) emulate sh;;
  zsh5) setopt shwordsplit;;
  [lm]ksh) set -o utf8-mode;;
esac

if [ -n "$test_shell" ]; then
  printf '%8s:' "$test_shell"
  testing() {
    n=$(($n + 1))
    test
    ret=$?
    [ "$ret" -eq "$na" ] && ret=-
    printf ' %2s' "$ret"
    [ "$ret" = "$1" ] && printf '\342\203\235'
  }
else
  testing() {
    n=$(($n + 1))
    printf '%2d: %s (expected: %d)\n' "$n" "$2" "$1"
  }
fi

na=99
if_accept_invalid() { [ "$o254" ] || exit "$na"; }
if_printf_builtin() {
  case $(type printf 2> /dev/null) in
    *builtin*);;
    (*) exit "$na"
  esac
}

################################################################
test() (
  exit "${#euro}"
)
testing 1 '${#utf8-character}'
################################################################
test() (
  if_accept_invalid
  exit "${#o254}"
)
testing 1 '${#single byte, invalid char}'
################################################################
test() (
  if_accept_invalid
  t=$o342$o202
  exit "${#t}"
)
testing 2 '${truncated character, 2 bytes}'
################################################################
test() (
  if_accept_invalid
  case $euro in
    *"$o254"*) true;;
    *) false
  esac
)
testing 0 '$char contains byte component'
################################################################
test() (
  if_accept_invalid
  t=+$euro-
  [ "${t##*"$o254"}" = - ]
)
testing 0 '${##} matching with byte components of $mbchar'
################################################################
test() (
  if_accept_invalid
  t=+$o254$euro-
  [ "${t##*"$euro"}" = - ]
)
testing 0 '${##} works in invalid strings'
################################################################
test() (
  if_accept_invalid
  IFS=$o254
  t=+$euro+
  set -- $t
  exit "$#"
)
testing 2 'byte component found in $mbchar by IFS'
################################################################
test() (
  if_accept_invalid
  IFS=$o342$o202
  t=+$euro+
  set -- $t
  exit "$#"
)
testing 3 'byte component found in $mbchar by IFS'
################################################################
test() (
  if_accept_invalid
  IFS=$o254
  t=+$o254+
  set -- $t
  exit "$#"
)
testing 2 'IFS works with bytes'
################################################################
test() (
  IFS=$euro
  t=+$euro+
  set -- $t
  exit "$#"
)
testing 2 'IFS works with mbchars'
################################################################
test() (
  if_accept_invalid
  IFS=$euro
  t=+$o254$o342$euro+
  set -- $t
  exit "$#"
)
testing 2 'IFS works with mbchars in invalid strings'
################################################################
test() (
  if_accept_invalid
  IFS=$o342$o202
  set / /
  [ "$*" = "/$o342/" ]
)
testing 0 'Joining $* with byte forming invalid char'
################################################################
test() (
  IFS=$euro
  set / /
  [ "$*" = "/$euro/" ]
)
testing 0 'Joining $* with mbchar'
################################################################
test() (
  if_printf_builtin
  exit "$(printf '%4s' "$euro" | wc -c | tr -cd 0-9)"
)
testing 6 'byte length of 4-padded euro'
################################################################
test() (
  if_accept_invalid
  if_printf_builtin
  exit "$(printf '%4s' "$o342$o202" | wc -c | tr -cd 0-9)"
)
testing 4 'byte length of 4-padded truncated mbchar'
################################################################
test() (
  case $euro in
    ?) true;;
    (*) false
  esac
)
testing 0 '? matches mbchar'
################################################################
test() (
  if_accept_invalid
  case $o342$o202 in
    ??) true;;
    (*) false
  esac
)
testing 0 '?? matches 2-byte truncated mbchar'
################################################################


echo
if [ -z "$test_shell" ]; then
  printf '%8s:' tests
  i=1; while [ "$i" -le "$n" ]; do
    printf ' %2d' "$i"
    i=$(($i + 1))
  done
  printf '\n\n'
  for test_shell in dash zsh yash bash lksh mksh ksh93 zsh5 posh; do
    "$test_shell" "$0" "$test_shell"
  done
fi

Re: [BUG] ${#var} returns length in bytes, not characters

[Herbert Xu]

Martijn Dekker <martijn@inlv.org> wrote:
>
> dash does not expand the length in characters; it expands the length in
> bytes instead. That is invalid for locales that include multi-byte
> characters, such as the now ubiquitous UTF-8 set.

dash doesn't handle locales at all.

Cheers,
-- 
Email: Herbert Xu <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt