core/char/
decode.rs

1//! UTF-8 and UTF-16 decoding iterators
2
3#[cfg(not(feature = "ferrocene_subset"))]
4use crate::error::Error;
5#[cfg(not(feature = "ferrocene_subset"))]
6use crate::fmt;
7#[cfg(not(feature = "ferrocene_subset"))]
8use crate::iter::FusedIterator;
9
10/// An iterator that decodes UTF-16 encoded code points from an iterator of `u16`s.
11///
12/// This `struct` is created by the [`decode_utf16`] method on [`char`]. See its
13/// documentation for more.
14///
15/// [`decode_utf16`]: char::decode_utf16
16#[stable(feature = "decode_utf16", since = "1.9.0")]
17#[cfg_attr(not(feature = "ferrocene_subset"), derive(Clone, Debug))]
18pub struct DecodeUtf16<I>
19where
20    I: Iterator<Item = u16>,
21{
22    iter: I,
23    buf: Option<u16>,
24}
25
26/// An error that can be returned when decoding UTF-16 code points.
27///
28/// This `struct` is created when using the [`DecodeUtf16`] type.
29#[stable(feature = "decode_utf16", since = "1.9.0")]
30#[cfg_attr(not(feature = "ferrocene_subset"), derive(Debug, Clone, Eq, PartialEq))]
31pub struct DecodeUtf16Error {
32    code: u16,
33}
34
35/// Creates an iterator over the UTF-16 encoded code points in `iter`,
36/// returning unpaired surrogates as `Err`s. See [`char::decode_utf16`].
37#[inline]
38pub(super) fn decode_utf16<I: IntoIterator<Item = u16>>(iter: I) -> DecodeUtf16<I::IntoIter> {
39    DecodeUtf16 { iter: iter.into_iter(), buf: None }
40}
41
42#[stable(feature = "decode_utf16", since = "1.9.0")]
43impl<I: Iterator<Item = u16>> Iterator for DecodeUtf16<I> {
44    type Item = Result<char, DecodeUtf16Error>;
45
46    fn next(&mut self) -> Option<Result<char, DecodeUtf16Error>> {
47        let u = match self.buf.take() {
48            Some(buf) => buf,
49            None => self.iter.next()?,
50        };
51
52        if !u.is_utf16_surrogate() {
53            // SAFETY: not a surrogate
54            Some(Ok(unsafe { char::from_u32_unchecked(u as u32) }))
55        } else if u >= 0xDC00 {
56            // a trailing surrogate
57            Some(Err(DecodeUtf16Error { code: u }))
58        } else {
59            let u2 = match self.iter.next() {
60                Some(u2) => u2,
61                // eof
62                None => return Some(Err(DecodeUtf16Error { code: u })),
63            };
64            if u2 < 0xDC00 || u2 > 0xDFFF {
65                // not a trailing surrogate so we're not a valid
66                // surrogate pair, so rewind to redecode u2 next time.
67                self.buf = Some(u2);
68                return Some(Err(DecodeUtf16Error { code: u }));
69            }
70
71            // all ok, so lets decode it.
72            let c = (((u & 0x3ff) as u32) << 10 | (u2 & 0x3ff) as u32) + 0x1_0000;
73            // SAFETY: we checked that it's a legal unicode value
74            Some(Ok(unsafe { char::from_u32_unchecked(c) }))
75        }
76    }
77
78    #[inline]
79    fn size_hint(&self) -> (usize, Option<usize>) {
80        let (low, high) = self.iter.size_hint();
81
82        let (low_buf, high_buf) = match self.buf {
83            // buf is empty, no additional elements from it.
84            None => (0, 0),
85            // `u` is a non surrogate, so it's always an additional character.
86            Some(u) if !u.is_utf16_surrogate() => (1, 1),
87            // `u` is a leading surrogate (it can never be a trailing surrogate and
88            // it's a surrogate due to the previous branch) and `self.iter` is empty.
89            //
90            // `u` can't be paired, since the `self.iter` is empty,
91            // so it will always become an additional element (error).
92            Some(_u) if high == Some(0) => (1, 1),
93            // `u` is a leading surrogate and `iter` may be non-empty.
94            //
95            // `u` can either pair with a trailing surrogate, in which case no additional elements
96            // are produced, or it can become an error, in which case it's an additional character (error).
97            Some(_u) => (0, 1),
98        };
99
100        // `self.iter` could contain entirely valid surrogates (2 elements per
101        // char), or entirely non-surrogates (1 element per char).
102        //
103        // On odd lower bound, at least one element must stay unpaired
104        // (with other elements from `self.iter`), so we round up.
105        let low = low.div_ceil(2) + low_buf;
106        let high = high.and_then(|h| h.checked_add(high_buf));
107
108        (low, high)
109    }
110}
111
112#[cfg(not(feature = "ferrocene_subset"))]
113#[stable(feature = "decode_utf16_fused_iterator", since = "1.75.0")]
114impl<I: Iterator<Item = u16> + FusedIterator> FusedIterator for DecodeUtf16<I> {}
115
116impl DecodeUtf16Error {
117    /// Returns the unpaired surrogate which caused this error.
118    #[must_use]
119    #[stable(feature = "decode_utf16", since = "1.9.0")]
120    pub fn unpaired_surrogate(&self) -> u16 {
121        self.code
122    }
123}
124
125#[cfg(not(feature = "ferrocene_subset"))]
126#[stable(feature = "decode_utf16", since = "1.9.0")]
127impl fmt::Display for DecodeUtf16Error {
128    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
129        write!(f, "unpaired surrogate found: {:x}", self.code)
130    }
131}
132
133#[cfg(not(feature = "ferrocene_subset"))]
134#[stable(feature = "decode_utf16", since = "1.9.0")]
135impl Error for DecodeUtf16Error {}
core/char/decode.rs

core/char/
decode.rs
