package console

  1. Overview
  2. Docs

Source file console_text.ml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
(*---------------------------------------------------------------------------
  Copyright (c) 2026 Thomas Gazagnaire. All rights reserved.
  SPDX-License-Identifier: ISC
  ---------------------------------------------------------------------------*)

(* The escape scanner follows the DEC parser of Paul Williams's state diagram
   (vt100.net/emu/dec_ansi_parser) over ECMA-48 and ECMA-35. *)

let removed u =
  let c = Uchar.to_int u in
  (c >= 0xfff9 && c <= 0xfffb)
  || Uucp.Gen.is_default_ignorable u
     && c <> 0x200c && c <> 0x200d
     && not (Uucp.Gen.is_variation_selector u)

let is_selector u = Uucp.Gen.is_variation_selector u

(* What [sanitize] has seen: [cr] after a CR, whose line end a line feed
   then completes, and [base] where a variation selector may follow, which
   the start of the text is, since its base may end the text before. A
   character removed leaves both as they were. *)
type seen = { mutable cr : bool; mutable base : bool }

(* [text buf seen u] adds what [sanitize] makes of the character [u]:
   itself, nothing if it is {!removed} or a variation selector with no base,
   a line end for CR or a line feed (LF, VT, FF and NEL, U+0085), or a space
   for another control. *)
let text buf ~keep_newlines seen u =
  let c = Uchar.to_int u in
  let line_end () = Buffer.add_char buf (if keep_newlines then '\n' else ' ') in
  let after_cr = seen.cr in
  let ascii = c < 0x80 in
  let dropped =
    (not ascii) && (removed u || (is_selector u && not seen.base))
  in
  if not dropped then begin
    seen.cr <- c = 0x0d;
    seen.base <-
      c > 0x20
      && (not (c >= 0x7f && c <= 0x9f))
      && (ascii || not (is_selector u))
      && c <> 0x200c && c <> 0x200d
  end;
  if c = 0x0d then line_end ()
  else if (c >= 0x0a && c <= 0x0c) || c = 0x85 then (
    if not after_cr then line_end ())
  else if dropped then ()
  else if c >= 0x20 && not (c >= 0x7f && c <= 0x9f) then
    Buffer.add_utf_8_uchar buf u
  else Buffer.add_char buf ' '

let subdivision_flags =
  let flag code =
    let b = Buffer.create 28 in
    Buffer.add_utf_8_uchar b (Uchar.of_int 0x1f3f4);
    String.iter
      (fun ch ->
        Buffer.add_utf_8_uchar b (Uchar.of_int (0xe0000 + Char.code ch)))
      code;
    Buffer.add_utf_8_uchar b (Uchar.of_int 0xe007f);
    Buffer.contents b
  in
  List.map flag [ "gbeng"; "gbsct"; "gbwls" ]

let flag_length = String.length (List.hd subdivision_flags)

(* [tag_sequence str i d] is where the flag at [i], whose first character
   [d] decodes, ends, if one of {!subdivision_flags} starts there. *)
let tag_sequence str i d =
  if
    Uchar.to_int (Uchar.utf_decode_uchar d) = 0x1f3f4
    && i + flag_length <= String.length str
    && List.mem (String.sub str i flag_length) subdivision_flags
  then Some (i + flag_length)
  else None

let sanitize_into buf ~keep_newlines seen str =
  let n = String.length str in
  let rec go i =
    if i < n then begin
      let d = String.get_utf_8_uchar str i in
      if not (Uchar.utf_decode_is_valid d) then begin
        seen.cr <- false;
        seen.base <- true;
        Buffer.add_utf_8_uchar buf Uchar.rep;
        go (i + Uchar.utf_decode_length d)
      end
      else
        match tag_sequence str i d with
        | Some stop ->
            seen.cr <- false;
            seen.base <- true;
            Buffer.add_string buf (String.sub str i (stop - i));
            go stop
        | None ->
            text buf ~keep_newlines seen (Uchar.utf_decode_uchar d);
            go (i + Uchar.utf_decode_length d)
    end
  in
  go 0

let seen () = { cr = false; base = true }

(* Whether [sanitize] gives [str] back as it is: valid UTF-8 with no
   control but, if kept, the line feed, nothing removed, and no variation
   selector without a base. *)
let rec is_clean ~keep_newlines str i base =
  i >= String.length str
  ||
  let d = String.get_utf_8_uchar str i in
  Uchar.utf_decode_is_valid d
  &&
  match tag_sequence str i d with
  | Some stop -> is_clean ~keep_newlines str stop true
  | None ->
      let u = Uchar.utf_decode_uchar d in
      let c = Uchar.to_int u in
      let next = i + Uchar.utf_decode_length d in
      if c = 0x0a then keep_newlines && is_clean ~keep_newlines str next false
      else if c < 0x20 || (c >= 0x7f && c <= 0x9f) then false
      else if c >= 0x80 && (removed u || (is_selector u && not base)) then false
      else
        is_clean ~keep_newlines str next
          (c > 0x20
          && (c < 0x80 || not (is_selector u))
          && c <> 0x200c && c <> 0x200d)

let rec ascii_from s i =
  i >= String.length s
  || (Char.Ascii.is_print (String.unsafe_get s i) && ascii_from s (i + 1))

let sanitize ?(keep_newlines = true) str =
  if ascii_from str 0 || is_clean ~keep_newlines str 0 true then str
  else
    let buf = Buffer.create (String.length str) in
    sanitize_into buf ~keep_newlines (seen ()) str;
    Buffer.contents buf

(* What the scan is inside of. ECMA-48 (5th ed.) 5.3: a C1 control is ESC Fe
   in a 7-bit code, Fe in 04/00 to 05/15, and 08/00 to 09/15 in an 8-bit code,
   which UTF-8 writes as U+0080 to U+009F. *)
type state =
  | Ground
  | Escape  (** After ESC (ECMA-35 13.2). *)
  | Escape_intermediate  (** After ESC and an intermediate byte, 02/00-02/15. *)
  | Sequence  (** After CSI (ECMA-48 5.4). *)
  | String of { bel : bool }
      (** Inside a control string (ECMA-48 5.6); [bel] when BEL closes it, as
          xterm's OSC. *)

let esc = 0x1b
let bel = 0x07
let can = 0x18
let sub = 0x1a
let del = 0x7f

(* The controls a terminal acts on without printing anything: DEL and the C0
   controls but HT, which leaves blank cells, the line ends LF, VT, FF and CR,
   and ESC. *)
let silent c = (c < 0x20 && c <> esc && not (c >= 0x09 && c <= 0x0d)) || c = del
let is_c1 c = c >= 0x80 && c <= 0x9f
let nel = 0x85

(* The state a C1 control [c] leads to: CSI opens a control sequence (8.3.16);
   OSC (8.3.89), DCS (8.3.27), SOS (8.3.128), PM (8.3.94) and APC (8.3.2) open
   a control string; every other one, ST (8.3.143) among them, is a control
   function of its own. *)
let c1 = function
  | 0x9b -> Sequence
  | 0x9d -> String { bel = true }
  | 0x90 | 0x98 | 0x9e | 0x9f -> String { bel = false }
  | _ -> Ground

(* ECMA-35 13.2: ESC, intermediate bytes 02/00-02/15, a final byte
   03/00-07/14; a final byte in 04/00-05/15 is ESC Fe, a C1 control. *)
let after_escape c =
  if c >= 0x40 && c <= 0x5f then Some (c1 (c + 0x40))
  else if c >= 0x20 && c <= 0x2f then Some Escape_intermediate
  else if c >= 0x30 && c <= 0x7e then Some Ground
  else None

let after_intermediate c =
  if c >= 0x20 && c <= 0x2f then Some Escape_intermediate
  else if c >= 0x30 && c <= 0x7e then Some Ground
  else None

(* ECMA-48 5.4: parameter bytes 03/00-03/15 and intermediate bytes
   02/00-02/15, then the final byte, 04/00-07/14. *)
let in_sequence c =
  if c >= 0x20 && c <= 0x3f then Some Sequence
  else if c >= 0x40 && c <= 0x7e then Some Ground
  else None

let continues = function
  | Escape -> after_escape
  | Escape_intermediate -> after_intermediate
  | Ground | Sequence | String _ -> in_sequence

(* Inside an escape or a control sequence, the DEC parser xterm follows acts on
   a C0 control and stays, ignores DEL, and lets ESC, CAN, SUB and a C1 control
   end the sequence and act on their own; any other character outside the
   sequence's ranges ends it in error, and is text. A control string ends at
   ST, ESC 05/12 or 09/12, an OSC also at BEL; CAN, SUB, ESC and a C1 control
   end it and act. A {!silent} control acts by being discarded. *)
(* What the character at [i], [len] bytes long, does from [state] in a token
   that began at [start]: the tokens it ends, and where the scan goes on. *)
type move = {
  ends : ([ `Text | `Escape ] * int * int) list;
  next : state;
  at : int;
  from : int;
}

let move ?(ends = []) next at from = { ends; next; at; from }

(* NEL is text: {!text} makes it a line end. A {!silent} control is an
   escape of its own. *)
let in_ground ~valid ~c ~i ~len ~start =
  if valid && silent c then
    move
      ~ends:[ (`Text, start, i); (`Escape, i, i + len) ]
      Ground (i + len) (i + len)
  else if valid && (c = esc || (is_c1 c && c <> nel)) then
    match if c = esc then Escape else c1 c with
    | Ground ->
        move
          ~ends:[ (`Text, start, i); (`Escape, i, i + len) ]
          Ground (i + len) (i + len)
    | next -> move ~ends:[ (`Text, start, i) ] next (i + len) i
  else move Ground (i + len) start

let in_string str ~bel:b state ~valid ~c ~i ~len ~start =
  let n = String.length str in
  if not valid then move state (i + len) start
  else if c = esc && i + 1 < n && str.[i + 1] = '\\' then
    move ~ends:[ (`Escape, start, i + 2) ] Ground (i + 2) (i + 2)
  else if c = esc || c = can || c = sub || is_c1 c then
    move ~ends:[ (`Escape, start, i) ] Ground i i
  else if b && c = bel then
    move ~ends:[ (`Escape, start, i + len) ] Ground (i + len) (i + len)
  else move state (i + len) start

let in_sequence_or_escape state ~valid ~c ~i ~len ~start =
  if not valid then move ~ends:[ (`Escape, start, i) ] Ground i i
  else if c < 0x20 && c <> esc && c <> can && c <> sub then
    let kind = if silent c then `Escape else `Text in
    move
      ~ends:[ (`Escape, start, i); (kind, i, i + len) ]
      state (i + len) (i + len)
  else if c = del then move state (i + len) start
  else
    match continues state c with
    | Some Ground ->
        move ~ends:[ (`Escape, start, i + len) ] Ground (i + len) (i + len)
    | Some next -> move next (i + len) start
    | None -> move ~ends:[ (`Escape, start, i) ] Ground i i

(* [fold_tokens f acc str] folds [f] over the tokens of [str], and gives
   where the escape code under way at its end starts, if one is. *)
let fold_tokens f acc str =
  let n = String.length str in
  let emit acc (kind, start, stop) =
    if stop > start then f acc kind start (stop - start) else acc
  in
  let rec go acc i state start =
    if i >= n then
      if state = Ground then (emit acc (`Text, start, n), None)
      else (emit acc (`Escape, start, n), Some start)
    else
      let d = String.get_utf_8_uchar str i in
      let len = Uchar.utf_decode_length d in
      let valid = Uchar.utf_decode_is_valid d in
      let c = Uchar.to_int (Uchar.utf_decode_uchar d) in
      let m =
        match state with
        | Ground -> in_ground ~valid ~c ~i ~len ~start
        | String { bel } -> in_string str ~bel state ~valid ~c ~i ~len ~start
        | Escape | Escape_intermediate | Sequence ->
            in_sequence_or_escape state ~valid ~c ~i ~len ~start
      in
      go (List.fold_left emit acc m.ends) m.at m.next m.from
  in
  go acc 0 Ground 0

(* ECMA-48 8.3.117 SELECT GRAPHIC RENDITION: CSI, parameter bytes, final byte
   'm'. The accepted parameter bytes are a strict subset of 5.4's -- the
   digits and ';' -- and at most 64 of them, which no real SGR reaches. *)
let sgr seq =
  let n = String.length seq in
  let params_ok =
    n >= 3
    && n - 3 <= 64
    && seq.[0] = '\027'
    && seq.[1] = '['
    && seq.[n - 1] = 'm'
    && String.for_all
         (function '0' .. '9' | ';' -> true | _ -> false)
         (String.sub seq 2 (n - 3))
  in
  if not params_ok then None
  else
    (* Parameters apply left to right and 0, or an empty one, turns every
       attribute off. 38, 48 and 58 take a colour as the parameters after
       them, [5;n] or [2;r;g;b], which are values rather than commands; with
       fewer, every one left is a value. *)
    let params = String.split_on_char ';' (String.sub seq 2 (n - 3)) in
    let rec state acc = function
      | [] -> acc
      | ("38" | "48" | "58") :: "5" :: _ :: rest -> state `Set rest
      | ("38" | "48" | "58") :: "2" :: _ :: _ :: _ :: rest -> state `Set rest
      | ("38" | "48" | "58") :: _ -> `Set
      | p :: rest ->
          let acc =
            if p = "" || int_of_string_opt p = Some 0 then `Reset else `Set
          in
          state acc rest
    in
    Some (state `Reset params)

let rewrite ~keep_newlines ~keep str =
  let buf = Buffer.create (String.length str) and seen = seen () in
  let open_style, _ =
    fold_tokens
      (fun open_style kind pos len ->
        let token = String.sub str pos len in
        match kind with
        | `Text ->
            sanitize_into buf ~keep_newlines seen token;
            open_style
        | `Escape when token = "\027E" ->
            sanitize_into buf ~keep_newlines seen "\u{0085}";
            open_style
        | `Escape -> (
            match keep token with
            | Some state ->
                Buffer.add_string buf token;
                state = `Set
            | None -> open_style))
      false str
  in
  if open_style then Buffer.add_string buf "\027[0m";
  Buffer.contents buf

let strip ?(keep_newlines = true) ?(keep_styles = false) str =
  rewrite ~keep_newlines ~keep:(if keep_styles then sgr else fun _ -> None) str

(* The length of [str] without its last character if that one is not
   valid UTF-8, which the bytes after [str] may complete. *)
let character_end str =
  let n = String.length str in
  let rec last i =
    if i >= n then n
    else
      let d = String.get_utf_8_uchar str i in
      let next = i + Uchar.utf_decode_length d in
      if next < n then last next
      else if Uchar.utf_decode_is_valid d then n
      else i
  in
  last 0

let boundary str =
  match snd (fold_tokens (fun () _ _ _ -> ()) () str) with
  | Some start when start > 0 -> start
  | Some _ | None -> character_end str