shithub: martian9

ref: 0560b9b189c123d48e91231a3adf15016c0a49d9
dir: /utils.ml/

exception Syntax_error of string
exception Runtime_error of string

let token_re = Str.regexp "~@\\|[][{}()'`~^@]\\|\"\\(\\\\.\\|[^\"]\\)*\"?\\|;.*\\$\\|[^][  \n{}('\"`,;)]*"

(* copied verbatim - must needs grok *)
let gsub re f str =
  String.concat ""
    (List.map
       (function
         | Str.Delim x -> f x
         | Str.Text x -> x )
       (Str.full_split re str) )

let tokenize str =
  List.map
    (function
      | Str.Delim x -> String.trim x (* move trim to regex for speed? *)
      | Str.Text _ -> "tokenize botch" )
    (List.filter
       (function
         | Str.Delim _ -> true
         | Str.Text _ -> false )
       (Str.full_split token_re str) )