changelog shortlog graph tags branches changeset files file revisions raw help

Mercurial > core / annotate lisp/ffi/tree-sitter/api.lisp

changeset 241: bc5832f97879
parent: 255000153a76
child: 48e671eac752
author: Richard Westhaver <ellis@rwest.io>
date: Sun, 24 Mar 2024 23:05:46 -0400
permissions: -rw-r--r--
description: more urings
140
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
1
 ;;; ffi/tree-sitter/api.lisp --- High-level API for Tree-sitter library
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
2
 
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
3
 ;; High-level Tree-sitter API
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
4
 
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
5
 ;;; Code:
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
6
 (in-package :tree-sitter)
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
7
 
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
8
 (defstruct (node (:type list))
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
9
   type range children)
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
10
 
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
11
 (defun make-lisp-name (string)
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
12
   (intern (string-upcase (substitute #\- #\_ string))
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
13
           (load-time-value (find-package "KEYWORD"))))
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
14
 
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
15
 (define-condition tree-sitter-error (error)
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
16
   ())
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
17
 
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
18
 (define-condition create-parser-error (tree-sitter-error)
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
19
   ())
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
20
 
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
21
 (define-condition set-language-error (tree-sitter-error)
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
22
   ((language :initarg :language :reader tree-sitter-error-language)))
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
23
 
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
24
 (define-condition parse-string-error (tree-sitter-error)
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
25
   ((string :initarg :string :reader tree-sitter-error-string)
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
26
    (string-start :initarg :string-start :reader tree-sitter-error-string-start)
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
27
    (string-end :initarg :string-end :reader tree-sitter-error-string-end)
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
28
    (language :initarg :language :reader tree-sitter-error-language)))
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
29
 
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
30
 (define-condition null-node-pointer (tree-sitter-error)
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
31
   ())
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
32
 
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
33
 (define-condition null-tree-cursor-pointer (tree-sitter-error)
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
34
   ())
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
35
 
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
36
 ;; util
141
ellis <ellis@rwest.io>
parents: 140
diff changeset
37
 (defmacro with-ts-node ((var node) &body forms)
140
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
38
   `(let ((,var ,node))
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
39
      (when (sb-alien:null-alien ,var)
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
40
        (error 'null-node-pointer))
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
41
      (unwind-protect
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
42
           (progn ,@forms)
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
43
        (sb-alien:free-alien ,var))))
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
44
 
141
ellis <ellis@rwest.io>
parents: 140
diff changeset
45
 (defmacro with-tree-cursor ((var tree) &body forms &aux (node (gensym)))
ellis <ellis@rwest.io>
parents: 140
diff changeset
46
   `(with-ts-node (,node (ts-tree-root-node ,tree))
ellis <ellis@rwest.io>
parents: 140
diff changeset
47
      (let ((,var (ts-tree-cursor-new ,node)))
140
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
48
        (when (sb-alien:null-alien ,var)
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
49
          (error 'null-tree-cursor-pointer))
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
50
        (unwind-protect
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
51
             (progn ,@forms)
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
52
          (ts-tree-cursor-delete ,var)))))
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
53
 
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
54
 (defun parse-string (language string &key (start 0) end produce-cst (name-generator #'make-lisp-name))
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
55
   "Parse a STRING that represents LANGUAGE code using tree-sitter. START is
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
56
 where to start parsing STRING. END is where to stop parsing STRING.
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
57
 When PRODUCE-CST is set, the full concrete syntax tree will be produced as
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
58
 opposed to the abstract syntax tree. See 'Named vs Anonymous Nodes':
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
59
 http://tree-sitter.github.io/tree-sitter/using-parsers#named-vs-anonymous-nodes
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
60
 NAME-GENERATOR is a function which converts a string from tree-sitter into a
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
61
 desired name for use in lisp."
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
62
   (let ((parser (ts-parser-new)))
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
63
     (when (sb-alien:null-alien parser)
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
64
       (error 'cant-create-parser))
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
65
     (unwind-protect (parse-string-with-language language string parser
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
66
                                                 :start start
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
67
                                                 :end end
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
68
                                                 :produce-cst produce-cst
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
69
                                                 :name-generator name-generator)
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
70
       (ts-parser-delete parser))))
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
71
 
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
72
 (defun parse-string-with-language (language string parser
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
73
                                    &key (start 0) end produce-cst name-generator)
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
74
   (unless (ts-parser-set-language parser (language-module language))
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
75
     (error 'cant-set-language :language language))
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
76
   (let* ((string-start start)
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
77
          (string-end (or end (length string)))
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
78
          ;; TODO: this might need to be +1 if it's actually a c-string for null
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
79
          (string-length (- string-end string-start))
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
80
          (string-to-pass (if (plusp string-start)
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
81
                              (subseq string string-start string-end)
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
82
                              string))
231
Richard Westhaver <ellis@rwest.io>
parents: 141
diff changeset
83
          (tree (ts-parser-parse-string parser string string-to-pass string-length)))
140
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
84
     (when (sb-alien:null-alien tree)
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
85
       (error 'cant-parse-string
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
86
              :string string
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
87
              :string-start start
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
88
              :string-end end
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
89
              :language language))
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
90
     (unwind-protect (convert-foreign-tree-to-list tree :produce-cst produce-cst
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
91
                                                        :name-generator name-generator)
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
92
       (ts-tree-delete tree))))
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
93
 
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
94
 (defun convert-foreign-tree-to-list (tree &key produce-cst name-generator
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
95
                                      &aux did-visit-children parse-stack)
141
ellis <ellis@rwest.io>
parents: 140
diff changeset
96
   (with-tree-cursor (cursor tree)
140
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
97
     ;; Closely follows tree-sitter-cli parse
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
98
     ;; implementation with a modification to
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
99
     ;; allow for production of the full CST.
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
100
     (loop
141
ellis <ellis@rwest.io>
parents: 140
diff changeset
101
       (with-ts-node (node (ts-tree-cursor-current-node cursor))
ellis <ellis@rwest.io>
parents: 140
diff changeset
102
         (let ((is-named (or produce-cst (ts-node-is-named node))))
140
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
103
           (cond (did-visit-children
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
104
                  (when (and is-named (second parse-stack))
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
105
                    (let ((item (pop parse-stack)))
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
106
                      (setf (node-children item)
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
107
                            (nreverse (node-children item)))
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
108
                      (push item (node-children (first parse-stack)))))
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
109
                  (cond ((ts-tree-cursor-goto-next-sibling cursor)
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
110
                         (setf did-visit-children nil))
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
111
                        ((ts-tree-cursor-goto-parent cursor)
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
112
                         (setf did-visit-children t))
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
113
                        (t
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
114
                         (let ((root (first parse-stack)))
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
115
                           (setf (node-children root)
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
116
                                 (nreverse (node-children root)))
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
117
                           (return root)))))
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
118
                 (t
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
119
                  (when is-named
141
ellis <ellis@rwest.io>
parents: 140
diff changeset
120
                    (let ((start-point (ts-node-start-point node))
ellis <ellis@rwest.io>
parents: 140
diff changeset
121
                          (end-point (ts-node-end-point node))
ellis <ellis@rwest.io>
parents: 140
diff changeset
122
                          (type (funcall name-generator (ts-node-type node)))
140
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
123
                          (field-name-ptr (ts-tree-cursor-current-field-name cursor)))
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
124
                      (unless (sb-alien:null-alien field-name-ptr)
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
125
                        ;; TODO
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
126
                        (let ((field-name (deref field-name-ptr)))
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
127
                          (setf type (list (funcall name-generator field-name) type))))
241
bc5832f97879 more urings
Richard Westhaver <ellis@rwest.io>
parents: 231
diff changeset
128
                      (push (make-node :type type :range (list start-point end-point))
140
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
129
                            parse-stack)))
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
130
                  (setf did-visit-children
9b7ec8636a2d syn init and tree sitter doodles
ellis <ellis@rwest.io>
parents:
diff changeset
131
                        (not (ts-tree-cursor-goto-first-child cursor))))))))))