3 (defvar *current-parser* nil)
5 (defclass table-column-meta ()
6 ((name :accessor meta-name)
7 (type :accessor meta-type)
8 (flags :accessor meta-flags)
9 (len :accessor meta-len)
10 (table :accessor meta-table)))
12 (defclass xdump-table ()
13 ((name :accessor table-name :initarg :name)
14 (index :accessor table-index :initarg :index)
15 (last-update :accessor table-last-update :initform 0)
16 (entry-class :accessor table-entry-class :initarg :entry-class)
17 (entries :accessor table-entries
18 :initform (make-array 8
21 :initial-element nil))
22 (has-uid-p :accessor has-uid-p :initarg :has-uid-p)))
24 (defclass xdump-parser ()
25 ((line-parser :accessor line-parser :initform nil)
26 (entry-buffer :accessor entry-buffer
27 :initform (make-array 1 :fill-pointer 0 :adjustable t))
28 (entry-index :accessor entry-index :initform 0)
29 (finalizer :accessor finalizer :initform nil)
30 (name :accessor table-name :initform nil)
31 (timestamp :accessor timestamp :initform nil)
32 (meta-by-index :accessor meta-by-index
33 :initform (make-array 128
35 :initial-element nil))
36 (table-by-index :accessor table-by-index
37 :initform (make-array 128
39 :initial-element nil))
40 (table-classes-package :accessor table-classes-package
41 :initform (make-package (gensym)))
42 (index-by-name :accessor index-by-name
43 :initform (make-hash-table :size 128 :test 'equal))
44 (meta-index :accessor meta-index :initform (make-array 5))
45 (meta-meta :accessor meta-meta :initform (make-array 5))
46 (user-log :accessor user-log :initarg :user-log)))
48 (defgeneric get-table (xdump-parser table-name))
49 (defgeneric get-table-entry (xdump-parser table-name index))
50 (defgeneric sym-by-value (xdump-parser table-name value))
51 (defgeneric array-p (xdump-parser column))
52 (defgeneric finish-table (xdump-parser number-of-records))
53 (defgeneric parse-entry (xdump-parser line))
54 (defgeneric parse-meta (xdump-parser name timestamp))
55 (defgeneric parse-table (xdump-parser name timestamp))
56 (defgeneric header-parser (xdump-parser line))
57 (defgeneric parse-line (xdump-parser line))
58 (defgeneric reset-table-parser (xdump-parser))
59 (defgeneric flush-log (xdump-parser))
60 (defgeneric entry-from-vector (xdump-parser class meta-table entry-vector
61 &key array-support-p))
63 (defmethod get-table ((parser xdump-parser) table-name)
64 (with-slots (index-by-name table-by-index) parser
65 (let* ((table-index (gethash table-name index-by-name))
66 (table (aref table-by-index table-index)))
69 (defmacro with-parser (parser &body body)
70 `(let ((*current-parser* ,parser))
73 (defun table (table-name &optional (parser *current-parser*))
74 (get-table parser table-name))
76 (defun checkpoint (&optional (parser *current-parser*))
79 (defmethod get-table-entry ((parser xdump-parser) table-name index)
80 (aref (table-entries (get-table parser table-name)) index))
82 (defmethod sym-by-value ((parser xdump-parser) table-name value)
83 (with-slots (table-classes-package) parser
84 (let ((value-slot (find-symbol "VALUE" table-classes-package))
85 (name-slot (find-symbol "NAME" table-classes-package))
86 (meta-type-table (get-table parser table-name)))
87 (loop for entry across (table-entries meta-type-table)
88 if (= (slot-value entry value-slot) value)
89 return (slot-value entry name-slot)))))
91 (defmethod reset-table-parser ((parser xdump-parser))
92 (with-slots (line-parser entry-buffer entry-index finalizer name timestamp)
94 (setf line-parser #'header-parser
95 entry-buffer (make-array 1 :fill-pointer 0 :adjustable t)
101 (defun define-meta-table (entry-buffer)
102 (with-slots (meta-index meta-meta user-log) *current-parser*
103 (print `(define-meta-table ,entry-buffer) user-log)
104 ; build meta-index from integer index to
107 for i = 0 then (+ i 1)
108 for e across entry-buffer
109 do (let* ((slot-name (string-upcase (car e)))
110 (slot (find-symbol slot-name :xdump)))
111 (setf (aref meta-index i) slot)))
112 ; build meta-meta table
113 (loop for entry across entry-buffer
114 for i = 0 then (+ i 1)
115 do (let ((meta-meta-column (make-instance 'table-column-meta)))
116 (loop for slot across meta-index
118 do (setf (slot-value meta-meta-column slot) field))
119 (setf (aref meta-meta i) meta-meta-column)))))
121 (defun meta-meta-finalizer (parser)
123 (define-meta-table (entry-buffer parser))))
125 (defun table-entry-defclass-form (name class-name package slot-list)
126 `(defclass ,(find-symbol class-name package) ()
127 ,(mapcar #'(lambda (raw-slot-name)
128 (let* ((slot-name (string-upcase raw-slot-name))
129 (accessor-name (string-upcase (format nil "~a-~a"
132 (list (intern slot-name package)
133 :accessor (intern accessor-name package))))
136 (defun define-table (name entry-buffer)
137 (let ((meta-table (make-array 1 :fill-pointer 0 :adjustable t)))
138 (with-slots (meta-index meta-by-index index-by-name table-by-index user-log)
140 (print `(define-table ,name ,entry-buffer) user-log)
141 (loop for entry across entry-buffer
142 do (let ((meta (make-instance 'table-column-meta)))
143 (loop for slot across meta-index
145 do (setf (slot-value meta slot) field))
146 (vector-push-extend meta meta-table)))
147 (let ((table-index (if (string= "table" name)
148 ; voodoo: the first column of the table of tables
150 (elt (aref entry-buffer 0) 4)
151 (gethash name index-by-name)))
153 (setf (aref meta-by-index table-index) meta-table)
155 ;; determine if this table has a uid column
156 (if (and (string= "uid" (elt (aref entry-buffer 0) 0))
157 (eql (elt (aref entry-buffer 0) 4) table-index))
160 ;; create xdump-table instance and entry class
161 (let* ((class-name (string-upcase (format nil "~a-table-entry" name)))
162 (package (table-classes-package *current-parser*))
163 (table-instance (make-instance 'xdump-table
166 :entry-class (intern class-name
168 :has-uid-p has-uid-p))
169 (slot-list (loop for entry across entry-buffer
170 collect (car entry))))
171 (setf (aref table-by-index table-index) table-instance)
172 (eval (table-entry-defclass-form name class-name package
175 (defun meta-finalizer (parser)
177 (with-slots (name entry-buffer) parser
178 (define-table name entry-buffer))))
180 (defmethod array-p ((parser xdump-parser) column)
181 (let ((column-len (meta-len column))
182 (column-type (sym-by-value parser "meta-type" (meta-type column))))
183 (and (> column-len 0)
184 (not (string= column-type "c")))))
186 (defmethod entry-from-vector (parser class meta-table entry-vector
187 &key (array-support-p t))
188 (with-slots (table-classes-package) parser
189 (let ((new-entry (make-instance class))
190 (entries entry-vector))
192 for column across meta-table do
193 (let ((slot (find-symbol (string-upcase (meta-name column))
194 table-classes-package)))
195 (if (and array-support-p (array-p parser column))
196 ;; then collect array
198 (dotimes (j (meta-len column))
199 (push (pop entries) array))
200 (setf (slot-value new-entry slot) array))
201 ;; else collect single entry
203 (setf (slot-value new-entry slot) (pop entries))))))
206 (defmethod table-entry-insert-at ((table xdump-table) entry index)
207 ; extend array if necessary
208 (let ((entries (table-entries table)))
209 (unless (> (fill-pointer entries) index)
210 (adjust-array entries (max 1 (* 2 index))
212 :initial-element nil))
213 (setf (aref entries index) entry)))
215 (defun load-table (name timestamp entry-buffer)
216 (with-slots (index-by-name meta-by-index table-by-index table-classes-package
219 (print `(load-table ,name ,timestamp ,entry-buffer) user-log)
220 (if (string= name "table") ; special magic: prefill index-by-name
221 (loop for entry across entry-buffer do
222 (setf (gethash (second entry) index-by-name) (first entry))))
223 (let* ((table-index (gethash name index-by-name))
224 (meta-table (aref meta-by-index table-index))
225 (table (aref table-by-index table-index))
226 (has-uid-p (has-uid-p table))
227 (array-support-p (not (or (string= name "table")
228 (string= name "meta-type")))))
229 (loop for entry across entry-buffer do
230 (let ((entry-instance (entry-from-vector *current-parser*
231 (table-entry-class table)
237 (let ((index (first entry)))
238 (table-entry-insert-at table entry-instance index))
239 (vector-push-extend entry-instance (table-entries table)))))
240 (setf (table-last-update table) timestamp))))
242 (defun table-finalizer (parser)
244 (with-slots (name timestamp entry-buffer) parser
245 (load-table name timestamp entry-buffer))))
247 (defmethod finish-table ((parser xdump-parser) number-of-records)
248 (if (not (equal number-of-records (length (entry-buffer parser))))
249 (error "Table row count mismatch"))
250 (funcall (finalizer parser) parser)
251 (reset-table-parser parser)
254 (defmethod parse-entry ((parser xdump-parser) line)
255 (if (char= #\/ (aref line 0))
256 (finish-table parser (parse-integer (subseq line 1)))
257 (let ((fields (mapcar #'read-from-string (cl-ppcre:split " " line))))
258 (with-slots (entry-buffer entry-index) parser
259 (vector-push-extend fields entry-buffer)
263 (defmethod parse-meta ((parser xdump-parser) input-name input-timestamp)
264 (with-slots (line-parser finalizer name timestamp) parser
265 (setf line-parser #'parse-entry
266 finalizer (if (string= "meta" input-name)
267 #'meta-meta-finalizer
270 timestamp input-timestamp)))
272 (defmethod parse-table ((parser xdump-parser) input-name input-timestamp)
273 "Parse a normal table.
276 parser: The current parser
277 input-name: Name of the current table
278 input-timestamp: Timestamp of the current table"
279 (with-slots (line-parser finalizer name timestamp) parser
280 (setf line-parser #'parse-entry
281 finalizer #'table-finalizer
283 timestamp input-timestamp)))
285 (defmethod header-parser ((parser xdump-parser) line)
286 (multiple-value-bind (fullmatch groups)
287 (cl-ppcre:scan-to-strings "^XDUMP meta (.*) ([0-9]+)$" line)
290 (let ((name (aref groups 0))
291 (timestamp (parse-integer (aref groups 1))))
292 (parse-meta parser name timestamp))
294 (multiple-value-bind (fullmatch groups)
295 (cl-ppcre:scan-to-strings "^XDUMP (.*) ([0-9]+)$" line)
296 (declare (ignorable fullmatch))
297 (let ((name (aref groups 0))
298 (timestamp (parse-integer (aref groups 1))))
299 (parse-table parser name timestamp)))))
302 (defun make-parser (&key user-log)
303 (let ((parser (make-instance 'xdump-parser :user-log user-log)))
304 (setf (line-parser parser) #'header-parser)
307 (defmethod parse-line ((parser xdump-parser) line)
308 (with-slots (line-parser) parser
309 (funcall line-parser parser line)))
311 (defmethod parse-stream ((parser xdump-parser) stream)
312 (loop for line = (read-line stream nil)
313 while line do (parse-line parser line)))
315 (defmethod parse-file ((parser xdump-parser) filename)
316 (with-open-file (s filename)
317 (parse-stream parser s)))
319 (defmethod flush-log ((parser xdump-parser))
320 (finish-output (user-log parser)))
323 (let ((parser (make-parser)))
324 (setf *current-parser* parser)
325 (parse-file parser "testdata/xdump-meta-meta.txt")))
328 (let ((parser (make-parser)))
329 (setf *current-parser* parser)
330 (parse-file parser "testdata/xdump-meta-meta.txt")
331 (parse-file parser "testdata/xdump-meta-table.txt")
332 (parse-file parser "testdata/xdump-table.txt")
333 (parse-file parser "testdata/xdump-meta-34.txt")
334 (parse-file parser "testdata/xdump-34.txt")))