changeset 277: | 10faf95f90dd |
parent: | 78ef6145e272 |
child: | c0fc6b87557f |
author: | Richard Westhaver <ellis@rwest.io> |
date: | Sun, 14 Apr 2024 01:19:10 -0400 |
permissions: | -rw-r--r-- |
description: | stream and basic type upgrades. fixed some bugs and improved csv parsing |
123
a4ed30cbe083
data testing, added ical and vcard formats
ellis <ellis@rwest.io>
parents:
122
diff
changeset
|
1 | ;;; lib/dat/csv.lisp --- CSV Data Format |
a4ed30cbe083
data testing, added ical and vcard formats
ellis <ellis@rwest.io>
parents:
122
diff
changeset
|
2 | |
275 | 3 | ;; Character Separated Values |
123
a4ed30cbe083
data testing, added ical and vcard formats
ellis <ellis@rwest.io>
parents:
122
diff
changeset
|
4 | |
a4ed30cbe083
data testing, added ical and vcard formats
ellis <ellis@rwest.io>
parents:
122
diff
changeset
|
5 | ;;; Code: |
122
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
6 | (in-package :dat/csv) |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
7 | |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
8 | (defun parse-number-no-error (string &optional default) |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
9 | (let ((result |
277
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
10 | (ignore-errors |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
11 | (parse-number string)))) |
122
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
12 | (if result |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
13 | result |
277
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
14 | default))) |
122
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
15 | |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
16 | (defparameter *csv-separator* #\,) |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
17 | (defparameter *csv-quote* #\") |
277
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
18 | (defparameter *csv-print-quote-p* t "print \" when the element is a string?") |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
19 | (defparameter *csv-default-external-format* :utf-8) |
122
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
20 | |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
21 | (defun write-csv-line (record &key stream (delimiter *csv-separator*)) |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
22 | "Accept a record and print it in one line as a csv record. |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
23 | |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
24 | A record is a sequence of element. A element can be of any type. |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
25 | If record is nil, nothing will be printed. |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
26 | If stream is nil (default case), it will return a string, otherwise it will return nil. |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
27 | For efficiency reason, no intermediate string will be constructed. " |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
28 | (let ((result |
277
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
29 | (with-output-to-string (s) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
30 | (let ((*standard-output* s) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
31 | (record-size (length record))) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
32 | (loop for e across record |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
33 | for i from 0 |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
34 | do (typecase e |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
35 | (string (progn |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
36 | (if *csv-print-quote-p* |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
37 | (progn |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
38 | (write-char *csv-quote*) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
39 | (write-string e) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
40 | (write-char *csv-quote*)) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
41 | (write-string e)))) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
42 | (t (princ e))) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
43 | (when (< i (1- record-size)) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
44 | (write-char delimiter))))))) |
122
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
45 | (format stream "~&~a" result))) |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
46 | |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
47 | (defun write-csv-stream (stream table &key (delimiter *csv-separator*)) |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
48 | "Accept a stream and a table and output the table as csv form to the stream. |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
49 | |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
50 | A table is a sequence of lines. A line is a sequence of elements. |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
51 | Elements can be any types" |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
52 | (loop for l across table |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
53 | do (write-csv-line l :stream stream :delimiter delimiter)) |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
54 | (write-char #\newline stream) |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
55 | '(ok)) |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
56 | |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
57 | (defun write-csv-file (filename table &key (external-format *csv-default-external-format*) (delimiter *csv-separator*)) |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
58 | "Accept a filename and a table and output the table as csv form to the file. |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
59 | |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
60 | A table is a sequence of lines. A line is a sequence of elements. |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
61 | Elements can be any types" |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
62 | (with-open-file (f filename :direction :output |
277
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
63 | :if-does-not-exist :create |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
64 | :if-exists :supersede |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
65 | :external-format external-format) |
122
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
66 | (write-csv-stream f table :delimiter delimiter))) |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
67 | |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
68 | (defun parse-csv-string (str &key (delimiter *csv-separator*)) ;; refer RFC4180 |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
69 | (coerce |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
70 | ;; (regexp:split-re "," str) |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
71 | (let ((q-count (count *csv-quote* str :test #'char-equal))) |
277
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
72 | (when (oddp q-count) (warn 'simple-warning :format-control "odd number of #\" in a line (~A)" |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
73 | :format-arguments (list q-count))) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
74 | (if (zerop q-count) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
75 | (cl-ppcre:split delimiter str) ;(cl-ppcre:split *csv-separator* str) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
76 | (macrolet ((push-f (fld flds) `(push (coerce (reverse ,fld) 'string) ,flds))) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
77 | (loop with state = :at-first ;; :at-first | :data-nq | :data-q | :q-in-nq | q-in-q |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
78 | with field with fields |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
79 | for chr of-type character across str |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
80 | do (cond ((eq state :at-first) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
81 | (setf field nil) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
82 | (cond ((char-equal chr *csv-quote*) (setf state :data-q)) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
83 | ((char-equal chr delimiter) (push "" fields)) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
84 | (t (setf state :data-nq) (push chr field)))) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
85 | ((eq state :data-nq) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
86 | (cond ((char-equal chr *csv-quote*) (setf state :q-in-nq)) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
87 | ((char-equal chr delimiter) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
88 | (push-f field fields) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
89 | (setf state :at-first)) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
90 | (t (push chr field)))) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
91 | ((eq state :q-in-nq) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
92 | (cond ((char-equal chr *csv-quote*) (setf state :q-in-q)) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
93 | ((char-equal chr delimiter) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
94 | (push-f field fields) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
95 | (setf state :at-first)) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
96 | (t (setf state :data-nq) (push chr field)))) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
97 | ((eq state :data-q) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
98 | (cond ((char-equal chr *csv-quote*) (setf state :q-in-q)) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
99 | ((char-equal chr delimiter) (push-f field fields) (setf state :at-first)) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
100 | (t (push chr field)))) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
101 | ((eq state :q-in-q) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
102 | (cond ((char-equal chr *csv-quote*) (setf state :data-q)) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
103 | ;; this should only be done conditionally - early escapes quotes |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
104 | ((char-equal chr delimiter) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
105 | (push-f field fields) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
106 | (setf state :at-first)) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
107 | (t |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
108 | ;; (error "illegal value ( ~A ) after quotation" chr) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
109 | (push chr field) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
110 | )))) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
111 | finally (return |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
112 | (progn (push-f field fields) (reverse fields))))))) |
122
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
113 | 'vector)) |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
114 | |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
115 | |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
116 | (defun read-csv-line (stream &key type-conv-fns map-fns (delimiter *csv-separator*) (start 0) end) |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
117 | "Read one line from stream and return a csv record. |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
118 | |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
119 | A CSV record is a vector of elements. |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
120 | |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
121 | type-conv-fns should be a list of functions. |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
122 | If type-conv-fns is nil (the default case), then all will be treated |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
123 | as string. |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
124 | |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
125 | map-fns is a list of functions of one argument and output one result. |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
126 | each function in it will be applied to the parsed element. |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
127 | If map-fns is nil, then nothing will be applied. |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
128 | |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
129 | start and end specifies how many elements per record will be included. |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
130 | If start or end is negative, it counts from the end. -1 is the last element. |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
131 | " |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
132 | (declare (type (or (simple-array function *) null) type-conv-fns map-fns)) |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
133 | (let* ((rline (read-line stream nil nil))) |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
134 | (when rline |
277
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
135 | (let* ((line (string-trim '(#\Space #\Newline #\Return) rline)) |
122
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
136 | (strs (parse-csv-string line :delimiter delimiter)) |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
137 | (strs-size (length strs))) |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
138 | (when (< start 0) |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
139 | (setf start (+ start strs-size))) |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
140 | (when (and end (< end 0)) |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
141 | (setf end (+ end strs-size))) |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
142 | (setf strs (subseq strs start end)) |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
143 | (when type-conv-fns |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
144 | (unless (= (length strs) (length type-conv-fns)) |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
145 | (error "Number of type specifier (~a) does not match the number of elements (~a)." |
277
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
146 | (length type-conv-fns) (length strs)))) |
122
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
147 | (when map-fns |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
148 | (unless (= (length strs) (length map-fns)) |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
149 | (error "Number of mapping functions (~a) does not match the number of elements (~a)." |
277
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
150 | (length map-fns) (length strs)))) |
122
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
151 | (let ((result strs)) |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
152 | ;; strs is not needed so we simply overwrite it |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
153 | (when type-conv-fns |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
154 | (setf result |
277
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
155 | (map 'vector #'funcall type-conv-fns result))) |
122
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
156 | (when map-fns |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
157 | (setf result |
277
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
158 | (map 'vector #'funcall map-fns result))) |
122
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
159 | result))))) |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
160 | |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
161 | (defun read-csv-stream (stream &key (header t) type-spec map-fns (delimiter *csv-separator*) (start 0) end) |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
162 | "Read from stream until eof and return a csv table. |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
163 | |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
164 | A csv table is a vector of csv records. |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
165 | A csv record is a vector of elements. |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
166 | |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
167 | Type spec should be a list of type specifier (symbols). |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
168 | If the type specifier is nil or t, it will be treated as string. |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
169 | If type-spec is nil (the default case), then all will be treated |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
170 | as string. |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
171 | |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
172 | map-fns is a list of functions of one argument and output one result. |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
173 | each function in it will be applied to the parsed element. |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
174 | If any function in the list is nil or t, it equals to #'identity. |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
175 | If map-fns is nil, then nothing will be applied. |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
176 | |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
177 | start and end specifies how many elements per record will be included. |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
178 | If start or end is negative, it counts from the end. -1 is the last element. |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
179 | " |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
180 | (let ((type-conv-fns |
277
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
181 | (when type-spec |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
182 | (macrolet ((make-num-specifier (specifier) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
183 | `(lambda (s) (let ((s (parse-number-no-error s s))) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
184 | (if (numberp s) (funcall ,specifier s) s))))) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
185 | (map 'vector |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
186 | (lambda (type) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
187 | (ecase type |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
188 | ((t nil string) #'identity) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
189 | (number #'(lambda (s) (parse-number-no-error s s))) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
190 | (float (make-num-specifier #'float)) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
191 | (single-float (make-num-specifier #'(lambda (s) (coerce s 'single-float)))) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
192 | (double-float (make-num-specifier #'(lambda (s) (coerce s 'double-float)))) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
193 | (integer (make-num-specifier #'round)) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
194 | (pathname #'pathname) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
195 | (symbol #'intern) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
196 | (keyword (lambda (s) (intern s :keyword))))) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
197 | type-spec)))) |
122
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
198 | (map-fns |
277
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
199 | (when map-fns |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
200 | (map 'vector |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
201 | (lambda (fn) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
202 | (cond ((or (eq fn t) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
203 | (eq fn nil)) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
204 | #'identity) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
205 | ((functionp fn) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
206 | fn) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
207 | ((and (symbolp fn) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
208 | (not (keywordp fn))) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
209 | (symbol-function fn)) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
210 | (t (error "~a is not a valid function specifier." fn)))) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
211 | map-fns))) |
122
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
212 | (header |
277
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
213 | (etypecase header |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
214 | (cons (coerce header 'vector)) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
215 | (boolean (when header |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
216 | (read-csv-line stream :delimiter delimiter)))))) |
122
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
217 | (loop for rec = (read-csv-line stream :type-conv-fns type-conv-fns :map-fns map-fns :delimiter delimiter |
277
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
218 | :start start :end end) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
219 | while rec |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
220 | collect rec into result |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
221 | finally (return |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
222 | (values |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
223 | (coerce result 'vector) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
224 | header))))) |
122
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
225 | |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
226 | (defun read-csv-file (filename &key (header t) type-spec map-fns (delimiter *csv-separator*) (external-format *csv-default-external-format*) |
277
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
227 | (start 0) end) |
122
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
228 | "Read from stream until eof and return a csv table. |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
229 | |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
230 | A csv table is a vector of csv records. |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
231 | A csv record is a vector of elements. |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
232 | |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
233 | Type spec should be a list of type specifier (symbols). |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
234 | If the type specifier is nil or t, it will be treated as string. |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
235 | If type-spec is nil (the default case), then all will be treated |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
236 | as string. |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
237 | |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
238 | map-fns is a list of functions of one argument and output one result. |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
239 | each function in it will be applied to the parsed element. |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
240 | If any function in the list is nil or t, it equals to #'identity. |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
241 | If map-fns is nil, then nothing will be applied. |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
242 | https://cgit.gentoo.org/proj/lisp.git/tree/dev-lisp/cl-rsm-finance/cl-rsm-finance-1.1.ebuild?h=old-portage&id=e9b71910b0d4f22aeb66f14e158a2451f9955b0d |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
243 | external-format (default is shift-jis) is a valid AllegroCL external-format type. |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
244 | |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
245 | OS is a set to eol-convention of the file stream. |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
246 | |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
247 | start and end specifies how many elements per record will be included. |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
248 | If start or end is negative, it counts from the end. -1 is the last element. |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
249 | " |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
250 | (with-open-file (f filename :external-format external-format) |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
251 | (read-csv-stream f :type-spec type-spec :map-fns map-fns |
277
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
252 | :delimiter delimiter |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
253 | :start start :end end |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
254 | :header header))) |
122
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
255 | |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
256 | |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
257 | (defun read-csv-file-and-sort (filename sort-order &key (header t) (order :ascend) type-spec map-fns (delimiter *csv-separator*) (external-format *csv-default-external-format*)) |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
258 | (let ((table (read-csv-file filename |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
259 | :header header |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
260 | :type-spec type-spec |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
261 | :map-fns map-fns |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
262 | :delimiter delimiter |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
263 | :external-format external-format))) |
4ba88cac5bc7
num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff
changeset
|
264 | (loop for i in (reverse sort-order) |
277
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
265 | do (setf table |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
266 | (stable-sort table (ecase order (:ascend #'string<=) (:descend #'string>=)) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
267 | :key (lambda (rec) (aref rec i)))) |
10faf95f90dd
stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents:
275
diff
changeset
|
268 | finally (return table)))) |